Following is an easy way to hide the modal popup dialog extender from the AJAX control toolkit using JavaScript.  The key is to set a BehaviorID on the ModalPopupExtender.  Then you can use this ID to call the hide() method via Javascript like this:
function HideModal() {
  $find('modalPopupBehavior').hide();
}
In the full example below I am closing the modal popup dialog box when the user clicks on a link.
Full ASPX example:
<%@ Register Assembly="AjaxControlToolkit" Namespace="AjaxControlToolkit" TagPrefix="ajaxtk" %>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" >
<head runat="server">
    <title>My Page</title>
    <link href="css/Test.css" rel="stylesheet" type="text/css" />
</head>
<body>
<form id="aspNetForm" runat="server">
    
<asp:ScriptManager ID="scriptMan" runat="server" />
<script type="text/javascript" language="javascript">
    function HideModal() {
        $find('modalPopupBehavior').hide();
    }
</script>
<asp:UpdatePanel ID="upnlMain" runat="server">
    <ContentTemplate>
    
        <div>
            <ajaxtk:ModalPopupExtender ID="modalPopup" runat="server" 
                BehaviorID="modalPopupBehavior"
                TargetControlID="btnPopup" PopupControlID="pnlPopup">
            </ajaxtk:ModalPopupExtender>
            
            <asp:Button ID="btnPopup" runat="server" Text="Show Popup" />
            
            <asp:Panel ID="pnlPopup" runat="server" CssClass="modalPopup" 
                style="width: 715px; display: none;">
                <div>
                    <p>
                    <a href="http://www.jonathanjungman.com/blog/" 
                        target="_blank" onclick="HideModal()">Hide Modal</a>
                    </p>
                </div>
            </asp:Panel>
        </div>
    
    </ContentTemplate>
</asp:UpdatePanel>
   
</form>
</body>
</html>