You must implement the IDocHostUIHandler interface and set it to the web browser with a code similar to this (extracted from the doc):

ComPtr<IDispatch> spDocument;
hr = spWebBrowser2->get_Document(&spDocument);
if (SUCCEEDED(hr) && (spDocument != nullptr))
{
    // Request default handler from MSHTML client site
    ComPtr<IOleObject> spOleObject;
    if (SUCCEEDED(spDocument.As(&spOleObject)))
    {
        ComPtr<IOleClientSite> spClientSite;
        hr = spOleObject->GetClientSite(&spClientSite);
        if (SUCCEEDED(hr) && spClientSite)
        {
            // Save pointer for delegation to default 
            m_spDefaultDocHostUIHandler = spClientSite;
        }
    }

    // Set the new custom IDocHostUIHandler
    ComPtr<ICustomDoc> spCustomDoc;
    if (SUCCEEDED(spDocument.As(&spCustomDoc)))
    {
        // NOTE: spHandler is user-defined class
        spCustomDoc->SetUIHandler(spHandler.Get());
    }
} 

You must specifically implement the GetExternal method

Now, in IE's javascript (or vbscript for that matter), you can access your host with a call like this:

var ext = window.external; // this will call your host's IDocHostUIHandler.GetExternal method
ext.SomeFunction(...); // implemented by your object

What you return in GetExternal must be an IDispatch object that you can design the way you want.