You want to use the IFolderView::SetCurrentViewMode method.

Here is a C++ (using Visual Studio's ATL) example:

int main()
{
    CoInitialize(NULL);
    {
        // get a shell item
        CComPtr<IShellItem> folder;
        ATLASSERT(SUCCEEDED(SHCreateItemFromParsingName(L"c:\\myPath1\myPath2", nullptr, IID_PPV_ARGS(&folder))));

        // get its PIDL
        CComHeapPtr<ITEMIDLIST> pidl;
        ATLASSERT(SUCCEEDED(CComQIPtr<IPersistIDList>(folder)->GetIDList(&pidl)));

        // open the item
        SHELLEXECUTEINFO info = { };
        info.cbSize = sizeof(info);
        info.fMask = SEE_MASK_IDLIST;
        info.nShow = SW_SHOW;
        info.lpIDList = pidl;
        ATLASSERT(ShellExecuteEx(&info));

        // build a variant from the PIDL
        UINT size = ILGetSize(pidl);
        SAFEARRAY* psa = SafeArrayCreateVector(VT_UI1, 0, size);
        CopyMemory(psa->pvData, pidl, size);
        CComVariant v;
        v.parray = psa;
        v.vt = VT_ARRAY | VT_UI1;

        // find the opened window
        CComPtr<IShellWindows> windows;
        ATLASSERT(SUCCEEDED(windows.CoCreateInstance(CLSID_ShellWindows)));

        CComVariant empty;
        long hwnd;
        CComPtr<IDispatch> disp;
        do
        {
            windows->FindWindowSW(&v, &empty, SWC_BROWSER, &hwnd, SWFO_NEEDDISPATCH, &disp);
            if (disp)
                break;

            // we sleep for a while but using events would be better
            // see https://stackoverflow.com/a/59974072/403671
            Sleep(500);
        } while (true);

        // get IFolderView
        CComPtr<IFolderView> view;
        ATLASSERT(SUCCEEDED(IUnknown_QueryService(disp, IID_IFolderView, IID_PPV_ARGS(&view))));

        // change view mode
        ATLASSERT(SUCCEEDED(view->SetCurrentViewMode(FOLDERVIEWMODE::FVM_THUMBNAIL)));
    }

    CoUninitialize();
    return 0;
}