You can't use IShellFolder to filter, but you can use the search facilities built into the Shell to do the same thing programmatically, as you would do using the Windows Explorer UI.

For example, you can type something like ext:.txt in the search box in the right corner, it means you want to filter for all files with a .txt extension:

This is some C++ sample code somewhat equivalent (I've removed error checks at each line but make sure you test all possible errors):

int main()
{
  CoInitialize(NULL);
  {
    CComPtr<ISearchFolderItemFactory> search;
    CComPtr<IShellItem> item;
    CComPtr<IShellItemArray> items;
    CComPtr<IQueryParserManager> mgr;
    CComPtr<IQueryParser> parser;
    CComPtr<IQuerySolution> solution;
    CComPtr<ICondition> condition;
    CComPtr<IShellItem> searchItem;
    CComPtr<IEnumShellItems> enumItems;

    // create search folder factory
    search.CoCreateInstance(CLSID_SearchFolderItemFactory);

    // create d:\temp shell item and set search folder scope to it
    SHCreateItemFromParsingName(L"d:\\temp", NULL, IID_PPV_ARGS(&item));
    SHCreateShellItemArrayFromShellItem(item, IID_PPV_ARGS(&items));
    search->SetScope(items);

    // create the query parser manager
    mgr.CoCreateInstance(CLSID_QueryParserManager);
    mgr->CreateLoadedParser(L"", 0, IID_PPV_ARGS(&parser));

    // parse an ms-search expression
    parser->Parse(L"ext:.txt", NULL, &solution);

    // get the condition the parser has built for us
    solution->GetQuery(&condition, NULL);

    // give the condition to the search folder factory
    search->SetCondition(condition);

    // get the search result back as a shell item (a virtual folder) and enumerates it
    search->GetShellItem(IID_PPV_ARGS(&searchItem));
    searchItem->BindToHandler(NULL, BHID_EnumItems, IID_PPV_ARGS(&enumItems));

    do
    {
      CComPtr<IShellItem> child;
      ULONG fetched;
      HRESULT hr2 = enumItems->Next(1, &child, &fetched);
      if (!fetched)
        break;

      // get the display name (for example)
      CComHeapPtr<WCHAR> name;
      child->GetDisplayName(SIGDN_NORMALDISPLAY, &name);
      wprintf(L"item: %s\n", name);

      CComHeapPtr<WCHAR> path;
      child->GetDisplayName(SIGDN_FILESYSPATH, &path);
      wprintf(L" path: %s\n", path);
    } while (TRUE);
  }
  CoUninitialize();
  return 0;
}

The search-ms language is quite powerful. Its syntax is available here: Querying the Index with the search-ms Protocol