I managed to do it using UI Automation. It's not the ideal solution but it seems to work. Here is the code with comments inline:

#include <stdio.h>
#include <windows.h>
#include <atlbase.h>
#include <atlcom.h>
#include <UIAutomationCore.h>
#include <UIAutomationClient.h>

// the main function
HRESULT OpenSetDefaultsByApp(LPCWSTR appName);

// helpers
HRESULT FindFirstChild(IUIAutomation *automation, IUIAutomationElement *element, PROPERTYID pid, VARIANT value, IUIAutomationElement **child);
HRESULT FindFirstChildInList(IUIAutomation *automation, IUIAutomationElement *list, PROPERTYID pid, VARIANT value, IUIAutomationElement **child);
HRESULT OpenSetDefaultsByApp();

// some useful macros for error handling
// uses wprintf so you might want to change it, if running in a non-console context
#define WIDEN2(x) L ## x
#define WIDEN(x) WIDEN2(x)
#define __WFILE__ WIDEN(__FILE__)
#define HRCHECK(__expr) {hr=(__expr);if(FAILED(hr)){wprintf(L"FAILURE 0x%08X (%i)\n\tline: %u file: '%s'\n\texpr: '" WIDEN(#__expr) L"'\n",hr, hr, __LINE__,__WFILE__);goto cleanup;}}

int main()
{
  CoInitialize(NULL);
  OpenSetDefaultsByApp(L"Google Chrome"); // pass the app name as it's displayed in app settings
  CoUninitialize();
}

HRESULT OpenSetDefaultsByApp(LPCWSTR appName)
{
  HRESULT hr = S_OK;
  CComBSTR name = appName;
  CComPtr<IUIAutomation> automation;
  CComPtr<IUIAutomationElement> root;
  CComPtr<IUIAutomationElement> settingsWindow;
  CComPtr<IUIAutomationElement> coreWindow;
  CComPtr<IUIAutomationElement> content;
  CComPtr<IUIAutomationElement> list;
  CComPtr<IUIAutomationElement> scrollViewer;
  CComPtr<IUIAutomationElement> appNameListItem;
  CComPtr<IUIAutomationElement> manageButton;
  CComPtr<IUIAutomationSelectionItemPattern> selection;
  CComPtr<IUIAutomationInvokePattern> invoke;

  // because setting windows and content are completely refreshed, we need two rounds
  // one to open the list of apps
  HRCHECK(OpenSetDefaultsByApp());

  // another one to select the app that starts now...
  // create UIA COM server and get root
  HRCHECK(automation.CoCreateInstance(CLSID_CUIAutomation8));
  HRCHECK(automation->GetRootElement(&root));

  // get hierarchy one by one. This is so it doesn't take too much time
  HRCHECK(FindFirstChild(automation, root, UIA_ClassNamePropertyId, CComVariant("ApplicationFrameWindow"), &settingsWindow));
  HRCHECK(FindFirstChild(automation, settingsWindow, UIA_ClassNamePropertyId, CComVariant("Windows.UI.Core.CoreWindow"), &coreWindow));
  HRCHECK(FindFirstChild(automation, coreWindow, UIA_AutomationIdPropertyId, CComVariant("pageContent"), &content));
  HRCHECK(FindFirstChild(automation, content, UIA_AutomationIdPropertyId, CComVariant("ItemsControlScrollViewer"), &scrollViewer));

  // now the list of app should be shown, get it
  HRCHECK(FindFirstChild(automation, scrollViewer, UIA_AutomationIdPropertyId, CComVariant("SystemSettings_DefaultApps_DefaultAppsList_ListView"), &list));

  // find the item by it's name
  // the list is virtualized so we use a helper method
  // note for some reason, the name is the name plus a space... 
  name.Append(" ");
  HRCHECK(FindFirstChildInList(automation, list, UIA_NamePropertyId, CComVariant(name), &appNameListItem));

  // we got the app item, select it so the 'Manage' button can appear
  HRCHECK(appNameListItem->GetCurrentPatternAs(UIA_SelectionItemPatternId, IID_PPV_ARGS(&selection)));
  if (!selection) HRCHECK(E_FAIL);
  HRCHECK(selection->Select());

  // get the 'Manage' button
  HRCHECK(FindFirstChild(automation, scrollViewer, UIA_ClassNamePropertyId, CComVariant("Button"), &manageButton));

  // press the 'Manage' button
  HRCHECK(manageButton->GetCurrentPatternAs(UIA_InvokePatternId, IID_PPV_ARGS(&invoke)));
  if (!invoke) HRCHECK(E_FAIL);
  HRCHECK(invoke->Invoke());

cleanup:
  return hr;
}

HRESULT OpenSetDefaultsByApp()
{
  HRESULT hr = S_OK;
  CComPtr<IUIAutomation> automation;
  CComPtr<IUIAutomationElement> root;
  CComPtr<IUIAutomationElement> settingsWindow;
  CComPtr<IUIAutomationElement> coreWindow;
  CComPtr<IUIAutomationElement> content;
  CComPtr<IUIAutomationElement> setDefaultsByAppLink;
  CComPtr<IUIAutomationInvokePattern> invoke;

  // create UIA COM server and get root
  HRCHECK(automation.CoCreateInstance(CLSID_CUIAutomation8));
  HRCHECK(automation->GetRootElement(&root));

  // show up to the deepest we can
  WinExec("control.exe /name Microsoft.DefaultPrograms /page pageDefaultProgram", SW_NORMAL);

  // find the 'Set defaults by app' link (button).
  HRCHECK(FindFirstChild(automation, root, UIA_ClassNamePropertyId, CComVariant("ApplicationFrameWindow"), &settingsWindow));
  HRCHECK(FindFirstChild(automation, settingsWindow, UIA_ClassNamePropertyId, CComVariant("Windows.UI.Core.CoreWindow"), &coreWindow));
  HRCHECK(FindFirstChild(automation, coreWindow, UIA_AutomationIdPropertyId, CComVariant("pageContent"), &content));
  HRCHECK(FindFirstChild(automation, content, UIA_AutomationIdPropertyId, CComVariant("SettingsPageAppsDefaultsDefaultAppsListView_HyperlinkButton"), &setDefaultsByAppLink));

  // yes, so press this button
  HRCHECK(setDefaultsByAppLink->GetCurrentPatternAs(UIA_InvokePatternId, IID_PPV_ARGS(&invoke)));
  if (!invoke) HRCHECK(E_FAIL);
  HRCHECK(invoke->Invoke());

cleanup:
  return hr;
}

// this method has retries with timeouts included, so it's much better than a raw call to FindFirst
HRESULT FindFirstChild(IUIAutomation *automation, IUIAutomationElement *element, PROPERTYID pid, VARIANT value, IUIAutomationElement **child)
{
  HRESULT hr = S_OK;
  int timeout = 5000; // max timeout is defined here as 5 sec. This should be ok for most machines
  int slice = 100; // time between too retries, defined as 100 ms.
  int time = 0;
  CComPtr<IUIAutomationCondition> condition;
  HRCHECK(automation->CreatePropertyCondition(pid, value, &condition));

  do
  {
    // I used SubTree here, this may not be appropriate in all context
    // for performance issues. In fact, this could be passed as a parameter...
    hr = element->FindFirst(TreeScope_Subtree, condition, child);
    if (*child) break;
    time += slice;
    if (time >= timeout) HRCHECK(E_FAIL);
    Sleep(slice);
  } while (TRUE);

cleanup:
  return hr;
}

// this helper supports virtualized list
HRESULT FindFirstChildInList(IUIAutomation *automation, IUIAutomationElement *list, PROPERTYID pid, VARIANT value, IUIAutomationElement **child)
{
  HRESULT hr = S_OK;
  CComBSTR lastName;
  int lastNameCount = 0;
  CComPtr<IUIAutomationCondition> trueCondition;
  HRCHECK(automation->CreateTrueCondition(&trueCondition));

  do
  {
    // get all children
    CComPtr<IUIAutomationElementArray> all;
    HRCHECK(list->FindAll(TreeScope_Children, trueCondition, &all));

    int count;
    HRCHECK(all->get_Length(&count));
    if (count == 0) continue; // there shouldn't be zero element, so go on scanning

    for (int i = 0; i < count; i++)
    {
      // test each element for the searched property
      CComPtr<IUIAutomationElement> element;
      HRCHECK(all->GetElement(i, &element));

      CComVariant v;
      HRCHECK(element->GetCurrentPropertyValue(pid, &v));
      if (VarCmp(&v, &value, 0) == 1)
      {
        HRCHECK(element.QueryInterface(child));
        goto cleanup;
      }
    }

    // not found in the current page/set, go to last element and scroll it into view to force list to load the next
    CComPtr<IUIAutomationElement> last;
    CComPtr<IUIAutomationScrollItemPattern> pattern;
    HRCHECK(all->GetElement(count - 1, &last));

    // check if we didn't progress (same name for 20 rounds)
    CComBSTR name;
    HRCHECK(last->get_CurrentName(&name));
    if (name == lastName)
    {
      lastNameCount++;
      if (lastNameCount > 20) HRCHECK(E_FAIL); // not found!
    }
    else
    {
      lastNameCount = 0;
    }
    lastName = name;

    HRCHECK(last->GetCurrentPatternAs(UIA_ScrollItemPatternId, IID_PPV_ARGS(&pattern)));
    if (!pattern) HRCHECK(E_FAIL);
    HRCHECK(pattern->ScrollIntoView());
  } while (TRUE);

cleanup:
  return hr;
}