You can use __VSPROPID7.VSPROPID_IsInOpenFolderMode on an IVsSolution reference, with a code like this:

var solution = (IVsSolution)ServiceProvider.GetService(typeof(SVsSolution));

// __VSPROPID7 needs Microsoft.VisualStudio.Shell.Interop.15.0.DesignTime.dll nuget
// folderMode will be a boolean
solution.GetProperty((int)__VSPROPID7.VSPROPID_IsInOpenFolderMode, out object folderMode);