Here is some utility code based on Windows API that enumerates file system entries without throwing/catching exceptions.

Sample usage:

...
foreach (var fi in Utilities.EnumerateFileSystemEntries(@"c:\windows\system32", DirectoryEnumerateOptions.Recursive))
{
    Console.WriteLine(fi.FullName);
}
...

Utility classes:

[Flags]
public enum DirectoryEnumerateOptions
{
    None = 0x0,
    Recursive = 0x1,
    ThrowErrors = 0x2, // if you really want it
    ExpandEnvironmentVariables = 0x4,
}

public static class Utilities
{
    public static IEnumerable<FileSystemInfo> EnumerateFileSystemEntries(string directoryPath, DirectoryEnumerateOptions options = DirectoryEnumerateOptions.None)
    {
        if (directoryPath == null)
            throw new ArgumentNullException(nameof(directoryPath));

        if (!Path.IsPathRooted(directoryPath))
        {
            directoryPath = Path.GetFullPath(directoryPath);
        }

        return EnumerateFileSystemEntriesPrivate(directoryPath, options);
    }

    private static IEnumerable<FileSystemInfo> EnumerateFileSystemEntriesPrivate(string directoryPath, DirectoryEnumerateOptions options = DirectoryEnumerateOptions.None)
    {
        if (!Directory.Exists(directoryPath))
            yield break;

        var findPath = Normalize(directoryPath, options.HasFlag(DirectoryEnumerateOptions.ExpandEnvironmentVariables));
        if (!findPath.EndsWith("*"))
        {
            findPath = Path.Combine(findPath, "*");
        }

        var h = FindFirstFile(findPath, out var data);
        if (h == INVALID_HANDLE_VALUE)
        {
            if (options.HasFlag(DirectoryEnumerateOptions.ThrowErrors))
                throw new Win32Exception(Marshal.GetLastWin32Error());

            yield break;
        }

        if (Include(ref data))
        {
            yield return ToInfo(ref data, directoryPath);
            if (options.HasFlag(DirectoryEnumerateOptions.Recursive) && data.fileAttributes.HasFlag(FileAttributes.Directory))
            {
                foreach (var wfd in EnumerateFileSystemEntriesPrivate(Path.Combine(directoryPath, data.cFileName), options))
                {
                    yield return wfd;
                }
            }
        }
        do
        {
            if (!FindNextFile(h, out data))
            {
                if (Marshal.GetLastWin32Error() == ERROR_NO_MORE_FILES)
                {
                    FindClose(h);
                    break;
                }
                continue;
            }

            if (Include(ref data))
            {
                yield return ToInfo(ref data, directoryPath);
                if (options.HasFlag(DirectoryEnumerateOptions.Recursive) && data.fileAttributes.HasFlag(FileAttributes.Directory))
                {
                    foreach (var wfd in EnumerateFileSystemEntriesPrivate(Path.Combine(directoryPath, data.cFileName), options))
                    {
                        yield return wfd;
                    }
                }
            }
        }
        while (true);
    }

    private static bool Include(ref WIN32_FIND_DATA data) => data.cFileName != "." && data.cFileName != "..";

    private static FileSystemInfo ToInfo(ref WIN32_FIND_DATA data, string directoryPath)
    {
        if (data.fileAttributes.HasFlag(FileAttributes.Directory))
            return new FileInfo(Path.Combine(directoryPath, data.cFileName));

        return new DirectoryInfo(Path.Combine(directoryPath, data.cFileName));
    }

    private static string Normalize(string path, bool expandEnvironmentVariables)
    {
        if (path == null)
            return null;

        string expanded;
        if (expandEnvironmentVariables)
        {
            expanded = Environment.ExpandEnvironmentVariables(path);
        }
        else
        {
            expanded = path;
        }

        if (expanded.StartsWith(_prefix))
            return expanded;

        if (expanded.StartsWith(@"\\"))
            return _uncPrefix + expanded.Substring(2);

        return _prefix + expanded;
    }

    [DllImport("kernel32", SetLastError = true, CharSet = CharSet.Unicode)]
    private static extern IntPtr FindFirstFile(string lpFileName, out WIN32_FIND_DATA lpFindFileData);

    [DllImport("kernel32", SetLastError = true)]
    private static extern bool FindClose(IntPtr hFindFile);

    [DllImport("kernel32", SetLastError = true, CharSet = CharSet.Unicode)]
    private static extern bool FindNextFile(IntPtr hFindFile, out WIN32_FIND_DATA lpFindFileData);

    #pragma warning disable IDE1006 // Naming Styles
    private const int ERROR_NO_MORE_FILES = 18;
    private static readonly IntPtr INVALID_HANDLE_VALUE = new IntPtr(-1);
    #pragma warning restore IDE1006 // Naming Styles

    private const string _prefix = @"\\?\";
    private const string _uncPrefix = _prefix + @"UNC\";

    [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)]
    internal struct WIN32_FIND_DATA
    {
        public FileAttributes fileAttributes;
        public uint ftCreationTimeLow;
        public uint ftCreationTimeHigh;
        public uint ftLastAccessTimeLow;
        public uint ftLastAccessTimeHigh;
        public uint ftLastWriteTimeLow;
        public uint ftLastWriteTimeHigh;
        public uint fileSizeHigh;
        public uint fileSizeLow;
        public uint dwReserved0;
        public uint dwReserved1;
        [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 260)]
        public string cFileName;
        [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 14)]
        public string cAlternateFileName;
    }
}