What I usually do is for API controller calls is:

• write .NET code natually, throw exceptions when I need too. The code that's not in controllers (but that controllers call) often has no knowledge of ASP.NET anyway. Note 3rd party code I use can throw exceptions too, so I need to somehow handle this.
• write a middleware that handle these exceptions and return corresponding nice JSON ProblemDetails instances, like ASP.NET does.

Here is an example of such a custom middleware, it does quite the same as the default one:

using System;
using System.Net;
using System.Text.Json;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;

namespace myStuff
{
    // this is abstract because we can't have multiple public 'Invoke' or 'InvokeAsync' methods on a middleware
    // some you'll have too derive it too add to the services
    public abstract class ExceptionHandlingMiddleware
    {
        private readonly RequestDelegate _next;

        protected ExceptionHandlingMiddleware(RequestDelegate next)
        {
            ArgumentNullException.ThrowIfNull(next);
            _next = next;
        }

        protected async Task DoInvoke(HttpContext context, ILogger? logger = null)
        {
            ArgumentNullException.ThrowIfNull(context);

            try
            {
                await _next(context);
            }
            catch (Exception ex)
            {
                logger?.LogError(ex);
                var handled = await HandleExceptionAsync(context, ex);
                if (!handled)
                    throw;
            }
        }

        protected virtual async Task<bool> HandleExceptionAsync(HttpContext context, Exception exception)
        {
            ArgumentNullException.ThrowIfNull(context);
            ArgumentNullException.ThrowIfNull(exception);

            if (context.Request.IsApiCall())
            {
                if (context.Response.HasStarted) // too late, no can do
                    return false;

                var details = await GetExceptionDetailsAsync(context, exception);
                if (details != null)
                {
                    context.Response.ContentType = "application/json";
                    context.Response.StatusCode = details.Status ?? StatusCodes.Status501NotImplemented;
                    await context.Response.WriteAsync(JsonSerializer.Serialize(details, JsonUtilities.SerializerOptions));
                    return true;
                }
            }
            return true;
        }

        protected virtual Task<ProblemDetails?> GetUnhandledExceptionDetailsAsync(HttpContext context, Exception exception)
        {
            ArgumentNullException.ThrowIfNull(context);
            ArgumentNullException.ThrowIfNull(exception);

            // the bad one (a real bug in our code or lower levels 3rd party code)
            return Task.FromResult<ProblemDetails?>(new ExceptionProblemDetails(HttpStatusCode.InternalServerError, exception, context.GetTraceId()));
        }

        // handle some exception with custom problem details
        protected virtual Task<ProblemDetails?> GetExceptionDetailsAsync(HttpContext context, Exception exception)
        {
            ArgumentNullException.ThrowIfNull(context);
            ArgumentNullException.ThrowIfNull(exception);

            if (exception is ArgumentException)
                return Task.FromResult<ProblemDetails?>(new ExceptionProblemDetails(HttpStatusCode.BadRequest, exception, context.GetTraceId()));

            if (exception is UnauthorizedAccessException)
                return Task.FromResult<ProblemDetails?>(new ExceptionProblemDetails(HttpStatusCode.Unauthorized, exception, context.GetTraceId()));

            if (exception is ForbiddenAccessException) // a new one
                return Task.FromResult<ProblemDetails?>(new ExceptionProblemDetails(HttpStatusCode.Forbidden, exception, context.GetTraceId()));

            // TODO: add some other special handling
            if (exception is SomeSpecialException)
                return Task.FromResult<ProblemDetails?>(new ExceptionProblemDetails(HttpStatusCode.UnprocessableEntity, exception, context.GetTraceId()));

            return GetUnhandledExceptionDetailsAsync(context, exception);
        }
    }

    // tools
    public static class HttpExtensions
    {
        // get trace id for correlation
        public static string? GetTraceId(this HttpContext? context)
        {
            if (context == null)
                return null;

            var feature = context.Features.Get<IHttpActivityFeature>();
            return feature?.Activity?.Id?.ToString() ?? context.TraceIdentifier;
        }

        // some utility class too determine if it's an API call
        // adapt to your context
        public static bool IsApiCall(this HttpRequest request)
        {
            if (request == null)
                return false;

            var isJson = request.GetTypedHeaders().Accept.Contains(new MediaTypeHeaderValue("application/json"));
            if (isJson)
                return true;

            if (request.Path.Value?.StartsWith("/api/") == true)
                return true;

            if (request.HttpContext != null)
            {
                var ep = request.HttpContext.GetEndpoint();
                if (ep != null)
                {
                    // check if class has the ApiController attribute
                    foreach (var metadata in ep.Metadata)
                    {
                        if (metadata is ApiControllerAttribute)
                            return true;
                    }
                }
            }
            return false;
        }
    }

    // our custom problem details
    public class ExceptionProblemDetails : ProblemDetails
    {
        public ExceptionProblemDetails(HttpStatusCode code, string? traceId = null)
        {
            Status = (int)code;
            Type = GetType(code);
            if (traceId != null)
            {
                Extensions["traceId"] = traceId;
            }
        }

        public ExceptionProblemDetails(HttpStatusCode code, Exception exception, string? traceId = null)
            : this(code, traceId)
        {
            ArgumentNullException.ThrowIfNull(exception);
            Title = exception.GetAllMessagesWithDots(); // concats messages
#if DEBUG
            Detail = exception.StackTrace;
#endif
        }

        public static string? GetType(HttpStatusCode code)
        {
            var webdav = false;
            if ((int)code >= 400)
            {
                string section;
                switch (code)
                {
                    case HttpStatusCode.BadRequest:
                        section = "6.5.1";
                        break;

                    case HttpStatusCode.PaymentRequired:
                        section = "6.5.2";
                        break;

                    case HttpStatusCode.Forbidden:
                        section = "6.5.3";
                        break;

                    case HttpStatusCode.NotFound:
                        section = "6.5.4";
                        break;

                    case HttpStatusCode.MethodNotAllowed:
                        section = "6.5.5";
                        break;

                    case HttpStatusCode.NotAcceptable:
                        section = "6.5.6";
                        break;

                    case HttpStatusCode.RequestTimeout:
                        section = "6.5.7";
                        break;

                    case HttpStatusCode.Conflict:
                        section = "6.5.8";
                        break;

                    case HttpStatusCode.Gone:
                        section = "6.5.9";
                        break;

                    case HttpStatusCode.LengthRequired:
                        section = "6.5.10";
                        break;

                    case HttpStatusCode.RequestEntityTooLarge:
                        section = "6.5.11";
                        break;

                    case HttpStatusCode.RequestUriTooLong:
                        section = "6.5.12";
                        break;

                    case HttpStatusCode.UnsupportedMediaType:
                        section = "6.5.13";
                        break;

                    case HttpStatusCode.ExpectationFailed:
                        section = "6.5.14";
                        break;

                    case HttpStatusCode.UpgradeRequired:
                        section = "6.5.15";
                        break;

                    case HttpStatusCode.UnprocessableEntity:
                        webdav = true;
                        section = "11.2";
                        break;

                    case HttpStatusCode.Locked:
                        webdav = true;
                        section = "11.3";
                        break;

                    case HttpStatusCode.FailedDependency:
                        webdav = true;
                        section = "11.4";
                        break;

                    case HttpStatusCode.InsufficientStorage:
                        webdav = true;
                        section = "11.5";
                        break;

                    default:
                        section = "6.5";
                        break;
                }

                if (webdav)
                    return "https://tools.ietf.org/html/rfc4918.html#section-" + section;

                return "https://tools.ietf.org/html/rfc7231#section-" + section;
            }
            return null;
        }
    }