覆盖句柄质询异步：
在下面的例子中，failReason是我的AuthenticationHandler实现中的私有字段。我不知道这是否是传递失败原因的最佳方式。但是在我的测试中，AuthenticateResult.Fail方法上的AuthenticationProperties没有传递到HandleChallengeAsync。

public class CustomAuthenticationHandler<TOptions> : AuthenticationHandler<TOptions> where TOptions : AuthenticationSchemeOptions, new()
{
    private string failReason;

    public CustomAuthenticationHandler(IOptionsMonitor<TOptions> options
        , ILoggerFactory logger
        , UrlEncoder encoder, ISystemClock clock) : base(options, logger, encoder, clock) { }

    protected override async Task<AuthenticateResult> HandleAuthenticateAsync()
    {
        failReason = "Reason for auth fail";
        return AuthenticateResult.Fail(failReason);
    }

    protected override Task HandleChallengeAsync(AuthenticationProperties properties)
    {
        Response.StatusCode = 401;

        if (failReason != null)
        {
            Response.HttpContext.Features.Get<IHttpResponseFeature>().ReasonPhrase = failReason;
        }

        return Task.CompletedTask;
    }
}

来自文档：https://learn.microsoft.com/en-us/dotnet/api/microsoft.aspnetcore.authentication.authenticationhandler-1?view=aspnetcore-2.2
如果所讨论的身份验证方案将身份验证交互作为其请求流的一部分进行处理，则重写此方法以处理401质询问题（例如添加响应头，或者将登录页面或外部登录位置的401结果更改为302）。
图片来源：https://github.com/aspnet/Security/blob/master/src/Microsoft.AspNetCore.Authentication/AuthenticationHandler.cs#L201

我在我的定制中间件中使用了这段代码来返回problemDetails响应。

public async Task Invoke(HttpContext httpContext)
{
    await this.Next(httpContext);

    if (httpContext.Response.StatusCode == StatusCodes.Status401Unauthorized)
    {
        var authenticateResult = await httpContext.AuthenticateAsync();

        if (authenticateResult.Failure != null)
        {
            var routeData = httpContext.GetRouteData() ?? new RouteData();
            var actionContext = new ActionContext(httpContext, routeData, new ActionDescriptor());

            var problemDetails = this.ProblemDetailsFactory.CreateProblemDetails(httpContext,
                statusCode: httpContext.Response.StatusCode,
                detail: authenticateResult.Failure.Message);

            var result = new ObjectResult(problemDetails)
            {
                ContentTypes = new MediaTypeCollection(),
                StatusCode = problemDetails.Status,
                DeclaredType = problemDetails.GetType()
            };

            await this.Executor.ExecuteAsync(actionContext, result);
        }
    }
}

要更改正文或Http状态，可以尝试Context.Response。
下面是一个演示代码：

using Microsoft.AspNetCore.Authentication;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using System.Text.Encodings.Web;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http;
namespace TestIdentity
{
    public class CustomAuthenticationHandler<TOptions> : AuthenticationHandler<TOptions> where TOptions : AuthenticationSchemeOptions, new()
    {
        public CustomAuthenticationHandler(IOptionsMonitor<TOptions> options
            , ILoggerFactory logger
            , UrlEncoder encoder, ISystemClock clock) : base(options, logger, encoder, clock)
        {

        }
        protected override async Task<AuthenticateResult> HandleAuthenticateAsync()
        {
            await Context.Response.WriteAsync("This is why you can't log in");
            return AuthenticateResult.Fail("This is why you can't log in");
        }
    }
}

下面是通过重写HandleChallengeAsync方法写入失败的自定义AuthenticationHandler的响应主体的方法：

using System.Security.Claims;
using System.Text;
using System.Text.Encodings.Web;
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Mvc;
using Microsoft.AspNetCore.Mvc.Abstractions;
using Microsoft.AspNetCore.Mvc.Formatters;
using Microsoft.AspNetCore.Mvc.Infrastructure;
using Microsoft.Extensions.Options;

namespace OpenAPISwaggerDoc.Web.Authentication;

public class BasicAuthenticationHandler : AuthenticationHandler<AuthenticationSchemeOptions>
{
    private string _failReason;

    public BasicAuthenticationHandler(
        IOptionsMonitor<AuthenticationSchemeOptions> options,
        ILoggerFactory logger,
        UrlEncoder encoder,
        ISystemClock clock)
        : base(options, logger, encoder, clock)
    {
    }

    protected override Task<AuthenticateResult> HandleAuthenticateAsync()
    {
        if (!Request.Headers.ContainsKey("Authorization"))
        {
            _failReason = "Missing Authorization header";
            return Task.FromResult(AuthenticateResult.Fail(_failReason));
        }

        try
        {
            var authenticationHeader = AuthenticationHeaderValue.Parse(Request.Headers["Authorization"]);
            var credentialBytes = Convert.FromBase64String(authenticationHeader.Parameter);
            var credentials = Encoding.UTF8.GetString(credentialBytes).Split(':');
            var username = credentials[0];
            var password = credentials[1];

            if (string.Equals(username, "DNT", StringComparison.Ordinal) &&
                string.Equals(password, "123", StringComparison.Ordinal))
            {
                var claims = new[] { new Claim(ClaimTypes.NameIdentifier, username) };
                var identity = new ClaimsIdentity(claims, Scheme.Name);
                var principal = new ClaimsPrincipal(identity);
                var ticket = new AuthenticationTicket(principal, Scheme.Name);
                return Task.FromResult(AuthenticateResult.Success(ticket));
            }

            _failReason = "Invalid username or password";
            return Task.FromResult(AuthenticateResult.Fail(_failReason));
        }
        catch
        {
            _failReason = "Invalid Authorization header";
            return Task.FromResult(AuthenticateResult.Fail(_failReason));
        }
    }

    protected override async Task HandleChallengeAsync(AuthenticationProperties properties)
    {
        await base.HandleChallengeAsync(properties);
        if (Response.StatusCode == StatusCodes.Status401Unauthorized &&
            !string.IsNullOrWhiteSpace(_failReason))
        {
            Response.Headers.Add("WWW-Authenticate", _failReason);
            Response.ContentType = "application/json";
            await WriteProblemDetailsAsync(_failReason);
        }
    }

    private Task WriteProblemDetailsAsync(string detail)
    {
        var problemDetails = new ProblemDetails { Detail = detail, Status = Context.Response.StatusCode };
        var result = new ObjectResult(problemDetails)
                     {
                         ContentTypes = new MediaTypeCollection(),
                         StatusCode = problemDetails.Status,
                         DeclaredType = problemDetails.GetType(),
                     };
        var executor = Context.RequestServices.GetRequiredService<IActionResultExecutor<ObjectResult>>();
        var routeData = Context.GetRouteData() ?? new RouteData();
        var actionContext = new ActionContext(Context, routeData, new ActionDescriptor());
        return executor.ExecuteAsync(actionContext, result);
    }
}