Fix admin panel hanging when the admin user storage is unreachable

src/Dapr/AdminPanel.Host/Dockerfile

@@ -18,5 +18,8 @@ RUN dotnet publish "MUnique.OpenMU.AdminPanel.Host.csproj" -c Release -o /app/pu
 FROM base AS final
 WORKDIR /app
 COPY --from=publish /app/publish .
+# The container runs as a non-root user, but /app belongs to root. The data protection
+# key ring, which protects the admin panel sessions, needs a writable directory.
+RUN mkdir -p /app/data-protection-keys && chmod 777 /app/data-protection-keys
 USER $APP_UID
 ENTRYPOINT ["dotnet", "MUnique.OpenMU.AdminPanel.Host.dll"]

src/Persistence/EntityFramework/AdminAuth/AdminUserRepository.cs

@@ -16,9 +16,25 @@ namespace MUnique.OpenMU.Persistence.EntityFramework.AdminAuth;
 /// </summary>
 public class AdminUserRepository : IAdminUserRepository
 {
+    /// <summary>
+    /// The time after which a connection attempt to the database server is given up.
+    /// </summary>
+    private static readonly TimeSpan ConnectTimeout = TimeSpan.FromSeconds(3);
+
+    /// <summary>
+    /// The time after which the creation of the schema is given up.
+    /// </summary>
+    private static readonly TimeSpan MigrationTimeout = TimeSpan.FromSeconds(30);
+
+    /// <summary>
+    /// The time to wait before the storage is probed again after a failed attempt.
+    /// </summary>
+    private static readonly TimeSpan RetryDelay = TimeSpan.FromSeconds(30);
+
     private readonly ILogger<AdminUserRepository> _logger;
     private readonly AsyncLock _storageLock = new();
     private bool _isStorageReady;
+    private DateTime _nextProbeAt = DateTime.MinValue;
 
     /// <summary>
     /// Initializes a new instance of the <see cref="AdminUserRepository"/> class.
@@ -37,24 +53,54 @@ public async ValueTask<bool> EnsureStorageAsync(CancellationToken cancellationTo
             return true;
         }
 
+        // The authorization of every request asks whether a user exists, so a database which is not
+        // reachable must not be retried on each of them - otherwise the whole panel waits for a
+        // connection which is going to time out anyway.
+        if (DateTime.UtcNow < this._nextProbeAt)
+        {
+            return false;
+        }
+
         using var l = await this._storageLock.LockAsync(cancellationToken).ConfigureAwait(false);
         if (this._isStorageReady)
         {
             return true;
         }
 
+        if (DateTime.UtcNow < this._nextProbeAt)
+        {
+            return false;
+        }
+
         try
         {
             await using var context = new AdminPanelContext();
-            await context.Database.MigrateAsync(cancellationToken).ConfigureAwait(false);
-            this._isStorageReady = true;
+
+            // Connecting is checked separately and with a short timeout, because the configured
+            // command timeout of the connection string is far too long to block a request on.
+            using var connectCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+            connectCts.CancelAfter(ConnectTimeout);
+            if (await context.Database.CanConnectAsync(connectCts.Token).ConfigureAwait(false))
+            {
+                using var migrationCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+                migrationCts.CancelAfter(MigrationTimeout);
+                await context.Database.MigrateAsync(migrationCts.Token).ConfigureAwait(false);
+                this._isStorageReady = true;
+            }
         }
         catch (Exception ex)
         {
             // This is an expected state before the database server is reachable or the database has been created.
             // The admin panel then falls back to the configured bootstrap user.
             this._logger.LogInformation(ex, "The admin user storage is not available (yet).");
         }
+        finally
+        {
+            if (!this._isStorageReady)
+            {
+                this._nextProbeAt = DateTime.UtcNow + RetryDelay;
+            }
+        }
 
         return this._isStorageReady;
     }

src/Startup/Dockerfile

@@ -45,7 +45,10 @@ WORKDIR /app
 
 COPY --from=publish /app/publish .
 
-RUN mkdir -p /app/logs && chmod 777 /app/logs
+# The container runs as a non-root user, but /app belongs to root. Directories the
+# application writes to have to be created and made writable beforehand - the data
+# protection key ring among them, which protects the admin panel sessions.
+RUN mkdir -p /app/logs /app/data-protection-keys && chmod 777 /app/logs /app/data-protection-keys
 
 ARG APP_UID=1000
 USER ${APP_UID}

src/Web/AdminPanel/Auth/AdminPanelAuthExtensions.cs

@@ -16,6 +16,7 @@ namespace MUnique.OpenMU.Web.AdminPanel.Auth;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.DependencyInjection.Extensions;
+using Microsoft.Extensions.Logging;
 using MUnique.OpenMU.Persistence.AdminAuth;
 
 /// <summary>
@@ -58,13 +59,7 @@ public static IServiceCollection AddAdminPanelAuth(this IServiceCollection servi
             options.BootstrapUser = authOptions.BootstrapUser;
         });
 
-        // The key ring protects the authentication cookies and the authenticator keys. It has to be
-        // persisted, otherwise a restart invalidates all sessions and makes all stored authenticator
-        // keys unreadable. In docker, the directory should be a mounted volume.
-        var keyPath = configuration["AdminPanel:Auth:DataProtectionKeyPath"] ?? "data-protection-keys";
-        services.AddDataProtection()
-            .SetApplicationName("MUnique.OpenMU.AdminPanel")
-            .PersistKeysToFileSystem(new DirectoryInfo(Path.Combine(Directory.GetCurrentDirectory(), keyPath)));
+        services.AddSingleton(ConfigureDataProtection(services, configuration));
 
         // The hosting application registers the real storage; this is just a fallback which lets
         // the panel start in its initial setup mode instead of failing to resolve its services.
@@ -131,6 +126,18 @@ public static IServiceCollection AddAdminPanelAuth(this IServiceCollection servi
     /// <returns>The same instance, to allow chaining of further calls.</returns>
     public static IApplicationBuilder UseAdminPanelAuth(this IApplicationBuilder app)
     {
+        if (app.ApplicationServices.GetService<DataProtectionKeyStorageStatus>() is { Error: { } error } status)
+        {
+            app.ApplicationServices.GetRequiredService<ILoggerFactory>()
+                .CreateLogger(typeof(AdminPanelAuthExtensions))
+                .LogWarning(
+                    error,
+                    "The data protection keys can't be stored at '{Path}', so they are only kept in memory: "
+                    + "everybody is signed out when the application restarts, and stored authenticator keys "
+                    + "become unreadable. Make sure the directory exists and is writable by the user which runs the application.",
+                    status.Path);
+        }
+
         app.UseAuthentication();
         app.UseAuthorization();
         return app;
@@ -170,6 +177,45 @@ public static IApplicationBuilder UseAuthorizedPath(this IApplicationBuilder app
         });
     }
 
+    /// <summary>
+    /// Sets the storage of the data protection key ring up.
+    /// </summary>
+    /// <param name="services">The service collection.</param>
+    /// <param name="configuration">The configuration.</param>
+    /// <returns>The result, which is logged when the request pipeline is built.</returns>
+    /// <remarks>
+    /// The key ring protects the authentication cookies and the authenticator keys, so it has to be
+    /// persisted - otherwise a restart invalidates all sessions and makes all stored authenticator
+    /// keys unreadable. In docker, the directory should be a mounted volume.
+    /// It's deliberately not fatal when the directory can't be used: the containers run as a
+    /// non-root user, so a directory which wasn't prepared in the image would otherwise make every
+    /// page of the panel fail with an error as soon as a key is needed.
+    /// </remarks>
+    private static DataProtectionKeyStorageStatus ConfigureDataProtection(IServiceCollection services, IConfiguration configuration)
+    {
+        var keyPath = configuration["AdminPanel:Auth:DataProtectionKeyPath"] ?? "data-protection-keys";
+        var directory = new DirectoryInfo(Path.Combine(Directory.GetCurrentDirectory(), keyPath));
+        var dataProtection = services.AddDataProtection().SetApplicationName("MUnique.OpenMU.AdminPanel");
+
+        try
+        {
+            directory.Create();
+
+            // Creating an existing directory succeeds even without write access to it, so the
+            // access is checked explicitly - a mounted volume may belong to another user.
+            var probeFilePath = Path.Combine(directory.FullName, ".write-probe");
+            File.WriteAllBytes(probeFilePath, Array.Empty<byte>());
+            File.Delete(probeFilePath);
+
+            dataProtection.PersistKeysToFileSystem(directory);
+            return new DataProtectionKeyStorageStatus(directory.FullName, null);
+        }
+        catch (Exception ex)
+        {
+            return new DataProtectionKeyStorageStatus(directory.FullName, ex);
+        }
+    }
+
     private static void ApplyEnvironmentVariables(AdminPanelAuthOptions options)
     {
         var loginName = Environment.GetEnvironmentVariable(BootstrapUserVariableName);

src/Web/AdminPanel/Auth/AdminUserAvailabilityService.cs

@@ -17,6 +17,11 @@ namespace MUnique.OpenMU.Web.AdminPanel.Auth;
 /// </remarks>
 public class AdminUserAvailabilityService
 {
+    /// <summary>
+    /// The time for which the answer is reused before the storage is asked again.
+    /// </summary>
+    private static readonly TimeSpan CheckInterval = TimeSpan.FromSeconds(10);
+
     private readonly IAdminUserRepository _repository;
     private readonly BootstrapAdminUserProvider _bootstrapUserProvider;
     private readonly SemaphoreSlim _semaphore = new(1, 1);
@@ -40,35 +45,39 @@ public AdminUserAvailabilityService(IAdminUserRepository repository, BootstrapAd
     /// </summary>
     /// <param name="cancellationToken">The cancellation token.</param>
     /// <returns><c>true</c>, if at least one user exists; otherwise, <c>false</c>.</returns>
+    /// <remarks>
+    /// This is called by the authorization of every request, so it must never wait for the
+    /// database: when another caller is already asking, or when the last answer is still fresh,
+    /// the known value is returned right away.
+    /// </remarks>
     public async ValueTask<bool> AnyUserExistsAsync(CancellationToken cancellationToken = default)
     {
-        if (this._bootstrapUserProvider.User is not null)
-        {
-            return true;
-        }
-
-        if (this._anyUserExists)
+        if (this._bootstrapUserProvider.User is not null || this._anyUserExists)
         {
             return true;
         }
 
-        if (DateTime.UtcNow < this._nextCheck)
+        if (DateTime.UtcNow < this._nextCheck || !await this._semaphore.WaitAsync(0, cancellationToken).ConfigureAwait(false))
         {
-            return false;
+            return this._anyUserExists;
         }
 
-        await this._semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
         try
         {
             if (this._anyUserExists || DateTime.UtcNow < this._nextCheck)
             {
                 return this._anyUserExists;
             }
 
-            this._anyUserExists = await this._repository.GetCountAsync(cancellationToken).ConfigureAwait(false) > 0;
+            if (await this._repository.EnsureStorageAsync(cancellationToken).ConfigureAwait(false))
+            {
+                this._anyUserExists = await this._repository.GetCountAsync(cancellationToken).ConfigureAwait(false) > 0;
+            }
 
-            // The database might not be reachable yet, so don't hammer it on every render.
-            this._nextCheck = DateTime.UtcNow.AddSeconds(5);
+            // When the storage isn't available, we can't tell - the previous answer is kept, which
+            // is the initial setup mode on a fresh installation. The installation needs the database
+            // as well, so there is nothing to protect at that point anyway.
+            this._nextCheck = DateTime.UtcNow.Add(CheckInterval);
             return this._anyUserExists;
         }
         finally

src/Web/AdminPanel/Auth/DataProtectionKeyStorageStatus.cs

@@ -0,0 +1,18 @@
+// <copyright file="DataProtectionKeyStorageStatus.cs" company="MUnique">
+// Licensed under the MIT License. See LICENSE file in the project root for full license information.
+// </copyright>
+
+namespace MUnique.OpenMU.Web.AdminPanel.Auth;
+
+/// <summary>
+/// The result of setting the storage of the data protection key ring up.
+/// </summary>
+/// <param name="Path">The path at which the keys should be stored.</param>
+/// <param name="Error">The error which prevented the usage of that path; <c>null</c>, if it works.</param>
+/// <remarks>
+/// The key ring protects the authentication cookies and the stored authenticator keys. When it
+/// can't be persisted, the admin panel still works - but everybody is signed out after a restart
+/// and the stored authenticator keys become unreadable. That's worth a warning, but it's not worth
+/// taking the whole panel down for, which is what an exception at this point would do.
+/// </remarks>
+public record DataProtectionKeyStorageStatus(string Path, Exception? Error);