diff --git a/i18n/en/Server.resx b/i18n/en/Server.resx index b9ba718d..32f9f1f6 100644 --- a/i18n/en/Server.resx +++ b/i18n/en/Server.resx @@ -5,4 +5,53 @@ System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 Not connected to a server + Too many failed sign-ins; the account is locked for a while.long + The request is not valid.long + This upload has no such chunk.long + The chunk has the wrong size.long + The file could not be stored in the archive; try again.long + The security token is missing; reload the page.long + Your account cannot do this.long + The file arrived damaged; upload it again.long + Open the admin panel over HTTPS to sign in.long + Some chunks are still missing.long + The server is out of disk space.long + Unexpected server error; try again.long + Wrong user name or password.long + The paging cursor is not valid; start from the first page.long + The file name is not valid.long + A filter value is not valid.long + The SHA-256 must be 64 hexadecimal characters.long + This job kind cannot be started here.long + The job parameters are not valid.long + The priority must be background, normal or user.long + The role must be admin, editor, uploader or viewer.long + The setting {0} is not valid.long + The settings must be valid JSON.long + The job cannot do that in its current state.long + The naming template is not valid.long + Use letters, digits, dot, dash or underscore.long + This job is no longer assigned to this node.long + This server cannot run that kind of job.long + The last administrator cannot be removed or disabled.long + This method is not allowed here.long + Not found.long + Change your password first.long + This upload would exceed your quota.long + Too many attempts; wait a minute and try again.long + Choose a password different from the current one.long + You cannot do this to your own account.long + An environment variable sets this; change it there.long + The sequence changed on the server; reload it and apply your edit again.long + The request is larger than the server accepts.long + Too many uploads at once; retry shortly.long + Sign in to continue.long + Send the body as JSON.long + This upload is not accepting that request now.long + That user name is taken.long + Use at least {0} characters.long + Connect with a WebSocket.long + The current password is wrong.long + This upload belongs to a render job.long + Titano time-lapse archive, uploads, jobs and rendering server.long diff --git a/i18n/it/Server.it.resx b/i18n/it/Server.it.resx index d80ac2f7..beeafe59 100644 --- a/i18n/it/Server.it.resx +++ b/i18n/it/Server.it.resx @@ -5,4 +5,53 @@ System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 Non connesso a un server + Troppi accessi falliti; l'account è bloccato per un po'. + La richiesta non è valida. + Questo caricamento non ha quel blocco. + Il blocco ha la dimensione sbagliata. + Il file non è stato salvato nell'archivio; riprova. + Manca il token di sicurezza; ricarica la pagina. + Il tuo account non può farlo. + Il file è arrivato danneggiato; caricalo di nuovo. + Apri il pannello di amministrazione in HTTPS per accedere. + Mancano ancora alcuni blocchi. + Il server ha esaurito lo spazio su disco. + Errore imprevisto del server; riprova. + Nome utente o password errati. + Il cursore di paginazione non è valido; ricomincia dalla prima pagina. + Il nome del file non è valido. + Un valore di filtro non è valido. + Lo SHA-256 deve avere 64 caratteri esadecimali. + Questo tipo di job non si può avviare qui. + I parametri del job non sono validi. + La priorità deve essere background, normal o user. + Il ruolo deve essere admin, editor, uploader o viewer. + L'impostazione {0} non è valida. + Le impostazioni devono essere JSON valido. + Il job non può farlo nello stato attuale. + Il modello dei nomi non è valido. + Usa lettere, cifre, punto, trattino o trattino basso. + Questo job non è più assegnato a questo nodo. + Questo server non può eseguire quel tipo di job. + L'ultimo amministratore non può essere rimosso o disattivato. + Metodo non consentito qui. + Non trovato. + Cambia prima la password. + Questo caricamento supererebbe la tua quota. + Troppi tentativi; attendi un minuto e riprova. + Scegli una password diversa da quella attuale. + Non puoi farlo sul tuo account. + È impostato da una variabile d'ambiente; modificala lì. + La sequenza è cambiata sul server; ricaricala e riapplica la modifica. + La richiesta supera la dimensione accettata dal server. + Troppi caricamenti insieme; riprova tra poco. + Accedi per continuare. + Invia il corpo come JSON. + Questo caricamento ora non accetta la richiesta. + Questo nome utente è già in uso. + Usa almeno {0} caratteri. + Connettiti con un WebSocket. + La password attuale è errata. + Questo caricamento appartiene a un job di rendering. + Server Titano per archivio time-lapse, caricamenti, job e rendering. diff --git a/src/Titano.Server/Api/AdminEndpoints.cs b/src/Titano.Server/Api/AdminEndpoints.cs new file mode 100644 index 00000000..45826700 --- /dev/null +++ b/src/Titano.Server/Api/AdminEndpoints.cs @@ -0,0 +1,112 @@ +using System.Text; +using Titano.Core.Model; +using Titano.Data; +using Titano.Protocol; +using Titano.Server.Configuration; +using Titano.Server.Jobs; +using Titano.Server.Security; +using Titano.Server.Storage; + +namespace Titano.Server.Api; + +/// Administration (BRIEF §14.8): dashboard figures, server settings, log tail and downloads, audit log. +public static class AdminEndpoints +{ + private const int MaxTailLines = 2000; + private const int TailWindowBytes = 1 << 20; + + public static void Map(RouteGroupBuilder api) + { + api.MapGet("/server/dashboard", DashboardAsync).Requires(Permissions.ManageArchive | Permissions.ManageUsers); + api.MapGet("/server/settings", (ServerSettingsStore store) => Results.Json(store.Current(), ProtocolJson.Default.ServerSettingsDto)) + .Requires(Permissions.ManageArchive); + api.MapPut("/server/settings", SaveSettingsAsync).Requires(Permissions.ManageArchive); + api.MapGet("/audit", AuditAsync).Requires(Permissions.ManageUsers | Permissions.ManageArchive); + + var logs = api.MapGroup("/logs").Requires(Permissions.ManageArchive); + logs.MapGet("", TailAsync); + logs.MapGet("/files", (ServerOptions options) => Results.Json(LogFiles(options).Select(f => new LogFileDto(f.Name, f.Length, f.LastWriteTimeUtc)).ToList(), + ProtocolJson.Default.ListLogFileDto)); + logs.MapGet("/files/{name}", (HttpContext context, string name, ServerOptions options) => + { + // Only names of existing log files are served: the name never becomes part of a path. + if (LogFiles(options).FirstOrDefault(f => f.Name == name) is not { } file) return ApiErrors.NotFound(context, "log"); + context.Response.Headers.CacheControl = "no-store"; + return Results.File(file.FullName, "text/plain; charset=utf-8", file.Name, file.LastWriteTimeUtc, enableRangeProcessing: true); + }); + } + + private static async Task DashboardAsync(HttpContext context, TitanoDatabase db, ServerArchive archive, ServerOptions options, ServerState state, + JobEngineHost jobs, TimeProvider time) + { + var ct = context.RequestAborted; + var now = time.GetUtcNow(); + var today = new DateTimeOffset(now.UtcDateTime.Date, TimeSpan.Zero); + var users = await db.Users.ListAsync(ct); + var names = users.ToDictionary(u => u.Id, u => u.Username); + var counts = await db.Jobs.CountAsync(today, ct); + var running = await db.Jobs.ListAsync(JobState.Running, null, 10, null, ct); + var queued = await db.Jobs.ListAsync(JobState.Queued, null, 10, null, ct); + var nodes = await db.RenderNodes.ListAsync(ct); + var uploads = await db.Uploads.CommittedPerDayAsync(today.AddDays(-29), ct); + var activity = await db.Audit.ListAsync(null, null, 12, null, ct); + return Results.Json(new DashboardDto( + options.ServerName, + SystemEndpoints.Version, + (long)(now - state.StartedAt).TotalSeconds, + state.Database, + options.TlsMode.ToString().ToLowerInvariant(), + options.Discovery, + users.Count, + await ArchiveEndpoints.StatsAsync(db, archive, ct), + new JobCountsDto(counts.Queued, counts.Running, counts.Paused, counts.DoneSince, counts.FailedSince), + jobs.Workers, + [.. nodes.Select(n => Dto.Node(n, names.GetValueOrDefault(n.UserId, string.Empty)))], + [.. uploads.Select(u => new UploadDayDto(u.Day, u.Count, u.Bytes))], + [.. activity.Items.Select(Dto.Audit)], + [.. running.Items.Concat(queued.Items.OrderByDescending(j => j.Priority).ThenBy(j => j.Created)).Take(10).Select(Dto.Job)]), + ProtocolJson.Default.DashboardDto); + } + + private static async Task SaveSettingsAsync(HttpContext context, Caller caller, ServerSettingsDto request, ServerSettingsStore store, AuditLog audit) + { + switch (store.Save(request)) + { + case ("setting_locked", var setting): + return ApiErrors.Conflict(context, "setting_locked", setting); + case (var code, var setting): + return ApiErrors.BadRequest(context, code, setting, setting); + } + await audit.WriteAsync(caller, context, "server_settings", "server", null, request.ServerName); + return Results.Json(store.Current(), ProtocolJson.Default.ServerSettingsDto); + } + + private static async Task AuditAsync(HttpContext context, TitanoDatabase db, Guid? user, string? action, string? cursor, int? limit) + { + var page = await db.Audit.ListAsync(user, string.IsNullOrWhiteSpace(action) ? null : action, limit ?? 100, + string.IsNullOrEmpty(cursor) ? null : cursor, context.RequestAborted); + return Results.Json(new Page([.. page.Items.Select(Dto.Audit)], page.NextCursor, null), ProtocolJson.Default.PageAuditEntryDto); + } + + /// The last lines lines (default 300) of the newest server log. + private static async Task TailAsync(HttpContext context, ServerOptions options, int? lines) + { + var take = Math.Clamp(lines ?? 300, 1, MaxTailLines); + if (LogFiles(options).FirstOrDefault() is not { } file) return Results.Json(new LogTailDto(string.Empty, []), ProtocolJson.Default.LogTailDto); + await using var stream = new FileStream(file.FullName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete); + var start = Math.Max(0, stream.Length - TailWindowBytes); + stream.Position = start; + using var reader = new StreamReader(stream, Encoding.UTF8); + var text = await reader.ReadToEndAsync(context.RequestAborted); + var all = text.Split('\n', StringSplitOptions.RemoveEmptyEntries); + // A window that starts mid-file begins with a partial line. + var complete = start > 0 && all.Length > 0 ? all[1..] : all; + return Results.Json(new LogTailDto(file.Name, [.. complete.TakeLast(take).Select(l => l.TrimEnd('\r'))]), ProtocolJson.Default.LogTailDto); + } + + /// Server log files, newest first. + private static List LogFiles(ServerOptions options) => + Directory.Exists(options.LogPath) + ? [.. new DirectoryInfo(options.LogPath).EnumerateFiles("*.log").OrderByDescending(f => f.LastWriteTimeUtc).ThenByDescending(f => f.Name)] + : []; +} diff --git a/src/Titano.Server/Api/ApiErrors.cs b/src/Titano.Server/Api/ApiErrors.cs new file mode 100644 index 00000000..372962c5 --- /dev/null +++ b/src/Titano.Server/Api/ApiErrors.cs @@ -0,0 +1,90 @@ +using System.Globalization; +using System.Text; +using Microsoft.Net.Http.Headers; +using Titano.Core.Localization; +using Titano.Protocol; +using Titano.Server.Configuration; + +namespace Titano.Server.Api; + +/// +/// Error responses of API v1: { code, message, details } (BRIEF §14.3). code is a stable snake_case +/// identifier for programs; message is one localised sentence (resource Server.Error.<Code>, language +/// from Accept-Language, else the server language); details is optional machine-readable context. +/// +public static class ApiErrors +{ + public static IResult Result(HttpContext context, int status, string code, string? details = null, params object?[] args) => + Results.Json(new ApiError(code, Message(context, code, args), details), ProtocolJson.Default.ApiError, statusCode: status); + + public static Task WriteAsync(HttpContext context, int status, string code, string? details = null) => + Result(context, status, code, details).ExecuteAsync(context); + + public static IResult NotFound(HttpContext context, string? what = null) => Result(context, StatusCodes.Status404NotFound, "not_found", what); + + public static IResult BadRequest(HttpContext context, string code, string? details = null, params object?[] args) => + Result(context, StatusCodes.Status400BadRequest, code, details, args); + + public static IResult Conflict(HttpContext context, string code, string? details = null) => Result(context, StatusCodes.Status409Conflict, code, details); + + public static IResult Forbidden(HttpContext context) => Result(context, StatusCodes.Status403Forbidden, "forbidden"); + + public static string Message(HttpContext context, string code, params object?[] args) + { + var culture = RequestCulture.Of(context); + var template = Loc.Get("Server.Error." + Pascal(code), culture) ?? code; + return args.Length == 0 ? template : string.Format(culture, template, args); + } + + private static string Pascal(string code) + { + var sb = new StringBuilder(code.Length); + var upper = true; + foreach (var c in code) + { + if (c == '_') + { + upper = true; + continue; + } + sb.Append(upper ? char.ToUpperInvariant(c) : c); + upper = false; + } + return sb.ToString(); + } +} + +/// The language of a request's messages: the best shipped match of Accept-Language, else the server language. +public static class RequestCulture +{ + public static CultureInfo Of(HttpContext context) + { + if (context.Items.TryGetValue(typeof(RequestCulture), out var cached) && cached is CultureInfo known) return known; + var fallback = context.RequestServices.GetRequiredService().Language; + var culture = FromHeader(context.Request.Headers.AcceptLanguage.ToString()) ?? Parse(fallback) ?? CultureInfo.GetCultureInfo("en"); + context.Items[typeof(RequestCulture)] = culture; + return culture; + } + + private static CultureInfo? FromHeader(string header) + { + if (string.IsNullOrWhiteSpace(header) || !StringWithQualityHeaderValue.TryParseList([header], out var values)) return null; + foreach (var value in values.OrderByDescending(v => v.Quality ?? 1)) + { + if (value.Value.Value is { } tag && tag != "*" && Parse(tag) is { } culture) return culture; + } + return null; + } + + private static CultureInfo? Parse(string tag) + { + try + { + return Languages.BestMatch(CultureInfo.GetCultureInfo(tag)); + } + catch (CultureNotFoundException) + { + return null; + } + } +} diff --git a/src/Titano.Server/Api/ArchiveEndpoints.cs b/src/Titano.Server/Api/ArchiveEndpoints.cs new file mode 100644 index 00000000..daf097d1 --- /dev/null +++ b/src/Titano.Server/Api/ArchiveEndpoints.cs @@ -0,0 +1,95 @@ +using Titano.Core.Model; +using Titano.Core.Naming; +using Titano.Data; +using Titano.Data.Repositories; +using Titano.Protocol; +using Titano.Server.Jobs; +using Titano.Server.Security; +using Titano.Server.Storage; + +namespace Titano.Server.Api; + +/// /api/v1/archive (admin): templates and proxy policy, scan/rebuild/verify/maintenance jobs, statistics. +public static class ArchiveEndpoints +{ + public static void Map(RouteGroupBuilder api) + { + var archive = api.MapGroup("/archive").Requires(Permissions.ManageArchive); + archive.MapGet("/settings", (ServerArchive a) => Results.Json(Settings(a.Current), ProtocolJson.Default.ArchiveSettingsDto)); + archive.MapPut("/settings", UpdateAsync); + archive.MapGet("/stats", async (HttpContext c, TitanoDatabase db, ServerArchive a) => + Results.Json(await StatsAsync(db, a, c.RequestAborted), ProtocolJson.Default.ArchiveStatsDto)); + foreach (var (path, kind) in new[] { ("scan", JobKind.Scan), ("rebuild", JobKind.Rebuild), ("verify", JobKind.Verify), ("maintenance", JobKind.Maintenance) }) + { + archive.MapPost("/" + path, (HttpContext c, Caller caller, ServerArchive a, JobEngineHost host, AuditLog audit) => EnqueueAsync(c, caller, a, host, audit, kind)); + } + } + + /// Totals, distributions, free space and health of the archive (also shown on the dashboard). + public static async Task StatsAsync(TitanoDatabase db, ServerArchive archive, CancellationToken cancellationToken) + { + var stats = await db.Sequences.GetStatsAsync(archive.Id, cancellationToken); + var withoutProxies = await db.Sequences.CountAsync( + new SequenceQuery { ArchiveId = archive.Id, WithoutStatus = SequenceStatus.ProxiesBuilt }, cancellationToken); + long free = 0, capacity = 0; + try + { + var drive = new DriveInfo(archive.Root); + free = drive.AvailableFreeSpace; + capacity = drive.TotalSize; + } + catch (Exception ex) when (ex is IOException or ArgumentException or UnauthorizedAccessException) + { + // An offline archive volume reports no space figures. + } + return new ArchiveStatsDto( + stats.Sequences, + stats.Photos, + stats.Bytes, + free, + capacity, + stats.ByCamera.Count(c => c.Model is not null), + stats.ByYear.Count > 0 ? stats.ByYear[0].Year : null, + stats.ByYear.Count > 0 ? stats.ByYear[^1].Year : null, + [.. stats.ByYear.Select(y => new KeyValueCount(y.Year.ToString(System.Globalization.CultureInfo.InvariantCulture), y.Photos, y.Bytes))], + [.. stats.ByMonth.Select(m => new KeyValueCount($"{m.Year:D4}-{m.Month:D2}", m.Photos, 0))], + [.. stats.ByCamera.Select(c => new KeyValueCount(string.Join(' ', new[] { c.Make, c.Model }.Where(v => !string.IsNullOrEmpty(v))), c.Photos, c.Bytes))], + [.. stats.ByFormat.Select(f => new KeyValueCount(f.Format.ToString().ToLowerInvariant(), f.Photos, f.Bytes))], + withoutProxies, + stats.Health.Unverified); + } + + private static async Task UpdateAsync(HttpContext context, Caller caller, ArchiveSettingsDto request, TitanoDatabase db, ServerArchive archive, AuditLog audit) + { + var name = request.Name?.Trim(); + if (string.IsNullOrEmpty(name) || name.Length > 100) return ApiErrors.BadRequest(context, "bad_request", "name"); + foreach (var (field, template) in new[] { ("folderTemplate", request.FolderTemplate), ("fileTemplate", request.FileTemplate) }) + { + if (string.IsNullOrWhiteSpace(template) || !PathTemplate.Parse(template).IsValid) return ApiErrors.BadRequest(context, "invalid_template", field); + } + if (request.ProxyLongEdge is not (-1 or 0) and (< 256 or > 8192)) return ApiErrors.BadRequest(context, "bad_request", "proxyLongEdge"); + + var updated = archive.Current with + { + Name = name, + FolderTemplate = request.FolderTemplate, + FileTemplate = request.FileTemplate, + ProxyLongEdge = request.ProxyLongEdge, + }; + await db.Archives.UpsertAsync(updated, context.RequestAborted); + archive.Current = updated; + await audit.WriteAsync(caller, context, "archive_settings", "archive", updated.Id.ToString(), $"{updated.FolderTemplate} · {updated.FileTemplate}"); + return Results.Json(Settings(updated), ProtocolJson.Default.ArchiveSettingsDto); + } + + private static async Task EnqueueAsync(HttpContext context, Caller caller, ServerArchive archive, JobEngineHost host, AuditLog audit, JobKind kind) + { + if (!host.CanRun(kind)) return ApiErrors.Conflict(context, "job_unavailable", JobWire.Kind(kind)); + var job = await host.Engine.EnqueueAsync(kind, [archive.Id], archive.Current.Name, priority: JobPriority.User, createdBy: caller.Id, + cancellationToken: context.RequestAborted); + await audit.WriteAsync(caller, context, "job_created", "job", job.Id.ToString(), JobWire.Kind(kind)); + return Results.Json(Dto.Job(job), ProtocolJson.Default.JobDto, statusCode: StatusCodes.Status202Accepted); + } + + private static ArchiveSettingsDto Settings(Archive a) => new(a.Id, a.Name, a.RootPath, a.FolderTemplate, a.FileTemplate, a.ProxyLongEdge); +} diff --git a/src/Titano.Server/Api/AuthEndpoints.cs b/src/Titano.Server/Api/AuthEndpoints.cs new file mode 100644 index 00000000..6f9d30f3 --- /dev/null +++ b/src/Titano.Server/Api/AuthEndpoints.cs @@ -0,0 +1,105 @@ +using System.Globalization; +using Titano.Core.Model; +using Titano.Data; +using Titano.Protocol; +using Titano.Server.Configuration; +using Titano.Server.Security; + +namespace Titano.Server.Api; + +/// /api/v1/auth: sign-in (token or admin cookie), sign-out, password change, current account. +public static class AuthEndpoints +{ + public static void Map(RouteGroupBuilder api) + { + api.MapPost("/auth/login", LoginAsync).AllowAnonymousCaller(); + api.MapPost("/auth/logout", LogoutAsync).AllowDuringPasswordChange(); + api.MapPost("/auth/password", ChangePasswordAsync).AllowDuringPasswordChange(); + api.MapGet("/auth/me", (Caller caller) => Results.Json(Dto.User(caller.User), ProtocolJson.Default.UserDto)).AllowDuringPasswordChange(); + } + + private static async Task LoginAsync(HttpContext context, LoginRequest request, TitanoDatabase db, TokenService tokens, LoginGuard guard, + AuditLog audit, TimeProvider time) + { + var ct = context.RequestAborted; + if (!guard.TryAttempt(context.Connection.RemoteIpAddress?.ToString() ?? "unknown", out var wait)) return TooMany(context, "rate_limited", wait); + var account = Accounts.Key(request.Username); + if (!Accounts.IsValidUsername(account) || string.IsNullOrEmpty(request.Password)) + return ApiErrors.Result(context, StatusCodes.Status401Unauthorized, "invalid_credentials"); + if (guard.IsLocked(account, out wait)) return TooMany(context, "account_locked", wait); + if (request.Cookie && !SessionCookies.IsSecureTransport(context)) return ApiErrors.BadRequest(context, "https_required"); + + var user = await db.Users.GetByUsernameAsync(account, ct); + bool valid; + if (user is null) + { + PasswordHasher.VerifyDummy(request.Password); + valid = false; + } + else + { + valid = PasswordHasher.Verify(request.Password, user.PasswordHash) && !user.Disabled; + } + if (!valid) + { + guard.RecordFailure(account); + await audit.WriteAsync(user?.Id, account, context, "login_failed", "user", user?.Id.ToString()); + return ApiErrors.Result(context, StatusCodes.Status401Unauthorized, "invalid_credentials"); + } + + guard.RecordSuccess(account); + var signedIn = user! with { LastLogin = time.GetUtcNow() }; + if (PasswordHasher.NeedsRehash(signedIn.PasswordHash)) + { + signedIn = signedIn with { PasswordHash = PasswordHasher.Hash(request.Password) }; + await db.Users.UpdateAsync(signedIn, ct); + } + else + { + await db.Users.RecordLoginAsync(signedIn.Id, signedIn.LastLogin.Value, ct); + } + var issued = await tokens.IssueAsync(signedIn, request.ClientName, ct); + await audit.WriteAsync(signedIn.Id, signedIn.Username, context, "login", "user", signedIn.Id.ToString(), issued.Stored.ClientName); + + string? secret = issued.Secret; + if (request.Cookie) + { + // The admin panel never sees the token itself; it travels only in the HttpOnly cookie. + SessionCookies.Issue(context, issued); + secret = null; + } + return Results.Json(new LoginResponse(secret, Dto.User(signedIn), signedIn.MustChangePassword, issued.Stored.Expires), ProtocolJson.Default.LoginResponse); + } + + private static async Task LogoutAsync(HttpContext context, Caller caller, TitanoDatabase db) + { + await db.Tokens.RevokeAsync(caller.TokenHash, context.RequestAborted); + if (caller.Source == CredentialSource.Cookie) SessionCookies.Clear(context); + return Results.NoContent(); + } + + private static async Task ChangePasswordAsync(HttpContext context, Caller caller, ChangePasswordRequest request, TitanoDatabase db, + AuditLog audit, ServerOptions options) + { + var ct = context.RequestAborted; + if (string.IsNullOrEmpty(request.CurrentPassword) || !PasswordHasher.Verify(request.CurrentPassword, caller.User.PasswordHash)) + return ApiErrors.BadRequest(context, "wrong_password"); + if (!PasswordHasher.IsAcceptable(request.NewPassword)) + return ApiErrors.BadRequest(context, "weak_password", PasswordHasher.MinimumLength.ToString(CultureInfo.InvariantCulture), PasswordHasher.MinimumLength); + if (request.NewPassword == request.CurrentPassword) return ApiErrors.BadRequest(context, "same_password"); + + var updated = caller.User with { PasswordHash = PasswordHasher.Hash(request.NewPassword), MustChangePassword = false }; + await db.Users.UpdateAsync(updated, ct); + await db.Tokens.RevokeAllExceptAsync(caller.Id, caller.TokenHash, ct); + await audit.WriteAsync(caller, context, "password_changed", "user", caller.Id.ToString()); + if (caller.User.Role == UserRole.Admin) AdminBootstrap.ForgetInitialPassword(options); + return Results.NoContent(); + } + + private static IResult TooMany(HttpContext context, string code, TimeSpan wait) + { + var seconds = Math.Max(1, (int)Math.Ceiling(wait.TotalSeconds)); + context.Response.Headers.RetryAfter = seconds.ToString(CultureInfo.InvariantCulture); + return ApiErrors.Result(context, StatusCodes.Status429TooManyRequests, code, seconds.ToString(CultureInfo.InvariantCulture)); + } +} diff --git a/src/Titano.Server/Api/Dto.cs b/src/Titano.Server/Api/Dto.cs new file mode 100644 index 00000000..7d02ecd3 --- /dev/null +++ b/src/Titano.Server/Api/Dto.cs @@ -0,0 +1,90 @@ +using System.Text; +using System.Text.Json; +using Titano.Core.Model; +using Titano.Protocol; + +namespace Titano.Server.Api; + +/// Domain entities → API v1 contracts. +public static class Dto +{ + public static UserDto User(User u) => new( + u.Id, u.Username, u.DisplayName, Role(u.Role), (long)u.Permissions, u.QuotaBytes, u.UsedBytes, u.Disabled, u.MustChangePassword, + u.Created, u.LastLogin); + + public static string Role(UserRole role) => role.ToString().ToLowerInvariant(); + + public static SequenceDto Sequence(Sequence s) => new( + s.Id, s.Title, s.DisplayTitle, s.Folder, s.CameraMake, s.CameraModel, s.Lens, s.FirstCapture, s.LastCapture, s.FrameCount, + s.MedianInterval, s.TotalBytes, (long)s.Status, s.RenderedCount, s.HeroFrameId, s.Tags, s.Rating, s.Notes, s.SettingsVersion, + s.FramesVersion, s.WarningCount, s.OwnerUserId, s.UpdatedAt); + + public static FrameDto Frame(Frame f) => new( + f.Id, f.SequenceId, f.Index, f.FileName, f.Size, f.Sha256, f.CapturedAt, f.UtcOffsetMinutes, f.ExposureTime, f.FNumber, f.Iso, + f.FocalLength, f.Orientation, f.Width, f.Height, f.Format.ToString().ToLowerInvariant(), (long)f.Flags); + + public static RenderDto Render(Render r) => new(r.Id, r.PresetName, r.Codec, r.Width, r.Height, r.Fps, r.DurationSeconds, r.Bytes, r.Created, r.ProducedBy); + + public static JobDto Job(Job j) => new( + j.Id, JobWire.Kind(j.Kind), j.TargetIds, j.TargetLabel, JobWire.State(j.State), JobWire.Priority(j.Priority), j.ProgressDone, + j.ProgressTotal, j.ItemsPerSecond, j.Eta?.TotalSeconds, j.LogTail, j.NodeId, j.Attempts, j.Error, j.Created, j.Started, j.Finished); + + public static AuditEntryDto Audit(AuditEntry a) => new(a.Id, a.At, a.UserId, a.Username, a.Action, a.TargetKind, a.TargetId, a.Details, a.RemoteAddress); + + public static RenderNodeDto Node(RenderNode node, string username) => + new(node.Id, node.ClientName, username, Capabilities(node)?.Encoders ?? [], node.LastHeartbeat, node.State); + + /// What a node announced at registration (stored as its capabilities JSON). + public static RegisterNodeRequest? Capabilities(RenderNode node) + { + if (node.CapabilitiesJson is null) return null; + try + { + return JsonSerializer.Deserialize(node.CapabilitiesJson, ProtocolJson.Default.RegisterNodeRequest); + } + catch (JsonException) + { + return null; + } + } +} + +/// Wire names of job kinds, states and priorities: kebab-case (deep-analyse) and lower-case (running). +public static class JobWire +{ + public static string Kind(JobKind kind) + { + var name = kind.ToString(); + var sb = new StringBuilder(name.Length + 4); + for (var i = 0; i < name.Length; i++) + { + if (i > 0 && char.IsUpper(name[i])) sb.Append('-'); + sb.Append(char.ToLowerInvariant(name[i])); + } + return sb.ToString(); + } + + public static bool TryParseKind(string? text, out JobKind kind) => TryParse(text, Kind, out kind); + + public static string State(JobState state) => state.ToString().ToLowerInvariant(); + + public static bool TryParseState(string? text, out JobState state) => TryParse(text, State, out state); + + public static string Priority(JobPriority priority) => priority.ToString().ToLowerInvariant(); + + public static bool TryParsePriority(string? text, out JobPriority priority) => TryParse(text, Priority, out priority); + + private static bool TryParse(string? text, Func name, out T value) where T : struct, Enum + { + foreach (var candidate in Enum.GetValues()) + { + if (string.Equals(name(candidate), text, StringComparison.OrdinalIgnoreCase)) + { + value = candidate; + return true; + } + } + value = default; + return false; + } +} diff --git a/src/Titano.Server/Api/FileEndpoints.cs b/src/Titano.Server/Api/FileEndpoints.cs new file mode 100644 index 00000000..71bfc53e --- /dev/null +++ b/src/Titano.Server/Api/FileEndpoints.cs @@ -0,0 +1,59 @@ +using System.Globalization; +using Titano.Core.Model; +using Titano.Data; +using Titano.Server.Security; +using Titano.Server.Storage; + +namespace Titano.Server.Api; + +/// +/// Streamed archive files with Range support (BRIEF §14.3): originals, proxies, thumbnails and exports. Paths come +/// from the database by id and are resolved under the archive root only. +/// +public static class FileEndpoints +{ + private const string Immutable = "private, max-age=31536000, immutable"; + private const string Regenerable = "private, max-age=3600"; + + public static void Map(RouteGroupBuilder api) + { + // Originals feed editing clients and render nodes; viewers browse proxies, thumbnails and exports. + api.MapGet("/frames/{id:guid}/original", OriginalAsync).Requires(Permissions.Edit | Permissions.RenderNode); + api.MapGet("/frames/{id:guid}/proxy", (HttpContext c, Guid id, TitanoDatabase db, ServerArchive a, IFrameAssetLocator l) => AssetAsync(c, id, FrameAsset.Proxy, db, a, l)) + .Requires(Permissions.Read | Permissions.RenderNode); + api.MapGet("/frames/{id:guid}/thumb", (HttpContext c, Guid id, TitanoDatabase db, ServerArchive a, IFrameAssetLocator l) => AssetAsync(c, id, FrameAsset.Thumbnail, db, a, l)) + .Requires(Permissions.Read); + api.MapGet("/renders/{id:guid}/file", RenderAsync).Requires(Permissions.Read); + } + + private static async Task OriginalAsync(HttpContext context, Guid id, TitanoDatabase db, ServerArchive archive) + { + if (await db.Frames.GetAsync(id, context.RequestAborted) is not { } frame) return ApiErrors.NotFound(context, "frame"); + if (ArchivePaths.ExistingFile(archive.Root, frame.RelativePath) is not { } file) return ApiErrors.NotFound(context, "file_missing"); + return FileResponses.Serve(context, file, 0, file.Length, FileResponses.ContentTypeOf(frame.FileName), Immutable, frame.Sha256, frame.FileName) + ?? ApiErrors.NotFound(context, "file_missing"); + } + + private static async Task AssetAsync(HttpContext context, Guid id, FrameAsset kind, TitanoDatabase db, ServerArchive archive, IFrameAssetLocator locator) + { + var ct = context.RequestAborted; + if (await db.Frames.GetAsync(id, ct) is not { } frame || await db.Sequences.GetAsync(frame.SequenceId, ct) is not { } sequence) + return ApiErrors.NotFound(context, "frame"); + var location = await locator.LocateAsync(archive.Current, sequence, frame, kind, ct); + if (location is null || !ArchivePaths.IsInside(archive.Root, location.Path)) return ApiErrors.NotFound(context, "asset_missing"); + var file = new FileInfo(location.Path); + if (!file.Exists || file.LinkTarget is not null) return ApiErrors.NotFound(context, "asset_missing"); + var etag = string.Create(CultureInfo.InvariantCulture, $"{kind}-{frame.Id:N}-{file.LastWriteTimeUtc.Ticks}-{location.Offset}"); + return FileResponses.Serve(context, file, location.Offset, location.Length, location.ContentType, Regenerable, etag) + ?? ApiErrors.NotFound(context, "asset_missing"); + } + + private static async Task RenderAsync(HttpContext context, Guid id, TitanoDatabase db, ServerArchive archive) + { + if (await db.Renders.GetAsync(id, context.RequestAborted) is not { } render) return ApiErrors.NotFound(context, "render"); + if (ArchivePaths.ExistingFile(archive.Root, render.OutputPath) is not { } file) return ApiErrors.NotFound(context, "file_missing"); + var etag = string.Create(CultureInfo.InvariantCulture, $"r-{render.Id:N}-{file.Length}-{file.LastWriteTimeUtc.Ticks}"); + return FileResponses.Serve(context, file, 0, file.Length, FileResponses.ContentTypeOf(file.Name), Immutable, etag, file.Name) + ?? ApiErrors.NotFound(context, "file_missing"); + } +} diff --git a/src/Titano.Server/Api/JobEndpoints.cs b/src/Titano.Server/Api/JobEndpoints.cs new file mode 100644 index 00000000..c07e5f00 --- /dev/null +++ b/src/Titano.Server/Api/JobEndpoints.cs @@ -0,0 +1,155 @@ +using System.Text.Json; +using Titano.Core.Model; +using Titano.Data; +using Titano.Protocol; +using Titano.Server.Jobs; +using Titano.Server.Security; +using Titano.Server.Storage; + +namespace Titano.Server.Api; + +/// +/// /api/v1/jobs (BRIEF §14.6): queue and history, creation, cancel, pause, resume, retry and priority. Changes +/// reach WebSocket clients as jobState and jobProgress events. +/// +public static class JobEndpoints +{ + /// Kinds that work on sequences (targets are sequence ids) and need the edit right. + private static readonly HashSet SequenceKinds = [JobKind.LightAnalyse, JobKind.DeepAnalyse, JobKind.Proxies, JobKind.Thumbs, JobKind.Verify]; + + /// Archive-wide maintenance (target: the archive) for administrators. + private static readonly HashSet ArchiveKinds = [JobKind.Scan, JobKind.Rebuild, JobKind.Maintenance, JobKind.Duplicates, JobKind.Orphans]; + + public static void Map(RouteGroupBuilder api) + { + var jobs = api.MapGroup("/jobs").Requires(Permissions.Read); + jobs.MapGet("", ListAsync); + jobs.MapPost("", CreateAsync); + jobs.MapGet("/{id:guid}", async (HttpContext context, Guid id, TitanoDatabase db) => + await db.Jobs.GetAsync(id, context.RequestAborted) is { } job + ? Results.Json(Dto.Job(job), ProtocolJson.Default.JobDto) + : ApiErrors.NotFound(context, "job")); + jobs.MapPost("/{id:guid}/cancel", (HttpContext c, Caller caller, Guid id, TitanoDatabase db, JobEngineHost host) => + ActAsync(c, caller, id, db, [JobState.Queued, JobState.Running, JobState.Paused], (job, ct) => host.Engine.CancelAsync(job.Id, ct))); + jobs.MapPost("/{id:guid}/pause", (HttpContext c, Caller caller, Guid id, TitanoDatabase db, JobEngineHost host) => + ActAsync(c, caller, id, db, [JobState.Queued, JobState.Running], (job, ct) => host.Engine.PauseAsync(job.Id, ct))); + jobs.MapPost("/{id:guid}/resume", (HttpContext c, Caller caller, Guid id, TitanoDatabase db, JobEngineHost host) => + ActAsync(c, caller, id, db, [JobState.Paused], (job, ct) => host.Engine.ResumeAsync(job.Id, ct))); + jobs.MapPost("/{id:guid}/retry", (HttpContext c, Caller caller, Guid id, TitanoDatabase db, JobEngineHost host) => + ActAsync(c, caller, id, db, [JobState.Failed, JobState.Cancelled], (job, ct) => host.Engine.ResumeAsync(job.Id, ct))); + jobs.MapPost("/{id:guid}/priority", PriorityAsync); + } + + private static async Task ListAsync(HttpContext context, TitanoDatabase db, string? state, string? kind, string? cursor, int? limit) + { + JobState? stateFilter = null; + JobKind? kindFilter = null; + if (!string.IsNullOrEmpty(state)) + { + if (!JobWire.TryParseState(state, out var s)) return ApiErrors.BadRequest(context, "invalid_filter", "state"); + stateFilter = s; + } + if (!string.IsNullOrEmpty(kind)) + { + if (!JobWire.TryParseKind(kind, out var k)) return ApiErrors.BadRequest(context, "invalid_filter", "kind"); + kindFilter = k; + } + var page = await db.Jobs.ListAsync(stateFilter, kindFilter, limit ?? 50, string.IsNullOrEmpty(cursor) ? null : cursor, context.RequestAborted); + return Results.Json(new Page([.. page.Items.Select(Dto.Job)], page.NextCursor, null), ProtocolJson.Default.PageJobDto); + } + + private static async Task CreateAsync(HttpContext context, Caller caller, CreateJobRequest request, TitanoDatabase db, JobEngineHost host, + ServerArchive archive, AuditLog audit) + { + var ct = context.RequestAborted; + if (!JobWire.TryParseKind(request.Kind, out var kind) || !(kind == JobKind.Render || SequenceKinds.Contains(kind) || ArchiveKinds.Contains(kind))) + return ApiErrors.BadRequest(context, "invalid_job_kind"); + var needed = kind == JobKind.Render ? Permissions.Render : SequenceKinds.Contains(kind) ? Permissions.Edit : Permissions.ManageArchive; + if (!caller.Has(needed)) return ApiErrors.Forbidden(context); + if (!host.CanRun(kind)) return ApiErrors.Conflict(context, "job_unavailable", JobWire.Kind(kind)); + + IReadOnlyList targets; + string label; + if (ArchiveKinds.Contains(kind)) + { + targets = [archive.Id]; + label = archive.Current.Name; + } + else + { + var ids = request.Target ?? []; + if (ids.Count == 0 || (kind == JobKind.Render && ids.Count != 1)) return ApiErrors.BadRequest(context, "invalid_parameters", "target"); + var titles = new List(ids.Count); + foreach (var id in ids) + { + if (await db.Sequences.GetAsync(id, ct) is not { } sequence) return ApiErrors.NotFound(context, "sequence"); + titles.Add(sequence.DisplayTitle); + } + targets = ids; + label = titles.Count == 1 ? titles[0] : string.Join(", ", titles.Take(3)) + (titles.Count > 3 ? " …" : string.Empty); + } + if (kind == JobKind.Render) + { + if (ParseRender(request.ParametersJson) is not { } render) return ApiErrors.BadRequest(context, "invalid_parameters", "parametersJson"); + label = $"{label} → {render.PresetName}"; + } + else if (request.ParametersJson is not null && !IsJsonObject(request.ParametersJson)) + { + return ApiErrors.BadRequest(context, "invalid_parameters", "parametersJson"); + } + + var job = await host.Engine.EnqueueAsync(kind, targets, label, request.ParametersJson, JobPriority.User, caller.Id, ct); + await audit.WriteAsync(caller, context, "job_created", "job", job.Id.ToString(), $"{JobWire.Kind(kind)} · {label}"); + return Results.Json(Dto.Job(job), ProtocolJson.Default.JobDto, statusCode: StatusCodes.Status202Accepted); + } + + private static async Task PriorityAsync(HttpContext context, Caller caller, Guid id, JobPriorityRequest request, TitanoDatabase db, JobEngineHost host) + { + if (!JobWire.TryParsePriority(request.Priority, out var priority)) return ApiErrors.BadRequest(context, "invalid_priority"); + return await ActAsync(context, caller, id, db, [JobState.Queued, JobState.Paused], async (job, ct) => + { + await db.Jobs.SetPriorityAsync(job.Id, priority, ct); + host.Publish(job with { Priority = priority }); + }); + } + + /// Applies when the job is in one of states and the caller created it or administers the archive. + private static async Task ActAsync(HttpContext context, Caller caller, Guid id, TitanoDatabase db, JobState[] allowed, Func action) + { + var ct = context.RequestAborted; + if (await db.Jobs.GetAsync(id, ct) is not { } job) return ApiErrors.NotFound(context, "job"); + if (job.CreatedBy != caller.Id && !caller.Has(Permissions.ManageArchive)) return ApiErrors.Forbidden(context); + if (!allowed.Contains(job.State)) return ApiErrors.Conflict(context, "invalid_state", JobWire.State(job.State)); + await action(job, ct); + var updated = await db.Jobs.GetAsync(id, ct) ?? job; + return Results.Json(Dto.Job(updated), ProtocolJson.Default.JobDto); + } + + /// Render parameters, or null when they are missing or incomplete. + public static RenderJobParameters? ParseRender(string? json) + { + if (string.IsNullOrWhiteSpace(json)) return null; + try + { + var p = JsonSerializer.Deserialize(json, ProtocolJson.Default.RenderJobParameters); + return p is { PresetName.Length: > 0, PresetJson.Length: > 0, Codec.Length: > 0, Width: > 0, Height: > 0 } && IsJsonObject(p.PresetJson) ? p : null; + } + catch (JsonException) + { + return null; + } + } + + private static bool IsJsonObject(string json) + { + try + { + using var document = JsonDocument.Parse(json); + return document.RootElement.ValueKind == JsonValueKind.Object; + } + catch (JsonException) + { + return false; + } + } +} diff --git a/src/Titano.Server/Api/NodeEndpoints.cs b/src/Titano.Server/Api/NodeEndpoints.cs new file mode 100644 index 00000000..24594b1d --- /dev/null +++ b/src/Titano.Server/Api/NodeEndpoints.cs @@ -0,0 +1,57 @@ +using Titano.Core.Model; +using Titano.Data; +using Titano.Protocol; +using Titano.Server.Events; +using Titano.Server.Jobs; +using Titano.Server.Security; + +namespace Titano.Server.Api; + +/// /api/v1/nodes (BRIEF §14.7): render node registration, heartbeat, claim, progress, failure and output upload. +public static class NodeEndpoints +{ + public static void Map(RouteGroupBuilder api) + { + api.MapGet("/nodes", ListAsync).Requires(Permissions.Read | Permissions.RenderNode); + api.MapDelete("/nodes/{id:guid}", DeleteAsync).Requires(Permissions.RenderNode | Permissions.ManageArchive); + + var nodes = api.MapGroup("/nodes").Requires(Permissions.RenderNode); + nodes.MapPost("/register", async (HttpContext c, Caller caller, RegisterNodeRequest request, RenderNodeService service) => + (await service.RegisterAsync(caller, request, c.RequestAborted)).ToResult(c, n => Results.Json(n, ProtocolJson.Default.RenderNodeDto))); + nodes.MapPost("/{id:guid}/heartbeat", async (HttpContext c, Caller caller, Guid id, RenderNodeService service) => + (await service.HeartbeatAsync(caller, id, c.RequestAborted)).ToResult(c, _ => Results.NoContent())); + nodes.MapPost("/{id:guid}/claim", async (HttpContext c, Caller caller, Guid id, RenderNodeService service) => + (await service.ClaimAsync(caller, id, c.RequestAborted)).ToResult(c, job => job is null ? Results.NoContent() : Results.Json(job, ProtocolJson.Default.ClaimedJob))); + nodes.MapPost("/{id:guid}/jobs/{jobId:guid}/progress", async (HttpContext c, Caller caller, Guid id, Guid jobId, NodeProgress progress, RenderNodeService service) => + (await service.ProgressAsync(caller, id, jobId, progress, c.RequestAborted)).ToResult(c, _ => Results.NoContent())); + nodes.MapPost("/{id:guid}/jobs/{jobId:guid}/failed", async (HttpContext c, Caller caller, Guid id, Guid jobId, NodeJobFailure failure, RenderNodeService service) => + (await service.FailAsync(caller, id, jobId, failure, c.RequestAborted)).ToResult(c, _ => Results.NoContent())); + nodes.MapPost("/{id:guid}/jobs/{jobId:guid}/result", async (HttpContext c, Caller caller, Guid id, Guid jobId, RenderResultRequest request, RenderNodeService service) => + (await service.StartResultAsync(caller, id, jobId, request, c.RequestAborted)) + .ToResult(c, r => Results.Json(r, ProtocolJson.Default.StartUploadResponse, statusCode: StatusCodes.Status201Created))); + nodes.MapPost("/{id:guid}/jobs/{jobId:guid}/result/complete", async (HttpContext c, Caller caller, Guid id, Guid jobId, RenderNodeService service, + IHostApplicationLifetime lifetime) => + // The commit runs to the end even if the node disconnects meanwhile. + (await service.CompleteResultAsync(caller, id, jobId, lifetime.ApplicationStopping)).ToResult(c, r => Results.Json(r, ProtocolJson.Default.RenderDto))); + } + + private static async Task ListAsync(HttpContext context, TitanoDatabase db) + { + var ct = context.RequestAborted; + var users = (await db.Users.ListAsync(ct)).ToDictionary(u => u.Id, u => u.Username); + var nodes = await db.RenderNodes.ListAsync(ct); + return Results.Json(nodes.Select(n => Dto.Node(n, users.GetValueOrDefault(n.UserId, string.Empty))).ToList(), ProtocolJson.Default.ListRenderNodeDto); + } + + /// Forgets a node (its owner or an archive admin); its running jobs go back to the queue. + private static async Task DeleteAsync(HttpContext context, Caller caller, Guid id, TitanoDatabase db, EventHub events) + { + var ct = context.RequestAborted; + if (await db.RenderNodes.GetAsync(id, ct) is not { } node) return ApiErrors.NotFound(context, "node"); + if (node.UserId != caller.Id && !caller.Has(Permissions.ManageArchive)) return ApiErrors.Forbidden(context); + await db.Jobs.RequeueRunningAsync(RenderNodeService.NodeKey(node), ct); + await db.RenderNodes.DeleteAsync(id, ct); + events.Publish(new NodeStateEvent(node.Id, node.ClientName, RenderNodeService.Offline)); + return Results.NoContent(); + } +} diff --git a/src/Titano.Server/Api/Outcome.cs b/src/Titano.Server/Api/Outcome.cs new file mode 100644 index 00000000..faf8d4df --- /dev/null +++ b/src/Titano.Server/Api/Outcome.cs @@ -0,0 +1,17 @@ +namespace Titano.Server.Api; + +/// Result of a server operation: a value, or an HTTP status with an API error code (and optional details). +public readonly record struct Outcome(T? Value, int Status, string? Code, string? Details) +{ + public bool Succeeded => Code is null; + + public static Outcome Ok(T value) => new(value, StatusCodes.Status200OK, null, null); + + public static Outcome Fail(int status, string code, string? details = null) => new(default, status, code, details); + + /// The API response: for a value, the error body otherwise. + public IResult ToResult(HttpContext context, Func ok) => + Succeeded ? ok(Value!) : ApiErrors.Result(context, Status, Code!, Details); + + public static implicit operator Outcome(T value) => Ok(value); +} diff --git a/src/Titano.Server/Api/SequenceEndpoints.cs b/src/Titano.Server/Api/SequenceEndpoints.cs new file mode 100644 index 00000000..dd687af4 --- /dev/null +++ b/src/Titano.Server/Api/SequenceEndpoints.cs @@ -0,0 +1,313 @@ +using System.Buffers.Text; +using System.Globalization; +using System.Text; +using System.Text.Json; +using Titano.Core.Model; +using Titano.Data; +using Titano.Data.Repositories; +using Titano.Protocol; +using Titano.Server.Configuration; +using Titano.Server.Events; +using Titano.Server.Security; +using Titano.Server.Storage; + +namespace Titano.Server.Api; + +/// +/// /api/v1/sequences: library listing with filters and keyset paging, detail with ETag, edits with +/// settings_version optimistic concurrency, the sync manifest, frames, analysis, statistics and renders. +/// +public static class SequenceEndpoints +{ + private const int MaxTitle = 200; + private const int MaxNotes = 2000; + private const int MaxTags = 50; + + public static void Map(RouteGroupBuilder api) + { + // Uploaders list sequences to pick an upload target. + var sequences = api.MapGroup("/sequences").Requires(Permissions.Read | Permissions.Upload); + sequences.MapGet("", ListAsync); + sequences.MapPost("", CreateAsync).Requires(Permissions.Upload | Permissions.Edit); + sequences.MapGet("/{id:guid}", GetAsync); + sequences.MapPatch("/{id:guid}", UpdateAsync).Requires(Permissions.Edit); + sequences.MapDelete("/{id:guid}", DeleteAsync).Requires(Permissions.Delete); + sequences.MapGet("/{id:guid}/manifest", ManifestAsync).Requires(Permissions.Read); + sequences.MapGet("/{id:guid}/frames", FramesAsync).Requires(Permissions.Read); + sequences.MapGet("/{id:guid}/analysis", AnalysisAsync).Requires(Permissions.Read); + sequences.MapGet("/{id:guid}/stats", StatsAsync).Requires(Permissions.Read); + sequences.MapGet("/{id:guid}/renders", RendersAsync).Requires(Permissions.Read); + } + + private static async Task ListAsync(HttpContext context, TitanoDatabase db, ServerArchive archive, int? year, string? camera, string? status, + string? tag, string? q, string? owner, string? sort, string? cursor, int? limit, Caller caller) + { + if (!TryParseStatus(status, out var with, out var without)) return ApiErrors.BadRequest(context, "invalid_filter", "status"); + if (!TryParseSort(sort, out var order)) return ApiErrors.BadRequest(context, "invalid_filter", "sort"); + Guid? ownerId = null; + if (owner == "me") ownerId = caller.Id; + else if (owner is not null && Guid.TryParse(owner, out var parsed)) ownerId = parsed; + else if (owner is not null) return ApiErrors.BadRequest(context, "invalid_filter", "owner"); + + var query = new SequenceQuery + { + ArchiveId = archive.Id, + Year = year, + Camera = string.IsNullOrWhiteSpace(camera) ? null : camera, + WithStatus = with, + WithoutStatus = without, + Tag = string.IsNullOrWhiteSpace(tag) ? null : tag, + Text = string.IsNullOrWhiteSpace(q) ? null : q, + OwnerUserId = ownerId, + Sort = order, + Limit = limit ?? 100, + Cursor = string.IsNullOrEmpty(cursor) ? null : cursor, + }; + var page = await db.Sequences.ListAsync(query, context.RequestAborted); + long? total = query.Cursor is null ? await db.Sequences.CountAsync(query, context.RequestAborted) : null; + return Results.Json(new Titano.Protocol.Page([.. page.Items.Select(Dto.Sequence)], page.NextCursor, total), ProtocolJson.Default.PageSequenceDto); + } + + private static async Task CreateAsync(HttpContext context, Caller caller, CreateSequenceRequest request, TitanoDatabase db, ServerArchive archive, + AuditLog audit, EventHub events, TimeProvider time) + { + var title = request.Title?.Trim(); + if (title is { Length: > MaxTitle }) return ApiErrors.BadRequest(context, "bad_request", "title"); + var hint = request.CameraHint?.Trim(); + var sequence = new Sequence + { + Id = Guid.CreateVersion7(), + ArchiveId = archive.Id, + Title = string.IsNullOrEmpty(title) ? null : title, + // Placed in the archive by the ingest when its first frame arrives. + Folder = string.Empty, + CameraModel = string.IsNullOrEmpty(hint) ? null : hint[..Math.Min(hint.Length, MaxTitle)], + Origin = SequenceOrigin.Server, + SyncState = SyncState.InSync, + OwnerUserId = caller.Id, + UpdatedAt = time.GetUtcNow(), + }; + await db.Sequences.UpsertAsync(sequence, context.RequestAborted); + await audit.WriteAsync(caller, context, "sequence_created", "sequence", sequence.Id.ToString(), sequence.DisplayTitle); + events.Publish(new SequenceChangedEvent(sequence.Id, "created", sequence.FramesVersion, sequence.SettingsVersion)); + context.Response.Headers.ETag = ETag(sequence); + return Results.Json(Dto.Sequence(sequence), ProtocolJson.Default.SequenceDto, statusCode: StatusCodes.Status201Created); + } + + private static async Task GetAsync(HttpContext context, Guid id, TitanoDatabase db) + { + if (await db.Sequences.GetAsync(id, context.RequestAborted) is not { } sequence) return ApiErrors.NotFound(context, "sequence"); + var etag = ETag(sequence); + if (context.Request.Headers.IfNoneMatch == etag) return Results.StatusCode(StatusCodes.Status304NotModified); + context.Response.Headers.ETag = etag; + return Results.Json(Dto.Sequence(sequence), ProtocolJson.Default.SequenceDto); + } + + private static async Task UpdateAsync(HttpContext context, Caller caller, Guid id, UpdateSequenceRequest request, TitanoDatabase db, + AuditLog audit, EventHub events) + { + if (request.Title is { Length: > MaxTitle }) return ApiErrors.BadRequest(context, "bad_request", "title"); + if (request.Notes is { Length: > MaxNotes }) return ApiErrors.BadRequest(context, "bad_request", "notes"); + if (request.Rating is < 0 or > 5) return ApiErrors.BadRequest(context, "bad_request", "rating"); + if (request.Tags is { } tags && (tags.Count > MaxTags || tags.Any(t => t is null || t.Trim().Length > TagRepository.MaxLength))) + return ApiErrors.BadRequest(context, "bad_request", "tags"); + if (request.SettingsJson is not null && !IsJson(request.SettingsJson)) return ApiErrors.BadRequest(context, "invalid_settings"); + + var expected = request.ExpectedSettingsVersion ?? IfMatchVersion(context); + var edit = new SequenceEdit { Title = request.Title, Tags = request.Tags, Rating = request.Rating, Notes = request.Notes, SettingsJson = request.SettingsJson }; + var result = await db.Sequences.EditAsync(id, edit, expected, context.RequestAborted); + switch (result.Outcome) + { + case SequenceEditOutcome.NotFound: + return ApiErrors.NotFound(context, "sequence"); + case SequenceEditOutcome.Conflict: + context.Response.Headers.ETag = ETag(result.Sequence!); + return ApiErrors.Conflict(context, "settings_conflict", result.Sequence!.SettingsVersion.ToString(CultureInfo.InvariantCulture)); + } + var sequence = result.Sequence!; + await audit.WriteAsync(caller, context, "sequence_updated", "sequence", id.ToString(), Changes(request)); + events.Publish(new SequenceChangedEvent(id, "settings", sequence.FramesVersion, sequence.SettingsVersion)); + context.Response.Headers.ETag = ETag(sequence); + return Results.Json(Dto.Sequence(sequence), ProtocolJson.Default.SequenceDto); + } + + /// + /// Removes a sequence from the index and moves its folder into the archive's trash (BRIEF §0.2 rule 8: nothing is + /// erased). ?dryRun=true only reports what would happen. Owners and admins only. + /// + private static async Task DeleteAsync(HttpContext context, Caller caller, Guid id, bool? dryRun, TitanoDatabase db, ServerArchive archive, + ServerOptions options, AuditLog audit, EventHub events) + { + if (await db.Sequences.GetAsync(id, context.RequestAborted) is not { } sequence) return ApiErrors.NotFound(context, "sequence"); + if (!caller.IsAdmin && sequence.OwnerUserId != caller.Id) return ApiErrors.Forbidden(context); + var folder = string.IsNullOrEmpty(sequence.Folder) ? null : ArchivePaths.Resolve(archive.Root, sequence.Folder); + var trash = Path.Combine(options.TrashPath, id.ToString("N")); + var hasFolder = folder is not null && Directory.Exists(folder); + var plan = new SequenceDeletePlan(id, sequence.Folder, hasFolder ? Path.GetRelativePath(archive.Root, trash).Replace('\\', '/') : null, + sequence.FrameCount, sequence.TotalBytes, sequence.RenderedCount, dryRun == true); + if (dryRun == true) return Results.Json(plan, ProtocolJson.Default.SequenceDeletePlan); + + if (hasFolder) + { + Directory.CreateDirectory(options.TrashPath); + Directory.Move(folder!, trash); + } + await db.Sequences.DeleteAsync(id, context.RequestAborted); + await audit.WriteAsync(caller, context, "sequence_deleted", "sequence", id.ToString(), sequence.DisplayTitle); + events.Publish(new SequenceChangedEvent(id, "deleted", sequence.FramesVersion, sequence.SettingsVersion)); + return Results.Json(plan, ProtocolJson.Default.SequenceDeletePlan); + } + + /// Everything a client needs to mirror a sequence and detect changes (BRIEF §14.5); ETag over the version counters. + private static async Task ManifestAsync(HttpContext context, Guid id, TitanoDatabase db) + { + var ct = context.RequestAborted; + if (await db.Sequences.GetAsync(id, ct) is not { } sequence) return ApiErrors.NotFound(context, "sequence"); + var etag = string.Create(CultureInfo.InvariantCulture, $"\"m{sequence.SettingsVersion}-{sequence.FramesVersion}-{sequence.RenderedCount}\""); + if (context.Request.Headers.IfNoneMatch == etag) return Results.StatusCode(StatusCodes.Status304NotModified); + + var hasProxy = sequence.Status.HasFlag(SequenceStatus.ProxiesBuilt); + var hasThumb = sequence.Status.HasFlag(SequenceStatus.ThumbsBuilt); + var frames = new List(sequence.FrameCount); + var after = -1; + while (true) + { + var batch = await db.Frames.ListBySequenceAsync(id, after, 1000, ct); + frames.AddRange(batch.Select(f => new ManifestFrame(f.Id, f.Index, f.FileName, f.Size, f.Sha256 ?? string.Empty, hasProxy, hasThumb))); + if (batch.Count < 1000) break; + after = batch[^1].Index; + } + var renders = await db.Renders.ListBySequenceAsync(id, ct); + context.Response.Headers.ETag = etag; + return Results.Json(new SequenceManifest(id, sequence.FramesVersion, sequence.SettingsVersion, sequence.SettingsJson, sequence.Tags, frames, + [.. renders.Select(Dto.Render)]), ProtocolJson.Default.SequenceManifest); + } + + private static async Task FramesAsync(HttpContext context, Guid id, string? cursor, int? limit, TitanoDatabase db) + { + var ct = context.RequestAborted; + if (await db.Sequences.GetAsync(id, ct) is null) return ApiErrors.NotFound(context, "sequence"); + var after = -1; + if (!string.IsNullOrEmpty(cursor) && !FrameCursor.TryDecode(cursor, out after)) return ApiErrors.BadRequest(context, "invalid_cursor"); + var take = Math.Clamp(limit ?? 500, 1, 1000); + var frames = await db.Frames.ListBySequenceAsync(id, after, take, ct); + var next = frames.Count == take ? FrameCursor.Encode(frames[^1].Index) : null; + return Results.Json(new Titano.Protocol.Page([.. frames.Select(Dto.Frame)], next, null), ProtocolJson.Default.PageFrameDto); + } + + private static async Task AnalysisAsync(HttpContext context, Guid id, TitanoDatabase db) => + await db.Sequences.GetAsync(id, context.RequestAborted) switch + { + null => ApiErrors.NotFound(context, "sequence"), + { AnalysisJson: { } json } => Results.Text(json, "application/json; charset=utf-8"), + _ => ApiErrors.NotFound(context, "analysis"), + }; + + private static async Task StatsAsync(HttpContext context, Guid id, TitanoDatabase db) + { + if (await db.Sequences.GetAsync(id, context.RequestAborted) is not { } s) return ApiErrors.NotFound(context, "sequence"); + var stats = await db.Frames.GetSequenceStatsAsync(id, context.RequestAborted); + return Results.Json(new SequenceStatsDto( + id, s.FrameCount, s.TotalBytes, s.FirstCapture, s.LastCapture, s.MedianInterval, + [.. stats.ByFormat.Select(f => new KeyValueCount(f.Format.ToString().ToLowerInvariant(), f.Photos, f.Bytes))], + stats.Missing, stats.Corrupt, stats.Duplicate, stats.Excluded), ProtocolJson.Default.SequenceStatsDto); + } + + private static async Task RendersAsync(HttpContext context, Guid id, TitanoDatabase db) + { + if (await db.Sequences.GetAsync(id, context.RequestAborted) is null) return ApiErrors.NotFound(context, "sequence"); + var renders = await db.Renders.ListBySequenceAsync(id, context.RequestAborted); + return Results.Json(renders.Select(Dto.Render).ToList(), ProtocolJson.Default.ListRenderDto); + } + + /// Changes whenever anything the sequence representation shows changes (versions, status, statistics). + private static string ETag(Sequence s) => + string.Create(CultureInfo.InvariantCulture, $"\"{s.SettingsVersion}-{s.FramesVersion}-{s.UpdatedAt.UtcTicks}\""); + + /// The settings version named by an If-Match ETag of , if any. + private static int? IfMatchVersion(HttpContext context) + { + var header = context.Request.Headers.IfMatch.ToString().Trim('"', ' '); + var dash = header.IndexOf('-', StringComparison.Ordinal); + return dash > 0 && int.TryParse(header.AsSpan(0, dash), NumberStyles.None, CultureInfo.InvariantCulture, out var version) ? version : null; + } + + private static bool TryParseStatus(string? text, out SequenceStatus with, out SequenceStatus without) + { + with = without = SequenceStatus.None; + if (string.IsNullOrWhiteSpace(text)) return true; + foreach (var part in text.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + var clear = part.StartsWith('-'); + var name = clear ? part[1..] : part; + var flag = Enum.GetValues().FirstOrDefault(v => v != SequenceStatus.None && v.ToString().Equals(name, StringComparison.OrdinalIgnoreCase)); + if (flag == SequenceStatus.None) return false; + if (clear) without |= flag; + else with |= flag; + } + return true; + } + + private static bool TryParseSort(string? text, out SequenceSort sort) + { + sort = SequenceSort.Newest; + if (string.IsNullOrEmpty(text)) return true; + foreach (var candidate in Enum.GetValues()) + { + if (candidate.ToString().Equals(text, StringComparison.OrdinalIgnoreCase)) + { + sort = candidate; + return true; + } + } + return false; + } + + private static bool IsJson(string text) + { + try + { + using var _ = JsonDocument.Parse(text); + return true; + } + catch (JsonException) + { + return false; + } + } + + private static string Changes(UpdateSequenceRequest r) + { + var fields = new List(); + if (r.Title is not null) fields.Add("title"); + if (r.Tags is not null) fields.Add("tags"); + if (r.Rating is not null) fields.Add("rating"); + if (r.Notes is not null) fields.Add("notes"); + if (r.SettingsJson is not null) fields.Add("settings"); + return string.Join(", ", fields); + } + + /// Opaque cursor of the frame listing: the index of the last frame returned. + private static class FrameCursor + { + private const string Prefix = "frames|"; + + public static string Encode(int index) => + Base64Url.EncodeToString(Encoding.UTF8.GetBytes(Prefix + index.ToString(CultureInfo.InvariantCulture))); + + public static bool TryDecode(string cursor, out int index) + { + index = -1; + try + { + var text = Encoding.UTF8.GetString(Base64Url.DecodeFromChars(cursor)); + return text.StartsWith(Prefix, StringComparison.Ordinal) + && int.TryParse(text.AsSpan(Prefix.Length), NumberStyles.None, CultureInfo.InvariantCulture, out index); + } + catch (FormatException) + { + return false; + } + } + } +} diff --git a/src/Titano.Server/Api/SystemEndpoints.cs b/src/Titano.Server/Api/SystemEndpoints.cs index a7da0fe4..d104bde0 100644 --- a/src/Titano.Server/Api/SystemEndpoints.cs +++ b/src/Titano.Server/Api/SystemEndpoints.cs @@ -1,40 +1,48 @@ using System.Globalization; using System.Reflection; using Titano.Core.Localization; +using Titano.Data; using Titano.Protocol; using Titano.Server.Configuration; +using Titano.Server.Security; namespace Titano.Server.Api; -/// /health, /api/v1/server/info and /api/v1/i18n/{culture}. +/// /health, /api/v1/server/info and /api/v1/i18n/{culture} (all anonymous). public static class SystemEndpoints { private static readonly string[] AdminAreas = ["Common", "Admin"]; + /// Languages with translated resources (English plus every satellite of the catalog), for the admin language menu. + private static readonly Lazy AdminLanguages = new(() => string.Join(',', Languages.Shipped.Where(code => + { + if (code == "en") return true; + try + { + typeof(Loc).Assembly.GetSatelliteAssembly(CultureInfo.GetCultureInfo(code)); + return true; + } + catch (FileNotFoundException) + { + return false; + } + }))); + public static string Version { get; } = typeof(SystemEndpoints).Assembly.GetCustomAttribute()?.InformationalVersion ?? "0.0.0"; - public static void Map(WebApplication app, ServerOptions options, ServerState state) + public static void Map(WebApplication app, RouteGroupBuilder api) { - app.MapGet("/health", () => - { - var checks = new Dictionary(StringComparer.Ordinal) - { - ["archive"] = Directory.Exists(options.ArchivePath) ? "ok" : "missing", - ["data"] = Directory.Exists(options.DataPath) ? "ok" : "missing", - }; - return Results.Json(new HealthStatus("ok", Version, checks), ProtocolJson.Default.HealthStatus); - }); + app.MapGet("/health", HealthAsync); - var api = app.MapGroup("/api/v1"); - api.MapGet("/server/info", () => Results.Json(new ServerInfo( + api.MapGet("/server/info", (ServerOptions options, ServerState state) => Results.Json(new ServerInfo( options.ServerName, Version, ApiVersion.Current, state.Capabilities, state.TlsFingerprint, ["jpeg", "png", "tiff", "dng", "cr2", "cr3", "nef", "arw", "raf", "orf", "rw2", "pef", "heic", "webp"], - new ServerLimits(options.MaxUploadBytes, options.ChunkSizeBytes)), ProtocolJson.Default.ServerInfo)); + new ServerLimits(options.MaxUploadBytes, options.ChunkSizeBytes, PasswordHasher.MinimumLength)), ProtocolJson.Default.ServerInfo)).AllowAnonymousCaller(); api.MapGet("/i18n/{culture}", (string culture) => { @@ -52,15 +60,46 @@ public static class SystemEndpoints foreach (var (k, v) in Loc.GetArea(area, ci)) strings[k] = v; strings["$culture"] = ci.Name; strings["$rtl"] = ci.TextInfo.IsRightToLeft ? "true" : "false"; + strings["$languages"] = AdminLanguages.Value; return Results.Json(strings, ProtocolJson.Default.DictionaryStringString); - }); + }).AllowAnonymousCaller(); + } + + /// Liveness for the container health check: 503 while the database cannot be reached. + private static async Task HealthAsync(HttpContext context, ServerOptions options, TitanoDatabase db) + { + var checks = new Dictionary(StringComparer.Ordinal) + { + ["archive"] = Directory.Exists(options.ArchivePath) ? "ok" : "missing", + ["data"] = Directory.Exists(options.DataPath) ? "ok" : "missing", + }; + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(context.RequestAborted); + timeout.CancelAfter(TimeSpan.FromSeconds(3)); + try + { + await using var session = await db.Database.OpenAsync(timeout.Token); + await session.ExecuteAsync("SELECT 1", [], timeout.Token); + checks["database"] = "ok"; + } + catch (Exception ex) when (ex is DatabaseException or OperationCanceledException or IOException) + { + checks["database"] = "unavailable"; + } + var healthy = checks["database"] == "ok"; + return Results.Json(new HealthStatus(healthy ? "ok" : "unavailable", Version, checks), ProtocolJson.Default.HealthStatus, + statusCode: healthy ? StatusCodes.Status200OK : StatusCodes.Status503ServiceUnavailable); } } /// Process-wide facts computed at startup. public sealed class ServerState { - public string? TlsFingerprint { get; set; } + public string? TlsFingerprint { get; init; } + + /// Engine and version of the database ("PostgreSQL 17.2", "SQLite 3.45.1"). + public string Database { get; init; } = string.Empty; + public List Capabilities { get; } = ["uploads", "sync", "jobs", "render-nodes", "events"]; + public DateTimeOffset StartedAt { get; } = DateTimeOffset.UtcNow; } diff --git a/src/Titano.Server/Api/UploadEndpoints.cs b/src/Titano.Server/Api/UploadEndpoints.cs new file mode 100644 index 00000000..92ccfa53 --- /dev/null +++ b/src/Titano.Server/Api/UploadEndpoints.cs @@ -0,0 +1,44 @@ +using Microsoft.AspNetCore.Http.Features; +using Titano.Core.Model; +using Titano.Protocol; +using Titano.Server.Configuration; +using Titano.Server.Security; +using Titano.Server.Storage; + +namespace Titano.Server.Api; + +/// +/// /api/v1/uploads (BRIEF §14.4). Chunk, state and cancel calls also serve render nodes uploading their output, +/// which start and complete through the node endpoints. +/// +public static class UploadEndpoints +{ + public static void Map(RouteGroupBuilder api) + { + var uploads = api.MapGroup("/uploads").Requires(Permissions.Upload | Permissions.RenderNode); + uploads.MapPost("", StartAsync).Requires(Permissions.Upload); + uploads.MapPut("/{id:guid}/chunks/{index:int}", ChunkAsync); + uploads.MapGet("/{id:guid}", async (HttpContext context, Caller caller, Guid id, UploadService service) => + (await service.GetStateAsync(caller, id, context.RequestAborted)).ToResult(context, s => Results.Json(s, ProtocolJson.Default.UploadState))); + uploads.MapDelete("/{id:guid}", async (HttpContext context, Caller caller, Guid id, UploadService service) => + (await service.CancelAsync(caller, id, context.RequestAborted)).ToResult(context, _ => Results.NoContent())); + uploads.MapPost("/{id:guid}/complete", async (HttpContext context, Caller caller, Guid id, UploadService service) => + (await service.CompleteAsync(caller, id)).ToResult(context, r => Results.Json(r, ProtocolJson.Default.CompleteUploadResponse))) + .Requires(Permissions.Upload); + } + + private static async Task StartAsync(HttpContext context, Caller caller, StartUploadRequest request, UploadService service) + { + var outcome = await service.StartAsync(caller, request, context.RequestAborted); + return outcome.ToResult(context, r => Results.Json(r, ProtocolJson.Default.StartUploadResponse, + statusCode: r.AlreadyExists ? StatusCodes.Status200OK : StatusCodes.Status201Created)); + } + + private static async Task ChunkAsync(HttpContext context, Caller caller, Guid id, int index, UploadService service, ServerOptions options) + { + // Chunks are the only large request bodies; every other endpoint keeps Kestrel's small default limit. + if (context.Features.Get() is { IsReadOnly: false } limit) limit.MaxRequestBodySize = options.ChunkSizeBytes; + var outcome = await service.ReceiveChunkAsync(caller, id, index, context.Request, context.RequestAborted); + return outcome.ToResult(context, r => Results.Json(r, ProtocolJson.Default.ChunkReceived)); + } +} diff --git a/src/Titano.Server/Api/UserEndpoints.cs b/src/Titano.Server/Api/UserEndpoints.cs new file mode 100644 index 00000000..2a4099fc --- /dev/null +++ b/src/Titano.Server/Api/UserEndpoints.cs @@ -0,0 +1,157 @@ +using System.Globalization; +using Titano.Core.Model; +using Titano.Data; +using Titano.Protocol; +using Titano.Server.Security; + +namespace Titano.Server.Api; + +/// /api/v1/users (admin): accounts, roles, quotas, render-node right, temporary passwords. +public static class UserEndpoints +{ + private const int MaxDisplayName = 100; + + public static void Map(RouteGroupBuilder api) + { + var users = api.MapGroup("/users").Requires(Permissions.ManageUsers); + users.MapGet("", ListAsync); + users.MapPost("", CreateAsync); + users.MapGet("/{id:guid}", GetAsync); + users.MapPatch("/{id:guid}", UpdateAsync); + users.MapDelete("/{id:guid}", DeleteAsync); + users.MapPost("/{id:guid}/reset-password", ResetPasswordAsync); + } + + private static async Task ListAsync(HttpContext context, TitanoDatabase db, TimeProvider time) + { + var ct = context.RequestAborted; + var users = await db.Users.ListAsync(ct); + var latest = await db.Tokens.LatestByUserAsync(ct); + var uploads = await db.Uploads.CommittedCountByUserAsync(time.GetUtcNow().AddDays(-30), ct); + var rows = users.Select(u => + { + var token = latest.GetValueOrDefault(u.Id); + return new UserSummaryDto(Dto.User(u), token?.LastUsed ?? u.LastLogin, token?.ClientName, uploads.GetValueOrDefault(u.Id)); + }).ToList(); + return Results.Json(rows, ProtocolJson.Default.ListUserSummaryDto); + } + + private static async Task GetAsync(HttpContext context, Guid id, TitanoDatabase db) => + await db.Users.GetAsync(id, context.RequestAborted) is { } user + ? Results.Json(Dto.User(user), ProtocolJson.Default.UserDto) + : ApiErrors.NotFound(context, "user"); + + private static async Task CreateAsync(HttpContext context, Caller caller, CreateUserRequest request, TitanoDatabase db, AuditLog audit, TimeProvider time) + { + var username = request.Username?.Trim(); + if (!Accounts.IsValidUsername(username)) return ApiErrors.BadRequest(context, "invalid_username"); + if (!Accounts.TryParseRole(request.Role, out var role)) return ApiErrors.BadRequest(context, "invalid_role"); + if (request.QuotaBytes < 0) return ApiErrors.BadRequest(context, "bad_request", "quotaBytes"); + var password = string.IsNullOrEmpty(request.Password) ? TemporaryPassword.Generate() : request.Password; + if (!PasswordHasher.IsAcceptable(password)) return WeakPassword(context); + + var user = new User + { + Id = Guid.CreateVersion7(), + Username = username!, + DisplayName = Clean(request.DisplayName), + PasswordHash = PasswordHasher.Hash(password), + Role = role, + Permissions = Accounts.PermissionsFor(role, request.AllowRenderNode), + QuotaBytes = request.QuotaBytes, + MustChangePassword = true, + Created = time.GetUtcNow(), + }; + try + { + await db.Users.CreateAsync(user, context.RequestAborted); + } + catch (DatabaseException ex) when (ex.IsUniqueViolation) + { + return ApiErrors.Conflict(context, "username_taken"); + } + await audit.WriteAsync(caller, context, "user_created", "user", user.Id.ToString(), $"{user.Username} · {Dto.Role(role)}"); + return Results.Json(new TemporaryPasswordResponse(Dto.User(user), password), ProtocolJson.Default.TemporaryPasswordResponse, statusCode: StatusCodes.Status201Created); + } + + private static async Task UpdateAsync(HttpContext context, Caller caller, Guid id, UpdateUserRequest request, TitanoDatabase db, AuditLog audit) + { + var ct = context.RequestAborted; + if (await db.Users.GetAsync(id, ct) is not { } user) return ApiErrors.NotFound(context, "user"); + var role = user.Role; + if (request.Role is not null && !Accounts.TryParseRole(request.Role, out role)) return ApiErrors.BadRequest(context, "invalid_role"); + if (request.QuotaBytes < 0) return ApiErrors.BadRequest(context, "bad_request", "quotaBytes"); + var updated = user with + { + DisplayName = request.DisplayName is null ? user.DisplayName : Clean(request.DisplayName), + Role = role, + Permissions = Accounts.PermissionsFor(role, request.AllowRenderNode ?? user.Permissions.HasFlag(Permissions.RenderNode)), + QuotaBytes = request.QuotaBytes ?? user.QuotaBytes, + Disabled = request.Disabled ?? user.Disabled, + }; + var losesAdmin = IsActiveAdmin(user) && !IsActiveAdmin(updated); + if (losesAdmin && user.Id == caller.Id) return ApiErrors.Conflict(context, "self"); + if (losesAdmin && !await HasOtherActiveAdminAsync(db, user.Id, ct)) return ApiErrors.Conflict(context, "last_admin"); + + await db.Users.UpdateAsync(updated, ct); + if (updated.Disabled && !user.Disabled) await db.Tokens.RevokeAllAsync(id, ct); + await audit.WriteAsync(caller, context, "user_updated", "user", id.ToString(), Changes(user, updated)); + return Results.Json(Dto.User(updated), ProtocolJson.Default.UserDto); + } + + private static async Task DeleteAsync(HttpContext context, Caller caller, Guid id, TitanoDatabase db, AuditLog audit) + { + var ct = context.RequestAborted; + if (id == caller.Id) return ApiErrors.Conflict(context, "self"); + if (await db.Users.GetAsync(id, ct) is not { } user) return ApiErrors.NotFound(context, "user"); + if (IsActiveAdmin(user) && !await HasOtherActiveAdminAsync(db, id, ct)) return ApiErrors.Conflict(context, "last_admin"); + + // Render jobs of the user's nodes go back to the queue before the nodes disappear with the account. + foreach (var node in (await db.RenderNodes.ListAsync(ct)).Where(n => n.UserId == id)) + await db.Jobs.RequeueRunningAsync(node.Id.ToString(), ct); + var reassigned = await db.Sequences.ReassignOwnerAsync(id, caller.Id, ct); + await db.Users.DeleteAsync(id, ct); + await audit.WriteAsync(caller, context, "user_deleted", "user", id.ToString(), + $"{user.Username} · {reassigned.ToString(CultureInfo.InvariantCulture)}"); + return Results.NoContent(); + } + + private static async Task ResetPasswordAsync(HttpContext context, Caller caller, Guid id, ResetPasswordRequest? request, TitanoDatabase db, AuditLog audit) + { + var ct = context.RequestAborted; + if (await db.Users.GetAsync(id, ct) is not { } user) return ApiErrors.NotFound(context, "user"); + var password = string.IsNullOrEmpty(request?.Password) ? TemporaryPassword.Generate() : request.Password; + if (!PasswordHasher.IsAcceptable(password)) return WeakPassword(context); + var updated = user with { PasswordHash = PasswordHasher.Hash(password), MustChangePassword = true }; + await db.Users.UpdateAsync(updated, ct); + await db.Tokens.RevokeAllAsync(id, ct); + await audit.WriteAsync(caller, context, "password_reset", "user", id.ToString(), user.Username); + return Results.Json(new TemporaryPasswordResponse(Dto.User(updated), password), ProtocolJson.Default.TemporaryPasswordResponse); + } + + private static bool IsActiveAdmin(User user) => user.Role == UserRole.Admin && !user.Disabled; + + private static async Task HasOtherActiveAdminAsync(TitanoDatabase db, Guid except, CancellationToken ct) => + (await db.Users.ListAsync(ct)).Any(u => u.Id != except && IsActiveAdmin(u)); + + private static IResult WeakPassword(HttpContext context) => + ApiErrors.BadRequest(context, "weak_password", PasswordHasher.MinimumLength.ToString(CultureInfo.InvariantCulture), PasswordHasher.MinimumLength); + + private static string? Clean(string? displayName) + { + var trimmed = displayName?.Trim(); + return string.IsNullOrEmpty(trimmed) ? null : trimmed[..Math.Min(trimmed.Length, MaxDisplayName)]; + } + + /// Audit detail listing the fields an edit changed (never secrets). + private static string Changes(User before, User after) + { + var changes = new List(); + if (before.DisplayName != after.DisplayName) changes.Add("displayName"); + if (before.Role != after.Role) changes.Add("role=" + Dto.Role(after.Role)); + if (before.Permissions != after.Permissions) changes.Add("permissions"); + if (before.QuotaBytes != after.QuotaBytes) changes.Add("quota=" + after.QuotaBytes.ToString(CultureInfo.InvariantCulture)); + if (before.Disabled != after.Disabled) changes.Add(after.Disabled ? "disabled" : "enabled"); + return string.Join(", ", changes); + } +} diff --git a/src/Titano.Server/Configuration/ServerOptions.cs b/src/Titano.Server/Configuration/ServerOptions.cs index d1bcbfa8..b6658bcc 100644 --- a/src/Titano.Server/Configuration/ServerOptions.cs +++ b/src/Titano.Server/Configuration/ServerOptions.cs @@ -1,5 +1,7 @@ +using System.Globalization; using System.Text.Json; using System.Text.Json.Serialization; +using Titano.Data; namespace Titano.Server.Configuration; @@ -23,15 +25,44 @@ public enum DatabaseKind /// /// Server configuration (BRIEF §14.9): environment variables override /data/server.json, which overrides defaults. +/// Settings applied from the environment are listed in ; the admin panel cannot change them. /// public sealed class ServerOptions { + /// Environment variable → setting it fixes, and how its text is applied. + private static readonly (string Variable, string Setting, Action Apply)[] Variables = + [ + ("TITANO_SERVER_NAME", nameof(ServerName), (o, v) => o.ServerName = v), + ("TITANO_ARCHIVE_PATH", nameof(ArchivePath), (o, v) => o.ArchivePath = v), + ("TITANO_ADMIN_USER", nameof(AdminUser), (o, v) => o.AdminUser = v), + ("TITANO_ADMIN_PASSWORD", nameof(AdminPassword), (o, v) => o.AdminPassword = v), + ("TITANO_DB", nameof(Database), (o, v) => o.ApplyDatabase(v)), + ("TITANO_DB_HOST", nameof(DbHost), (o, v) => o.DbHost = v), + ("TITANO_DB_PORT", nameof(DbPort), (o, v) => o.DbPort = Int("TITANO_DB_PORT", v)), + ("TITANO_DB_NAME", nameof(DbName), (o, v) => o.DbName = v), + ("TITANO_DB_USER", nameof(DbUser), (o, v) => o.DbUser = v), + ("TITANO_DB_PASSWORD", nameof(DbPassword), (o, v) => o.DbPassword = v), + ("TITANO_DB_SSLMODE", nameof(DbSslMode), (o, v) => o.DbSslMode = v), + ("TITANO_HTTP_PORT", nameof(HttpPort), (o, v) => o.HttpPort = Int("TITANO_HTTP_PORT", v)), + ("TITANO_HTTPS_PORT", nameof(HttpsPort), (o, v) => o.HttpsPort = Int("TITANO_HTTPS_PORT", v)), + ("TITANO_TLS_MODE", nameof(TlsMode), (o, v) => o.TlsMode = Enum.Parse(v, ignoreCase: true)), + ("TITANO_TLS_CERT", nameof(TlsCert), (o, v) => o.TlsCert = v), + ("TITANO_TLS_KEY", nameof(TlsKey), (o, v) => o.TlsKey = v), + ("TITANO_LANG", nameof(Language), (o, v) => o.Language = v), + ("TITANO_WORKERS", nameof(Workers), (o, v) => o.Workers = Math.Max(1, Int("TITANO_WORKERS", v))), + ("TITANO_DISCOVERY", nameof(Discovery), (o, v) => o.Discovery = bool.Parse(v)), + ]; + public string ServerName { get; set; } = "Titano Server"; public string ArchivePath { get; set; } = OperatingSystem.IsWindows() ? @"C:\TitanoArchive" : "/archive"; public string DataPath { get; set; } = OperatingSystem.IsWindows() ? Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), "Titano Server") : "/data"; public string? AdminUser { get; set; } public string? AdminPassword { get; set; } public DatabaseKind Database { get; set; } = OperatingSystem.IsWindows() ? DatabaseKind.Sqlite : DatabaseKind.Postgres; + + /// Complete connection string (from TITANO_DB when it holds one); overrides the Db* parts. + public string? DbConnectionString { get; set; } + public string DbHost { get; set; } = "postgres"; public int DbPort { get; set; } = 5432; public string DbName { get; set; } = "titano"; @@ -51,6 +82,27 @@ public sealed class ServerOptions public int JobHistoryDays { get; set; } = 90; public int TokenLifetimeDays { get; set; } = 30; + /// Uploads untouched for this long are abandoned: their partial files are deleted. + public int UploadRetentionHours { get; set; } = 48; + + /// Sign-in attempts accepted per minute and client address (BRIEF §14.2: 5). + public int LoginAttemptsPerMinute { get; set; } = 5; + + /// Consecutive failed sign-ins that lock an account name for . + public int LockoutThreshold { get; set; } = 10; + + public int LockoutMinutes { get; set; } = 15; + + /// A render node silent for this long is offline and its jobs go back to the queue. + public int NodeTimeoutSeconds { get; set; } = 90; + + /// Chunk uploads one user may run at the same time; more are answered 429. + public int ConcurrentChunksPerUser { get; set; } = 8; + + /// Settings fixed by environment variables (names of the properties above). + [JsonIgnore] + public IReadOnlyCollection Locked { get; private set; } = []; + [JsonIgnore] public string LogPath => Path.Combine(DataPath, "logs"); @@ -60,6 +112,17 @@ public sealed class ServerOptions [JsonIgnore] public string UploadsPath => Path.Combine(DataPath, "uploads"); + [JsonIgnore] + public string SettingsFile => Path.Combine(DataPath, "server.json"); + + /// Verified uploads wait here, on the archive volume, until the archive ingest moves them into place. + [JsonIgnore] + public string StagingPath => Path.Combine(ArchivePath, ".titano", "staging"); + + /// Deleted sequences are moved here (never erased by the server). + [JsonIgnore] + public string TrashPath => Path.Combine(ArchivePath, ".titano", "trash"); + /// Loads {DataPath}/server.json (if present) then applies TITANO_* environment variables. public static ServerOptions Load(IReadOnlyDictionary? environment = null) { @@ -82,35 +145,38 @@ public sealed class ServerOptions } options.DataPath = dataPath; - options.ServerName = Env("TITANO_SERVER_NAME") ?? options.ServerName; - options.ArchivePath = Env("TITANO_ARCHIVE_PATH") ?? options.ArchivePath; - options.AdminUser = Env("TITANO_ADMIN_USER") ?? options.AdminUser; - options.AdminPassword = Env("TITANO_ADMIN_PASSWORD") ?? options.AdminPassword; - if (Env("TITANO_DB") is { } db) options.Database = Enum.Parse(db, ignoreCase: true); - options.DbHost = Env("TITANO_DB_HOST") ?? options.DbHost; - if (Env("TITANO_DB_PORT") is { } port) options.DbPort = int.Parse(port, System.Globalization.CultureInfo.InvariantCulture); - options.DbName = Env("TITANO_DB_NAME") ?? options.DbName; - options.DbUser = Env("TITANO_DB_USER") ?? options.DbUser; - options.DbPassword = Env("TITANO_DB_PASSWORD") ?? options.DbPassword; - options.DbSslMode = Env("TITANO_DB_SSLMODE") ?? options.DbSslMode; - if (Env("TITANO_HTTP_PORT") is { } http) options.HttpPort = int.Parse(http, System.Globalization.CultureInfo.InvariantCulture); - if (Env("TITANO_HTTPS_PORT") is { } https) options.HttpsPort = int.Parse(https, System.Globalization.CultureInfo.InvariantCulture); - if (Env("TITANO_TLS_MODE") is { } tls) options.TlsMode = Enum.Parse(tls, ignoreCase: true); - options.TlsCert = Env("TITANO_TLS_CERT") ?? options.TlsCert; - options.TlsKey = Env("TITANO_TLS_KEY") ?? options.TlsKey; - options.Language = Env("TITANO_LANG") ?? options.Language; - if (Env("TITANO_WORKERS") is { } workers) options.Workers = Math.Max(1, int.Parse(workers, System.Globalization.CultureInfo.InvariantCulture)); - if (Env("TITANO_DISCOVERY") is { } discovery) options.Discovery = bool.Parse(discovery); + var locked = new List(); + foreach (var (variable, setting, apply) in Variables) + { + if (Env(variable) is not { } value) continue; + try + { + apply(options, value); + } + catch (Exception ex) when (ex is FormatException or ArgumentException) + { + throw new InvalidOperationException($"{variable} has an invalid value.", ex); + } + locked.Add(setting); + } + options.Locked = locked; options.Validate(); return options; } - private static Dictionary ReadEnvironment() + /// The database to open: , the SQLite file in the data folder, or PostgreSQL from the parts. + public DatabaseOptions ToDatabaseOptions() { - // Variables may differ only by case on Linux (no_proxy/NO_PROXY); the last one wins. - var result = new Dictionary(StringComparer.OrdinalIgnoreCase); - foreach (System.Collections.DictionaryEntry e in Environment.GetEnvironmentVariables()) result[(string)e.Key] = (string?)e.Value; - return result; + if (DbConnectionString is { Length: > 0 } cs) return DatabaseOptions.Parse(cs); + if (Database == DatabaseKind.Sqlite) return DatabaseOptions.Sqlite(Path.Combine(DataPath, "titano.db")); + var builder = new System.Text.StringBuilder() + .Append("Host=").Append(Quote(DbHost)) + .Append(";Port=").Append(DbPort.ToString(CultureInfo.InvariantCulture)) + .Append(";Database=").Append(Quote(DbName)) + .Append(";Username=").Append(Quote(DbUser)) + .Append(";SslMode=").Append(Quote(DbSslMode)); + if (DbPassword is not null) builder.Append(";Password=").Append(Quote(DbPassword)); + return DatabaseOptions.Postgres(builder.ToString()); } public void Validate() @@ -122,6 +188,35 @@ public sealed class ServerOptions // Port 0 asks the OS for a free port (used by the in-process test host). if (HttpPort is < 0 or > 65535 || HttpsPort is < 0 or > 65535) throw new InvalidOperationException("Ports must be between 0 and 65535."); + if (ChunkSizeBytes is < 64 * 1024 or > 64 * 1024 * 1024) + throw new InvalidOperationException("The chunk size must be between 64 KiB and 64 MiB."); + if (MaxUploadBytes < 1 || TokenLifetimeDays < 1 || JobHistoryDays < 1 || UploadRetentionHours < 1) + throw new InvalidOperationException("Upload size, token lifetime, job history and upload retention must be positive."); + } + + /// TITANO_DB is postgres, sqlite or a complete connection string (Host=… / Data Source=…). + private void ApplyDatabase(string value) + { + if (value.Contains('=', StringComparison.Ordinal)) + { + DbConnectionString = value; + Database = DatabaseOptions.Parse(value).Dialect == Dialect.Postgres ? DatabaseKind.Postgres : DatabaseKind.Sqlite; + return; + } + Database = Enum.Parse(value, ignoreCase: true); + } + + private static int Int(string variable, string value) => + int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var n) ? n : throw new FormatException($"{variable} must be a number."); + + private static string Quote(string value) => "'" + value.Replace("'", "''", StringComparison.Ordinal) + "'"; + + private static Dictionary ReadEnvironment() + { + // Variables may differ only by case on Linux (no_proxy/NO_PROXY); the last one wins. + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (System.Collections.DictionaryEntry e in Environment.GetEnvironmentVariables()) result[(string)e.Key] = (string?)e.Value; + return result; } } diff --git a/src/Titano.Server/Configuration/ServerSettingsStore.cs b/src/Titano.Server/Configuration/ServerSettingsStore.cs new file mode 100644 index 00000000..5d830471 --- /dev/null +++ b/src/Titano.Server/Configuration/ServerSettingsStore.cs @@ -0,0 +1,101 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using Titano.Core.IO; +using Titano.Core.Localization; +using Titano.Protocol; + +namespace Titano.Server.Configuration; + +/// +/// The admin panel's view of the configuration (BRIEF §14.8 Settings). Saving writes /data/server.json (only the +/// edited keys; secrets and environment values are never written) and applies what can change while running; ports, +/// TLS, workers, chunk size and job history wait for a restart. +/// +public sealed class ServerSettingsStore(ServerOptions options) +{ + private readonly Lock _gate = new(); + private ServerSettingsDto? _saved; + + public ServerSettingsDto Current() + { + lock (_gate) return _saved ?? Snapshot(options, restartRequired: false); + } + + /// Validates and saves; returns the error code and the offending setting, or null on success. + public (string Code, string Setting)? Save(ServerSettingsDto request) + { + if (Validate(request) is { } invalid) return ("invalid_setting", invalid); + lock (_gate) + { + var current = _saved ?? Snapshot(options, restartRequired: false); + var changes = Changes(current, request); + if (changes.FirstOrDefault(c => options.Locked.Contains(c.Name)) is { Name: { } locked }) return ("setting_locked", locked); + if (changes.Count == 0) return null; + + var file = File.Exists(options.SettingsFile) ? JsonNode.Parse(File.ReadAllText(options.SettingsFile)) as JsonObject ?? [] : []; + foreach (var (name, value) in changes) file[name] = value; + Directory.CreateDirectory(options.DataPath); + AtomicFile.WriteAllText(options.SettingsFile, file.ToJsonString(new JsonSerializerOptions { WriteIndented = true })); + + options.ServerName = request.ServerName.Trim(); + options.Discovery = request.Discovery; + options.Language = request.Language; + options.MaxUploadBytes = request.MaxUploadBytes; + options.TokenLifetimeDays = request.TokenLifetimeDays; + options.UploadRetentionHours = request.UploadRetentionHours; + var restart = request.HttpPort != options.HttpPort || request.HttpsPort != options.HttpsPort + || !string.Equals(request.TlsMode, options.TlsMode.ToString(), StringComparison.OrdinalIgnoreCase) + || request.TlsCert != options.TlsCert || request.TlsKey != options.TlsKey || request.Workers != options.Workers + || request.ChunkSizeBytes != options.ChunkSizeBytes || request.JobHistoryDays != options.JobHistoryDays; + _saved = request with { ServerName = request.ServerName.Trim(), Locked = [.. options.Locked], RestartRequired = restart }; + return null; + } + } + + private static ServerSettingsDto Snapshot(ServerOptions o, bool restartRequired) => new( + o.ServerName, o.HttpPort, o.HttpsPort, o.TlsMode.ToString().ToLowerInvariant(), o.TlsCert, o.TlsKey, o.Language, o.Workers, o.Discovery, + o.MaxUploadBytes, o.ChunkSizeBytes, o.JobHistoryDays, o.TokenLifetimeDays, o.UploadRetentionHours, [.. o.Locked], restartRequired); + + private static string? Validate(ServerSettingsDto r) + { + if (string.IsNullOrWhiteSpace(r.ServerName) || r.ServerName.Trim().Length > 60) return nameof(ServerOptions.ServerName); + if (r.HttpPort is < 1 or > 65535) return nameof(ServerOptions.HttpPort); + if (r.HttpsPort is < 1 or > 65535 || r.HttpsPort == r.HttpPort) return nameof(ServerOptions.HttpsPort); + if (!Enum.TryParse(r.TlsMode, ignoreCase: true, out var tls) || !Enum.IsDefined(tls)) return nameof(ServerOptions.TlsMode); + if (tls == TlsMode.Pem && (string.IsNullOrWhiteSpace(r.TlsCert) || !File.Exists(r.TlsCert))) return nameof(ServerOptions.TlsCert); + if (tls == TlsMode.Pem && (string.IsNullOrWhiteSpace(r.TlsKey) || !File.Exists(r.TlsKey))) return nameof(ServerOptions.TlsKey); + if (!Languages.Shipped.Contains(r.Language, StringComparer.OrdinalIgnoreCase)) return nameof(ServerOptions.Language); + if (r.Workers is < 1 or > 64) return nameof(ServerOptions.Workers); + if (r.MaxUploadBytes < 1024 * 1024) return nameof(ServerOptions.MaxUploadBytes); + if (r.ChunkSizeBytes is < 64 * 1024 or > 64 * 1024 * 1024) return nameof(ServerOptions.ChunkSizeBytes); + if (r.JobHistoryDays is < 1 or > 3650) return nameof(ServerOptions.JobHistoryDays); + if (r.TokenLifetimeDays is < 1 or > 365) return nameof(ServerOptions.TokenLifetimeDays); + if (r.UploadRetentionHours is < 1 or > 8760) return nameof(ServerOptions.UploadRetentionHours); + return null; + } + + /// Settings whose value differs, as server.json keys and JSON values. + private static List<(string Name, JsonNode? Value)> Changes(ServerSettingsDto a, ServerSettingsDto b) + { + var changes = new List<(string, JsonNode?)>(); + void Add(string name, T before, T after, Func json) + { + if (!EqualityComparer.Default.Equals(before, after)) changes.Add((name, json(after))); + } + Add(nameof(ServerOptions.ServerName), a.ServerName, b.ServerName.Trim(), v => JsonValue.Create(v)); + Add(nameof(ServerOptions.HttpPort), a.HttpPort, b.HttpPort, v => JsonValue.Create(v)); + Add(nameof(ServerOptions.HttpsPort), a.HttpsPort, b.HttpsPort, v => JsonValue.Create(v)); + Add(nameof(ServerOptions.TlsMode), a.TlsMode.ToLowerInvariant(), b.TlsMode.ToLowerInvariant(), v => JsonValue.Create(v)); + Add(nameof(ServerOptions.TlsCert), a.TlsCert, b.TlsCert, v => v is null ? null : JsonValue.Create(v)); + Add(nameof(ServerOptions.TlsKey), a.TlsKey, b.TlsKey, v => v is null ? null : JsonValue.Create(v)); + Add(nameof(ServerOptions.Language), a.Language, b.Language, v => JsonValue.Create(v)); + Add(nameof(ServerOptions.Workers), a.Workers, b.Workers, v => JsonValue.Create(v)); + Add(nameof(ServerOptions.Discovery), a.Discovery, b.Discovery, v => JsonValue.Create(v)); + Add(nameof(ServerOptions.MaxUploadBytes), a.MaxUploadBytes, b.MaxUploadBytes, v => JsonValue.Create(v)); + Add(nameof(ServerOptions.ChunkSizeBytes), a.ChunkSizeBytes, b.ChunkSizeBytes, v => JsonValue.Create(v)); + Add(nameof(ServerOptions.JobHistoryDays), a.JobHistoryDays, b.JobHistoryDays, v => JsonValue.Create(v)); + Add(nameof(ServerOptions.TokenLifetimeDays), a.TokenLifetimeDays, b.TokenLifetimeDays, v => JsonValue.Create(v)); + Add(nameof(ServerOptions.UploadRetentionHours), a.UploadRetentionHours, b.UploadRetentionHours, v => JsonValue.Create(v)); + return changes; + } +} diff --git a/src/Titano.Server/Events/EventEndpoints.cs b/src/Titano.Server/Events/EventEndpoints.cs new file mode 100644 index 00000000..aa315548 --- /dev/null +++ b/src/Titano.Server/Events/EventEndpoints.cs @@ -0,0 +1,103 @@ +using System.Net.WebSockets; +using System.Text.Json; +using Titano.Core.Model; +using Titano.Protocol; +using Titano.Server.Api; +using Titano.Server.Security; + +namespace Titano.Server.Events; + +/// +/// WS /api/v1/events: server events as one JSON text message each (, discriminated by +/// type). The handshake is authenticated like any API call (bearer header, or the admin cookie from this origin); +/// the token is checked again every minute and the socket closes once it is revoked or expires. +/// +public static class EventEndpoints +{ + private static readonly TimeSpan Revalidate = TimeSpan.FromSeconds(60); + + public static void Map(RouteGroupBuilder api) => api.MapGet("/events", RunAsync).Requires(Permissions.Read); + + private static async Task RunAsync(HttpContext context, Caller caller, EventHub hub, TokenService tokens, IHostApplicationLifetime lifetime) + { + if (!context.WebSockets.IsWebSocketRequest) return ApiErrors.BadRequest(context, "websocket_required"); + using var socket = await context.WebSockets.AcceptWebSocketAsync(); + using var subscription = hub.Subscribe(); + using var stop = CancellationTokenSource.CreateLinkedTokenSource(context.RequestAborted, lifetime.ApplicationStopping); + var receiving = ReceiveUntilClosedAsync(socket, stop); + var status = WebSocketCloseStatus.NormalClosure; + try + { + status = await SendAsync(socket, subscription, caller, tokens, stop.Token); + } + catch (Exception ex) when (ex is OperationCanceledException or WebSocketException) + { + status = lifetime.ApplicationStopping.IsCancellationRequested ? WebSocketCloseStatus.EndpointUnavailable : WebSocketCloseStatus.NormalClosure; + } + finally + { + await stop.CancelAsync(); + await receiving; + } + if (socket.State is WebSocketState.Open or WebSocketState.CloseReceived) + { + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + try + { + await socket.CloseOutputAsync(status, null, timeout.Token); + } + catch (Exception ex) when (ex is OperationCanceledException or WebSocketException) + { + // The peer is gone. + } + } + return Results.Empty; + } + + /// Forwards events until cancelled; returns the close status when the caller's token stops being valid. + private static async Task SendAsync(WebSocket socket, EventHub.Subscription subscription, Caller caller, TokenService tokens, + CancellationToken cancellationToken) + { + while (true) + { + using (var wait = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)) + { + wait.CancelAfter(Revalidate); + try + { + while (await subscription.Reader.WaitToReadAsync(wait.Token)) + { + while (subscription.Reader.TryRead(out var e)) + { + var payload = JsonSerializer.SerializeToUtf8Bytes(e, ProtocolJson.Default.ServerEvent); + await socket.SendAsync(payload, WebSocketMessageType.Text, endOfMessage: true, cancellationToken); + } + } + return WebSocketCloseStatus.NormalClosure; + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + // Quiet minute: fall through to the token check. + } + } + if (!await tokens.IsStillValidAsync(caller, cancellationToken)) return WebSocketCloseStatus.PolicyViolation; + } + } + + private static async Task ReceiveUntilClosedAsync(WebSocket socket, CancellationTokenSource stop) + { + var buffer = new byte[1024]; + try + { + while (!stop.IsCancellationRequested) + { + var result = await socket.ReceiveAsync(buffer, stop.Token); + if (result.MessageType == WebSocketMessageType.Close) break; + } + } + catch (Exception ex) when (ex is OperationCanceledException or WebSocketException) + { + } + await stop.CancelAsync(); + } +} diff --git a/src/Titano.Server/Events/EventHub.cs b/src/Titano.Server/Events/EventHub.cs new file mode 100644 index 00000000..3240ad4d --- /dev/null +++ b/src/Titano.Server/Events/EventHub.cs @@ -0,0 +1,47 @@ +using System.Threading.Channels; +using Titano.Protocol; + +namespace Titano.Server.Events; + +/// +/// Fan-out of server events to WebSocket subscribers (BRIEF §14.3). Each subscriber has a bounded queue that drops +/// its oldest events when it falls behind, so a slow client never holds up the server. +/// +public sealed class EventHub +{ + private const int QueueLength = 512; + private readonly Lock _gate = new(); + private Channel[] _subscribers = []; + + public int SubscriberCount => Volatile.Read(ref _subscribers).Length; + + public Subscription Subscribe() + { + var channel = Channel.CreateBounded(new BoundedChannelOptions(QueueLength) + { + FullMode = BoundedChannelFullMode.DropOldest, + SingleReader = true, + }); + lock (_gate) _subscribers = [.. _subscribers, channel]; + return new Subscription(this, channel); + } + + public void Publish(ServerEvent e) + { + foreach (var subscriber in Volatile.Read(ref _subscribers)) subscriber.Writer.TryWrite(e); + } + + private void Remove(Channel channel) + { + lock (_gate) _subscribers = [.. _subscribers.Where(c => c != channel)]; + channel.Writer.TryComplete(); + } + + /// One subscriber's queue; dispose to unsubscribe. + public sealed class Subscription(EventHub hub, Channel channel) : IDisposable + { + public ChannelReader Reader => channel.Reader; + + public void Dispose() => hub.Remove(channel); + } +} diff --git a/src/Titano.Server/Hosting/AdminPanel.cs b/src/Titano.Server/Hosting/AdminPanel.cs new file mode 100644 index 00000000..d92fde09 --- /dev/null +++ b/src/Titano.Server/Hosting/AdminPanel.cs @@ -0,0 +1,48 @@ +using System.Net; +using Microsoft.AspNetCore.Hosting.Server; +using Microsoft.AspNetCore.Hosting.Server.Features; +using Titano.Server.Configuration; + +namespace Titano.Server.Hosting; + +/// +/// Serves the embedded admin panel at /admin. Requests over plain HTTP from another machine are redirected to +/// the HTTPS listener, because the session cookie is Secure (BRIEF §14.2) and would not survive plain HTTP. +/// +public static class AdminPanel +{ + public static void Map(WebApplication app, ServerOptions options) + { + var files = new EmbeddedStaticFiles(typeof(AdminPanel).Assembly, "admin/"); + var server = app.Services.GetRequiredService(); + app.MapGet("/", () => Results.Redirect("/admin/")); + app.MapMethods("/admin/{**path}", ["GET", "HEAD"], async (HttpContext context, string? path) => + { + if (HttpsLocation(context, options, server) is { } secure) + { + context.Response.Redirect(secure); + return; + } + // Relative asset URLs need the trailing slash. + if (!context.Request.Path.Value!.EndsWith('/') && string.IsNullOrEmpty(path)) + { + context.Response.Redirect("/admin/"); + return; + } + if (!await files.TryServeAsync(context, path ?? string.Empty)) context.Response.StatusCode = StatusCodes.Status404NotFound; + }); + } + + private static string? HttpsLocation(HttpContext context, ServerOptions options, IServer server) + { + if (context.Request.IsHttps || options.TlsMode == TlsMode.Proxy) return null; + if (context.Connection.RemoteIpAddress is { } remote && IPAddress.IsLoopback(remote)) return null; + foreach (var address in server.Features.Get()?.Addresses ?? []) + { + if (!address.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) continue; + var port = address[(address.LastIndexOf(':') + 1)..].TrimEnd('/'); + return $"https://{context.Request.Host.Host}:{port}{context.Request.PathBase}{context.Request.Path}{context.Request.QueryString}"; + } + return null; + } +} diff --git a/src/Titano.Server/Hosting/DiscoveryBeaconService.cs b/src/Titano.Server/Hosting/DiscoveryBeaconService.cs new file mode 100644 index 00000000..3f2e8f29 --- /dev/null +++ b/src/Titano.Server/Hosting/DiscoveryBeaconService.cs @@ -0,0 +1,99 @@ +using System.Net; +using System.Net.NetworkInformation; +using System.Net.Sockets; +using Microsoft.AspNetCore.Hosting.Server; +using Microsoft.AspNetCore.Hosting.Server.Features; +using Titano.Protocol; +using Titano.Server.Api; +using Titano.Server.Configuration; + +namespace Titano.Server.Hosting; + +/// +/// LAN discovery (BRIEF §14.9, TITANO_DISCOVERY): every few seconds a is broadcast on +/// UDP port of every IPv4 network, so clients can offer this server on first run. +/// Switching discovery off in the settings stops it without a restart. +/// +public sealed class DiscoveryBeaconService(ServerOptions options, ServerState state, IServer server, ILogger logger) : BackgroundService +{ + private static readonly TimeSpan Interval = TimeSpan.FromSeconds(5); + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + using var udp = new UdpClient(AddressFamily.InterNetwork) { EnableBroadcast = true }; + var warned = false; + while (!stoppingToken.IsCancellationRequested) + { + if (options.Discovery) + { + try + { + await SendOnceAsync(udp, BroadcastTargets(), stoppingToken); + } + catch (SocketException ex) when (!warned) + { + warned = true; + logger.LogInformation("LAN discovery beacon cannot be sent: {Reason}", ex.Message); + } + catch (SocketException) + { + // Already reported once; networks may come and go. + } + } + try + { + await Task.Delay(Interval, stoppingToken); + } + catch (OperationCanceledException) + { + return; + } + } + } + + /// The beacon for the ports Kestrel actually listens on (HTTPS 0 when TLS is terminated elsewhere). + public DiscoveryBeacon CreateBeacon() => new( + options.ServerName, + SystemEndpoints.Version, + BoundPort(Uri.UriSchemeHttp) ?? options.HttpPort, + state.TlsFingerprint is null ? 0 : BoundPort(Uri.UriSchemeHttps) ?? options.HttpsPort, + state.TlsFingerprint); + + public async Task SendOnceAsync(UdpClient udp, IEnumerable targets, CancellationToken cancellationToken) + { + var datagram = CreateBeacon().ToBytes(); + foreach (var target in targets) await udp.SendAsync(datagram, target, cancellationToken); + } + + /// The limited broadcast address plus the directed broadcast of each IPv4 interface that is up. + private static List BroadcastTargets() + { + var targets = new List { new(IPAddress.Broadcast, DiscoveryBeacon.Port) }; + foreach (var nic in NetworkInterface.GetAllNetworkInterfaces()) + { + if (nic.OperationalStatus != OperationalStatus.Up || nic.NetworkInterfaceType == NetworkInterfaceType.Loopback) continue; + foreach (var unicast in nic.GetIPProperties().UnicastAddresses) + { + if (unicast.Address.AddressFamily != AddressFamily.InterNetwork || unicast.IPv4Mask is not { } mask) continue; + var address = unicast.Address.GetAddressBytes(); + var bits = mask.GetAddressBytes(); + for (var i = 0; i < 4; i++) address[i] = (byte)(address[i] | ~bits[i]); + var broadcast = new IPEndPoint(new IPAddress(address), DiscoveryBeacon.Port); + if (!targets.Contains(broadcast)) targets.Add(broadcast); + } + } + return targets; + } + + private int? BoundPort(string scheme) + { + var addresses = server.Features.Get()?.Addresses ?? []; + foreach (var address in addresses) + { + if (Uri.TryCreate(address.Replace("[::]", "localhost", StringComparison.Ordinal).Replace("0.0.0.0", "localhost", StringComparison.Ordinal), + UriKind.Absolute, out var uri) && uri.Scheme == scheme) + return uri.Port; + } + return null; + } +} diff --git a/src/Titano.Server/Hosting/FileLoggerProvider.cs b/src/Titano.Server/Hosting/FileLoggerProvider.cs index 7052813b..8f5c4f63 100644 --- a/src/Titano.Server/Hosting/FileLoggerProvider.cs +++ b/src/Titano.Server/Hosting/FileLoggerProvider.cs @@ -38,7 +38,16 @@ public sealed class LoggerSink(ILoggerFactory factory) : ILogSink { public void Write(CoreLogLevel level, string category, string message, Exception? exception) { - var logger = factory.CreateLogger("Titano." + category); + ILogger logger; + try + { + logger = factory.CreateLogger("Titano." + category); + } + catch (ObjectDisposedException) + { + // The host that installed this sink has shut down (engine code may outlive it); the line is dropped. + return; + } var ms = level switch { CoreLogLevel.Trace => MsLogLevel.Trace, diff --git a/src/Titano.Server/Hosting/ResponseMiddleware.cs b/src/Titano.Server/Hosting/ResponseMiddleware.cs new file mode 100644 index 00000000..5536bf57 --- /dev/null +++ b/src/Titano.Server/Hosting/ResponseMiddleware.cs @@ -0,0 +1,74 @@ +using Titano.Protocol; +using Titano.Server.Api; + +namespace Titano.Server.Hosting; + +/// +/// First middleware of the pipeline: API version and security headers on every response, and the +/// { code, message, details } error body for failures that did not produce one (bad requests, unhandled +/// exceptions, unknown methods). +/// +public sealed class ResponseMiddleware(RequestDelegate next, ILogger logger) +{ + /// The admin panel loads nothing from elsewhere and is never framed. + private const string AdminPolicy = + "default-src 'self'; img-src 'self' data:; style-src 'self'; script-src 'self'; connect-src 'self'; " + + "frame-ancestors 'none'; base-uri 'none'; form-action 'self'"; + + public async Task InvokeAsync(HttpContext context) + { + var headers = context.Response.Headers; + headers[ApiVersion.Header] = ApiVersion.Current.ToString(System.Globalization.CultureInfo.InvariantCulture); + headers.XContentTypeOptions = "nosniff"; + headers["Referrer-Policy"] = "no-referrer"; + var api = context.Request.Path.StartsWithSegments("/api"); + if (api) headers.CacheControl = "no-store"; + if (context.Request.Path.StartsWithSegments("/admin")) + { + headers.ContentSecurityPolicy = AdminPolicy; + headers.XFrameOptions = "DENY"; + } + + try + { + await next(context); + } + catch (BadHttpRequestException ex) when (!context.Response.HasStarted) + { + await ApiErrors.WriteAsync(context, ex.StatusCode, CodeFor(ex.StatusCode)); + return; + } + catch (FormatException) when (!context.Response.HasStarted && context.Request.Query.ContainsKey("cursor")) + { + // Repositories reject cursors that do not belong to the listing. + await ApiErrors.WriteAsync(context, StatusCodes.Status400BadRequest, "invalid_cursor"); + return; + } + catch (OperationCanceledException) when (context.RequestAborted.IsCancellationRequested) + { + return; + } + catch (Exception ex) when (!context.Response.HasStarted) + { + logger.LogError(ex, "Unhandled error on {Method} {Path}", context.Request.Method, context.Request.Path); + await ApiErrors.WriteAsync(context, StatusCodes.Status500InternalServerError, "internal"); + return; + } + + var response = context.Response; + if (api && response.StatusCode >= 400 && !response.HasStarted && response.ContentType is null) + await ApiErrors.WriteAsync(context, response.StatusCode, CodeFor(response.StatusCode)); + } + + private static string CodeFor(int status) => status switch + { + StatusCodes.Status400BadRequest => "bad_request", + StatusCodes.Status401Unauthorized => "unauthorized", + StatusCodes.Status403Forbidden => "forbidden", + StatusCodes.Status404NotFound => "not_found", + StatusCodes.Status405MethodNotAllowed => "method_not_allowed", + StatusCodes.Status413PayloadTooLarge => "too_large", + StatusCodes.Status415UnsupportedMediaType => "unsupported_media_type", + _ => "internal", + }; +} diff --git a/src/Titano.Server/Hosting/ServerCommandLine.cs b/src/Titano.Server/Hosting/ServerCommandLine.cs new file mode 100644 index 00000000..a4751919 --- /dev/null +++ b/src/Titano.Server/Hosting/ServerCommandLine.cs @@ -0,0 +1,68 @@ +namespace Titano.Server.Hosting; + +/// What Titano.Server was asked to do. +public enum ServerCommand +{ + /// Run in the console (Docker, Linux, development). + Run, + + /// Run under the Windows Service Control Manager (the command line the installed service uses). + Service, + + InstallService, + UninstallService, + + /// Probe /health of the local server and exit 0/1 (container health check). + HealthCheck, + + Help, +} + +/// Command-line switches of the server; anything else is passed on to the web host. +public sealed record ServerCommandLine(ServerCommand Command, IReadOnlyList HostArguments, string? Error) +{ + public const string Usage = """ + Titano Server — time-lapse archive, uploads, jobs and rendering. + + Usage: Titano.Server [--service | --install-service | --uninstall-service | --healthcheck | --help] + + (no switch) run in the console until Ctrl+C + --service run under the Windows Service Control Manager + --install-service register the "TitanoServer" Windows service (automatic start, LocalSystem) + --uninstall-service stop and remove the Windows service + --healthcheck exit 0 when the local server answers /health, 1 otherwise + + Configuration: /data/server.json (TITANO_DATA_PATH) overridden by environment variables: + TITANO_ARCHIVE_PATH TITANO_DATA_PATH TITANO_SERVER_NAME TITANO_ADMIN_USER TITANO_ADMIN_PASSWORD + TITANO_DB (postgres | sqlite | connection string) TITANO_DB_HOST TITANO_DB_PORT TITANO_DB_NAME TITANO_DB_USER + TITANO_DB_PASSWORD TITANO_DB_SSLMODE TITANO_HTTP_PORT TITANO_HTTPS_PORT TITANO_TLS_MODE (auto | pem | proxy) + TITANO_TLS_CERT TITANO_TLS_KEY TITANO_LANG TITANO_WORKERS TITANO_DISCOVERY + """; + + public static ServerCommandLine Parse(IReadOnlyList args) + { + var command = ServerCommand.Run; + var rest = new List(); + string? error = null; + foreach (var arg in args) + { + ServerCommand? parsed = arg switch + { + "--service" => ServerCommand.Service, + "--install-service" => ServerCommand.InstallService, + "--uninstall-service" => ServerCommand.UninstallService, + "--healthcheck" => ServerCommand.HealthCheck, + "--help" or "-h" or "-?" or "/?" => ServerCommand.Help, + _ => null, + }; + if (parsed is not { } switchCommand) + { + rest.Add(arg); + continue; + } + if (command != ServerCommand.Run && command != switchCommand) error = $"{arg} cannot be combined with another command switch."; + command = switchCommand; + } + return new ServerCommandLine(command, rest, error); + } +} diff --git a/src/Titano.Server/Hosting/ServerDatabase.cs b/src/Titano.Server/Hosting/ServerDatabase.cs new file mode 100644 index 00000000..19db1ce9 --- /dev/null +++ b/src/Titano.Server/Hosting/ServerDatabase.cs @@ -0,0 +1,41 @@ +using Titano.Data; +using Titano.Server.Configuration; + +namespace Titano.Server.Hosting; + +/// Opens and migrates the server database at startup (BRIEF §6.1), waiting for a PostgreSQL container that is still starting. +public static class ServerDatabase +{ + private const int Attempts = 10; + private static readonly TimeSpan RetryDelay = TimeSpan.FromSeconds(3); + + public static async Task OpenAsync(ServerOptions options, ILogger logger, CancellationToken cancellationToken) + { + var settings = options.ToDatabaseOptions(); + for (var attempt = 1; ; attempt++) + { + try + { + var db = await TitanoDatabase.OpenAsync(settings, cancellationToken).ConfigureAwait(false); + logger.LogInformation("Database {Dialect} ready, schema version {Version}", db.Dialect, db.SchemaVersion); + return db; + } + catch (DatabaseConnectionException ex) when (attempt < Attempts) + { + logger.LogWarning("Database not reachable yet ({Reason}); retry {Attempt} of {Attempts} in {Delay} s", + ex.Message, attempt, Attempts - 1, RetryDelay.TotalSeconds); + await Task.Delay(RetryDelay, cancellationToken).ConfigureAwait(false); + } + } + } + + /// Engine and version, e.g. "PostgreSQL 17.2" or "SQLite 3.45.1". + public static async Task DescribeAsync(TitanoDatabase db, CancellationToken cancellationToken) + { + await using var session = await db.Database.OpenAsync(cancellationToken).ConfigureAwait(false); + var postgres = db.Dialect == Dialect.Postgres; + var version = await session.ScalarAsync(postgres ? "SHOW server_version" : "SELECT sqlite_version()", [], cancellationToken).ConfigureAwait(false); + var number = version?.Split(' ', 2)[0] ?? string.Empty; + return (postgres ? "PostgreSQL " : "SQLite ") + number; + } +} diff --git a/src/Titano.Server/Hosting/ServerHost.cs b/src/Titano.Server/Hosting/ServerHost.cs index 592a15d7..cda06dfa 100644 --- a/src/Titano.Server/Hosting/ServerHost.cs +++ b/src/Titano.Server/Hosting/ServerHost.cs @@ -1,21 +1,37 @@ using Microsoft.AspNetCore.HttpOverrides; using Microsoft.AspNetCore.Server.Kestrel.Core; using Titano.Core.Diagnostics; +using Titano.Data; +using Titano.Data.Archiving; using Titano.Protocol; using Titano.Server.Api; using Titano.Server.Configuration; +using Titano.Server.Events; +using Titano.Server.Jobs; +using Titano.Server.Security; +using Titano.Server.Storage; namespace Titano.Server.Hosting; -/// Builds the Titano Server web application from (used by Program and tests). +/// +/// Builds the Titano Server web application from (used by Program, the Windows service +/// host and tests). The archive module plugs in through of : +/// it must register and , and may add +/// s; the server refuses to start without the first two. +/// public static class ServerHost { - public static WebApplication Build(ServerOptions options, string[] args) + /// Request bodies other than upload chunks stay small (BRIEF §14.2 limits). + public const long JsonBodyLimit = 1024 * 1024; + + public static WebApplication Build(ServerOptions options, string[] args, Action? configureServices = null) { Directory.CreateDirectory(options.DataPath); Directory.CreateDirectory(options.LogPath); + Directory.CreateDirectory(options.UploadsPath); + Directory.CreateDirectory(options.ArchivePath); - var builder = WebApplication.CreateSlimBuilder(args); + var builder = WebApplication.CreateSlimBuilder(new WebApplicationOptions { Args = args, ContentRootPath = AppContext.BaseDirectory }); builder.Logging.ClearProviders(); builder.Logging.AddSimpleConsole(c => c.SingleLine = true); builder.Logging.AddProvider(new FileLoggerProvider(new FileLogSink(options.LogPath, "server"))); @@ -23,26 +39,67 @@ public static class ServerHost builder.Logging.AddFilter("Microsoft.Hosting.Lifetime", Microsoft.Extensions.Logging.LogLevel.Information); using var bootLoggerFactory = LoggerFactory.Create(l => l.AddSimpleConsole(c => c.SingleLine = true)); - var certificate = CertificateStore.Load(options, bootLoggerFactory.CreateLogger("Titano.Server")); - var state = new ServerState { TlsFingerprint = certificate is null ? null : CertificateStore.Fingerprint(certificate) }; - - builder.WebHost.ConfigureKestrel(k => + var bootLogger = bootLoggerFactory.CreateLogger("Titano.Server"); + var certificate = CertificateStore.Load(options, bootLogger); + var database = ServerDatabase.OpenAsync(options, bootLogger, CancellationToken.None).GetAwaiter().GetResult(); + try { - k.AddServerHeader = false; - k.Limits.MaxRequestBodySize = options.ChunkSizeBytes + 64 * 1024; - k.ListenAnyIP(options.HttpPort); - if (certificate is not null) k.ListenAnyIP(options.HttpsPort, l => + var archive = ServerArchive.OpenAsync(database, options, TimeProvider.System, CancellationToken.None).GetAwaiter().GetResult(); + AdminBootstrap.RunAsync(database, options, TimeProvider.System, bootLogger, CancellationToken.None).GetAwaiter().GetResult(); + var state = new ServerState { - l.Protocols = HttpProtocols.Http1AndHttp2; - l.UseHttps(certificate); + TlsFingerprint = certificate is null ? null : CertificateStore.Fingerprint(certificate), + Database = ServerDatabase.DescribeAsync(database, CancellationToken.None).GetAwaiter().GetResult(), + }; + + builder.WebHost.ConfigureKestrel(k => + { + k.AddServerHeader = false; + k.Limits.MaxRequestBodySize = JsonBodyLimit; + k.Limits.RequestHeadersTimeout = TimeSpan.FromSeconds(30); + k.Limits.KeepAliveTimeout = TimeSpan.FromMinutes(2); + k.ListenAnyIP(options.HttpPort); + if (certificate is not null) k.ListenAnyIP(options.HttpsPort, l => + { + l.Protocols = HttpProtocols.Http1AndHttp2; + l.UseHttps(certificate); + }); }); - }); - builder.Services.AddSingleton(options); - builder.Services.AddSingleton(state); - builder.Services.ConfigureHttpJsonOptions(j => j.SerializerOptions.TypeInfoResolverChain.Insert(0, ProtocolJson.Default)); + ConfigureServices(builder.Services, options, state, database, archive); + configureServices?.Invoke(builder.Services); + + var app = builder.Build(); + RequireArchiveModule(app.Services); + Log.SetSink(new LoggerSink(app.Services.GetRequiredService())); + ConfigurePipeline(app, options); + app.Logger.LogInformation("Titano Server {Version} · archive {Archive} · data {Data} · {Database} · TLS {Tls}{Fingerprint}", + SystemEndpoints.Version, options.ArchivePath, options.DataPath, state.Database, options.TlsMode, + state.TlsFingerprint is null ? string.Empty : " · SHA-256 " + state.TlsFingerprint); + app.Lifetime.ApplicationStopping.Register(() => app.Logger.LogInformation("Titano Server stopping")); + app.Lifetime.ApplicationStopped.Register(() => database.DisposeAsync().AsTask().GetAwaiter().GetResult()); + return app; + } + catch + { + database.DisposeAsync().AsTask().GetAwaiter().GetResult(); + throw; + } + } + + /// The single composition root of the server's services. + public static void ConfigureServices(IServiceCollection services, ServerOptions options, ServerState state, TitanoDatabase database, ServerArchive archive) + { + services.AddSingleton(options); + services.AddSingleton(state); + services.AddSingleton(database); + services.AddSingleton(archive); + services.AddSingleton(TimeProvider.System); + services.ConfigureHttpJsonOptions(j => j.SerializerOptions.TypeInfoResolverChain.Insert(0, ProtocolJson.Default)); + services.Configure(o => o.ThrowOnBadRequest = true); + services.Configure(o => o.ShutdownTimeout = TimeSpan.FromSeconds(30)); if (options.TlsMode == TlsMode.Proxy) { - builder.Services.Configure(f => + services.Configure(f => { f.ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto | ForwardedHeaders.XForwardedHost; f.KnownIPNetworks.Clear(); @@ -50,49 +107,62 @@ public static class ServerHost }); } - var app = builder.Build(); - Log.SetSink(new LoggerSink(app.Services.GetRequiredService())); - var logger = app.Logger; - logger.LogInformation("Titano Server {Version} · archive {Archive} · data {Data} · TLS {Tls}{Fingerprint}", - SystemEndpoints.Version, options.ArchivePath, options.DataPath, options.TlsMode, - state.TlsFingerprint is null ? string.Empty : " · SHA-256 " + state.TlsFingerprint); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddHostedService(sp => sp.GetRequiredService()); + services.AddSingleton(); + services.AddHostedService(sp => sp.GetRequiredService()); + services.AddSingleton(); + services.AddHostedService(sp => sp.GetRequiredService()); + services.AddSingleton(); + services.AddHostedService(sp => sp.GetRequiredService()); + } + private static void ConfigurePipeline(WebApplication app, ServerOptions options) + { if (options.TlsMode == TlsMode.Proxy) app.UseForwardedHeaders(); + app.UseMiddleware(); + app.UseWebSockets(new WebSocketOptions { KeepAliveInterval = TimeSpan.FromSeconds(30) }); + app.UseRouting(); + app.UseMiddleware(); - app.Use(async (context, next) => + AdminPanel.Map(app, options); + var api = app.MapGroup("/api/v1"); + SystemEndpoints.Map(app, api); + AuthEndpoints.Map(api); + UserEndpoints.Map(api); + SequenceEndpoints.Map(api); + FileEndpoints.Map(api); + UploadEndpoints.Map(api); + JobEndpoints.Map(api); + NodeEndpoints.Map(api); + ArchiveEndpoints.Map(api); + AdminEndpoints.Map(api); + EventEndpoints.Map(api); + app.MapFallback("/api/{**path}", (HttpContext context) => ApiErrors.NotFound(context)).AllowAnonymousCaller(); + } + + /// Fails fast when the archive module's services are missing (the server would accept uploads it cannot commit). + private static void RequireArchiveModule(IServiceProvider services) + { + var registry = services.GetRequiredService(); + var missing = new List(); + if (!registry.IsService(typeof(IArchiveIngest))) missing.Add($"{typeof(IArchiveIngest).FullName} (commits verified uploads into the archive layout)"); + if (!registry.IsService(typeof(IFrameAssetLocator))) missing.Add($"{typeof(IFrameAssetLocator).FullName} (finds proxies and thumbnails)"); + if (missing.Count > 0) { - context.Response.Headers[ApiVersion.Header] = ApiVersion.Current.ToString(System.Globalization.CultureInfo.InvariantCulture); - context.Response.Headers.XContentTypeOptions = "nosniff"; - context.Response.Headers["Referrer-Policy"] = "no-referrer"; - try - { - await next(context); - } - catch (Exception ex) when (!context.Response.HasStarted && ex is not OperationCanceledException) - { - logger.LogError(ex, "Unhandled error on {Method} {Path}", context.Request.Method, context.Request.Path); - context.Response.StatusCode = StatusCodes.Status500InternalServerError; - await context.Response.WriteAsJsonAsync(new ApiError("internal", "Unexpected server error."), ProtocolJson.Default.ApiError); - } - }); - - var admin = new EmbeddedStaticFiles(typeof(ServerHost).Assembly, "admin/"); - app.MapGet("/", () => Results.Redirect("/admin/")); - app.MapMethods("/admin/{**path}", ["GET", "HEAD"], async (HttpContext context, string? path) => - { - // Relative asset URLs need the trailing slash. - if (!context.Request.Path.Value!.EndsWith('/') && string.IsNullOrEmpty(path)) - { - context.Response.Redirect("/admin/"); - return; - } - if (!await admin.TryServeAsync(context, path ?? string.Empty)) context.Response.StatusCode = StatusCodes.Status404NotFound; - }); - - SystemEndpoints.Map(app, options, state); - - app.MapFallback("/api/{**path}", () => Results.Json(new ApiError("not_found", "No such endpoint."), ProtocolJson.Default.ApiError, statusCode: 404)); - app.Lifetime.ApplicationStopping.Register(() => logger.LogInformation("Titano Server stopping")); - return app; + throw new InvalidOperationException( + "Titano Server cannot start: no implementation is registered for " + string.Join(" and ", missing) + + ". Register them in the services callback of ServerHost.Build."); + } } } diff --git a/src/Titano.Server/Hosting/Windows/ServiceInstaller.cs b/src/Titano.Server/Hosting/Windows/ServiceInstaller.cs new file mode 100644 index 00000000..7d3adc8f --- /dev/null +++ b/src/Titano.Server/Hosting/Windows/ServiceInstaller.cs @@ -0,0 +1,83 @@ +using System.ComponentModel; +using System.Runtime.InteropServices; +using System.Runtime.Versioning; +using Titano.Core.Localization; +using static Titano.Server.Hosting.Windows.ServiceNative; + +namespace Titano.Server.Hosting.Windows; + +/// +/// --install-service / --uninstall-service: registers the server with the Service Control Manager +/// (automatic start, LocalSystem, command line "…\Titano.Server.exe" --service) or stops and removes it. +/// Both need an elevated prompt; failures surface as . +/// +[SupportedOSPlatform("windows")] +public static unsafe class ServiceInstaller +{ + private const string DisplayName = "Titano Server"; + + public static void Install() + { + var manager = OpenSCManagerW(null, null, ScManagerConnect | ScManagerCreateService); + if (manager == 0) throw new Win32Exception(Marshal.GetLastPInvokeError()); + try + { + var service = CreateServiceW(manager, WindowsServiceHost.ServiceName, DisplayName, ServiceAllAccess, ServiceWin32OwnProcess, ServiceAutoStart, + ServiceErrorNormal, CommandLine(), null, 0, null, null, null); + if (service == 0) throw new Win32Exception(Marshal.GetLastPInvokeError()); + try + { + fixed (char* text = Loc.L("Server.Service.Description")) + { + var description = new ServiceDescription { Description = text }; + ChangeServiceConfig2W(service, ServiceConfigDescription, &description); + } + } + finally + { + CloseServiceHandle(service); + } + } + finally + { + CloseServiceHandle(manager); + } + } + + public static void Uninstall() + { + var manager = OpenSCManagerW(null, null, ScManagerConnect); + if (manager == 0) throw new Win32Exception(Marshal.GetLastPInvokeError()); + try + { + var service = OpenServiceW(manager, WindowsServiceHost.ServiceName, Delete | ServiceStop | ServiceQueryStatus); + if (service == 0) throw new Win32Exception(Marshal.GetLastPInvokeError()); + try + { + ServiceStatus status; + if (!ControlService(service, ControlStop, &status) && Marshal.GetLastPInvokeError() is var stopError && stopError != ErrorServiceNotActive) + throw new Win32Exception(stopError); + if (!DeleteService(service) && Marshal.GetLastPInvokeError() is var deleteError && deleteError != ErrorServiceMarkedForDelete) + throw new Win32Exception(deleteError); + } + finally + { + CloseServiceHandle(service); + } + } + finally + { + CloseServiceHandle(manager); + } + } + + /// The service's command line: this executable (or dotnet plus this assembly) with --service. + private static string CommandLine() + { + var process = Environment.ProcessPath ?? throw new InvalidOperationException("The executable path is unknown."); + var assembly = typeof(ServiceInstaller).Assembly.Location; + return Path.GetFileNameWithoutExtension(process).Equals("dotnet", StringComparison.OrdinalIgnoreCase) + ? $"\"{process}\" \"{assembly}\" --service" + : $"\"{process}\" --service"; + } +} diff --git a/src/Titano.Server/Hosting/Windows/ServiceNative.cs b/src/Titano.Server/Hosting/Windows/ServiceNative.cs new file mode 100644 index 00000000..1b89fd1e --- /dev/null +++ b/src/Titano.Server/Hosting/Windows/ServiceNative.cs @@ -0,0 +1,93 @@ +using System.Runtime.InteropServices; +using System.Runtime.Versioning; + +namespace Titano.Server.Hosting.Windows; + +/// Service Control Manager functions of advapi32 (no Windows-only package needed). +[SupportedOSPlatform("windows")] +internal static unsafe partial class ServiceNative +{ + public const uint ServiceWin32OwnProcess = 0x10; + public const uint ServiceAutoStart = 2; + public const uint ServiceErrorNormal = 1; + public const uint ScManagerConnect = 0x1; + public const uint ScManagerCreateService = 0x2; + public const uint ServiceAllAccess = 0xF01FF; + public const uint ServiceQueryStatus = 0x4; + public const uint ServiceStop = 0x20; + public const uint Delete = 0x10000; + public const uint ServiceConfigDescription = 1; + + public const uint ControlStop = 1; + public const uint ControlInterrogate = 4; + public const uint ControlShutdown = 5; + + public const int ErrorFailedServiceControllerConnect = 1063; + public const int ErrorServiceNotActive = 1062; + public const int ErrorServiceMarkedForDelete = 1072; + public const uint NoError = 0; + public const uint ErrorCallNotImplemented = 120; + public const uint ErrorExceptionInService = 1064; + + [StructLayout(LayoutKind.Sequential)] + public struct ServiceStatus + { + public uint ServiceType; + public uint CurrentState; + public uint ControlsAccepted; + public uint Win32ExitCode; + public uint ServiceSpecificExitCode; + public uint CheckPoint; + public uint WaitHint; + } + + [StructLayout(LayoutKind.Sequential)] + public struct ServiceTableEntry + { + public char* ServiceName; + public delegate* unmanaged[Stdcall] ServiceProc; + } + + [StructLayout(LayoutKind.Sequential)] + public struct ServiceDescription + { + public char* Description; + } + + [LibraryImport("advapi32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static partial bool StartServiceCtrlDispatcherW(ServiceTableEntry* table); + + [LibraryImport("advapi32.dll", SetLastError = true, StringMarshalling = StringMarshalling.Utf16)] + public static partial nint RegisterServiceCtrlHandlerExW(string serviceName, delegate* unmanaged[Stdcall] handler, nint context); + + [LibraryImport("advapi32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static partial bool SetServiceStatus(nint statusHandle, ServiceStatus* status); + + [LibraryImport("advapi32.dll", SetLastError = true, StringMarshalling = StringMarshalling.Utf16)] + public static partial nint OpenSCManagerW(string? machineName, string? databaseName, uint desiredAccess); + + [LibraryImport("advapi32.dll", SetLastError = true, StringMarshalling = StringMarshalling.Utf16)] + public static partial nint CreateServiceW(nint manager, string serviceName, string displayName, uint desiredAccess, uint serviceType, uint startType, + uint errorControl, string binaryPathName, string? loadOrderGroup, nint tagId, string? dependencies, string? serviceStartName, string? password); + + [LibraryImport("advapi32.dll", SetLastError = true, StringMarshalling = StringMarshalling.Utf16)] + public static partial nint OpenServiceW(nint manager, string serviceName, uint desiredAccess); + + [LibraryImport("advapi32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static partial bool ChangeServiceConfig2W(nint service, uint infoLevel, void* info); + + [LibraryImport("advapi32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static partial bool ControlService(nint service, uint control, ServiceStatus* status); + + [LibraryImport("advapi32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static partial bool DeleteService(nint service); + + [LibraryImport("advapi32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static partial bool CloseServiceHandle(nint handle); +} diff --git a/src/Titano.Server/Hosting/Windows/ServiceStatusTracker.cs b/src/Titano.Server/Hosting/Windows/ServiceStatusTracker.cs new file mode 100644 index 00000000..01d55ad4 --- /dev/null +++ b/src/Titano.Server/Hosting/Windows/ServiceStatusTracker.cs @@ -0,0 +1,68 @@ +namespace Titano.Server.Hosting.Windows; + +/// SERVICE_STATUS.dwCurrentState values the server reports. +public enum ServiceState : uint +{ + Stopped = 1, + StartPending = 2, + StopPending = 3, + Running = 4, +} + +/// One status report for SetServiceStatus. +public readonly record struct ServiceStatusReport(ServiceState State, uint ControlsAccepted, uint CheckPoint, uint WaitHintMilliseconds, uint ExitCode); + +/// +/// The service's state machine and the rules of SERVICE_STATUS: no controls are accepted while pending, the check point +/// grows with every pending report and is zero otherwise, a stopped service carries its exit code. +/// +public sealed class ServiceStatusTracker +{ + public const uint AcceptStop = 0x1; + public const uint AcceptShutdown = 0x4; + + private readonly Lock _gate = new(); + private uint _checkPoint; + + public ServiceStatusReport Current { get; private set; } = new(ServiceState.Stopped, 0, 0, 0, 0); + + public ServiceStatusReport StartPending(uint waitHintMilliseconds) => Pending(ServiceState.StartPending, waitHintMilliseconds, ServiceState.Stopped, ServiceState.StartPending); + + public ServiceStatusReport Running() + { + lock (_gate) + { + Require(ServiceState.StartPending); + _checkPoint = 0; + return Current = new ServiceStatusReport(ServiceState.Running, AcceptStop | AcceptShutdown, 0, 0, 0); + } + } + + /// Stopping from start-up or from running; repeating it reports progress (a higher check point). + public ServiceStatusReport StopPending(uint waitHintMilliseconds) => + Pending(ServiceState.StopPending, waitHintMilliseconds, ServiceState.StartPending, ServiceState.Running, ServiceState.StopPending); + + public ServiceStatusReport Stopped(uint exitCode) + { + lock (_gate) + { + _checkPoint = 0; + return Current = new ServiceStatusReport(ServiceState.Stopped, 0, 0, 0, exitCode); + } + } + + private ServiceStatusReport Pending(ServiceState state, uint waitHintMilliseconds, params ServiceState[] from) + { + lock (_gate) + { + Require(from); + _checkPoint = Current.State == state ? _checkPoint + 1 : 1; + return Current = new ServiceStatusReport(state, 0, _checkPoint, waitHintMilliseconds, 0); + } + } + + private void Require(params ServiceState[] allowed) + { + if (!allowed.Contains(Current.State)) throw new InvalidOperationException($"A service cannot go from {Current.State} to that state."); + } +} diff --git a/src/Titano.Server/Hosting/Windows/WindowsServiceHost.cs b/src/Titano.Server/Hosting/Windows/WindowsServiceHost.cs new file mode 100644 index 00000000..d08a87e7 --- /dev/null +++ b/src/Titano.Server/Hosting/Windows/WindowsServiceHost.cs @@ -0,0 +1,117 @@ +using System.ComponentModel; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Versioning; +using static Titano.Server.Hosting.Windows.ServiceNative; + +namespace Titano.Server.Hosting.Windows; + +/// +/// Runs the server as a Windows service with the SCM API directly (StartServiceCtrlDispatcherW, +/// RegisterServiceCtrlHandlerExW, SetServiceStatus). Stop and shutdown requests stop the web host gracefully: running +/// jobs keep their checkpoints and resume at the next start. +/// +[SupportedOSPlatform("windows")] +public static class WindowsServiceHost +{ + public const string ServiceName = "TitanoServer"; + private const uint StartWaitHint = 60_000; + private const uint StopWaitHint = 35_000; + + private static readonly ServiceStatusTracker Status = new(); + private static Func? _build; + private static WebApplication? _app; + private static nint _statusHandle; + + /// + /// Hands the process to the Service Control Manager and returns when the service has stopped. False when the + /// process was not started by the SCM (for example from a console). + /// + public static unsafe bool Run(Func build) + { + _build = build; + fixed (char* name = ServiceName) + { + var table = stackalloc ServiceTableEntry[2]; + table[0] = new ServiceTableEntry { ServiceName = name, ServiceProc = &ServiceMain }; + table[1] = default; + if (StartServiceCtrlDispatcherW(table)) return true; + } + var error = Marshal.GetLastPInvokeError(); + if (error == ErrorFailedServiceControllerConnect) return false; + throw new Win32Exception(error); + } + + [UnmanagedCallersOnly(CallConvs = [typeof(CallConvStdcall)])] + private static unsafe void ServiceMain(uint argc, char** argv) + { + _statusHandle = RegisterServiceCtrlHandlerExW(ServiceName, &Handler, 0); + if (_statusHandle == 0) return; + Report(Status.StartPending(StartWaitHint)); + try + { + _app = _build!(); + _app.StartAsync().GetAwaiter().GetResult(); + Report(Status.Running()); + } + catch (Exception ex) + { + Console.Error.WriteLine(ex); + Report(Status.Stopped(ErrorExceptionInService)); + } + } + + [UnmanagedCallersOnly(CallConvs = [typeof(CallConvStdcall)])] + private static unsafe uint Handler(uint control, uint eventType, nint eventData, nint context) + { + switch (control) + { + case ControlStop or ControlShutdown: + if (Status.Current.State is ServiceState.Running or ServiceState.StartPending) + { + Report(Status.StopPending(StopWaitHint)); + // The handler must return at once; the host stops on the thread pool. + _ = Task.Run(StopAsync); + } + return NoError; + case ControlInterrogate: + Report(Status.Current); + return NoError; + default: + return ErrorCallNotImplemented; + } + } + + private static async Task StopAsync() + { + uint exitCode = 0; + try + { + if (_app is { } app) + { + await app.StopAsync(); + await app.DisposeAsync(); + } + } + catch (Exception ex) + { + Console.Error.WriteLine(ex); + exitCode = ErrorExceptionInService; + } + Report(Status.Stopped(exitCode)); + } + + private static unsafe void Report(ServiceStatusReport report) + { + var status = new ServiceStatus + { + ServiceType = ServiceWin32OwnProcess, + CurrentState = (uint)report.State, + ControlsAccepted = report.ControlsAccepted, + Win32ExitCode = report.ExitCode, + CheckPoint = report.CheckPoint, + WaitHint = report.WaitHintMilliseconds, + }; + SetServiceStatus(_statusHandle, &status); + } +} diff --git a/src/Titano.Server/Jobs/JobEngineHost.cs b/src/Titano.Server/Jobs/JobEngineHost.cs new file mode 100644 index 00000000..eda9eac4 --- /dev/null +++ b/src/Titano.Server/Jobs/JobEngineHost.cs @@ -0,0 +1,87 @@ +using System.Diagnostics; +using Titano.Core.Model; +using Titano.Data; +using Titano.Data.Jobs; +using Titano.Protocol; +using Titano.Server.Api; +using Titano.Server.Configuration; +using Titano.Server.Events; + +namespace Titano.Server.Jobs; + +/// +/// Hosts the server's (BRIEF §14.6) with the s registered in DI and +/// relays its state changes and progress as WebSocket events. Render jobs are never run here: render nodes claim them +/// (BRIEF §14.7). On shutdown running jobs stay marked running and are requeued from their checkpoints at the next start. +/// +public sealed class JobEngineHost : IHostedService +{ + private static readonly long ProgressInterval = Stopwatch.Frequency / 4; + + private readonly TitanoDatabase _db; + private readonly EventHub _events; + private readonly HashSet _kinds; + private readonly Dictionary _lastProgress = []; + private readonly Lock _gate = new(); + + public JobEngineHost(TitanoDatabase db, IEnumerable handlers, ServerOptions options, EventHub events) + { + _db = db; + _events = events; + var local = handlers.Where(h => h.Kind != JobKind.Render).ToList(); + _kinds = [.. local.Select(h => h.Kind)]; + // Without handlers no worker starts: an engine with an empty kind list would claim every kind. + Workers = _kinds.Count == 0 ? 0 : options.Workers; + Engine = new JobEngine(db, local, new JobEngineOptions + { + Workers = Workers, + WorkerId = WorkerId, + Kinds = _kinds, + HistoryRetention = TimeSpan.FromDays(options.JobHistoryDays), + }); + Engine.JobChanged += OnJobChanged; + Engine.Progress += OnProgress; + } + + /// Node id stamped on jobs this server runs (its workers use WorkerId#n). + public static string WorkerId { get; } = "server:" + Environment.MachineName; + + public JobEngine Engine { get; } + + public int Workers { get; } + + /// True when a job of has somewhere to run: a local handler, or render nodes for renders. + public bool CanRun(JobKind kind) => kind == JobKind.Render || _kinds.Contains(kind); + + public async Task StartAsync(CancellationToken cancellationToken) + { + // The engine requeues only its exact worker id; numbered workers of the previous run are requeued here. + await _db.Jobs.RequeueRunningWorkersAsync(WorkerId, cancellationToken); + await Engine.StartAsync(cancellationToken); + } + + public async Task StopAsync(CancellationToken cancellationToken) => await Engine.DisposeAsync(); + + /// Publishes a state change made outside the engine (render nodes, cancellation of node jobs). + public void Publish(Job job) => OnJobChanged(job); + + private void OnJobChanged(Job job) + { + if (job.State != JobState.Running) + { + lock (_gate) _lastProgress.Remove(job.Id); + } + _events.Publish(new JobStateEvent(job.Id, JobWire.State(job.State), job.Error)); + } + + private void OnProgress(JobProgressInfo progress) + { + var now = Stopwatch.GetTimestamp(); + lock (_gate) + { + if (_lastProgress.TryGetValue(progress.JobId, out var last) && now - last < ProgressInterval && progress.Done < progress.Total) return; + _lastProgress[progress.JobId] = now; + } + _events.Publish(new JobProgressEvent(progress.JobId, progress.Done, progress.Total, progress.ItemsPerSecond, progress.Eta?.TotalSeconds)); + } +} diff --git a/src/Titano.Server/Jobs/RenderNodeMonitor.cs b/src/Titano.Server/Jobs/RenderNodeMonitor.cs new file mode 100644 index 00000000..f8d5b73d --- /dev/null +++ b/src/Titano.Server/Jobs/RenderNodeMonitor.cs @@ -0,0 +1,72 @@ +using Titano.Core.Model; +using Titano.Data; +using Titano.Protocol; +using Titano.Server.Configuration; +using Titano.Server.Events; + +namespace Titano.Server.Jobs; + +/// +/// Marks render nodes offline when they miss heartbeats for and gives +/// their running jobs back to the queue for another node; a job that lost its node times fails. +/// +public sealed class RenderNodeMonitor(TitanoDatabase db, ServerOptions options, JobEngineHost jobs, EventHub events, TimeProvider time, + ILogger logger) : BackgroundService +{ + public const int MaxAttempts = 5; + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + var interval = TimeSpan.FromSeconds(Math.Clamp(options.NodeTimeoutSeconds / 3.0, 1, 30)); + while (!stoppingToken.IsCancellationRequested) + { + try + { + await SweepAsync(stoppingToken); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogWarning(ex, "Render node check failed"); + } + try + { + await Task.Delay(interval, time, stoppingToken); + } + catch (OperationCanceledException) + { + return; + } + } + } + + /// One pass: silent nodes go offline and their jobs are requeued (or failed after too many attempts). + public async Task SweepAsync(CancellationToken cancellationToken) + { + var silentSince = time.GetUtcNow() - TimeSpan.FromSeconds(options.NodeTimeoutSeconds); + foreach (var node in await db.RenderNodes.ListSilentAsync(silentSince, RenderNodeService.Offline, cancellationToken)) + { + await db.RenderNodes.SetStateAsync(node.Id, RenderNodeService.Offline, cancellationToken); + events.Publish(new NodeStateEvent(node.Id, node.ClientName, RenderNodeService.Offline)); + var key = RenderNodeService.NodeKey(node); + var lost = await db.Jobs.ListRunningByNodeAsync(key, cancellationToken); + foreach (var job in lost) + { + var line = RenderNodeService.AppendLog(job.LogTail, $"Render node {node.ClientName} stopped responding."); + if (job.Attempts >= MaxAttempts) + { + await db.Jobs.FinishNodeJobAsync(job.Id, key, JobState.Failed, $"Lost its render node {job.Attempts} times.", line, cancellationToken); + } + else + { + await db.Jobs.UpdateNodeProgressAsync(job.Id, key, job.ProgressDone, job.ProgressTotal, null, null, line, cancellationToken); + } + } + var requeued = await db.Jobs.RequeueRunningAsync(key, cancellationToken); + logger.LogWarning("Render node {Node} is offline; {Count} job(s) back in the queue", node.ClientName, requeued); + foreach (var job in lost) + { + if (await db.Jobs.GetAsync(job.Id, cancellationToken) is { } current) jobs.Publish(current); + } + } + } +} diff --git a/src/Titano.Server/Jobs/RenderNodeService.cs b/src/Titano.Server/Jobs/RenderNodeService.cs new file mode 100644 index 00000000..1e6f5e6a --- /dev/null +++ b/src/Titano.Server/Jobs/RenderNodeService.cs @@ -0,0 +1,240 @@ +using System.Globalization; +using System.Text.Json; +using Titano.Core.Model; +using Titano.Data; +using Titano.Data.Repositories; +using Titano.Protocol; +using Titano.Server.Api; +using Titano.Server.Configuration; +using Titano.Server.Events; +using Titano.Server.Security; +using Titano.Server.Storage; +using UploadState = Titano.Data.Repositories.UploadState; + +namespace Titano.Server.Jobs; + +/// +/// Render nodes (BRIEF §14.7): clients with the render-node right register their encoders, heartbeat, claim queued +/// render jobs they can encode, report progress, and upload the output in resumable chunks that become a render of +/// the sequence. A node that stops heart-beating loses its jobs to the queue (). +/// +public sealed class RenderNodeService( + TitanoDatabase db, + ServerOptions options, + UploadService uploads, + UploadStore store, + RenderStore renders, + JobEngineHost jobs, + EventHub events, + TimeProvider time) +{ + public const string Idle = "idle"; + public const string Busy = "busy"; + public const string Offline = "offline"; + private const int LogTailLines = 40; + + public async Task> RegisterAsync(Caller caller, RegisterNodeRequest request, CancellationToken cancellationToken) + { + var name = request.ClientName?.Trim(); + if (string.IsNullOrEmpty(name) || name.Length > 100) return Outcome.Fail(400, "bad_request", "clientName"); + if (request.Encoders is null || request.Encoders.Count > 64 || request.Encoders.Any(e => string.IsNullOrWhiteSpace(e?.Codec))) + return Outcome.Fail(400, "bad_request", "encoders"); + + var existing = (await db.RenderNodes.ListAsync(cancellationToken)).FirstOrDefault(n => n.UserId == caller.Id && n.ClientName == name); + var id = existing?.Id ?? Guid.CreateVersion7(); + var running = await db.Jobs.ListRunningByNodeAsync(id.ToString(), cancellationToken); + var node = new RenderNode + { + Id = id, + UserId = caller.Id, + ClientName = name, + CapabilitiesJson = JsonSerializer.Serialize(request with { ClientName = name }, ProtocolJson.Default.RegisterNodeRequest), + LastHeartbeat = time.GetUtcNow(), + State = running.Count > 0 ? Busy : Idle, + }; + await db.RenderNodes.RegisterAsync(node, cancellationToken); + events.Publish(new NodeStateEvent(node.Id, node.ClientName, node.State)); + return Dto.Node(node, caller.User.Username); + } + + public async Task> HeartbeatAsync(Caller caller, Guid nodeId, CancellationToken cancellationToken) + { + if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome.Fail(404, "not_found", "node"); + await TouchAsync(node, cancellationToken); + return true; + } + + /// The next queued render job this node can encode (highest priority, then oldest), or null when none fits. + public async Task> ClaimAsync(Caller caller, Guid nodeId, CancellationToken cancellationToken) + { + if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome.Fail(404, "not_found", "node"); + var capabilities = Dto.Capabilities(node); + foreach (var queued in await db.Jobs.ListQueuedAsync(JobKind.Render, 50, cancellationToken)) + { + if (JobEndpoints.ParseRender(queued.ParametersJson) is not { } parameters || capabilities is null || !CanEncode(capabilities, parameters)) continue; + if (await db.Jobs.TryClaimAsync(queued.Id, NodeKey(node), cancellationToken) is not { } job) continue; + jobs.Publish(job); + if (job.TargetIds.Count != 1 || await db.Sequences.GetAsync(job.TargetIds[0], cancellationToken) is not { } sequence) + { + await FinishAsync(node, job, JobState.Failed, "The sequence no longer exists.", cancellationToken); + continue; + } + await TouchAsync(node, cancellationToken); + return new ClaimedJob(Dto.Job(job), sequence.Id, parameters.SettingsJson ?? sequence.SettingsJson ?? "{}", parameters.PresetJson, + parameters.UseProxies, await ManifestFramesAsync(sequence, cancellationToken)); + } + await TouchAsync(node, cancellationToken); + return (ClaimedJob?)null; + } + + public async Task> ProgressAsync(Caller caller, Guid nodeId, Guid jobId, NodeProgress progress, CancellationToken cancellationToken) + { + if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome.Fail(404, "not_found", "node"); + if (await AssignedJobAsync(node, jobId, cancellationToken) is not { } job) return Outcome.Fail(409, "job_not_assigned"); + double? eta = progress.ItemsPerSecond is > 0 && progress.Total > progress.Done ? (progress.Total - progress.Done) / progress.ItemsPerSecond.Value : null; + var tail = AppendLog(job.LogTail, progress.LogLine); + if (!await db.Jobs.UpdateNodeProgressAsync(jobId, NodeKey(node), progress.Done, progress.Total, progress.ItemsPerSecond, + eta is { } seconds ? TimeSpan.FromSeconds(seconds) : null, tail, cancellationToken)) + return Outcome.Fail(409, "job_not_assigned"); + await TouchAsync(node, cancellationToken); + events.Publish(new JobProgressEvent(jobId, progress.Done, progress.Total, progress.ItemsPerSecond, eta)); + return true; + } + + public async Task> FailAsync(Caller caller, Guid nodeId, Guid jobId, NodeJobFailure failure, CancellationToken cancellationToken) + { + if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome.Fail(404, "not_found", "node"); + if (await AssignedJobAsync(node, jobId, cancellationToken) is not { } job) return Outcome.Fail(409, "job_not_assigned"); + var error = string.IsNullOrWhiteSpace(failure.Error) ? "Render failed on the node." : failure.Error.Trim()[..Math.Min(failure.Error.Trim().Length, 500)]; + return await FinishAsync(node, job, JobState.Failed, error, cancellationToken) ? true : Outcome.Fail(409, "job_not_assigned"); + } + + /// Starts (or resumes) the chunked upload of a job's output. + public async Task> StartResultAsync(Caller caller, Guid nodeId, Guid jobId, RenderResultRequest request, CancellationToken cancellationToken) + { + if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome.Fail(404, "not_found", "node"); + if (await AssignedJobAsync(node, jobId, cancellationToken) is not { } job) return Outcome.Fail(409, "job_not_assigned"); + if (UploadRules.SafeFileName(request.FileName) is not { } fileName) return Outcome.Fail(400, "invalid_file_name"); + if (!UploadRules.IsSha256(request.Sha256)) return Outcome.Fail(400, "invalid_hash"); + if (request.Size <= 0) return Outcome.Fail(400, "bad_request", "size"); + if (request.Size > options.MaxUploadBytes) + return Outcome.Fail(413, "too_large", options.MaxUploadBytes.ToString(CultureInfo.InvariantCulture)); + if (string.IsNullOrWhiteSpace(request.PresetName)) return Outcome.Fail(400, "bad_request", "presetName"); + + var sha = request.Sha256.ToLowerInvariant(); + if (await db.Uploads.FindByJobAsync(jobId, cancellationToken) is { State: UploadState.Receiving } open && open.Sha256 == sha && open.Size == request.Size + && store.Exists(open.Id)) + return Started(open); + if (UploadRules.FreeBytes(options.UploadsPath) is { } free && free < request.Size) return Outcome.Fail(507, "insufficient_storage"); + + var now = time.GetUtcNow(); + var upload = await db.Uploads.CreateAsync(new Upload + { + Id = Guid.CreateVersion7(), + UserId = caller.Id, + SequenceId = job.TargetIds.Count == 1 ? job.TargetIds[0] : null, + FileName = fileName, + Size = request.Size, + Sha256 = sha, + ChunkSize = options.ChunkSizeBytes, + State = UploadState.Receiving, + Created = now, + UpdatedAt = now, + JobId = jobId, + RenderJson = JsonSerializer.Serialize(request with { FileName = fileName, Sha256 = sha }, ProtocolJson.Default.RenderResultRequest), + }, cancellationToken); + store.Create(upload.Id, upload.Size); + return Started(upload); + } + + /// Verifies the uploaded output, files it as a render of the sequence and marks the job done. + public async Task> CompleteResultAsync(Caller caller, Guid nodeId, Guid jobId, CancellationToken cancellationToken) + { + if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome.Fail(404, "not_found", "node"); + if (await AssignedJobAsync(node, jobId, cancellationToken) is not { } job) return Outcome.Fail(409, "job_not_assigned"); + var upload = await db.Uploads.FindByJobAsync(jobId, cancellationToken); + if (upload is not { State: UploadState.Receiving } || upload.RenderJson is null) return Outcome.Fail(409, "upload_state"); + if (!upload.IsComplete) + return Outcome.Fail(409, "incomplete", (upload.ChunkCount - upload.ReceivedCount).ToString(CultureInfo.InvariantCulture)); + if (!await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Receiving, UploadState.Committing, cancellationToken: cancellationToken)) + return Outcome.Fail(409, "upload_state"); + + if (await uploads.StageVerifiedAsync(upload, cancellationToken) is not { } staged) return Outcome.Fail(422, "hash_mismatch"); + var output = JsonSerializer.Deserialize(upload.RenderJson, ProtocolJson.Default.RenderResultRequest)!; + var sequence = await db.Sequences.GetAsync(job.TargetIds[0], cancellationToken); + var render = sequence is null ? null : await renders.CommitAsync(sequence, staged, output, node.ClientName, cancellationToken); + if (render is null) + { + UploadStore.TryDelete(staged); + await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Failed, cancellationToken: CancellationToken.None); + store.Delete(upload.Id); + await FinishAsync(node, job, JobState.Failed, "The sequence has no folder in the archive.", CancellationToken.None); + return Outcome.Fail(409, "upload_state", "sequence"); + } + await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Committed, cancellationToken: CancellationToken.None); + store.Delete(upload.Id); + await FinishAsync(node, job, JobState.Done, null, CancellationToken.None); + if (await db.Sequences.GetAsync(sequence!.Id, CancellationToken.None) is { } updated) + events.Publish(new SequenceChangedEvent(updated.Id, "renders", updated.FramesVersion, updated.SettingsVersion)); + return Dto.Render(render); + } + + /// Running-job owner key stored in jobs.node_id. + public static string NodeKey(RenderNode node) => node.Id.ToString(); + + private async Task OwnedNodeAsync(Caller caller, Guid nodeId, CancellationToken cancellationToken) => + await db.RenderNodes.GetAsync(nodeId, cancellationToken) is { } node && node.UserId == caller.Id ? node : null; + + private async Task AssignedJobAsync(RenderNode node, Guid jobId, CancellationToken cancellationToken) => + await db.Jobs.GetAsync(jobId, cancellationToken) is { State: JobState.Running } job && job.NodeId == NodeKey(node) ? job : null; + + private async Task FinishAsync(RenderNode node, Job job, JobState state, string? error, CancellationToken cancellationToken) + { + var tail = error is null ? null : AppendLog(job.LogTail, error); + if (!await db.Jobs.FinishNodeJobAsync(job.Id, NodeKey(node), state, error, tail, cancellationToken)) return false; + if (await db.Jobs.GetAsync(job.Id, cancellationToken) is { } finished) jobs.Publish(finished); + await TouchAsync(node, cancellationToken); + return true; + } + + /// Records a sign of life and publishes the node's state when it changed. + private async Task TouchAsync(RenderNode node, CancellationToken cancellationToken) + { + var state = (await db.Jobs.ListRunningByNodeAsync(NodeKey(node), cancellationToken)).Count > 0 ? Busy : Idle; + await db.RenderNodes.HeartbeatAsync(node.Id, state, time.GetUtcNow(), cancellationToken); + if (state != node.State) events.Publish(new NodeStateEvent(node.Id, node.ClientName, state)); + } + + private async Task> ManifestFramesAsync(Sequence sequence, CancellationToken cancellationToken) + { + var hasProxy = sequence.Status.HasFlag(SequenceStatus.ProxiesBuilt); + var hasThumb = sequence.Status.HasFlag(SequenceStatus.ThumbsBuilt); + var frames = new List(sequence.FrameCount); + var after = -1; + while (true) + { + var batch = await db.Frames.ListBySequenceAsync(sequence.Id, after, 1000, cancellationToken); + frames.AddRange(batch.Where(f => !f.Flags.HasFlag(FrameFlags.Excluded)) + .Select(f => new ManifestFrame(f.Id, f.Index, f.FileName, f.Size, f.Sha256 ?? string.Empty, hasProxy, hasThumb))); + if (batch.Count < 1000) return frames; + after = batch[^1].Index; + } + } + + /// The node has an encoder for the job's codec and handles its resolution (0 = no announced limit). + private static bool CanEncode(RegisterNodeRequest node, RenderJobParameters job) => + node.Encoders.Any(e => string.Equals(e.Codec, job.Codec, StringComparison.OrdinalIgnoreCase)) + && (node.MaxWidth <= 0 || job.Width <= node.MaxWidth) + && (node.MaxHeight <= 0 || job.Height <= node.MaxHeight); + + internal static string? AppendLog(string? tail, string? line) + { + if (string.IsNullOrWhiteSpace(line)) return tail; + var lines = (tail ?? string.Empty).Split('\n', StringSplitOptions.RemoveEmptyEntries).ToList(); + lines.Add(string.Create(CultureInfo.InvariantCulture, $"{DateTime.UtcNow:HH:mm:ss} {line.Trim()[..Math.Min(line.Trim().Length, 300)]}")); + return string.Join('\n', lines.TakeLast(LogTailLines)); + } + + private static StartUploadResponse Started(Upload upload) => + new(upload.Id, upload.ChunkSize, UploadRules.ReceivedChunks(upload), AlreadyExists: false, FrameId: null); +} diff --git a/src/Titano.Server/Jobs/SequenceJobs.cs b/src/Titano.Server/Jobs/SequenceJobs.cs new file mode 100644 index 00000000..f53c962f --- /dev/null +++ b/src/Titano.Server/Jobs/SequenceJobs.cs @@ -0,0 +1,32 @@ +using Titano.Core.Model; +using Titano.Data; + +namespace Titano.Server.Jobs; + +/// +/// Follow-up work after frames arrive (BRIEF §14.4): thumbnails, proxies and light analysis of the sequence, queued at +/// background priority once per sequence while one is still waiting, and only for kinds this server can run. +/// +public sealed class SequenceJobs(TitanoDatabase db, JobEngineHost jobs) +{ + private static readonly JobKind[] AfterNewFrames = [JobKind.Thumbs, JobKind.Proxies, JobKind.LightAnalyse]; + + private readonly SemaphoreSlim _gate = new(1, 1); + + public async Task FramesAddedAsync(Sequence sequence, CancellationToken cancellationToken) + { + await _gate.WaitAsync(cancellationToken); + try + { + foreach (var kind in AfterNewFrames) + { + if (!jobs.CanRun(kind) || await db.Jobs.IsQueuedAsync(kind, sequence.Id, cancellationToken)) continue; + await jobs.Engine.EnqueueAsync(kind, [sequence.Id], sequence.DisplayTitle, priority: JobPriority.Background, cancellationToken: cancellationToken); + } + } + finally + { + _gate.Release(); + } + } +} diff --git a/src/Titano.Server/Program.cs b/src/Titano.Server/Program.cs index 9bfebd2c..bb9e439f 100644 --- a/src/Titano.Server/Program.cs +++ b/src/Titano.Server/Program.cs @@ -1,11 +1,53 @@ +using System.ComponentModel; using Titano.Server.Configuration; using Titano.Server.Hosting; +using Titano.Server.Hosting.Windows; -if (args.Contains("--healthcheck")) return await HealthProbe.RunAsync(); +var command = ServerCommandLine.Parse(args); +if (command.Error is not null) +{ + await Console.Error.WriteLineAsync(command.Error); + return 2; +} -var app = ServerHost.Build(ServerOptions.Load(), args); -await app.RunAsync(); -return 0; +switch (command.Command) +{ + case ServerCommand.Help: + Console.WriteLine(ServerCommandLine.Usage); + return 0; + case ServerCommand.HealthCheck: + return await HealthProbe.RunAsync(); + case ServerCommand.Service or ServerCommand.InstallService or ServerCommand.UninstallService when !OperatingSystem.IsWindows(): + await Console.Error.WriteLineAsync("Windows services are available on Windows only."); + return 2; + case ServerCommand.InstallService when OperatingSystem.IsWindows(): + return ServiceAction(ServiceInstaller.Install, $"Installed the {WindowsServiceHost.ServiceName} service."); + case ServerCommand.UninstallService when OperatingSystem.IsWindows(): + return ServiceAction(ServiceInstaller.Uninstall, $"Removed the {WindowsServiceHost.ServiceName} service."); + case ServerCommand.Service when OperatingSystem.IsWindows(): + if (WindowsServiceHost.Run(() => ServerHost.Build(ServerOptions.Load(), [.. command.HostArguments]))) return 0; + await Console.Error.WriteLineAsync("--service is used by the Service Control Manager; start the service with 'sc start TitanoServer'."); + return 1; + default: + var app = ServerHost.Build(ServerOptions.Load(), [.. command.HostArguments]); + await app.RunAsync(); + return 0; +} + +static int ServiceAction(Action action, string done) +{ + try + { + action(); + Console.WriteLine(done); + return 0; + } + catch (Win32Exception ex) + { + Console.Error.WriteLine(ex.Message); + return 1; + } +} /// Entry point type, visible to the in-process test host. public partial class Program; diff --git a/src/Titano.Server/Security/AccessRule.cs b/src/Titano.Server/Security/AccessRule.cs new file mode 100644 index 00000000..923829a7 --- /dev/null +++ b/src/Titano.Server/Security/AccessRule.cs @@ -0,0 +1,38 @@ +using Titano.Core.Model; + +namespace Titano.Server.Security; + +/// +/// Endpoint metadata read by . Every /api endpoint requires a signed-in +/// caller unless it carries ; the most specific rule (endpoint over group) wins whole. +/// +public sealed class AccessRule +{ + public static readonly AccessRule Anonymous = new() { AllowAnonymous = true }; + + public static readonly AccessRule SignedIn = new(); + + public bool AllowAnonymous { get; private init; } + + /// The caller needs at least one of these permissions; = any signed-in caller. + public Permissions AnyOf { get; private init; } + + /// Reachable while the account must still replace its temporary password (me, password, logout). + public bool DuringPasswordChange { get; private init; } + + public static AccessRule Requiring(Permissions anyOf) => new() { AnyOf = anyOf }; + + public static AccessRule PasswordChange { get; } = new() { DuringPasswordChange = true }; +} + +public static class AccessRuleExtensions +{ + public static TBuilder AllowAnonymousCaller(this TBuilder builder) where TBuilder : IEndpointConventionBuilder => + builder.WithMetadata(AccessRule.Anonymous); + + public static TBuilder Requires(this TBuilder builder, Permissions anyOf) where TBuilder : IEndpointConventionBuilder => + builder.WithMetadata(AccessRule.Requiring(anyOf)); + + public static TBuilder AllowDuringPasswordChange(this TBuilder builder) where TBuilder : IEndpointConventionBuilder => + builder.WithMetadata(AccessRule.PasswordChange); +} diff --git a/src/Titano.Server/Security/Accounts.cs b/src/Titano.Server/Security/Accounts.cs new file mode 100644 index 00000000..96ce6969 --- /dev/null +++ b/src/Titano.Server/Security/Accounts.cs @@ -0,0 +1,35 @@ +using Titano.Core.Model; + +namespace Titano.Server.Security; + +/// Account rules shared by bootstrap, sign-in and user management. +public static class Accounts +{ + public const int MaxUsernameLength = 64; + + /// Letters, digits, dot, dash and underscore; 1–64 characters. + public static bool IsValidUsername(string? name) => + name is { Length: > 0 and <= MaxUsernameLength } && name.All(c => char.IsAsciiLetterOrDigit(c) || c is '.' or '-' or '_'); + + /// The case-insensitive key of a user name (lock-out bookkeeping). + public static string Key(string? name) => (name ?? string.Empty).Trim().ToLowerInvariant(); + + /// The role's default permissions with the render-node right granted or withheld explicitly. + public static Permissions PermissionsFor(UserRole role, bool renderNode) => + (RolePermissions.Default(role) & ~Permissions.RenderNode) | (renderNode ? Permissions.RenderNode : Permissions.None); + + /// Parses a role name (admin, editor, uploader, viewer); numbers are not accepted. + public static bool TryParseRole(string? text, out UserRole role) + { + foreach (var candidate in Enum.GetValues()) + { + if (string.Equals(candidate.ToString(), text, StringComparison.OrdinalIgnoreCase)) + { + role = candidate; + return true; + } + } + role = default; + return false; + } +} diff --git a/src/Titano.Server/Security/AdminBootstrap.cs b/src/Titano.Server/Security/AdminBootstrap.cs new file mode 100644 index 00000000..027c94ef --- /dev/null +++ b/src/Titano.Server/Security/AdminBootstrap.cs @@ -0,0 +1,85 @@ +using Titano.Core.Model; +using Titano.Data; +using Titano.Server.Configuration; + +namespace Titano.Server.Security; + +/// +/// First-run administrator (BRIEF §14.9): when no admin account exists, one is created from +/// TITANO_ADMIN_USER/TITANO_ADMIN_PASSWORD and must change its password at first sign-in. Without a +/// configured password a temporary one is generated and written to /data/initial-admin-password.txt (owner-only), +/// never to the log; the file is removed once an administrator changes their password. +/// +public static class AdminBootstrap +{ + public const string DefaultUsername = "admin"; + private const string PasswordFileName = "initial-admin-password.txt"; + + public static async Task RunAsync(TitanoDatabase db, ServerOptions options, TimeProvider time, ILogger logger, CancellationToken cancellationToken) + { + var users = await db.Users.ListAsync(cancellationToken).ConfigureAwait(false); + if (users.Any(u => u.Role == UserRole.Admin)) + { + if (!string.IsNullOrEmpty(options.AdminPassword)) + logger.LogInformation("An administrator exists; TITANO_ADMIN_USER and TITANO_ADMIN_PASSWORD are ignored"); + return; + } + + var username = string.IsNullOrWhiteSpace(options.AdminUser) ? DefaultUsername : options.AdminUser.Trim(); + if (!Accounts.IsValidUsername(username)) + throw new InvalidOperationException("TITANO_ADMIN_USER may only contain letters, digits, dot, dash and underscore (up to 64 characters)."); + if (users.Any(u => Accounts.Key(u.Username) == Accounts.Key(username))) + throw new InvalidOperationException($"No administrator exists and TITANO_ADMIN_USER '{username}' names an existing account; choose another name."); + + var password = options.AdminPassword; + string? passwordFile = null; + if (string.IsNullOrEmpty(password)) + { + password = TemporaryPassword.Generate(); + passwordFile = Path.Combine(options.DataPath, PasswordFileName); + await File.WriteAllTextAsync(passwordFile, password + Environment.NewLine, cancellationToken).ConfigureAwait(false); + if (!OperatingSystem.IsWindows()) File.SetUnixFileMode(passwordFile, UnixFileMode.UserRead | UnixFileMode.UserWrite); + } + + var admin = new User + { + Id = Guid.CreateVersion7(), + Username = username, + DisplayName = null, + PasswordHash = PasswordHasher.Hash(password), + Role = UserRole.Admin, + Permissions = Permissions.All, + MustChangePassword = true, + Created = time.GetUtcNow(), + }; + await db.Users.CreateAsync(admin, cancellationToken).ConfigureAwait(false); + await db.Audit.AppendAsync(new AuditEntry + { + Id = Guid.CreateVersion7(), + At = admin.Created, + Action = "user_created", + TargetKind = "user", + TargetId = admin.Id.ToString(), + Details = "admin", + Username = username, + }, cancellationToken).ConfigureAwait(false); + + if (passwordFile is null) + logger.LogWarning("Created administrator '{User}' from TITANO_ADMIN_USER/TITANO_ADMIN_PASSWORD; the password must be changed at first sign-in", username); + else + logger.LogWarning("Created administrator '{User}'; its temporary password is in {File} and must be changed at first sign-in", username, passwordFile); + } + + /// Removes the generated password file (called when an administrator sets their own password). + public static void ForgetInitialPassword(ServerOptions options) + { + try + { + File.Delete(Path.Combine(options.DataPath, PasswordFileName)); + } + catch (IOException) + { + // Best effort: the file holds a password that no longer works. + } + } +} diff --git a/src/Titano.Server/Security/AuditLog.cs b/src/Titano.Server/Security/AuditLog.cs new file mode 100644 index 00000000..7afadcca --- /dev/null +++ b/src/Titano.Server/Security/AuditLog.cs @@ -0,0 +1,26 @@ +using Titano.Core.Model; +using Titano.Data; + +namespace Titano.Server.Security; + +/// Appends who did what to the audit log (BRIEF §14.2): sign-ins, user, sequence, archive and setting changes. +public sealed class AuditLog(TitanoDatabase db, TimeProvider time) +{ + public Task WriteAsync(Caller caller, HttpContext context, string action, string? targetKind = null, string? targetId = null, string? details = null) => + WriteAsync(caller.Id, caller.User.Username, context, action, targetKind, targetId, details); + + public Task WriteAsync(Guid? userId, string? username, HttpContext? context, string action, string? targetKind = null, string? targetId = null, + string? details = null) => + db.Audit.AppendAsync(new AuditEntry + { + Id = Guid.CreateVersion7(), + At = time.GetUtcNow(), + UserId = userId, + Username = username, + Action = action, + TargetKind = targetKind, + TargetId = targetId, + Details = details, + RemoteAddress = context?.Connection.RemoteIpAddress?.ToString(), + }); +} diff --git a/src/Titano.Server/Security/AuthenticationMiddleware.cs b/src/Titano.Server/Security/AuthenticationMiddleware.cs new file mode 100644 index 00000000..b0e18541 --- /dev/null +++ b/src/Titano.Server/Security/AuthenticationMiddleware.cs @@ -0,0 +1,79 @@ +using Titano.Core.Model; +using Titano.Server.Api; + +namespace Titano.Server.Security; + +/// +/// Resolves the caller of every /api request (bearer token or admin cookie) and enforces the endpoint's +/// : 401 without a valid token, 403 for a missing permission, a missing CSRF header on a +/// cookie request, or a pending password change. Runs after routing, so the endpoint's metadata is known. +/// +public sealed class AuthenticationMiddleware(RequestDelegate next, TokenService tokens) +{ + public async Task InvokeAsync(HttpContext context) + { + var rule = context.GetEndpoint()?.Metadata.GetMetadata(); + if (rule is null && !context.Request.Path.StartsWithSegments("/api")) + { + await next(context); + return; + } + rule ??= AccessRule.SignedIn; + if (rule.AllowAnonymous) + { + await next(context); + return; + } + + var caller = await AuthenticateAsync(context); + if (caller is null) + { + context.Response.Headers.WWWAuthenticate = "Bearer"; + await ApiErrors.WriteAsync(context, StatusCodes.Status401Unauthorized, "unauthorized"); + return; + } + if (caller.Source == CredentialSource.Cookie && !IsSameSiteRequest(context, caller)) + { + await ApiErrors.WriteAsync(context, StatusCodes.Status403Forbidden, "csrf_failed"); + return; + } + if (caller.User.MustChangePassword && !rule.DuringPasswordChange) + { + await ApiErrors.WriteAsync(context, StatusCodes.Status403Forbidden, "password_change_required"); + return; + } + if (rule.AnyOf != Permissions.None && !caller.Has(rule.AnyOf)) + { + await ApiErrors.WriteAsync(context, StatusCodes.Status403Forbidden, "forbidden"); + return; + } + context.Features.Set(caller); + await next(context); + } + + private Task AuthenticateAsync(HttpContext context) + { + var authorization = context.Request.Headers.Authorization.ToString(); + if (authorization.StartsWith("Bearer ", StringComparison.OrdinalIgnoreCase)) + return tokens.ValidateAsync(authorization[7..].Trim(), CredentialSource.Bearer, context.RequestAborted); + if (context.Request.Cookies.TryGetValue(SessionCookies.Session, out var cookie)) + return tokens.ValidateAsync(cookie, CredentialSource.Cookie, context.RequestAborted); + return Task.FromResult(null); + } + + /// + /// Cookie requests must prove they come from the admin panel: unsafe methods carry the CSRF header, WebSocket + /// upgrades (which cannot) must come from this origin. + /// + private static bool IsSameSiteRequest(HttpContext context, Caller caller) + { + if (context.WebSockets.IsWebSocketRequest) + { + var origin = context.Request.Headers.Origin.ToString(); + return origin.Equals($"{context.Request.Scheme}://{context.Request.Host}", StringComparison.OrdinalIgnoreCase); + } + var method = context.Request.Method; + if (HttpMethods.IsGet(method) || HttpMethods.IsHead(method) || HttpMethods.IsOptions(method)) return true; + return SessionCookies.HasValidCsrf(context, caller); + } +} diff --git a/src/Titano.Server/Security/LoginGuard.cs b/src/Titano.Server/Security/LoginGuard.cs new file mode 100644 index 00000000..8e1e49f9 --- /dev/null +++ b/src/Titano.Server/Security/LoginGuard.cs @@ -0,0 +1,92 @@ +using Titano.Server.Configuration; + +namespace Titano.Server.Security; + +/// +/// Sign-in throttling (BRIEF §14.2): at most attempts per client +/// address and minute, and an account name locked for after +/// consecutive failures. Names without an account are tracked the same +/// way, so a lockout reveals nothing about which names exist. State is per process and resets at restart. +/// +public sealed class LoginGuard(ServerOptions options, TimeProvider time) +{ + private static readonly TimeSpan Window = TimeSpan.FromMinutes(1); + private const int PruneAbove = 10_000; + + private readonly Lock _gate = new(); + private readonly Dictionary> _attempts = new(StringComparer.Ordinal); + private readonly Dictionary _accounts = new(StringComparer.Ordinal); + + /// Counts an attempt from ; false, with the time to wait, once its budget is spent. + public bool TryAttempt(string address, out TimeSpan retryAfter) + { + var now = time.GetUtcNow(); + lock (_gate) + { + if (_attempts.Count > PruneAbove) Prune(now); + if (!_attempts.TryGetValue(address, out var recent)) _attempts[address] = recent = new Queue(); + while (recent.Count > 0 && now - recent.Peek() >= Window) recent.Dequeue(); + if (recent.Count >= options.LoginAttemptsPerMinute) + { + retryAfter = Window - (now - recent.Peek()); + return false; + } + recent.Enqueue(now); + retryAfter = TimeSpan.Zero; + return true; + } + } + + /// True while (a normalised user name) is locked. + public bool IsLocked(string account, out TimeSpan retryAfter) + { + var now = time.GetUtcNow(); + lock (_gate) + { + if (_accounts.TryGetValue(account, out var state) && state.LockedUntil is { } until && until > now) + { + retryAfter = until - now; + return true; + } + retryAfter = TimeSpan.Zero; + return false; + } + } + + public void RecordFailure(string account) + { + var now = time.GetUtcNow(); + var span = TimeSpan.FromMinutes(options.LockoutMinutes); + lock (_gate) + { + if (_accounts.Count > PruneAbove) Prune(now); + var state = _accounts.GetValueOrDefault(account); + if (state?.LockedUntil is { } until && until > now) return; + if (state is null || state.LockedUntil is not null || now - state.FirstFailure > span) state = new AccountState(0, now, null); + var failures = state.Failures + 1; + _accounts[account] = failures >= options.LockoutThreshold + ? new AccountState(0, now, now + span) + : state with { Failures = failures }; + } + } + + public void RecordSuccess(string account) + { + lock (_gate) _accounts.Remove(account); + } + + private void Prune(DateTimeOffset now) + { + foreach (var (address, recent) in _attempts.ToList()) + { + if (recent.Count == 0 || now - recent.Last() >= Window) _attempts.Remove(address); + } + var span = TimeSpan.FromMinutes(options.LockoutMinutes); + foreach (var (account, state) in _accounts.ToList()) + { + if ((state.LockedUntil ?? state.FirstFailure + span) < now) _accounts.Remove(account); + } + } + + private sealed record AccountState(int Failures, DateTimeOffset FirstFailure, DateTimeOffset? LockedUntil); +} diff --git a/src/Titano.Server/Security/PasswordHasher.cs b/src/Titano.Server/Security/PasswordHasher.cs new file mode 100644 index 00000000..5f66b357 --- /dev/null +++ b/src/Titano.Server/Security/PasswordHasher.cs @@ -0,0 +1,69 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text; + +namespace Titano.Server.Security; + +/// +/// Password hashes (BRIEF §14.2): PBKDF2-SHA512, 210,000 iterations, 16-byte random salt per hash, stored as +/// pbkdf2-sha512$<iterations>$<salt>$<hash> (base64). The format names its algorithm and cost, so +/// older hashes keep verifying and are upgraded at the next sign-in. +/// +public static class PasswordHasher +{ + public const int MinimumLength = 10; + public const int Iterations = 210_000; + private const string Algorithm = "pbkdf2-sha512"; + private const int SaltBytes = 16; + private const int HashBytes = 64; + + /// Verified against unknown user names so a miss costs as much time as a wrong password. + private static readonly Lazy DummyHash = new(() => Hash(Convert.ToBase64String(RandomNumberGenerator.GetBytes(18)))); + + public static string Hash(string password) + { + var salt = RandomNumberGenerator.GetBytes(SaltBytes); + var hash = Derive(password, salt, Iterations); + return string.Create(CultureInfo.InvariantCulture, $"{Algorithm}${Iterations}${Convert.ToBase64String(salt)}${Convert.ToBase64String(hash)}"); + } + + /// Constant-time check of against a stored hash; malformed hashes never match. + public static bool Verify(string password, string storedHash) + { + var parts = storedHash.Split('$'); + if (parts.Length != 4 || parts[0] != Algorithm + || !int.TryParse(parts[1], NumberStyles.None, CultureInfo.InvariantCulture, out var iterations) || iterations < 1) + return false; + byte[] salt, expected; + try + { + salt = Convert.FromBase64String(parts[2]); + expected = Convert.FromBase64String(parts[3]); + } + catch (FormatException) + { + return false; + } + if (salt.Length == 0 || expected.Length == 0) return false; + var actual = Rfc2898DeriveBytes.Pbkdf2(Encoding.UTF8.GetBytes(password), salt, iterations, HashAlgorithmName.SHA512, expected.Length); + return CryptographicOperations.FixedTimeEquals(actual, expected); + } + + /// Spends the same time as for a name that has no account. + public static void VerifyDummy(string password) => Verify(password, DummyHash.Value); + + /// True when the hash was made with fewer iterations than the current policy. + public static bool NeedsRehash(string storedHash) + { + var parts = storedHash.Split('$'); + return parts.Length != 4 || parts[0] != Algorithm + || !int.TryParse(parts[1], NumberStyles.None, CultureInfo.InvariantCulture, out var iterations) || iterations < Iterations; + } + + /// Policy check for passwords users choose: at least characters, not only spaces. + public static bool IsAcceptable(string? password) => + password is not null && password.Length >= MinimumLength && !string.IsNullOrWhiteSpace(password); + + private static byte[] Derive(string password, byte[] salt, int iterations) => + Rfc2898DeriveBytes.Pbkdf2(Encoding.UTF8.GetBytes(password), salt, iterations, HashAlgorithmName.SHA512, HashBytes); +} diff --git a/src/Titano.Server/Security/SessionCookies.cs b/src/Titano.Server/Security/SessionCookies.cs new file mode 100644 index 00000000..8e49a4df --- /dev/null +++ b/src/Titano.Server/Security/SessionCookies.cs @@ -0,0 +1,55 @@ +using System.Buffers.Text; +using System.Net; +using System.Security.Cryptography; +using System.Text; + +namespace Titano.Server.Security; + +/// +/// The web admin's cookie session (BRIEF §14.2): the token in an HttpOnly; Secure; SameSite=Strict cookie, plus a +/// script-readable CSRF cookie whose value must come back in on every unsafe request. The CSRF +/// value is derived from the session token, so a planted cookie pair cannot forge it. +/// +public static class SessionCookies +{ + public const string Session = "titano_session"; + public const string Csrf = "titano_csrf"; + public const string CsrfHeader = "X-CSRF-Token"; + + public static void Issue(HttpContext context, IssuedToken token) + { + context.Response.Cookies.Append(Session, token.Secret, Options(httpOnly: true)); + context.Response.Cookies.Append(Csrf, CsrfFor(token.Stored.TokenHash), Options(httpOnly: false)); + } + + public static void Clear(HttpContext context) + { + context.Response.Cookies.Delete(Session, Options(httpOnly: true)); + context.Response.Cookies.Delete(Csrf, Options(httpOnly: false)); + } + + /// Constant-time check of the request's CSRF header against the caller's session. + public static bool HasValidCsrf(HttpContext context, Caller caller) + { + var header = context.Request.Headers[CsrfHeader].ToString(); + var expected = CsrfFor(caller.TokenHash); + return header.Length == expected.Length + && CryptographicOperations.FixedTimeEquals(Encoding.ASCII.GetBytes(header), Encoding.ASCII.GetBytes(expected)); + } + + /// Secure cookies only travel over HTTPS (or to the machine itself, which browsers treat as secure). + public static bool IsSecureTransport(HttpContext context) => + context.Request.IsHttps || (context.Connection.RemoteIpAddress is { } remote && IPAddress.IsLoopback(remote)); + + private static string CsrfFor(string tokenHash) => + Base64Url.EncodeToString(SHA256.HashData(Encoding.UTF8.GetBytes("titano-csrf|" + tokenHash))); + + private static CookieOptions Options(bool httpOnly) => new() + { + HttpOnly = httpOnly, + Secure = true, + SameSite = SameSiteMode.Strict, + Path = "/", + IsEssential = true, + }; +} diff --git a/src/Titano.Server/Security/TemporaryPassword.cs b/src/Titano.Server/Security/TemporaryPassword.cs new file mode 100644 index 00000000..bf6874f1 --- /dev/null +++ b/src/Titano.Server/Security/TemporaryPassword.cs @@ -0,0 +1,15 @@ +using System.Security.Cryptography; + +namespace Titano.Server.Security; + +/// One-time passwords for new accounts and resets: three groups of four unambiguous characters (≈ 70 bits). +public static class TemporaryPassword +{ + private const string Alphabet = "abcdefghjkmnpqrstuvwxyz23456789"; + + public static string Generate() + { + var chars = RandomNumberGenerator.GetItems(Alphabet, 12); + return $"{new string(chars, 0, 4)}-{new string(chars, 4, 4)}-{new string(chars, 8, 4)}"; + } +} diff --git a/src/Titano.Server/Security/TokenService.cs b/src/Titano.Server/Security/TokenService.cs new file mode 100644 index 00000000..d364f51d --- /dev/null +++ b/src/Titano.Server/Security/TokenService.cs @@ -0,0 +1,91 @@ +using System.Buffers.Text; +using System.Security.Cryptography; +using System.Text; +using Titano.Core.Model; +using Titano.Data; +using Titano.Server.Configuration; + +namespace Titano.Server.Security; + +/// How a request presented its token. +public enum CredentialSource +{ + /// Authorization: Bearer (desktop client, uploader, render node). + Bearer, + + /// HttpOnly session cookie of the web admin; unsafe methods also need the CSRF header. + Cookie, +} + +/// The signed-in account behind a request. +public sealed record Caller(User User, string TokenHash, CredentialSource Source, string? ClientName) +{ + public Guid Id => User.Id; + + public bool IsAdmin => User.Role == UserRole.Admin; + + /// True when the account holds at least one of . + public bool Has(Permissions any) => (User.Permissions & any) != 0; + + /// Minimal-API binding: endpoint handlers take the caller resolved by . + public static ValueTask BindAsync(HttpContext context) => ValueTask.FromResult(context.Features.Get()); +} + +/// A freshly issued token: the secret (returned once) and what is stored (its hash). +public sealed record IssuedToken(string Secret, SessionToken Stored); + +/// +/// Opaque bearer tokens (BRIEF §14.2): 32 random bytes, stored only as their SHA-256, sliding expiry, revocation. +/// The same tokens back the admin panel's cookie session. +/// +public sealed class TokenService(TitanoDatabase db, ServerOptions options, TimeProvider time) +{ + /// Sliding expiry is written at most this often per token, not on every request. + private static readonly TimeSpan TouchInterval = TimeSpan.FromMinutes(1); + + private TimeSpan Lifetime => TimeSpan.FromDays(options.TokenLifetimeDays); + + public async Task IssueAsync(User user, string? clientName, CancellationToken cancellationToken) + { + var secret = Base64Url.EncodeToString(RandomNumberGenerator.GetBytes(32)); + var now = time.GetUtcNow(); + var name = clientName?.Trim(); + var token = new SessionToken + { + TokenHash = HashOf(secret), + UserId = user.Id, + ClientName = string.IsNullOrEmpty(name) ? null : name[..Math.Min(name.Length, 100)], + Created = now, + Expires = now + Lifetime, + LastUsed = now, + }; + await db.Tokens.InsertAsync(token, cancellationToken).ConfigureAwait(false); + return new IssuedToken(secret, token); + } + + /// The caller a token belongs to, or null when it is unknown, revoked, expired or its account is disabled. + public async Task ValidateAsync(string secret, CredentialSource source, CancellationToken cancellationToken) + { + if (secret.Length is 0 or > 128) return null; + var hash = HashOf(secret); + var token = await db.Tokens.GetAsync(hash, cancellationToken).ConfigureAwait(false); + var now = time.GetUtcNow(); + if (token is null || token.Revoked || token.Expires <= now) return null; + var user = await db.Users.GetAsync(token.UserId, cancellationToken).ConfigureAwait(false); + if (user is null || user.Disabled) return null; + if (now - token.LastUsed >= TouchInterval && !await db.Tokens.TouchAsync(hash, now, now + Lifetime, cancellationToken).ConfigureAwait(false)) + return null; + return new Caller(user, hash, source, token.ClientName); + } + + /// True while the caller's token is still live and its account enabled (long-lived WebSocket sessions). + public async Task IsStillValidAsync(Caller caller, CancellationToken cancellationToken) + { + var token = await db.Tokens.GetAsync(caller.TokenHash, cancellationToken).ConfigureAwait(false); + if (token is null || token.Revoked || token.Expires <= time.GetUtcNow()) return false; + return await db.Users.GetAsync(caller.Id, cancellationToken).ConfigureAwait(false) is { Disabled: false }; + } + + /// Stored form of a token: lower-case hex SHA-256 of its text. + public static string HashOf(string secret) => Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(secret))); +} diff --git a/src/Titano.Server/Storage/ArchivePaths.cs b/src/Titano.Server/Storage/ArchivePaths.cs new file mode 100644 index 00000000..bb8e16f2 --- /dev/null +++ b/src/Titano.Server/Storage/ArchivePaths.cs @@ -0,0 +1,45 @@ +namespace Titano.Server.Storage; + +/// +/// Archive-relative paths from the database → absolute paths, refusing anything that would leave the archive root +/// (BRIEF §14.2: path traversal impossible). Clients never send paths; only ids that the database maps to these. +/// +public static class ArchivePaths +{ + private static readonly StringComparison PathComparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; + + /// The absolute path of under , or null when it escapes the root. + public static string? Resolve(string root, string? relative) + { + if (string.IsNullOrWhiteSpace(relative) || relative.Contains('\0', StringComparison.Ordinal)) return null; + var portable = relative.Replace('\\', '/'); + if (Path.IsPathRooted(portable) || portable.StartsWith('/')) return null; + string full; + try + { + full = Path.GetFullPath(Path.Combine(Root(root), portable)); + } + catch (Exception ex) when (ex is ArgumentException or NotSupportedException or PathTooLongException) + { + return null; + } + return IsInside(root, full) ? full : null; + } + + /// True when lies strictly inside . + public static bool IsInside(string root, string path) + { + var full = Path.GetFullPath(path); + return full.StartsWith(Root(root) + Path.DirectorySeparatorChar, PathComparison); + } + + /// A file of the archive that may be served: inside the root, existing, and not a symbolic link. + public static FileInfo? ExistingFile(string root, string? relative) + { + if (Resolve(root, relative) is not { } full) return null; + var file = new FileInfo(full); + return file.Exists && file.LinkTarget is null ? file : null; + } + + private static string Root(string root) => Path.TrimEndingDirectorySeparator(Path.GetFullPath(root)); +} diff --git a/src/Titano.Server/Storage/FileResponses.cs b/src/Titano.Server/Storage/FileResponses.cs new file mode 100644 index 00000000..c105d428 --- /dev/null +++ b/src/Titano.Server/Storage/FileResponses.cs @@ -0,0 +1,123 @@ +using System.Collections.Frozen; +using Microsoft.Net.Http.Headers; + +namespace Titano.Server.Storage; + +/// Streams archive files (originals, proxies, thumbnails, exports) with Range, If-Range and conditional GET support. +public static class FileResponses +{ + private static readonly FrozenDictionary Types = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + [".jpg"] = "image/jpeg", + [".jpeg"] = "image/jpeg", + [".png"] = "image/png", + [".gif"] = "image/gif", + [".tif"] = "image/tiff", + [".tiff"] = "image/tiff", + [".dng"] = "image/x-adobe-dng", + [".heic"] = "image/heic", + [".heif"] = "image/heif", + [".webp"] = "image/webp", + [".avif"] = "image/avif", + [".mp4"] = "video/mp4", + [".mov"] = "video/quicktime", + [".avi"] = "video/x-msvideo", + }.ToFrozenDictionary(StringComparer.OrdinalIgnoreCase); + + public static string ContentTypeOf(string fileName) => + Types.TryGetValue(Path.GetExtension(fileName), out var type) ? type : "application/octet-stream"; + + /// + /// The byte range [, offset + ) of as a + /// range-capable response; null when the range does not fit the file (the asset changed under the index). + /// + public static IResult? Serve(HttpContext context, FileInfo file, long offset, long length, string contentType, string cacheControl, + string? etag = null, string? downloadName = null) + { + if (offset < 0 || length < 0 || offset > file.Length - length) return null; + context.Response.Headers.CacheControl = cacheControl; + return Results.Stream(new FileSlice(file.FullName, offset, length), contentType, downloadName, file.LastWriteTimeUtc, + etag is null ? null : new EntityTagHeaderValue("\"" + etag + "\""), enableRangeProcessing: true); + } + + /// Read-only, seekable view of part of a file (one thumbnail inside a pack, or the whole file). + private sealed class FileSlice : Stream + { + private readonly FileStream _file; + private readonly long _offset; + private readonly long _length; + private long _position; + + public FileSlice(string path, long offset, long length) + { + _file = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, 1 << 16, FileOptions.Asynchronous | FileOptions.SequentialScan); + _offset = offset; + _length = length; + } + + public override bool CanRead => true; + public override bool CanSeek => true; + public override bool CanWrite => false; + public override long Length => _length; + + public override long Position + { + get => _position; + set => _position = Math.Clamp(value, 0, _length); + } + + public override int Read(byte[] buffer, int offset, int count) => Read(buffer.AsSpan(offset, count)); + + public override int Read(Span buffer) + { + var count = Available(buffer.Length); + if (count == 0) return 0; + _file.Position = _offset + _position; + var read = _file.Read(buffer[..count]); + _position += read; + return read; + } + + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => + ReadAsync(buffer.AsMemory(offset, count), cancellationToken).AsTask(); + + public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + var count = Available(buffer.Length); + if (count == 0) return 0; + _file.Position = _offset + _position; + var read = await _file.ReadAsync(buffer[..count], cancellationToken).ConfigureAwait(false); + _position += read; + return read; + } + + public override long Seek(long offset, SeekOrigin origin) => Position = origin switch + { + SeekOrigin.Begin => offset, + SeekOrigin.Current => _position + offset, + _ => _length + offset, + }; + + public override void Flush() + { + } + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + protected override void Dispose(bool disposing) + { + if (disposing) _file.Dispose(); + base.Dispose(disposing); + } + + public override async ValueTask DisposeAsync() + { + await _file.DisposeAsync().ConfigureAwait(false); + await base.DisposeAsync().ConfigureAwait(false); + } + + private int Available(int requested) => (int)Math.Min(requested, _length - _position); + } +} diff --git a/src/Titano.Server/Storage/IFrameAssetLocator.cs b/src/Titano.Server/Storage/IFrameAssetLocator.cs new file mode 100644 index 00000000..3148f286 --- /dev/null +++ b/src/Titano.Server/Storage/IFrameAssetLocator.cs @@ -0,0 +1,26 @@ +using Titano.Core.Model; + +namespace Titano.Server.Storage; + +/// Regenerable images derived from an original (BRIEF §8.5). +public enum FrameAsset +{ + /// JPEG proxy under the sequence's proxies/ folder. + Proxy, + + /// Thumbnail inside the sequence's packed thumbnail file. + Thumbnail, +} + +/// Where a derived image lives: a byte range of a file inside the archive (a whole proxy file or one thumbnail of a pack). +public sealed record AssetLocation(string Path, long Offset, long Length, string ContentType); + +/// +/// Finds proxies and thumbnails in the archive layout, which the archive module owns (BRIEF §7). The server streams +/// what it returns with Range support after checking the path stays inside the archive root; null means not built yet. +/// Registered by the host composition alongside . +/// +public interface IFrameAssetLocator +{ + ValueTask LocateAsync(Archive archive, Sequence sequence, Frame frame, FrameAsset asset, CancellationToken cancellationToken); +} diff --git a/src/Titano.Server/Storage/RenderStore.cs b/src/Titano.Server/Storage/RenderStore.cs new file mode 100644 index 00000000..9748accd --- /dev/null +++ b/src/Titano.Server/Storage/RenderStore.cs @@ -0,0 +1,49 @@ +using Titano.Core.IO; +using Titano.Core.Model; +using Titano.Data; +using Titano.Protocol; + +namespace Titano.Server.Storage; + +/// Files a verified render output under the sequence's exports/ folder (BRIEF §7) and records it. +public sealed class RenderStore(TitanoDatabase db, ServerArchive archive, TimeProvider time) +{ + /// + /// Moves (on the archive volume) to <sequence folder>/exports/, never + /// replacing an existing export, and inserts the render row. Null when the sequence has no folder in the archive yet. + /// + public async Task CommitAsync(Sequence sequence, string stagedFile, RenderResultRequest output, string producedBy, CancellationToken cancellationToken) + { + if (string.IsNullOrEmpty(sequence.Folder) || ArchivePaths.Resolve(archive.Root, sequence.Folder.TrimEnd('/') + "/exports") is not { } exports) + return null; + Directory.CreateDirectory(exports); + var target = Unique(exports, output.FileName); + File.Move(stagedFile, target); + var render = new Render + { + Id = Guid.CreateVersion7(), + SequenceId = sequence.Id, + PresetName = output.PresetName, + OutputPath = LongPath.ToPortableRelative(archive.Root, target), + Codec = output.Codec, + Width = output.Width, + Height = output.Height, + Fps = output.Fps, + DurationSeconds = output.DurationSeconds, + Bytes = output.Size, + Created = time.GetUtcNow(), + ProducedBy = producedBy, + }; + await db.Renders.InsertAsync(render, cancellationToken); + return render; + } + + private static string Unique(string folder, string fileName) + { + var candidate = Path.Combine(folder, fileName); + var stem = Path.GetFileNameWithoutExtension(fileName); + var extension = Path.GetExtension(fileName); + for (var n = 2; File.Exists(candidate); n++) candidate = Path.Combine(folder, $"{stem} ({n}){extension}"); + return candidate; + } +} diff --git a/src/Titano.Server/Storage/ServerArchive.cs b/src/Titano.Server/Storage/ServerArchive.cs new file mode 100644 index 00000000..d7e1ae8d --- /dev/null +++ b/src/Titano.Server/Storage/ServerArchive.cs @@ -0,0 +1,37 @@ +using Titano.Core.Model; +using Titano.Data; +using Titano.Server.Configuration; + +namespace Titano.Server.Storage; + +/// +/// The archive this server serves () and its database row, created at first +/// start. The admin panel can change its name, templates and proxy policy at runtime. +/// +public sealed class ServerArchive(Archive archive) +{ + private volatile Archive _current = archive; + + public Archive Current + { + get => _current; + set => _current = value; + } + + public Guid Id => _current.Id; + + public string Root => _current.RootPath; + + /// Finds the archive row for the configured root, or creates it with the default templates. + public static async Task OpenAsync(TitanoDatabase db, ServerOptions options, TimeProvider time, CancellationToken cancellationToken) + { + var root = Path.TrimEndingDirectorySeparator(Path.GetFullPath(options.ArchivePath)); + var comparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; + var existing = (await db.Archives.ListAsync(cancellationToken).ConfigureAwait(false)) + .FirstOrDefault(a => string.Equals(Path.TrimEndingDirectorySeparator(a.RootPath), root, comparison)); + if (existing is not null) return new ServerArchive(existing); + var created = new Archive { Id = Guid.CreateVersion7(), Name = options.ServerName, RootPath = root, Created = time.GetUtcNow() }; + await db.Archives.UpsertAsync(created, cancellationToken).ConfigureAwait(false); + return new ServerArchive(created); + } +} diff --git a/src/Titano.Server/Storage/UploadMaintenance.cs b/src/Titano.Server/Storage/UploadMaintenance.cs new file mode 100644 index 00000000..3abe04c1 --- /dev/null +++ b/src/Titano.Server/Storage/UploadMaintenance.cs @@ -0,0 +1,102 @@ +using Titano.Data; +using Titano.Data.Repositories; +using Titano.Server.Configuration; + +namespace Titano.Server.Storage; + +/// +/// Upload housekeeping (BRIEF §14.4): at start, uploads a crash left half-committed are settled; then every half hour +/// abandoned uploads expire, orphaned partial and staging files are removed, old upload records and dead tokens purged. +/// +public sealed class UploadMaintenance(TitanoDatabase db, ServerOptions options, UploadStore store, TimeProvider time, ILogger logger) + : BackgroundService +{ + private static readonly TimeSpan Interval = TimeSpan.FromMinutes(30); + private static readonly TimeSpan FinishedRetention = TimeSpan.FromDays(90); + private static readonly TimeSpan StagingRetention = TimeSpan.FromDays(1); + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + try + { + await RecoverAsync(stoppingToken); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogError(ex, "Upload recovery failed"); + } + while (!stoppingToken.IsCancellationRequested) + { + try + { + await SweepAsync(stoppingToken); + await Task.Delay(Interval, time, stoppingToken); + } + catch (OperationCanceledException) + { + return; + } + catch (Exception ex) + { + logger.LogWarning(ex, "Upload cleanup failed"); + } + } + } + + /// + /// Uploads left committing by a crash: committed when the frame with their hash exists, otherwise back to receiving + /// (the partial file is intact, so the client simply completes again). + /// + public async Task RecoverAsync(CancellationToken cancellationToken) + { + foreach (var upload in await db.Uploads.ListByStateAsync(UploadState.Committing, cancellationToken)) + { + var frame = upload.JobId is null ? (await db.Frames.FindBySha256Async(upload.Sha256, cancellationToken)).FirstOrDefault() : null; + if (frame is not null) + { + await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Committed, frame.Id, cancellationToken); + store.Delete(upload.Id); + } + else + { + await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Receiving, cancellationToken: cancellationToken); + } + logger.LogInformation("Recovered interrupted upload {Upload} ({File})", upload.Id, upload.FileName); + } + } + + public async Task SweepAsync(CancellationToken cancellationToken) + { + var now = time.GetUtcNow(); + var abandonedBefore = now - TimeSpan.FromHours(options.UploadRetentionHours); + var expired = 0; + foreach (var upload in await db.Uploads.ListStaleAsync(abandonedBefore, cancellationToken)) + { + if (!await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Receiving, UploadState.Cancelled, cancellationToken: cancellationToken)) continue; + store.Delete(upload.Id); + expired++; + } + + var active = new HashSet(); + foreach (var state in new[] { UploadState.Receiving, UploadState.Committing }) + active.UnionWith((await db.Uploads.ListByStateAsync(state, cancellationToken)).Select(u => u.Id)); + foreach (var (id, _) in store.List().ToList()) + { + if (!active.Contains(id)) store.Delete(id); + } + if (Directory.Exists(options.StagingPath)) + { + // Only this server's staging files (named by upload id) are touched; clients may share the archive. + foreach (var file in new DirectoryInfo(options.StagingPath).EnumerateFiles()) + { + if (Guid.TryParseExact(Path.GetFileNameWithoutExtension(file.Name), "N", out var id) && !active.Contains(id) + && file.LastWriteTimeUtc < now.UtcDateTime - StagingRetention) + UploadStore.TryDelete(file.FullName); + } + } + + await db.Uploads.PurgeFinishedAsync(now - FinishedRetention, cancellationToken); + await db.Tokens.PurgeAsync(now, cancellationToken); + if (expired > 0) logger.LogInformation("Expired {Count} abandoned upload(s)", expired); + } +} diff --git a/src/Titano.Server/Storage/UploadRules.cs b/src/Titano.Server/Storage/UploadRules.cs new file mode 100644 index 00000000..9dd31eef --- /dev/null +++ b/src/Titano.Server/Storage/UploadRules.cs @@ -0,0 +1,45 @@ +namespace Titano.Server.Storage; + +/// Validation shared by frame and render uploads. +public static class UploadRules +{ + /// A plain file name (no folders, no characters Windows or Linux refuse), or null. + public static string? SafeFileName(string? name) + { + if (string.IsNullOrWhiteSpace(name)) return null; + var trimmed = name.Trim(); + if (trimmed.Length > 255 || trimmed is "." or ".." || trimmed.EndsWith('.')) return null; + foreach (var c in trimmed) + { + if (char.IsControl(c) || c is '/' or '\\' or '<' or '>' or ':' or '"' or '|' or '?' or '*') return null; + } + return trimmed; + } + + /// 64 hexadecimal characters. + public static bool IsSha256(string? value) => value is { Length: 64 } && value.All(char.IsAsciiHexDigit); + + /// Indices of the chunks set in a received-chunk bitmap. + public static List ReceivedChunks(Titano.Data.Repositories.Upload upload) + { + var received = new List(upload.ReceivedCount); + for (var i = 0; i < upload.ChunkCount; i++) + { + if (upload.HasChunk(i)) received.Add(i); + } + return received; + } + + /// Free bytes on the volume holding , or null when the platform cannot tell. + public static long? FreeBytes(string path) + { + try + { + return new DriveInfo(Path.GetFullPath(path)).AvailableFreeSpace; + } + catch (Exception ex) when (ex is IOException or ArgumentException or UnauthorizedAccessException) + { + return null; + } + } +} diff --git a/src/Titano.Server/Storage/UploadService.cs b/src/Titano.Server/Storage/UploadService.cs new file mode 100644 index 00000000..cef9f2ff --- /dev/null +++ b/src/Titano.Server/Storage/UploadService.cs @@ -0,0 +1,242 @@ +using System.Collections.Concurrent; +using System.Globalization; +using Titano.Data; +using Titano.Data.Archiving; +using Titano.Data.Repositories; +using Titano.Protocol; +using Titano.Server.Api; +using Titano.Server.Configuration; +using Titano.Server.Events; +using Titano.Server.Jobs; +using Titano.Server.Security; +using UploadState = Titano.Data.Repositories.UploadState; +using UploadStateDto = Titano.Protocol.UploadState; + +namespace Titano.Server.Storage; + +/// +/// Chunked, resumable, hash-verified frame uploads (BRIEF §14.4): start (short-circuit on a known SHA-256, resume an +/// unfinished upload, quota and disk checks), idempotent chunks, and completion that stages the file on the archive +/// volume, verifies its hash and hands it to . +/// +public sealed class UploadService( + TitanoDatabase db, + ServerOptions options, + ServerArchive archive, + UploadStore store, + IArchiveIngest ingest, + SequenceJobs sequenceJobs, + EventHub events, + IHostApplicationLifetime lifetime, + TimeProvider time, + ILogger logger) +{ + /// Space kept free on the upload volume beyond the announced size. + private const long DiskReserve = 64L * 1024 * 1024; + + private readonly ConcurrentDictionary _chunksInFlight = new(); + + public async Task> StartAsync(Caller caller, StartUploadRequest request, CancellationToken cancellationToken) + { + if (UploadRules.SafeFileName(request.FileName) is not { } fileName) return Fail(400, "invalid_file_name"); + if (!UploadRules.IsSha256(request.Sha256)) return Fail(400, "invalid_hash"); + if (request.Size <= 0) return Fail(400, "bad_request", "size"); + if (request.Size > options.MaxUploadBytes) return Fail(413, "too_large", options.MaxUploadBytes.ToString(CultureInfo.InvariantCulture)); + if (request.SessionKey is { Length: > 200 }) return Fail(400, "bad_request", "sessionKey"); + if (request.SequenceId is { } target && await db.Sequences.GetAsync(target, cancellationToken) is null) + return Fail(404, "not_found", "sequence"); + + var sha = request.Sha256.ToLowerInvariant(); + var known = await db.Frames.FindBySha256Async(sha, cancellationToken); + if (known.Count > 0) + { + var frame = known.FirstOrDefault(f => f.SequenceId == request.SequenceId) ?? known[0]; + return new StartUploadResponse(null, options.ChunkSizeBytes, [], AlreadyExists: true, frame.Id); + } + + var resumable = await db.Uploads.FindReceivingAsync(caller.Id, sha, cancellationToken); + if (resumable is not null && resumable.Size == request.Size && resumable.SequenceId == request.SequenceId && store.Exists(resumable.Id)) + return Started(resumable); + + if (caller.User.QuotaBytes > 0) + { + var pending = await db.Uploads.PendingBytesAsync(caller.Id, cancellationToken); + if (caller.User.UsedBytes + pending + request.Size > caller.User.QuotaBytes) + return Fail(403, "quota_exceeded", caller.User.QuotaBytes.ToString(CultureInfo.InvariantCulture)); + } + if (UploadRules.FreeBytes(options.UploadsPath) is { } free && free < request.Size + DiskReserve) + return Fail(507, "insufficient_storage"); + + var now = time.GetUtcNow(); + var upload = await db.Uploads.CreateAsync(new Upload + { + Id = Guid.CreateVersion7(), + UserId = caller.Id, + SequenceId = request.SequenceId, + FileName = fileName, + Size = request.Size, + Sha256 = sha, + CapturedAt = request.CapturedAt, + ChunkSize = options.ChunkSizeBytes, + State = UploadState.Receiving, + Created = now, + UpdatedAt = now, + SessionKey = string.IsNullOrWhiteSpace(request.SessionKey) ? null : request.SessionKey.Trim(), + }, cancellationToken); + store.Create(upload.Id, upload.Size); + return Started(upload); + } + + /// Stores chunk of an upload the caller owns; repeating a received chunk changes nothing. + public async Task> ReceiveChunkAsync(Caller caller, Guid id, int index, HttpRequest request, CancellationToken cancellationToken) + { + var upload = await db.Uploads.GetAsync(id, cancellationToken); + if (upload is null || upload.UserId != caller.Id) return Fail(404, "not_found", "upload"); + if (upload.State != UploadState.Receiving) return Fail(409, "upload_state"); + if (index < 0 || index >= upload.ChunkCount) return Fail(400, "chunk_out_of_range"); + var expected = Math.Min(upload.ChunkSize, upload.Size - (long)index * upload.ChunkSize); + if (request.ContentLength is { } announced && announced != expected) return Fail(400, "chunk_size", expected.ToString(CultureInfo.InvariantCulture)); + if (upload.HasChunk(index)) return new ChunkReceived(index, UploadRules.ReceivedChunks(upload)); + + if (_chunksInFlight.AddOrUpdate(caller.Id, 1, (_, n) => n + 1) > options.ConcurrentChunksPerUser) + { + Release(caller.Id); + return Fail(429, "too_many_uploads"); + } + try + { + if (!await store.WriteChunkAsync(id, (long)index * upload.ChunkSize, request.Body, expected, cancellationToken)) + return Fail(400, "chunk_size", expected.ToString(CultureInfo.InvariantCulture)); + } + finally + { + Release(caller.Id); + } + var updated = await db.Uploads.MarkChunkReceivedAsync(id, index, cancellationToken); + return updated is null ? Fail(404, "not_found", "upload") : new ChunkReceived(index, UploadRules.ReceivedChunks(updated)); + } + + public async Task> GetStateAsync(Caller caller, Guid id, CancellationToken cancellationToken) + { + var upload = await db.Uploads.GetAsync(id, cancellationToken); + if (upload is null || upload.UserId != caller.Id) return Fail(404, "not_found", "upload"); + return new UploadStateDto(upload.Id, upload.FileName, upload.Size, upload.Sha256, upload.ChunkSize, UploadRules.ReceivedChunks(upload), + upload.State.ToString().ToLowerInvariant()); + } + + /// Abandons an upload that is still receiving and deletes its partial file. + public async Task> CancelAsync(Caller caller, Guid id, CancellationToken cancellationToken) + { + var upload = await db.Uploads.GetAsync(id, cancellationToken); + if (upload is null || upload.UserId != caller.Id) return Fail(404, "not_found", "upload"); + if (!await db.Uploads.TryTransitionAsync(id, UploadState.Receiving, UploadState.Cancelled, cancellationToken: cancellationToken)) + return Fail(409, "upload_state"); + store.Delete(id); + return true; + } + + /// + /// Verifies and commits a frame upload. Completing an already committed upload answers its frame again, so a client + /// may retry a completion whose response it lost. Runs to the end even if the client disconnects. + /// + public async Task> CompleteAsync(Caller caller, Guid id) + { + var stopping = lifetime.ApplicationStopping; + var upload = await db.Uploads.GetAsync(id, stopping); + if (upload is null || upload.UserId != caller.Id) return Fail(404, "not_found", "upload"); + if (upload.JobId is not null) return Fail(409, "wrong_upload_kind"); + if (upload is { State: UploadState.Committed, FrameId: { } committed } && await db.Frames.GetAsync(committed, stopping) is { } done) + return new CompleteUploadResponse(done.Id, done.SequenceId, done.Index); + if (upload.State != UploadState.Receiving) return Fail(409, "upload_state"); + if (!upload.IsComplete) + return Fail(409, "incomplete", (upload.ChunkCount - upload.ReceivedCount).ToString(CultureInfo.InvariantCulture)); + if (!await db.Uploads.TryTransitionAsync(id, UploadState.Receiving, UploadState.Committing, cancellationToken: stopping)) + return Fail(409, "upload_state"); + + var user = await db.Users.GetAsync(caller.Id, stopping); + if (user is { QuotaBytes: > 0 } && user.UsedBytes + upload.Size > user.QuotaBytes) + { + await FailAsync(upload); + return Fail(403, "quota_exceeded", user.QuotaBytes.ToString(CultureInfo.InvariantCulture)); + } + + var staged = await StageVerifiedAsync(upload, stopping); + if (staged is null) return Fail(422, "hash_mismatch"); + + IngestResult result; + try + { + result = await ingest.CommitAsync(new IngestRequest( + archive.Id, + upload.SequenceId, + NewSequenceTitle: null, + upload.SequenceId is null ? upload.SessionKey ?? "user-" + caller.Id.ToString("N") : null, + staged, + upload.FileName, + upload.Size, + upload.Sha256, + upload.CapturedAt, + caller.Id), stopping); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + // The part file is intact: the client may complete again. + logger.LogError(ex, "Archive ingest of upload {Upload} ({File}) failed", id, upload.FileName); + UploadStore.TryDelete(staged); + await db.Uploads.TryTransitionAsync(id, UploadState.Committing, UploadState.Receiving, cancellationToken: CancellationToken.None); + return Fail(503, "commit_failed"); + } + + await db.Uploads.TryTransitionAsync(id, UploadState.Committing, UploadState.Committed, result.FrameId, CancellationToken.None); + store.Delete(id); + if (File.Exists(staged)) UploadStore.TryDelete(staged); + if (!result.Duplicate) await db.Users.AddUsedBytesAsync(caller.Id, upload.Size, CancellationToken.None); + if (await db.Sequences.GetAsync(result.SequenceId, CancellationToken.None) is { } sequence) + { + await sequenceJobs.FramesAddedAsync(sequence, CancellationToken.None); + events.Publish(new SequenceChangedEvent(sequence.Id, "frames", sequence.FramesVersion, sequence.SettingsVersion)); + } + return new CompleteUploadResponse(result.FrameId, result.SequenceId, result.Index); + } + + /// + /// Copies a fully received upload to the archive staging folder and checks its SHA-256. On a mismatch the upload + /// fails, its files are deleted and null is returned; the client starts over. + /// + public async Task StageVerifiedAsync(Upload upload, CancellationToken cancellationToken) + { + var staged = Path.Combine(options.StagingPath, upload.Id.ToString("N") + Path.GetExtension(upload.FileName).ToLowerInvariant()); + string hash; + try + { + hash = await store.StageAsync(upload.Id, staged, cancellationToken); + } + catch + { + UploadStore.TryDelete(staged); + await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Receiving, cancellationToken: CancellationToken.None); + throw; + } + if (hash == upload.Sha256) return staged; + logger.LogWarning("Upload {Upload} ({File}) arrived damaged: SHA-256 {Actual} instead of {Expected}", upload.Id, upload.FileName, hash, upload.Sha256); + UploadStore.TryDelete(staged); + await FailAsync(upload); + return null; + } + + private async Task FailAsync(Upload upload) + { + await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Failed, cancellationToken: CancellationToken.None); + store.Delete(upload.Id); + } + + private StartUploadResponse Started(Upload upload) => + new(upload.Id, upload.ChunkSize, UploadRules.ReceivedChunks(upload), AlreadyExists: false, FrameId: null); + + private void Release(Guid userId) + { + if (_chunksInFlight.AddOrUpdate(userId, 0, (_, n) => n - 1) <= 0) _chunksInFlight.TryRemove(new KeyValuePair(userId, 0)); + } + + private static Outcome Fail(int status, string code, string? details = null) => Outcome.Fail(status, code, details); +} diff --git a/src/Titano.Server/Storage/UploadStore.cs b/src/Titano.Server/Storage/UploadStore.cs new file mode 100644 index 00000000..7e5c447b --- /dev/null +++ b/src/Titano.Server/Storage/UploadStore.cs @@ -0,0 +1,109 @@ +using System.Buffers; +using System.Security.Cryptography; +using Titano.Server.Configuration; + +namespace Titano.Server.Storage; + +/// +/// Files behind resumable uploads (BRIEF §14.4): /data/uploads/{id}.part receives chunks at their offsets in any +/// order; at completion it is copied, hashing on the way, to a staging file on the archive volume. +/// +public sealed class UploadStore(ServerOptions options) +{ + private const string Extension = ".part"; + + public string PartPath(Guid id) => Path.Combine(options.UploadsPath, id.ToString("N") + Extension); + + public bool Exists(Guid id) => File.Exists(PartPath(id)); + + /// Creates the part file at its final size, so chunks can land at their offsets. + public void Create(Guid id, long size) + { + Directory.CreateDirectory(options.UploadsPath); + using var file = new FileStream(PartPath(id), FileMode.CreateNew, FileAccess.Write, FileShare.ReadWrite); + file.SetLength(size); + } + + /// + /// Writes exactly bytes of at and flushes + /// them to disk; false when the body is shorter or longer (nothing is marked received then). + /// + public async Task WriteChunkAsync(Guid id, long offset, Stream body, long length, CancellationToken cancellationToken) + { + var buffer = ArrayPool.Shared.Rent(1 << 16); + try + { + await using var file = new FileStream(PartPath(id), FileMode.Open, FileAccess.Write, FileShare.ReadWrite, 1 << 16, FileOptions.Asynchronous); + file.Position = offset; + long total = 0; + int read; + while ((read = await body.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) > 0) + { + total += read; + if (total > length) return false; + await file.WriteAsync(buffer.AsMemory(0, read), cancellationToken).ConfigureAwait(false); + } + if (total != length) return false; + await file.FlushAsync(cancellationToken).ConfigureAwait(false); + file.Flush(flushToDisk: true); + return true; + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + /// Copies the assembled upload to (synced to disk) and returns its lower-case hex SHA-256. + public async Task StageAsync(Guid id, string stagingPath, CancellationToken cancellationToken) + { + Directory.CreateDirectory(Path.GetDirectoryName(stagingPath)!); + using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + var buffer = ArrayPool.Shared.Rent(1 << 20); + try + { + await using var source = new FileStream(PartPath(id), FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 16, FileOptions.Asynchronous | FileOptions.SequentialScan); + await using var target = new FileStream(stagingPath, FileMode.Create, FileAccess.Write, FileShare.None, 1 << 16, FileOptions.Asynchronous); + int read; + while ((read = await source.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) > 0) + { + hash.AppendData(buffer, 0, read); + await target.WriteAsync(buffer.AsMemory(0, read), cancellationToken).ConfigureAwait(false); + } + await target.FlushAsync(cancellationToken).ConfigureAwait(false); + target.Flush(flushToDisk: true); + } + finally + { + ArrayPool.Shared.Return(buffer); + } + return Convert.ToHexStringLower(hash.GetHashAndReset()); + } + + public void Delete(Guid id) => TryDelete(PartPath(id)); + + /// Every part file with its last write time (cleanup of uploads the database no longer knows). + public IEnumerable<(Guid Id, DateTime LastWriteUtc)> List() + { + if (!Directory.Exists(options.UploadsPath)) yield break; + foreach (var path in Directory.EnumerateFiles(options.UploadsPath, "*" + Extension)) + { + if (Guid.TryParseExact(Path.GetFileNameWithoutExtension(path), "N", out var id)) yield return (id, File.GetLastWriteTimeUtc(path)); + } + } + + public static void TryDelete(string path) + { + try + { + File.Delete(path); + } + catch (IOException) + { + // Retried by the next cleanup pass. + } + catch (UnauthorizedAccessException) + { + } + } +} diff --git a/tests/Titano.Server.Tests/Fakes.cs b/tests/Titano.Server.Tests/Fakes.cs new file mode 100644 index 00000000..d4f3d080 --- /dev/null +++ b/tests/Titano.Server.Tests/Fakes.cs @@ -0,0 +1,147 @@ +using System.Collections.Concurrent; +using Titano.Core.Model; +using Titano.Data; +using Titano.Data.Archiving; +using Titano.Data.Jobs; +using Titano.Server.Storage; + +namespace Titano.Server.Tests; + +/// +/// Stand-in for the archive module's ingest: files staged uploads as <folder>/originals/<name>, one +/// sequence per session key, and records every request it received. +/// +public sealed class FakeArchiveIngest(TitanoDatabase db) : IArchiveIngest +{ + private readonly ConcurrentDictionary _sessions = new(StringComparer.Ordinal); + private readonly SemaphoreSlim _gate = new(1, 1); + + public ConcurrentQueue Requests { get; } = new(); + + public async Task CommitAsync(IngestRequest request, CancellationToken cancellationToken = default) + { + Requests.Enqueue(request); + await _gate.WaitAsync(cancellationToken); + try + { + var archive = (await db.Archives.GetAsync(request.ArchiveId, cancellationToken))!; + var sequence = await SequenceForAsync(request, archive, cancellationToken); + var existing = (await db.Frames.FindBySha256Async(request.Sha256, cancellationToken)).FirstOrDefault(f => f.SequenceId == sequence.Id); + if (existing is not null) + { + File.Delete(request.StagedPath); + return new IngestResult(sequence.Id, existing.Id, existing.Index, existing.RelativePath, Duplicate: true); + } + var index = sequence.FrameCount; + var relative = $"{sequence.Folder}/originals/{index:D4}_{request.OriginalFileName}"; + var target = Path.Combine(archive.RootPath, relative); + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + File.Move(request.StagedPath, target); + var frame = new Frame + { + Id = Guid.CreateVersion7(), + SequenceId = sequence.Id, + Index = index, + RelativePath = relative, + FileName = Path.GetFileName(relative), + OriginalFileName = request.OriginalFileName, + Size = request.Size, + Sha256 = request.Sha256, + CapturedAt = request.ClientCapturedAt, + }; + await db.Frames.InsertAsync(frame, cancellationToken); + await db.Sequences.RefreshStatsAsync(sequence.Id, cancellationToken); + return new IngestResult(sequence.Id, frame.Id, index, relative, Duplicate: false); + } + finally + { + _gate.Release(); + } + } + + private async Task SequenceForAsync(IngestRequest request, Archive archive, CancellationToken cancellationToken) + { + Sequence? sequence = null; + if (request.SequenceId is { } id) sequence = await db.Sequences.GetAsync(id, cancellationToken); + else if (request.SessionKey is { } key && _sessions.TryGetValue(key, out var known)) sequence = await db.Sequences.GetAsync(known, cancellationToken); + if (sequence is null) + { + sequence = new Sequence { Id = Guid.CreateVersion7(), ArchiveId = archive.Id, Folder = string.Empty, Title = request.NewSequenceTitle, OwnerUserId = request.UserId }; + if (request.SessionKey is { } key) _sessions[key] = sequence.Id; + } + if (string.IsNullOrEmpty(sequence.Folder)) + { + sequence = sequence with { Folder = $"2026/{sequence.Id:N}", UpdatedAt = DateTimeOffset.UtcNow }; + await db.Sequences.UpsertAsync(sequence, cancellationToken); + } + return sequence; + } +} + +/// Proxies at <folder>/proxies/NNNN.jpg; thumbnails as 64-byte slots of <folder>/.titano/thumbs.pack. +public sealed class FakeAssetLocator : IFrameAssetLocator +{ + public const int ThumbnailSize = 64; + + /// When set, every lookup answers this location (path traversal tests). + public AssetLocation? Override { get; set; } + + public ValueTask LocateAsync(Archive archive, Sequence sequence, Frame frame, FrameAsset asset, CancellationToken cancellationToken) + { + if (Override is not null) return ValueTask.FromResult(Override); + var folder = Path.Combine(archive.RootPath, sequence.Folder); + AssetLocation? location = asset == FrameAsset.Proxy + ? new AssetLocation(Path.Combine(folder, "proxies", $"{frame.Index:D4}.jpg"), 0, -1, "image/jpeg") + : new AssetLocation(Path.Combine(folder, ".titano", "thumbs.pack"), (long)frame.Index * ThumbnailSize, ThumbnailSize, "image/jpeg"); + if (!File.Exists(location.Path)) return ValueTask.FromResult(null); + if (location.Length < 0) location = location with { Length = new FileInfo(location.Path).Length }; + return ValueTask.FromResult(location); + } +} + +/// A clock tests move by hand (token expiry, heartbeat timeouts). +public sealed class ManualTimeProvider : TimeProvider +{ + private long _ticks = DateTimeOffset.UtcNow.UtcTicks; + + public override DateTimeOffset GetUtcNow() => new(Interlocked.Read(ref _ticks), TimeSpan.Zero); + + public void Advance(TimeSpan by) => Interlocked.Add(ref _ticks, by.Ticks); +} + +/// +/// A job handler that reports progress and then waits until (or cancellation), so tests can +/// observe running, paused and cancelled states. +/// +public sealed class BlockingJobHandler(JobKind kind) : IJobHandler +{ + private TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public JobKind Kind => kind; + + public TaskCompletionSource Started { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + /// When set, the next run fails with this message. + public string? FailWith { get; set; } + + public void Release() => _release.TrySetResult(); + + public async Task RunAsync(JobContext context) + { + await context.ReportAsync(1, 4, "started"); + Started.TrySetResult(); + if (FailWith is { } message) + { + FailWith = null; + throw new InvalidOperationException(message); + } + for (var i = 2; i <= 3; i++) + { + await Task.Delay(50, context.Cancellation); + await context.ReportAsync(i, 4); + } + await _release.Task.WaitAsync(context.Cancellation); + _release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await context.ReportAsync(4, 4); + } +} diff --git a/tests/Titano.Server.Tests/TestServer.cs b/tests/Titano.Server.Tests/TestServer.cs index 2e099360..0a528bbc 100644 --- a/tests/Titano.Server.Tests/TestServer.cs +++ b/tests/Titano.Server.Tests/TestServer.cs @@ -1,17 +1,33 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Net.Http.Json; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting.Server; using Microsoft.AspNetCore.Hosting.Server.Features; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Titano.Data; +using Titano.Data.Archiving; +using Titano.Protocol; using Titano.Server.Configuration; using Titano.Server.Hosting; +using Titano.Server.Storage; using Titano.Testing; namespace Titano.Server.Tests; -/// In-process Kestrel on ephemeral ports with temporary archive/data folders (BRIEF §20.1). +/// +/// In-process Kestrel on ephemeral ports with temporary archive/data folders and a SQLite database (BRIEF §20.1). +/// The archive module is replaced by and . +/// public sealed class TestServer : IAsyncDisposable { + public const string AdminPassword = "first-admin-pass"; + public const string Password = "a-good-password-1"; + private readonly WebApplication _app; + private readonly List _clients = []; + private string _adminPassword = AdminPassword; private TestServer(WebApplication app, ServerOptions options, Uri http, Uri? https) { @@ -19,16 +35,21 @@ public sealed class TestServer : IAsyncDisposable Options = options; HttpBase = http; HttpsBase = https; - Client = new HttpClient { BaseAddress = http, Timeout = TimeSpan.FromSeconds(30) }; + Client = NewClient(); } public ServerOptions Options { get; } public Uri HttpBase { get; } public Uri? HttpsBase { get; } + + /// An anonymous client. public HttpClient Client { get; } + public IServiceProvider Services => _app.Services; - public static async Task StartAsync(TestContext context, Action? configure = null) + public TitanoDatabase Database => Services.GetRequiredService(); + + public static async Task StartAsync(TestContext context, Action? configure = null, Action? services = null) { var root = context.CreateTempDirectory("server"); var options = new ServerOptions @@ -39,10 +60,21 @@ public sealed class TestServer : IAsyncDisposable HttpsPort = 0, TlsMode = TlsMode.Auto, Database = DatabaseKind.Sqlite, + Discovery = false, + Workers = 2, + AdminUser = "admin", + AdminPassword = AdminPassword, }; Directory.CreateDirectory(options.ArchivePath); configure?.Invoke(options); - var app = ServerHost.Build(options, []); + var app = ServerHost.Build(options, [], s => + { + s.AddSingleton(); + s.TryAddSingleton(sp => sp.GetRequiredService()); + s.AddSingleton(); + s.TryAddSingleton(sp => sp.GetRequiredService()); + services?.Invoke(s); + }); await app.StartAsync(context.Cancellation); var addresses = app.Services.GetRequiredService().Features.Get()!.Addresses .Select(a => new Uri(a.Replace("[::]", "127.0.0.1", StringComparison.Ordinal).Replace("0.0.0.0", "127.0.0.1", StringComparison.Ordinal))) @@ -52,9 +84,65 @@ public sealed class TestServer : IAsyncDisposable return new TestServer(app, options, http, https); } + /// A plain HTTP client (owned by the server, disposed with it). + public HttpClient NewClient(string? token = null) + { + var client = new HttpClient { BaseAddress = HttpBase, Timeout = TimeSpan.FromSeconds(60) }; + if (token is not null) client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token); + _clients.Add(client); + return client; + } + + /// An HTTPS client with a cookie jar that trusts the self-signed certificate (admin panel sessions). + public HttpClient NewCookieClient(CookieContainer cookies) + { + var handler = new HttpClientHandler { CookieContainer = cookies, ServerCertificateCustomValidationCallback = (_, _, _, _) => true }; + var client = new HttpClient(handler) { BaseAddress = HttpsBase, Timeout = TimeSpan.FromSeconds(60) }; + _clients.Add(client); + return client; + } + + public async Task LoginAsync(string username, string password, string clientName = "tests") + { + using var response = await Client.PostAsJsonAsync("/api/v1/auth/login", new LoginRequest(username, password, clientName), ProtocolJson.Default.LoginRequest); + if (!response.IsSuccessStatusCode) throw new InvalidOperationException($"Login of {username} failed: {(int)response.StatusCode} {await response.Content.ReadAsStringAsync()}"); + return (await response.Content.ReadFromJsonAsync(ProtocolJson.Default.LoginResponse))!; + } + + /// The bootstrap administrator, signed in; its first sign-in replaces the bootstrap password with . + public async Task AdminAsync() + { + var login = await LoginAsync("admin", _adminPassword); + var client = NewClient(login.Token); + if (login.MustChangePassword) + { + using var changed = await client.PostAsJsonAsync("/api/v1/auth/password", new ChangePasswordRequest(_adminPassword, Password), + ProtocolJson.Default.ChangePasswordRequest); + changed.EnsureSuccessStatusCode(); + _adminPassword = Password; + } + return client; + } + + /// Creates a user (through the admin API) who has set , and returns a client signed in as them. + public async Task<(HttpClient Client, UserDto User, string Token)> UserAsync(string username, string role, long quotaBytes = 0, bool renderNode = false) + { + var admin = await AdminAsync(); + using var created = await admin.PostAsJsonAsync("/api/v1/users", new CreateUserRequest(username, null, role, quotaBytes, null, renderNode), + ProtocolJson.Default.CreateUserRequest); + created.EnsureSuccessStatusCode(); + var temporary = (await created.Content.ReadFromJsonAsync(ProtocolJson.Default.TemporaryPasswordResponse))!; + var first = await LoginAsync(username, temporary.TemporaryPassword); + var client = NewClient(first.Token); + using var changed = await client.PostAsJsonAsync("/api/v1/auth/password", new ChangePasswordRequest(temporary.TemporaryPassword, Password), + ProtocolJson.Default.ChangePasswordRequest); + changed.EnsureSuccessStatusCode(); + return (client, temporary.User, first.Token!); + } + public async ValueTask DisposeAsync() { - Client.Dispose(); + foreach (var client in _clients) client.Dispose(); await _app.StopAsync(); await _app.DisposeAsync(); }