diff --git a/src/Titano.Data/Migrations/MigrationRunner.cs b/src/Titano.Data/Migrations/MigrationRunner.cs new file mode 100644 index 00000000..b100ce03 --- /dev/null +++ b/src/Titano.Data/Migrations/MigrationRunner.cs @@ -0,0 +1,90 @@ +using System.Globalization; +using Titano.Core.Diagnostics; + +namespace Titano.Data.Migrations; + +/// A numbered schema change shipped as Migrations/<dialect>/NNNN_name.sql. +public sealed record Migration(int Version, string Name, string Sql); + +/// +/// Applies the embedded migrations of a dialect at startup (BRIEF §6.1): all pending ones in a single +/// transaction, each recorded in schema_migrations(version, name, applied_at). A database that already +/// holds a version this build does not know is refused with . +/// +public static class MigrationRunner +{ + private const string ResourcePrefix = "Titano.Data.Migrations."; + + /// Serialises concurrent migrators on one PostgreSQL database (several server instances starting). + private const long AdvisoryLockKey = 0x5469_7461_6E6F; + + /// The embedded migrations of , by ascending version. + public static IReadOnlyList Load(Dialect dialect) + { + var folder = ResourcePrefix + (dialect == Dialect.Postgres ? "postgres" : "sqlite"); + var assembly = typeof(MigrationRunner).Assembly; + var migrations = new List(); + foreach (var resource in assembly.GetManifestResourceNames()) + { + // The logical name keeps the folder separator of the build host ('/' or '\'). + if (resource.Length <= folder.Length + 1 || !resource.StartsWith(folder, StringComparison.Ordinal) + || resource[folder.Length] is not ('/' or '\\')) + continue; + var file = resource[(folder.Length + 1)..]; + var underscore = file.IndexOf('_', StringComparison.Ordinal); + if (underscore <= 0 || !file.EndsWith(".sql", StringComparison.Ordinal) + || !int.TryParse(file.AsSpan(0, underscore), NumberStyles.None, CultureInfo.InvariantCulture, out var version)) + throw new InvalidOperationException($"Migration resource '{resource}' is not named NNNN_name.sql."); + using var reader = new StreamReader(assembly.GetManifestResourceStream(resource)!); + migrations.Add(new Migration(version, file[(underscore + 1)..^4], reader.ReadToEnd())); + } + migrations.Sort((a, b) => a.Version.CompareTo(b.Version)); + for (var i = 1; i < migrations.Count; i++) + { + if (migrations[i].Version == migrations[i - 1].Version) + throw new InvalidOperationException($"Two migrations share version {migrations[i].Version}."); + } + return migrations; + } + + /// Migrates to the newest embedded version and returns that version. + public static Task MigrateAsync(IDatabase database, CancellationToken cancellationToken = default) => + MigrateAsync(database, Load(database.Dialect), cancellationToken); + + /// Applies every migration of not yet recorded; returns the resulting version. + public static async Task MigrateAsync(IDatabase database, IReadOnlyList migrations, CancellationToken cancellationToken = default) + { + var postgres = database.Dialect == Dialect.Postgres; + var session = await database.OpenAsync(cancellationToken).ConfigureAwait(false); + await using (session.ConfigureAwait(false)) + { + var transaction = await session.BeginAsync(cancellationToken).ConfigureAwait(false); + await using (transaction.ConfigureAwait(false)) + { + if (postgres) await session.ExecuteAsync("SELECT pg_advisory_xact_lock($1)", [AdvisoryLockKey], cancellationToken).ConfigureAwait(false); + await session.ExecuteScriptAsync(postgres + ? "CREATE TABLE IF NOT EXISTS schema_migrations (version integer PRIMARY KEY, name text NOT NULL, applied_at timestamptz NOT NULL)" + : "CREATE TABLE IF NOT EXISTS schema_migrations (version INTEGER PRIMARY KEY, name TEXT NOT NULL, applied_at TEXT NOT NULL)", + cancellationToken).ConfigureAwait(false); + + var applied = new HashSet(await session.QueryListAsync( + "SELECT version FROM schema_migrations", [], r => r.GetInt32(0), cancellationToken).ConfigureAwait(false)); + var supported = migrations.Count == 0 ? 0 : migrations[^1].Version; + var current = applied.Count == 0 ? 0 : applied.Max(); + if (current > supported) throw new SchemaTooNewException(current, supported); + + foreach (var migration in migrations) + { + if (applied.Contains(migration.Version)) continue; + await session.ExecuteScriptAsync(migration.Sql, cancellationToken).ConfigureAwait(false); + await session.ExecuteAsync( + "INSERT INTO schema_migrations (version, name, applied_at) VALUES ($1, $2, $3)", + [migration.Version, migration.Name, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false); + Log.Info("Data", $"Applied {database.Dialect} migration {migration.Version:0000} {migration.Name}"); + } + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + return supported; + } + } + } +} diff --git a/src/Titano.Data/Migrations/postgres/0001_initial.sql b/src/Titano.Data/Migrations/postgres/0001_initial.sql new file mode 100644 index 00000000..f62a74fd --- /dev/null +++ b/src/Titano.Data/Migrations/postgres/0001_initial.sql @@ -0,0 +1,250 @@ +-- Titano schema v1 (PostgreSQL). Instants are timestamptz, camera wall-clock times timestamp, JSON jsonb. +-- Sort keys use the "C" collation so ordering equals SQLite's binary UTF-8 order. Mirrors sqlite/0001_initial.sql. + +CREATE TABLE archives ( + id uuid NOT NULL PRIMARY KEY, + name text NOT NULL, + root_path text NOT NULL, + folder_template text NOT NULL, + file_template text NOT NULL, + schema_version integer NOT NULL DEFAULT 1, + created timestamptz NOT NULL, + proxy_long_edge integer NOT NULL DEFAULT 1920 +); + +CREATE TABLE users ( + id uuid NOT NULL PRIMARY KEY, + username text NOT NULL, + username_key text NOT NULL UNIQUE, + display_name text, + password_hash text NOT NULL, + role integer NOT NULL, + permissions bigint NOT NULL, + quota_bytes bigint NOT NULL DEFAULT 0, + used_bytes bigint NOT NULL DEFAULT 0, + disabled boolean NOT NULL DEFAULT FALSE, + must_change_password boolean NOT NULL DEFAULT FALSE, + created timestamptz NOT NULL, + last_login timestamptz +); + +CREATE TABLE sequences ( + id uuid NOT NULL PRIMARY KEY, + archive_id uuid NOT NULL REFERENCES archives (id) ON DELETE CASCADE, + title text, + title_key text COLLATE "C" NOT NULL, + search_text text NOT NULL, + folder text NOT NULL, + camera_make text, + camera_model text, + camera_serial text, + lens text, + first_capture timestamp, + last_capture timestamp, + frame_count integer NOT NULL DEFAULT 0, + median_interval double precision, + total_bytes bigint NOT NULL DEFAULT 0, + status integer NOT NULL DEFAULT 0, + rendered_count integer NOT NULL DEFAULT 0, + verified_at timestamptz, + hero_frame_id uuid, + rating integer NOT NULL DEFAULT 0, + notes text, + settings_json jsonb, + analysis_json jsonb, + origin integer NOT NULL DEFAULT 0, + sync_state integer NOT NULL DEFAULT 0, + settings_version integer NOT NULL DEFAULT 0, + frames_version integer NOT NULL DEFAULT 0, + owner_user_id uuid, + warning_count integer NOT NULL DEFAULT 0, + updated_at timestamptz NOT NULL +); +CREATE INDEX ix_sequences_archive_capture ON sequences (archive_id, first_capture, id); +CREATE INDEX ix_sequences_capture ON sequences (first_capture, id); +CREATE INDEX ix_sequences_camera ON sequences (camera_model); + +CREATE TABLE frames ( + id uuid NOT NULL PRIMARY KEY, + sequence_id uuid NOT NULL REFERENCES sequences (id) ON DELETE CASCADE, + idx integer NOT NULL, + relative_path text NOT NULL, + file_name text NOT NULL, + original_file_name text, + size bigint NOT NULL DEFAULT 0, + sha256 text, + captured_at timestamp, + utc_offset_minutes integer, + timestamp_source integer NOT NULL DEFAULT 0, + exposure_time double precision, + f_number double precision, + iso integer, + focal_length double precision, + exposure_bias double precision, + white_balance integer, + orientation integer NOT NULL DEFAULT 1, + width integer NOT NULL DEFAULT 0, + height integer NOT NULL DEFAULT 0, + format integer NOT NULL DEFAULT 0, + preview_offset bigint, + preview_length bigint, + mean_log_luminance double precision, + clip_low double precision, + clip_high double precision, + flags integer NOT NULL DEFAULT 0 +); +CREATE UNIQUE INDEX ux_frames_sequence_idx ON frames (sequence_id, idx); +CREATE INDEX ix_frames_sha256 ON frames (sha256); +CREATE INDEX ix_frames_captured_at ON frames (captured_at); + +CREATE TABLE renders ( + id uuid NOT NULL PRIMARY KEY, + sequence_id uuid NOT NULL REFERENCES sequences (id) ON DELETE CASCADE, + preset_name text NOT NULL, + settings_json jsonb, + output_path text NOT NULL, + codec text, + width integer NOT NULL DEFAULT 0, + height integer NOT NULL DEFAULT 0, + fps double precision NOT NULL DEFAULT 0, + duration_seconds double precision NOT NULL DEFAULT 0, + bytes bigint NOT NULL DEFAULT 0, + created timestamptz NOT NULL, + produced_by text +); +CREATE INDEX ix_renders_sequence ON renders (sequence_id, created); + +CREATE TABLE jobs ( + id uuid NOT NULL PRIMARY KEY, + kind integer NOT NULL, + target_ids text NOT NULL DEFAULT '', + target_label text, + parameters_json jsonb, + state integer NOT NULL, + priority integer NOT NULL, + progress_done bigint NOT NULL DEFAULT 0, + progress_total bigint NOT NULL DEFAULT 0, + items_per_second double precision, + eta_seconds double precision, + checkpoint text, + log_tail text, + created_by uuid, + node_id text, + attempts integer NOT NULL DEFAULT 0, + error text, + created timestamptz NOT NULL, + started timestamptz, + finished timestamptz +); +CREATE INDEX ix_jobs_queue ON jobs (state, priority DESC, created, id); +CREATE INDEX ix_jobs_state_created ON jobs (state, created, id); +CREATE INDEX ix_jobs_finished ON jobs (finished); + +CREATE TABLE media_cards ( + id uuid NOT NULL PRIMARY KEY, + identity text NOT NULL UNIQUE, + volume_serial bigint NOT NULL, + label text, + capacity bigint NOT NULL DEFAULT 0, + nickname text, + camera_make text, + camera_model text, + camera_serial text, + profile_json jsonb, + auto_import boolean NOT NULL DEFAULT FALSE, + first_seen timestamptz NOT NULL, + last_seen timestamptz NOT NULL +); + +CREATE TABLE import_history ( + card_id uuid NOT NULL REFERENCES media_cards (id) ON DELETE CASCADE, + path text NOT NULL, + size bigint NOT NULL, + mtime timestamptz NOT NULL, + sha256 text, + frame_id uuid REFERENCES frames (id) ON DELETE SET NULL, + job_id uuid, + imported_at timestamptz NOT NULL, + PRIMARY KEY (card_id, path, size, mtime) +); +CREATE INDEX ix_import_history_frame ON import_history (frame_id); + +CREATE TABLE tokens ( + token_hash text NOT NULL PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE, + client_name text, + created timestamptz NOT NULL, + expires timestamptz NOT NULL, + last_used timestamptz NOT NULL, + revoked boolean NOT NULL DEFAULT FALSE +); +CREATE INDEX ix_tokens_user ON tokens (user_id); +CREATE INDEX ix_tokens_expires ON tokens (expires); + +CREATE TABLE render_nodes ( + id uuid NOT NULL PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE, + client_name text NOT NULL, + capabilities_json jsonb, + last_heartbeat timestamptz NOT NULL, + state text NOT NULL +); +CREATE INDEX ix_render_nodes_user ON render_nodes (user_id); +CREATE INDEX ix_render_nodes_heartbeat ON render_nodes (last_heartbeat); + +CREATE TABLE settings ( + scope text NOT NULL, + scope_id text NOT NULL DEFAULT '', + key text NOT NULL, + value_json jsonb NOT NULL, + updated_at timestamptz NOT NULL, + PRIMARY KEY (scope, scope_id, key) +); + +CREATE TABLE tags ( + id uuid NOT NULL PRIMARY KEY, + name text NOT NULL, + name_key text COLLATE "C" NOT NULL UNIQUE +); + +CREATE TABLE sequence_tags ( + sequence_id uuid NOT NULL REFERENCES sequences (id) ON DELETE CASCADE, + tag_id uuid NOT NULL REFERENCES tags (id) ON DELETE CASCADE, + PRIMARY KEY (sequence_id, tag_id) +); +CREATE INDEX ix_sequence_tags_tag ON sequence_tags (tag_id); + +CREATE TABLE audit_log ( + id uuid NOT NULL PRIMARY KEY, + at timestamptz NOT NULL, + user_id uuid, + username text, + action text NOT NULL, + target_kind text, + target_id text, + details text, + remote_address text +); +CREATE INDEX ix_audit_log_at ON audit_log (at, id); +CREATE INDEX ix_audit_log_user ON audit_log (user_id, at); + +CREATE TABLE uploads ( + id uuid NOT NULL PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE, + sequence_id uuid REFERENCES sequences (id) ON DELETE SET NULL, + file_name text NOT NULL, + size bigint NOT NULL, + sha256 text NOT NULL, + captured_at timestamp, + chunk_size integer NOT NULL, + received bytea NOT NULL, + received_count integer NOT NULL DEFAULT 0, + state integer NOT NULL, + frame_id uuid, + created timestamptz NOT NULL, + updated_at timestamptz NOT NULL +); +CREATE INDEX ix_uploads_user ON uploads (user_id, created); +CREATE INDEX ix_uploads_state ON uploads (state, updated_at); +CREATE INDEX ix_uploads_sha256 ON uploads (sha256); +CREATE INDEX ix_uploads_sequence ON uploads (sequence_id); diff --git a/src/Titano.Data/Migrations/sqlite/0001_initial.sql b/src/Titano.Data/Migrations/sqlite/0001_initial.sql new file mode 100644 index 00000000..6d2c442b --- /dev/null +++ b/src/Titano.Data/Migrations/sqlite/0001_initial.sql @@ -0,0 +1,250 @@ +-- Titano schema v1 (SQLite). Ids are UUID text, instants ISO-8601 UTC text ('…Z'), camera wall-clock times +-- ISO-8601 text without zone, JSON as text, booleans as 0/1. Mirrors postgres/0001_initial.sql. + +CREATE TABLE archives ( + id TEXT NOT NULL PRIMARY KEY, + name TEXT NOT NULL, + root_path TEXT NOT NULL, + folder_template TEXT NOT NULL, + file_template TEXT NOT NULL, + schema_version INTEGER NOT NULL DEFAULT 1, + created TEXT NOT NULL, + proxy_long_edge INTEGER NOT NULL DEFAULT 1920 +); + +CREATE TABLE users ( + id TEXT NOT NULL PRIMARY KEY, + username TEXT NOT NULL, + username_key TEXT NOT NULL UNIQUE, + display_name TEXT, + password_hash TEXT NOT NULL, + role INTEGER NOT NULL, + permissions INTEGER NOT NULL, + quota_bytes INTEGER NOT NULL DEFAULT 0, + used_bytes INTEGER NOT NULL DEFAULT 0, + disabled INTEGER NOT NULL DEFAULT 0, + must_change_password INTEGER NOT NULL DEFAULT 0, + created TEXT NOT NULL, + last_login TEXT +); + +CREATE TABLE sequences ( + id TEXT NOT NULL PRIMARY KEY, + archive_id TEXT NOT NULL REFERENCES archives (id) ON DELETE CASCADE, + title TEXT, + title_key TEXT NOT NULL, + search_text TEXT NOT NULL, + folder TEXT NOT NULL, + camera_make TEXT, + camera_model TEXT, + camera_serial TEXT, + lens TEXT, + first_capture TEXT, + last_capture TEXT, + frame_count INTEGER NOT NULL DEFAULT 0, + median_interval REAL, + total_bytes INTEGER NOT NULL DEFAULT 0, + status INTEGER NOT NULL DEFAULT 0, + rendered_count INTEGER NOT NULL DEFAULT 0, + verified_at TEXT, + hero_frame_id TEXT, + rating INTEGER NOT NULL DEFAULT 0, + notes TEXT, + settings_json TEXT, + analysis_json TEXT, + origin INTEGER NOT NULL DEFAULT 0, + sync_state INTEGER NOT NULL DEFAULT 0, + settings_version INTEGER NOT NULL DEFAULT 0, + frames_version INTEGER NOT NULL DEFAULT 0, + owner_user_id TEXT, + warning_count INTEGER NOT NULL DEFAULT 0, + updated_at TEXT NOT NULL +); +CREATE INDEX ix_sequences_archive_capture ON sequences (archive_id, first_capture, id); +CREATE INDEX ix_sequences_capture ON sequences (first_capture, id); +CREATE INDEX ix_sequences_camera ON sequences (camera_model); + +CREATE TABLE frames ( + id TEXT NOT NULL PRIMARY KEY, + sequence_id TEXT NOT NULL REFERENCES sequences (id) ON DELETE CASCADE, + idx INTEGER NOT NULL, + relative_path TEXT NOT NULL, + file_name TEXT NOT NULL, + original_file_name TEXT, + size INTEGER NOT NULL DEFAULT 0, + sha256 TEXT, + captured_at TEXT, + utc_offset_minutes INTEGER, + timestamp_source INTEGER NOT NULL DEFAULT 0, + exposure_time REAL, + f_number REAL, + iso INTEGER, + focal_length REAL, + exposure_bias REAL, + white_balance INTEGER, + orientation INTEGER NOT NULL DEFAULT 1, + width INTEGER NOT NULL DEFAULT 0, + height INTEGER NOT NULL DEFAULT 0, + format INTEGER NOT NULL DEFAULT 0, + preview_offset INTEGER, + preview_length INTEGER, + mean_log_luminance REAL, + clip_low REAL, + clip_high REAL, + flags INTEGER NOT NULL DEFAULT 0 +); +CREATE UNIQUE INDEX ux_frames_sequence_idx ON frames (sequence_id, idx); +CREATE INDEX ix_frames_sha256 ON frames (sha256); +CREATE INDEX ix_frames_captured_at ON frames (captured_at); + +CREATE TABLE renders ( + id TEXT NOT NULL PRIMARY KEY, + sequence_id TEXT NOT NULL REFERENCES sequences (id) ON DELETE CASCADE, + preset_name TEXT NOT NULL, + settings_json TEXT, + output_path TEXT NOT NULL, + codec TEXT, + width INTEGER NOT NULL DEFAULT 0, + height INTEGER NOT NULL DEFAULT 0, + fps REAL NOT NULL DEFAULT 0, + duration_seconds REAL NOT NULL DEFAULT 0, + bytes INTEGER NOT NULL DEFAULT 0, + created TEXT NOT NULL, + produced_by TEXT +); +CREATE INDEX ix_renders_sequence ON renders (sequence_id, created); + +CREATE TABLE jobs ( + id TEXT NOT NULL PRIMARY KEY, + kind INTEGER NOT NULL, + target_ids TEXT NOT NULL DEFAULT '', + target_label TEXT, + parameters_json TEXT, + state INTEGER NOT NULL, + priority INTEGER NOT NULL, + progress_done INTEGER NOT NULL DEFAULT 0, + progress_total INTEGER NOT NULL DEFAULT 0, + items_per_second REAL, + eta_seconds REAL, + checkpoint TEXT, + log_tail TEXT, + created_by TEXT, + node_id TEXT, + attempts INTEGER NOT NULL DEFAULT 0, + error TEXT, + created TEXT NOT NULL, + started TEXT, + finished TEXT +); +CREATE INDEX ix_jobs_queue ON jobs (state, priority DESC, created, id); +CREATE INDEX ix_jobs_state_created ON jobs (state, created, id); +CREATE INDEX ix_jobs_finished ON jobs (finished); + +CREATE TABLE media_cards ( + id TEXT NOT NULL PRIMARY KEY, + identity TEXT NOT NULL UNIQUE, + volume_serial INTEGER NOT NULL, + label TEXT, + capacity INTEGER NOT NULL DEFAULT 0, + nickname TEXT, + camera_make TEXT, + camera_model TEXT, + camera_serial TEXT, + profile_json TEXT, + auto_import INTEGER NOT NULL DEFAULT 0, + first_seen TEXT NOT NULL, + last_seen TEXT NOT NULL +); + +CREATE TABLE import_history ( + card_id TEXT NOT NULL REFERENCES media_cards (id) ON DELETE CASCADE, + path TEXT NOT NULL, + size INTEGER NOT NULL, + mtime TEXT NOT NULL, + sha256 TEXT, + frame_id TEXT REFERENCES frames (id) ON DELETE SET NULL, + job_id TEXT, + imported_at TEXT NOT NULL, + PRIMARY KEY (card_id, path, size, mtime) +); +CREATE INDEX ix_import_history_frame ON import_history (frame_id); + +CREATE TABLE tokens ( + token_hash TEXT NOT NULL PRIMARY KEY, + user_id TEXT NOT NULL REFERENCES users (id) ON DELETE CASCADE, + client_name TEXT, + created TEXT NOT NULL, + expires TEXT NOT NULL, + last_used TEXT NOT NULL, + revoked INTEGER NOT NULL DEFAULT 0 +); +CREATE INDEX ix_tokens_user ON tokens (user_id); +CREATE INDEX ix_tokens_expires ON tokens (expires); + +CREATE TABLE render_nodes ( + id TEXT NOT NULL PRIMARY KEY, + user_id TEXT NOT NULL REFERENCES users (id) ON DELETE CASCADE, + client_name TEXT NOT NULL, + capabilities_json TEXT, + last_heartbeat TEXT NOT NULL, + state TEXT NOT NULL +); +CREATE INDEX ix_render_nodes_user ON render_nodes (user_id); +CREATE INDEX ix_render_nodes_heartbeat ON render_nodes (last_heartbeat); + +CREATE TABLE settings ( + scope TEXT NOT NULL, + scope_id TEXT NOT NULL DEFAULT '', + key TEXT NOT NULL, + value_json TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (scope, scope_id, key) +); + +CREATE TABLE tags ( + id TEXT NOT NULL PRIMARY KEY, + name TEXT NOT NULL, + name_key TEXT NOT NULL UNIQUE +); + +CREATE TABLE sequence_tags ( + sequence_id TEXT NOT NULL REFERENCES sequences (id) ON DELETE CASCADE, + tag_id TEXT NOT NULL REFERENCES tags (id) ON DELETE CASCADE, + PRIMARY KEY (sequence_id, tag_id) +); +CREATE INDEX ix_sequence_tags_tag ON sequence_tags (tag_id); + +CREATE TABLE audit_log ( + id TEXT NOT NULL PRIMARY KEY, + at TEXT NOT NULL, + user_id TEXT, + username TEXT, + action TEXT NOT NULL, + target_kind TEXT, + target_id TEXT, + details TEXT, + remote_address TEXT +); +CREATE INDEX ix_audit_log_at ON audit_log (at, id); +CREATE INDEX ix_audit_log_user ON audit_log (user_id, at); + +CREATE TABLE uploads ( + id TEXT NOT NULL PRIMARY KEY, + user_id TEXT NOT NULL REFERENCES users (id) ON DELETE CASCADE, + sequence_id TEXT REFERENCES sequences (id) ON DELETE SET NULL, + file_name TEXT NOT NULL, + size INTEGER NOT NULL, + sha256 TEXT NOT NULL, + captured_at TEXT, + chunk_size INTEGER NOT NULL, + received BLOB NOT NULL, + received_count INTEGER NOT NULL DEFAULT 0, + state INTEGER NOT NULL, + frame_id TEXT, + created TEXT NOT NULL, + updated_at TEXT NOT NULL +); +CREATE INDEX ix_uploads_user ON uploads (user_id, created); +CREATE INDEX ix_uploads_state ON uploads (state, updated_at); +CREATE INDEX ix_uploads_sha256 ON uploads (sha256); +CREATE INDEX ix_uploads_sequence ON uploads (sequence_id); diff --git a/src/Titano.Data/Repositories/ArchiveRepository.cs b/src/Titano.Data/Repositories/ArchiveRepository.cs new file mode 100644 index 00000000..717eb4f9 --- /dev/null +++ b/src/Titano.Data/Repositories/ArchiveRepository.cs @@ -0,0 +1,43 @@ +using Titano.Core.Model; + +namespace Titano.Data.Repositories; + +/// Archive roots (BRIEF §5, §7). +public sealed class ArchiveRepository +{ + private const string Columns = "id, name, root_path, folder_template, file_template, schema_version, created, proxy_long_edge"; + + private static readonly string UpsertSql = SqlDialect.Upsert("archives", + ["id", "name", "root_path", "folder_template", "file_template", "schema_version", "created", "proxy_long_edge"], 1); + + private readonly DbAccess _db; + + internal ArchiveRepository(DbAccess db) => _db = db; + + public async Task UpsertAsync(Archive archive, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync(UpsertSql, + [archive.Id, archive.Name, archive.RootPath, archive.FolderTemplate, archive.FileTemplate, archive.SchemaVersion, archive.Created, archive.ProxyLongEdge], + cancellationToken).ConfigureAwait(false); + + public Task GetAsync(Guid id, CancellationToken cancellationToken = default) => + _db.SingleAsync($"SELECT {Columns} FROM archives WHERE id = $1", [id], Map, cancellationToken); + + public Task> ListAsync(CancellationToken cancellationToken = default) => + _db.ListAsync($"SELECT {Columns} FROM archives ORDER BY name, id", [], Map, cancellationToken); + + /// Deletes the archive and everything indexed under it (the files on disk are untouched). + public async Task DeleteAsync(Guid id, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync("DELETE FROM archives WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) > 0; + + private static Archive Map(DbRow r) => new() + { + Id = r.GetGuid(0), + Name = r.GetString(1), + RootPath = r.GetString(2), + FolderTemplate = r.GetString(3), + FileTemplate = r.GetString(4), + SchemaVersion = r.GetInt32(5), + Created = r.GetDateTimeOffset(6), + ProxyLongEdge = r.GetInt32(7), + }; +} diff --git a/src/Titano.Data/Repositories/AuditRepository.cs b/src/Titano.Data/Repositories/AuditRepository.cs new file mode 100644 index 00000000..46e13c31 --- /dev/null +++ b/src/Titano.Data/Repositories/AuditRepository.cs @@ -0,0 +1,58 @@ +using Titano.Core.Model; + +namespace Titano.Data.Repositories; + +/// Append-only record of who did what (BRIEF §5 AuditLog, §14.2). +public sealed class AuditRepository +{ + private const string Columns = "id, at, user_id, username, action, target_kind, target_id, details, remote_address"; + + private readonly DbAccess _db; + + internal AuditRepository(DbAccess db) => _db = db; + + public async Task AppendAsync(AuditEntry entry, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync($"INSERT INTO audit_log ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", + [entry.Id, entry.At, entry.UserId, entry.Username, entry.Action, entry.TargetKind, entry.TargetId, entry.Details, entry.RemoteAddress], + cancellationToken).ConfigureAwait(false); + + /// Entries newest first, optionally of one user. + public Task> ListAsync(Guid? userId = null, int limit = 100, string? cursor = null, CancellationToken cancellationToken = default) + { + var p = new SqlParameters(); + var where = new List(); + if (userId is { } user) where.Add($"user_id = {p.Add(user)}"); + if (cursor is not null) + { + var (value, id) = Cursor.Decode("audit", cursor); + where.Add(Cursor.After("at", "id", descending: true, value, id, p.Add)); + } + var take = Cursor.ClampLimit(limit); + var sql = $"SELECT {Columns} FROM audit_log{(where.Count > 0 ? " WHERE " + string.Join(" AND ", where) : string.Empty)} " + + $"ORDER BY at DESC, id DESC LIMIT {p.Add(take + 1)}"; + return _db.RunAsync(async session => + { + var entries = await session.QueryListAsync(sql, p.ToArray(), Map, cancellationToken).ConfigureAwait(false); + string? next = null; + if (entries.Count > take) + { + entries.RemoveAt(take); + next = Cursor.Encode("audit", entries[^1].At, entries[^1].Id); + } + return new Page(entries, next); + }, cancellationToken); + } + + private static AuditEntry Map(DbRow r) => new() + { + Id = r.GetGuid(0), + At = r.GetDateTimeOffset(1), + UserId = r.GetGuidOrNull(2), + Username = r.GetStringOrNull(3), + Action = r.GetString(4), + TargetKind = r.GetStringOrNull(5), + TargetId = r.GetStringOrNull(6), + Details = r.GetStringOrNull(7), + RemoteAddress = r.GetStringOrNull(8), + }; +} diff --git a/src/Titano.Data/Repositories/DbAccess.cs b/src/Titano.Data/Repositories/DbAccess.cs new file mode 100644 index 00000000..77a419cf --- /dev/null +++ b/src/Titano.Data/Repositories/DbAccess.cs @@ -0,0 +1,145 @@ +using System.Buffers.Text; +using System.Globalization; +using System.Text; + +namespace Titano.Data.Repositories; + +/// One page of a keyset-paginated listing; pass back to get the following page. +public sealed record Page(IReadOnlyList Items, string? NextCursor); + +/// +/// Where repository commands run: on a caller's session (so they join its transaction) or on a session +/// borrowed from the pool for the duration of one call. +/// +internal sealed class DbAccess +{ + private readonly IDatabase? _database; + private readonly IDbSession? _session; + + public DbAccess(IDatabase database) + { + _database = database; + Sql = SqlDialect.For(database.Dialect); + } + + public DbAccess(IDbSession session) + { + _session = session; + Sql = SqlDialect.For(session.Dialect); + } + + public SqlDialect Sql { get; } + + public async Task RunAsync(Func> work, CancellationToken cancellationToken) + { + if (_session is not null) return await work(_session).ConfigureAwait(false); + var session = await _database!.OpenAsync(cancellationToken).ConfigureAwait(false); + await using (session.ConfigureAwait(false)) return await work(session).ConfigureAwait(false); + } + + /// Runs atomically: inside the caller's transaction, or in a new one. + public Task InTransactionAsync(Func> work, CancellationToken cancellationToken) => + RunAsync(async session => + { + if (session.InTransaction) return await work(session).ConfigureAwait(false); + var transaction = await session.BeginAsync(cancellationToken).ConfigureAwait(false); + await using (transaction.ConfigureAwait(false)) + { + var result = await work(session).ConfigureAwait(false); + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + return result; + } + }, cancellationToken); + + public Task ExecuteAsync(string sql, DbParam[] parameters, CancellationToken cancellationToken) => + RunAsync(s => s.ExecuteAsync(sql, parameters, cancellationToken), cancellationToken); + + public async Task> ListAsync(string sql, DbParam[] parameters, Func map, CancellationToken cancellationToken) => + await RunAsync(s => s.QueryListAsync(sql, parameters, map, cancellationToken), cancellationToken).ConfigureAwait(false); + + public Task SingleAsync(string sql, DbParam[] parameters, Func map, CancellationToken cancellationToken) => + RunAsync(s => s.QuerySingleOrDefaultAsync(sql, parameters, map, cancellationToken), cancellationToken); + + public Task ScalarAsync(string sql, DbParam[] parameters, CancellationToken cancellationToken) => + RunAsync(s => s.ScalarAsync(sql, parameters, cancellationToken), cancellationToken); +} + +/// +/// Opaque keyset cursors: the sort value and id of the last row of a page, tagged with the listing they belong +/// to so a cursor cannot be replayed against a different ordering. +/// +internal static class Cursor +{ + public static string Encode(string scope, object? value, Guid id) + { + var payload = value switch + { + null => "n", + DateTime d => "t" + d.Ticks.ToString(CultureInfo.InvariantCulture), + DateTimeOffset o => "o" + o.UtcTicks.ToString(CultureInfo.InvariantCulture), + long l => "i" + l.ToString(CultureInfo.InvariantCulture), + int i => "i" + i.ToString(CultureInfo.InvariantCulture), + string s => "s" + s, + _ => throw new NotSupportedException($"Cursor values of type {value.GetType().Name} are not supported."), + }; + return Base64Url.EncodeToString(Encoding.UTF8.GetBytes($"{scope}|{id:N}|{payload}")); + } + + /// Decodes a cursor of ; anything else is a . + public static (object? Value, Guid Id) Decode(string scope, string cursor) + { + string text; + try + { + text = Encoding.UTF8.GetString(Base64Url.DecodeFromChars(cursor)); + } + catch (FormatException ex) + { + throw new FormatException("Invalid paging cursor.", ex); + } + var parts = text.Split('|', 3); + if (parts.Length != 3 || parts[0] != scope || !Guid.TryParseExact(parts[1], "N", out var id) || parts[2].Length == 0) + throw new FormatException("The paging cursor does not belong to this listing."); + var payload = parts[2][1..]; + object? value = parts[2][0] switch + { + 'n' => null, + 't' => new DateTime(long.Parse(payload, CultureInfo.InvariantCulture), DateTimeKind.Unspecified), + 'o' => new DateTimeOffset(long.Parse(payload, CultureInfo.InvariantCulture), TimeSpan.Zero), + 'i' => long.Parse(payload, CultureInfo.InvariantCulture), + 's' => payload, + _ => throw new FormatException("Invalid paging cursor."), + }; + return (value, id); + } + + /// + /// Keyset condition "after (value, id)" for ORDER BY column dir NULLS LAST, id dir; adds its parameters + /// through , which returns the placeholder text. + /// + public static string After(string column, string idColumn, bool descending, object? value, Guid id, Func parameter) + { + var op = descending ? "<" : ">"; + var idParam = parameter(id); + if (value is null) return $"({column} IS NULL AND {idColumn} {op} {idParam})"; + var v = parameter(new DbParam(value)); + return $"({column} {op} {v} OR ({column} = {v} AND {idColumn} {op} {idParam}) OR {column} IS NULL)"; + } + + public static int ClampLimit(int limit) => Math.Clamp(limit, 1, 1000); +} + +/// Builds positional parameter lists alongside SQL text. +internal sealed class SqlParameters +{ + private readonly List _values = []; + + /// Adds a value and returns its placeholder ($n). + public string Add(DbParam value) + { + _values.Add(value); + return "$" + _values.Count.ToString(CultureInfo.InvariantCulture); + } + + public DbParam[] ToArray() => [.. _values]; +} diff --git a/src/Titano.Data/Repositories/FrameRepository.cs b/src/Titano.Data/Repositories/FrameRepository.cs new file mode 100644 index 00000000..30350e5e --- /dev/null +++ b/src/Titano.Data/Repositories/FrameRepository.cs @@ -0,0 +1,157 @@ +using System.Text; +using Titano.Core.Imaging; +using Titano.Core.Model; + +namespace Titano.Data.Repositories; + +/// Light statistics of one frame, written by the analysis job. +public readonly record struct FrameLightStats(Guid FrameId, double? MeanLogLuminance, double? ClipLow, double? ClipHigh); + +/// Frames of sequences: bulk import, paged listing, duplicate lookup and bulk updates. +public sealed class FrameRepository +{ + private static readonly string[] ColumnNames = + [ + "id", "sequence_id", "idx", "relative_path", "file_name", "original_file_name", "size", "sha256", "captured_at", + "utc_offset_minutes", "timestamp_source", "exposure_time", "f_number", "iso", "focal_length", "exposure_bias", + "white_balance", "orientation", "width", "height", "format", "preview_offset", "preview_length", + "mean_log_luminance", "clip_low", "clip_high", "flags", + ]; + + private static readonly string Columns = string.Join(", ", ColumnNames); + private static readonly string InsertSql = + $"INSERT INTO frames ({Columns}) VALUES ({string.Join(", ", ColumnNames.Select((_, i) => "$" + (i + 1)))})"; + + /// Rows per statement of the batched PostgreSQL updates (4 or fewer parameters per row). + private const int Batch = 1000; + + private readonly DbAccess _db; + + internal FrameRepository(DbAccess db) => _db = db; + + public async Task InsertAsync(Frame frame, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync(InsertSql, Array.ConvertAll(Values(frame), v => new DbParam(v)), cancellationToken).ConfigureAwait(false); + + /// Inserts many frames at once (COPY on PostgreSQL, batched INSERTs in one transaction on SQLite). + public Task BulkInsertAsync(IEnumerable frames, CancellationToken cancellationToken = default) => + _db.RunAsync(session => session.BulkInsertAsync("frames", ColumnNames, frames.Select(Values).ToAsyncEnumerable(), cancellationToken), cancellationToken); + + public Task GetAsync(Guid id, CancellationToken cancellationToken = default) => + _db.SingleAsync($"SELECT {Columns} FROM frames WHERE id = $1", [id], Map, cancellationToken); + + /// Frames of a sequence by index, starting after (keyset paging). + public Task> ListBySequenceAsync(Guid sequenceId, int afterIndex = -1, int limit = 1000, CancellationToken cancellationToken = default) => + _db.ListAsync($"SELECT {Columns} FROM frames WHERE sequence_id = $1 AND idx > $2 ORDER BY idx LIMIT $3", + [sequenceId, afterIndex, Cursor.ClampLimit(limit)], Map, cancellationToken); + + /// Every frame with this content hash (duplicate detection, upload short-circuit). + public Task> FindBySha256Async(string sha256, CancellationToken cancellationToken = default) => + _db.ListAsync($"SELECT {Columns} FROM frames WHERE sha256 = $1 ORDER BY sequence_id, idx", [sha256.ToLowerInvariant()], Map, cancellationToken); + + /// Writes analysis results for many frames in one transaction; returns the number of frames updated. + public Task UpdateLightStatsAsync(IReadOnlyList stats, CancellationToken cancellationToken = default) => + _db.InTransactionAsync(async session => + { + var updated = 0; + if (_db.Sql.IsPostgres) + { + // One UPDATE … FROM (VALUES …) per batch instead of a round trip per frame. + for (var start = 0; start < stats.Count; start += Batch) + { + var count = Math.Min(Batch, stats.Count - start); + var sql = new StringBuilder("UPDATE frames AS f SET mean_log_luminance = v.m, clip_low = v.lo, clip_high = v.hi FROM (VALUES "); + var args = new DbParam[count * 4]; + for (var i = 0; i < count; i++) + { + var s = stats[start + i]; + sql.Append(i > 0 ? ", (" : "(").Append('$').Append(4 * i + 1).Append(", $").Append(4 * i + 2) + .Append(", $").Append(4 * i + 3).Append(", $").Append(4 * i + 4).Append(')'); + args[4 * i] = s.FrameId; + args[4 * i + 1] = s.MeanLogLuminance; + args[4 * i + 2] = s.ClipLow; + args[4 * i + 3] = s.ClipHigh; + } + sql.Append(") AS v(id, m, lo, hi) WHERE f.id = v.id"); + updated += await session.ExecuteAsync(sql.ToString(), args, cancellationToken).ConfigureAwait(false); + } + } + else + { + foreach (var s in stats) + { + updated += await session.ExecuteAsync( + "UPDATE frames SET mean_log_luminance = $2, clip_low = $3, clip_high = $4 WHERE id = $1", + [s.FrameId, s.MeanLogLuminance, s.ClipLow, s.ClipHigh], cancellationToken).ConfigureAwait(false); + } + } + return updated; + }, cancellationToken); + + /// Sets and clears flags on many frames in one transaction; returns the number of frames updated. + public Task UpdateFlagsAsync(IReadOnlyList frameIds, FrameFlags set, FrameFlags clear, CancellationToken cancellationToken = default) => + _db.InTransactionAsync(async session => + { + // 500 ids per statement stays below SQLite's historical 999-variable limit. + const int idsPerStatement = 500; + var updated = 0; + for (var start = 0; start < frameIds.Count; start += idsPerStatement) + { + var count = Math.Min(idsPerStatement, frameIds.Count - start); + var args = new DbParam[count + 2]; + args[0] = (int)set; + args[1] = (int)clear; + var placeholders = new StringBuilder(); + for (var i = 0; i < count; i++) + { + args[i + 2] = frameIds[start + i]; + placeholders.Append(i > 0 ? ", $" : "$").Append(i + 3); + } + updated += await session.ExecuteAsync($"UPDATE frames SET flags = (flags | $1) & ~$2 WHERE id IN ({placeholders})", + args, cancellationToken).ConfigureAwait(false); + } + return updated; + }, cancellationToken); + + /// Removes every frame of a sequence; returns how many. + public Task DeleteBySequenceAsync(Guid sequenceId, CancellationToken cancellationToken = default) => + _db.ExecuteAsync("DELETE FROM frames WHERE sequence_id = $1", [sequenceId], cancellationToken); + + private static object?[] Values(Frame f) => + [ + f.Id, f.SequenceId, f.Index, f.RelativePath, f.FileName, f.OriginalFileName, f.Size, f.Sha256?.ToLowerInvariant(), + SequenceRepository.Wall(f.CapturedAt), f.UtcOffsetMinutes, (int)f.TimestampSource, f.ExposureTime, f.FNumber, f.Iso, + f.FocalLength, f.ExposureBias, f.WhiteBalance, f.Orientation, f.Width, f.Height, (int)f.Format, f.PreviewOffset, + f.PreviewLength, f.MeanLogLuminance, f.ClipLow, f.ClipHigh, (int)f.Flags, + ]; + + private static Frame Map(DbRow r) => new() + { + Id = r.GetGuid(0), + SequenceId = r.GetGuid(1), + Index = r.GetInt32(2), + RelativePath = r.GetString(3), + FileName = r.GetString(4), + OriginalFileName = r.GetStringOrNull(5), + Size = r.GetInt64(6), + Sha256 = r.GetStringOrNull(7), + CapturedAt = r.GetDateTimeOrNull(8), + UtcOffsetMinutes = r.GetInt32OrNull(9), + TimestampSource = (TimestampSource)r.GetInt32(10), + ExposureTime = r.GetDoubleOrNull(11), + FNumber = r.GetDoubleOrNull(12), + Iso = r.GetInt32OrNull(13), + FocalLength = r.GetDoubleOrNull(14), + ExposureBias = r.GetDoubleOrNull(15), + WhiteBalance = r.GetInt32OrNull(16), + Orientation = r.GetInt32(17), + Width = r.GetInt32(18), + Height = r.GetInt32(19), + Format = (ImageFileFormat)r.GetInt32(20), + PreviewOffset = r.GetInt64OrNull(21), + PreviewLength = r.GetInt64OrNull(22), + MeanLogLuminance = r.GetDoubleOrNull(23), + ClipLow = r.GetDoubleOrNull(24), + ClipHigh = r.GetDoubleOrNull(25), + Flags = (FrameFlags)r.GetInt32(26), + }; +} diff --git a/src/Titano.Data/Repositories/JobRepository.cs b/src/Titano.Data/Repositories/JobRepository.cs new file mode 100644 index 00000000..228a8159 --- /dev/null +++ b/src/Titano.Data/Repositories/JobRepository.cs @@ -0,0 +1,142 @@ +using System.Globalization; +using Titano.Core.Model; + +namespace Titano.Data.Repositories; + +/// +/// The persistent job queue (BRIEF §14.6): workers claim the oldest job of the highest priority atomically, +/// write checkpoints while running, and on startup requeue what a crash left running. +/// +public sealed class JobRepository +{ + private const string Columns = + "id, kind, target_ids, target_label, parameters_json, state, priority, progress_done, progress_total, items_per_second, " + + "eta_seconds, checkpoint, log_tail, created_by, node_id, attempts, error, created, started, finished"; + + private readonly DbAccess _db; + + internal JobRepository(DbAccess db) => _db = db; + + public async Task EnqueueAsync(Job job, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync( + $"INSERT INTO jobs ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20)", + [ + job.Id, (int)job.Kind, string.Join(',', job.TargetIds), job.TargetLabel, DbParam.Json(job.ParametersJson), (int)job.State, + (int)job.Priority, job.ProgressDone, job.ProgressTotal, job.ItemsPerSecond, job.Eta?.TotalSeconds, job.Checkpoint, job.LogTail, + job.CreatedBy, job.NodeId, job.Attempts, job.Error, job.Created, job.Started, job.Finished, + ], cancellationToken).ConfigureAwait(false); + + public Task GetAsync(Guid id, CancellationToken cancellationToken = default) => + _db.SingleAsync($"SELECT {Columns} FROM jobs WHERE id = $1", [id], Map, cancellationToken); + + /// + /// Marks the next queued job (highest priority, then oldest) of the given kinds as running for + /// and returns it, or null when none waits. Concurrent claimers never get the same + /// job: PostgreSQL skips rows locked by other claimers, SQLite holds its write lock across select and update. + /// + public Task ClaimNextAsync(string workerId, IReadOnlyCollection? kinds = null, CancellationToken cancellationToken = default) + { + var kindFilter = kinds is { Count: > 0 } + ? " AND kind IN (" + string.Join(", ", kinds.Select(k => ((int)k).ToString(CultureInfo.InvariantCulture))) + ")" + : string.Empty; + var pick = $"SELECT id FROM jobs WHERE state = {(int)JobState.Queued}{kindFilter} ORDER BY priority DESC, created, id LIMIT 1"; + var claim = $"UPDATE jobs SET state = {(int)JobState.Running}, node_id = $1, started = $2, finished = NULL, error = NULL, attempts = attempts + 1"; + DbParam[] args = [workerId, DateTimeOffset.UtcNow]; + if (_db.Sql.IsPostgres) + return _db.SingleAsync($"{claim} WHERE id = ({pick} FOR UPDATE SKIP LOCKED) RETURNING {Columns}", args, Map, cancellationToken); + + return _db.InTransactionAsync(async session => + { + var id = await session.ScalarAsync(pick, [], cancellationToken).ConfigureAwait(false); + if (id is null) return null; + await session.ExecuteAsync($"{claim} WHERE id = $3", [.. args, id], cancellationToken).ConfigureAwait(false); + return await session.QuerySingleOrDefaultAsync($"SELECT {Columns} FROM jobs WHERE id = $1", [id], Map, cancellationToken).ConfigureAwait(false); + }, cancellationToken); + } + + /// + /// Checkpoint of a running job. Returns false when the job is no longer running (cancelled or paused meanwhile), + /// which tells the worker to stop. + /// + public async Task UpdateProgressAsync(Guid id, long done, long total, double? itemsPerSecond, TimeSpan? eta, + string? checkpoint, string? logTail, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync( + "UPDATE jobs SET progress_done = $2, progress_total = $3, items_per_second = $4, eta_seconds = $5, checkpoint = $6, log_tail = $7 " + + $"WHERE id = $1 AND state = {(int)JobState.Running}", + [id, done, total, itemsPerSecond, eta?.TotalSeconds, checkpoint, logTail], cancellationToken).ConfigureAwait(false) > 0; + + /// Moves a job to ; done/failed/cancelled stamp the finish time, queued clears it for a retry. + public async Task SetStateAsync(Guid id, JobState state, string? error = null, CancellationToken cancellationToken = default) + { + var terminal = state is JobState.Done or JobState.Failed or JobState.Cancelled; + return await _db.ExecuteAsync("UPDATE jobs SET state = $2, error = $3, finished = $4 WHERE id = $1", + [id, (int)state, error, terminal ? DateTimeOffset.UtcNow : (DateTimeOffset?)null], cancellationToken).ConfigureAwait(false) > 0; + } + + /// Jobs, newest first, optionally of one state. + public Task> ListAsync(JobState? state = null, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default) + { + var p = new SqlParameters(); + var where = new List(); + if (state is { } s) where.Add($"state = {p.Add((int)s)}"); + if (cursor is not null) + { + var (value, id) = Cursor.Decode("jobs", cursor); + where.Add(Cursor.After("created", "id", descending: true, value, id, p.Add)); + } + var take = Cursor.ClampLimit(limit); + var sql = $"SELECT {Columns} FROM jobs{(where.Count > 0 ? " WHERE " + string.Join(" AND ", where) : string.Empty)} " + + $"ORDER BY created DESC, id DESC LIMIT {p.Add(take + 1)}"; + return _db.RunAsync(async session => + { + var jobs = await session.QueryListAsync(sql, p.ToArray(), Map, cancellationToken).ConfigureAwait(false); + string? next = null; + if (jobs.Count > take) + { + jobs.RemoveAt(take); + next = Cursor.Encode("jobs", jobs[^1].Created, jobs[^1].Id); + } + return new Page(jobs, next); + }, cancellationToken); + } + + /// Deletes finished jobs (done, failed, cancelled) that finished before . + public Task PurgeHistoryAsync(DateTimeOffset finishedBefore, CancellationToken cancellationToken = default) => + _db.ExecuteAsync( + $"DELETE FROM jobs WHERE state IN ({(int)JobState.Done}, {(int)JobState.Failed}, {(int)JobState.Cancelled}) AND finished < $1", + [finishedBefore], cancellationToken); + + /// + /// Crash recovery: puts running jobs back in the queue, keeping their checkpoints. With + /// only that worker's jobs are requeued (other server instances keep theirs). + /// + public Task RequeueRunningAsync(string? nodeId = null, CancellationToken cancellationToken = default) => + _db.ExecuteAsync( + $"UPDATE jobs SET state = {(int)JobState.Queued}, node_id = NULL, started = NULL WHERE state = {(int)JobState.Running}" + + (nodeId is null ? string.Empty : " AND node_id = $1"), + nodeId is null ? [] : [nodeId], cancellationToken); + + private static Job Map(DbRow r) => new() + { + Id = r.GetGuid(0), + Kind = (JobKind)r.GetInt32(1), + TargetIds = r.GetString(2) is { Length: > 0 } ids ? Array.ConvertAll(ids.Split(','), Guid.Parse) : [], + TargetLabel = r.GetStringOrNull(3), + ParametersJson = r.GetStringOrNull(4), + State = (JobState)r.GetInt32(5), + Priority = (JobPriority)r.GetInt32(6), + ProgressDone = r.GetInt64(7), + ProgressTotal = r.GetInt64(8), + ItemsPerSecond = r.GetDoubleOrNull(9), + Eta = r.GetDoubleOrNull(10) is { } seconds ? TimeSpan.FromSeconds(seconds) : null, + Checkpoint = r.GetStringOrNull(11), + LogTail = r.GetStringOrNull(12), + CreatedBy = r.GetGuidOrNull(13), + NodeId = r.GetStringOrNull(14), + Attempts = r.GetInt32(15), + Error = r.GetStringOrNull(16), + Created = r.GetDateTimeOffset(17), + Started = r.GetDateTimeOffsetOrNull(18), + Finished = r.GetDateTimeOffsetOrNull(19), + }; +} diff --git a/src/Titano.Data/Repositories/MediaCardRepository.cs b/src/Titano.Data/Repositories/MediaCardRepository.cs new file mode 100644 index 00000000..ab95cc5f --- /dev/null +++ b/src/Titano.Data/Repositories/MediaCardRepository.cs @@ -0,0 +1,125 @@ +using Titano.Core.Model; + +namespace Titano.Data.Repositories; + +/// +/// Identity of a file on a card as seen at import: path on the card, size and modification time. Times are +/// truncated to whole microseconds, the precision the database keeps, so keys built from the file system match +/// stored ones. +/// +public readonly record struct ImportKey +{ + public ImportKey(string path, long size, DateTimeOffset modified) + { + Path = path; + Size = size; + Modified = new DateTimeOffset(modified.UtcTicks - modified.UtcTicks % 10, TimeSpan.Zero); + } + + public string Path { get; } + + public long Size { get; } + + public DateTimeOffset Modified { get; } +} + +/// One file imported from a card and where it went. +public sealed record ImportRecord(Guid CardId, ImportKey Key, string? Sha256, Guid? FrameId, Guid? JobId, DateTimeOffset ImportedAt); + +/// Known memory cards and their import history (BRIEF §13), used to skip files already imported. +public sealed class MediaCardRepository +{ + private const string Columns = + "id, identity, volume_serial, label, capacity, nickname, camera_make, camera_model, camera_serial, profile_json, auto_import, first_seen, last_seen"; + + private static readonly string UpsertSql = SqlDialect.Upsert("media_cards", + [ + "id", "identity", "volume_serial", "label", "capacity", "nickname", "camera_make", "camera_model", "camera_serial", + "profile_json", "auto_import", "first_seen", "last_seen", + ], 1); + + private static readonly string RecordSql = SqlDialect.Upsert("import_history", + ["card_id", "path", "size", "mtime", "sha256", "frame_id", "job_id", "imported_at"], 4); + + private readonly DbAccess _db; + + internal MediaCardRepository(DbAccess db) => _db = db; + + public async Task UpsertAsync(MediaCard card, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync(UpsertSql, + [ + card.Id, card.Identity, (long)card.VolumeSerial, card.Label, card.Capacity, card.Nickname, card.CameraMake, card.CameraModel, + card.CameraSerial, DbParam.Json(card.ProfileJson), card.AutoImport, card.FirstSeen, card.LastSeen, + ], cancellationToken).ConfigureAwait(false); + + public Task GetAsync(Guid id, CancellationToken cancellationToken = default) => + _db.SingleAsync($"SELECT {Columns} FROM media_cards WHERE id = $1", [id], Map, cancellationToken); + + /// The card with this {volumeSerial}-{label}-{capacity} identity, if seen before. + public Task FindByIdentityAsync(string identity, CancellationToken cancellationToken = default) => + _db.SingleAsync($"SELECT {Columns} FROM media_cards WHERE identity = $1", [identity], Map, cancellationToken); + + /// Cards, most recently seen first. + public Task> ListAsync(CancellationToken cancellationToken = default) => + _db.ListAsync($"SELECT {Columns} FROM media_cards ORDER BY last_seen DESC, id", [], Map, cancellationToken); + + public async Task TouchAsync(Guid id, DateTimeOffset seen, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync("UPDATE media_cards SET last_seen = $2 WHERE id = $1", [id, seen], cancellationToken).ConfigureAwait(false) > 0; + + /// Forgets a card and its import history. + public async Task DeleteAsync(Guid id, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync("DELETE FROM media_cards WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) > 0; + + /// Records imported files in one transaction; re-recording a key updates its hash, frame and job. + public Task RecordImportsAsync(IEnumerable records, CancellationToken cancellationToken = default) => + _db.InTransactionAsync(async session => + { + var count = 0; + foreach (var r in records) + { + count += await session.ExecuteAsync(RecordSql, + [r.CardId, r.Key.Path, r.Key.Size, r.Key.Modified, r.Sha256?.ToLowerInvariant(), r.FrameId, r.JobId, r.ImportedAt], + cancellationToken).ConfigureAwait(false); + } + return count; + }, cancellationToken); + + /// Keys of every file already imported from the card, for skipping them on re-insert. + public async Task> GetImportedKeysAsync(Guid cardId, CancellationToken cancellationToken = default) + { + var keys = await _db.ListAsync("SELECT path, size, mtime FROM import_history WHERE card_id = $1", [cardId], + r => new ImportKey(r.GetString(0), r.GetInt64(1), r.GetDateTimeOffset(2)), cancellationToken).ConfigureAwait(false); + return keys.ToHashSet(); + } + + public async Task IsImportedAsync(Guid cardId, ImportKey key, CancellationToken cancellationToken = default) => + await _db.ScalarAsync("SELECT 1 FROM import_history WHERE card_id = $1 AND path = $2 AND size = $3 AND mtime = $4", + [cardId, key.Path, key.Size, key.Modified], cancellationToken).ConfigureAwait(false) is not null; + + /// Most recent imports from a card. + public Task> ListImportsAsync(Guid cardId, int limit = 1000, CancellationToken cancellationToken = default) => + _db.ListAsync( + "SELECT card_id, path, size, mtime, sha256, frame_id, job_id, imported_at FROM import_history WHERE card_id = $1 " + + "ORDER BY imported_at DESC, path LIMIT $2", + [cardId, Cursor.ClampLimit(limit)], + r => new ImportRecord(r.GetGuid(0), new ImportKey(r.GetString(1), r.GetInt64(2), r.GetDateTimeOffset(3)), + r.GetStringOrNull(4), r.GetGuidOrNull(5), r.GetGuidOrNull(6), r.GetDateTimeOffset(7)), + cancellationToken); + + private static MediaCard Map(DbRow r) => new() + { + Id = r.GetGuid(0), + Identity = r.GetString(1), + VolumeSerial = checked((uint)r.GetInt64(2)), + Label = r.GetStringOrNull(3), + Capacity = r.GetInt64(4), + Nickname = r.GetStringOrNull(5), + CameraMake = r.GetStringOrNull(6), + CameraModel = r.GetStringOrNull(7), + CameraSerial = r.GetStringOrNull(8), + ProfileJson = r.GetStringOrNull(9), + AutoImport = r.GetBool(10), + FirstSeen = r.GetDateTimeOffset(11), + LastSeen = r.GetDateTimeOffset(12), + }; +} diff --git a/src/Titano.Data/Repositories/RenderNodeRepository.cs b/src/Titano.Data/Repositories/RenderNodeRepository.cs new file mode 100644 index 00000000..898f1500 --- /dev/null +++ b/src/Titano.Data/Repositories/RenderNodeRepository.cs @@ -0,0 +1,49 @@ +using Titano.Core.Model; + +namespace Titano.Data.Repositories; + +/// Clients offering their PC as render node (BRIEF §14.7). +public sealed class RenderNodeRepository +{ + private const string Columns = "id, user_id, client_name, capabilities_json, last_heartbeat, state"; + + private static readonly string UpsertSql = SqlDialect.Upsert("render_nodes", + ["id", "user_id", "client_name", "capabilities_json", "last_heartbeat", "state"], 1); + + private readonly DbAccess _db; + + internal RenderNodeRepository(DbAccess db) => _db = db; + + /// Registers a node or refreshes its client name, capabilities and state. + public async Task RegisterAsync(RenderNode node, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync(UpsertSql, + [node.Id, node.UserId, node.ClientName, DbParam.Json(node.CapabilitiesJson), node.LastHeartbeat, node.State], + cancellationToken).ConfigureAwait(false); + + public async Task HeartbeatAsync(Guid id, string state, DateTimeOffset at, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync("UPDATE render_nodes SET last_heartbeat = $2, state = $3 WHERE id = $1", [id, at, state], cancellationToken) + .ConfigureAwait(false) > 0; + + public Task GetAsync(Guid id, CancellationToken cancellationToken = default) => + _db.SingleAsync($"SELECT {Columns} FROM render_nodes WHERE id = $1", [id], Map, cancellationToken); + + public Task> ListAsync(CancellationToken cancellationToken = default) => + _db.ListAsync($"SELECT {Columns} FROM render_nodes ORDER BY client_name, id", [], Map, cancellationToken); + + /// Nodes whose last heartbeat is at or after , most recent first. + public Task> ListOnlineAsync(DateTimeOffset since, CancellationToken cancellationToken = default) => + _db.ListAsync($"SELECT {Columns} FROM render_nodes WHERE last_heartbeat >= $1 ORDER BY last_heartbeat DESC, id", [since], Map, cancellationToken); + + public async Task DeleteAsync(Guid id, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync("DELETE FROM render_nodes WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) > 0; + + private static RenderNode Map(DbRow r) => new() + { + Id = r.GetGuid(0), + UserId = r.GetGuid(1), + ClientName = r.GetString(2), + CapabilitiesJson = r.GetStringOrNull(3), + LastHeartbeat = r.GetDateTimeOffset(4), + State = r.GetString(5), + }; +} diff --git a/src/Titano.Data/Repositories/RenderRepository.cs b/src/Titano.Data/Repositories/RenderRepository.cs new file mode 100644 index 00000000..98448d28 --- /dev/null +++ b/src/Titano.Data/Repositories/RenderRepository.cs @@ -0,0 +1,62 @@ +using Titano.Core.Model; + +namespace Titano.Data.Repositories; + +/// Exports of sequences; keeps sequences.rendered_count in step. +public sealed class RenderRepository +{ + private const string Columns = + "id, sequence_id, preset_name, settings_json, output_path, codec, width, height, fps, duration_seconds, bytes, created, produced_by"; + + private const string RecountSql = + "UPDATE sequences SET rendered_count = (SELECT count(*) FROM renders WHERE sequence_id = $1) WHERE id = $1"; + + private readonly DbAccess _db; + + internal RenderRepository(DbAccess db) => _db = db; + + public Task InsertAsync(Render render, CancellationToken cancellationToken = default) => + _db.InTransactionAsync(async session => + { + await session.ExecuteAsync($"INSERT INTO renders ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)", + [ + render.Id, render.SequenceId, render.PresetName, DbParam.Json(render.SettingsJson), render.OutputPath, render.Codec, + render.Width, render.Height, render.Fps, render.DurationSeconds, render.Bytes, render.Created, render.ProducedBy, + ], cancellationToken).ConfigureAwait(false); + return await session.ExecuteAsync(RecountSql, [render.SequenceId], cancellationToken).ConfigureAwait(false); + }, cancellationToken); + + public Task GetAsync(Guid id, CancellationToken cancellationToken = default) => + _db.SingleAsync($"SELECT {Columns} FROM renders WHERE id = $1", [id], Map, cancellationToken); + + /// Renders of a sequence, newest first. + public Task> ListBySequenceAsync(Guid sequenceId, CancellationToken cancellationToken = default) => + _db.ListAsync($"SELECT {Columns} FROM renders WHERE sequence_id = $1 ORDER BY created DESC, id DESC", [sequenceId], Map, cancellationToken); + + public Task DeleteAsync(Guid id, CancellationToken cancellationToken = default) => + _db.InTransactionAsync(async session => + { + var sequenceId = await session.ScalarAsync("SELECT sequence_id FROM renders WHERE id = $1", [id], cancellationToken).ConfigureAwait(false); + if (sequenceId is null) return false; + await session.ExecuteAsync("DELETE FROM renders WHERE id = $1", [id], cancellationToken).ConfigureAwait(false); + await session.ExecuteAsync(RecountSql, [sequenceId], cancellationToken).ConfigureAwait(false); + return true; + }, cancellationToken); + + private static Render Map(DbRow r) => new() + { + Id = r.GetGuid(0), + SequenceId = r.GetGuid(1), + PresetName = r.GetString(2), + SettingsJson = r.GetStringOrNull(3), + OutputPath = r.GetString(4), + Codec = r.GetStringOrNull(5), + Width = r.GetInt32(6), + Height = r.GetInt32(7), + Fps = r.GetDouble(8), + DurationSeconds = r.GetDouble(9), + Bytes = r.GetInt64(10), + Created = r.GetDateTimeOffset(11), + ProducedBy = r.GetStringOrNull(12), + }; +} diff --git a/src/Titano.Data/Repositories/SequenceRepository.cs b/src/Titano.Data/Repositories/SequenceRepository.cs new file mode 100644 index 00000000..422ad364 --- /dev/null +++ b/src/Titano.Data/Repositories/SequenceRepository.cs @@ -0,0 +1,369 @@ +using Titano.Core.Imaging; +using Titano.Core.Model; + +namespace Titano.Data.Repositories; + +/// Orderings of the Library; every one is keyset-paginated with the sequence id as tie-breaker. +public enum SequenceSort +{ + /// Latest first capture first; sequences without captures last. + Newest, + Oldest, + + /// Display title, case-insensitive. + Title, + FrameCount, + Size, +} + +/// Library filters (all optional, combined with AND) plus sort and page. +public sealed record SequenceQuery +{ + public Guid? ArchiveId { get; init; } + + /// Calendar year of the first capture. + public int? Year { get; init; } + + /// Exact camera model. + public string? Camera { get; init; } + + /// Status flags that must all be set. + public SequenceStatus WithStatus { get; init; } + + /// Status flags that must all be clear. + public SequenceStatus WithoutStatus { get; init; } + + public string? Tag { get; init; } + + /// Case-insensitive substring of the title, camera make/model or a tag. + public string? Text { get; init; } + + public Guid? OwnerUserId { get; init; } + + public SequenceSort Sort { get; init; } = SequenceSort.Newest; + + /// Page size, 1–1000. + public int Limit { get; init; } = 100; + + /// of the previous page, or null for the first page. + public string? Cursor { get; init; } +} + +/// Figures of the Archive screen (BRIEF §12.5). +public sealed record ArchiveStats( + int Sequences, + long Photos, + long Bytes, + IReadOnlyList ByYear, + IReadOnlyList ByMonth, + IReadOnlyList ByCamera, + IReadOnlyList ByFormat, + ArchiveHealth Health); + +public sealed record YearBucket(int Year, int Sequences, long Photos, long Bytes); + +/// One heatmap cell: sequences and photos whose first capture falls in the month. +public sealed record MonthBucket(int Year, int Month, int Sequences, long Photos); + +public sealed record CameraBucket(string? Make, string? Model, int Sequences, long Photos, long Bytes); + +public sealed record FormatBucket(ImageFileFormat Format, long Photos, long Bytes); + +/// Health counters: sequences needing attention and flagged frames. +public sealed record ArchiveHealth( + int Unverified, + int WithWarnings, + int WithErrors, + int NotAnalyzed, + long MissingFrames, + long CorruptFrames, + long DuplicateFrames); + +/// Sequences (projects) with their tags; Library listing and Archive statistics. +public sealed class SequenceRepository +{ + private const string Columns = + "s.id, s.archive_id, s.title, s.folder, s.camera_make, s.camera_model, s.camera_serial, s.lens, s.first_capture, s.last_capture, " + + "s.frame_count, s.median_interval, s.total_bytes, s.status, s.rendered_count, s.verified_at, s.hero_frame_id, s.rating, s.notes, " + + "s.settings_json, s.analysis_json, s.origin, s.sync_state, s.settings_version, s.frames_version, s.owner_user_id, s.warning_count, s.updated_at"; + + private static readonly string UpsertSql = SqlDialect.Upsert("sequences", + [ + "id", "archive_id", "title", "title_key", "search_text", "folder", "camera_make", "camera_model", "camera_serial", "lens", + "first_capture", "last_capture", "frame_count", "median_interval", "total_bytes", "status", "rendered_count", "verified_at", + "hero_frame_id", "rating", "notes", "settings_json", "analysis_json", "origin", "sync_state", "settings_version", + "frames_version", "owner_user_id", "warning_count", "updated_at", + ], 1); + + /// Ordinal of the tag list after , and of the sort key a listing appends after it. + private const int TagsOrdinal = 28; + private const int SortKeyOrdinal = 29; + + private readonly DbAccess _db; + private readonly string _select; + + internal SequenceRepository(DbAccess db) + { + _db = db; + _select = SelectSql(string.Empty); + } + + private string SelectSql(string extraColumns) => + $"SELECT {Columns}, (SELECT {_db.Sql.StringAgg("t.name")} FROM sequence_tags st JOIN tags t ON t.id = st.tag_id " + + $"WHERE st.sequence_id = s.id) AS tags{extraColumns} FROM sequences s"; + + /// Inserts or replaces the sequence row and its tag set. + public Task UpsertAsync(Sequence sequence, CancellationToken cancellationToken = default) => + _db.InTransactionAsync(async session => + { + await UpsertRowAsync(session, sequence, cancellationToken).ConfigureAwait(false); + await ReplaceTagsAsync(session, sequence.Id, sequence.Tags, cancellationToken).ConfigureAwait(false); + return true; + }, cancellationToken); + + public Task GetAsync(Guid id, CancellationToken cancellationToken = default) => + _db.SingleAsync($"{_select} WHERE s.id = $1", [id], Map, cancellationToken); + + /// Deletes the sequence with its frames, renders and tag links. + public async Task DeleteAsync(Guid id, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync("DELETE FROM sequences WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) > 0; + + /// Replaces the tags of a sequence (trimmed, de-duplicated case-insensitively). + public async Task SetTagsAsync(Guid id, IEnumerable tags, CancellationToken cancellationToken = default) => + await _db.InTransactionAsync(async session => + { + if (await session.ScalarAsync("SELECT id FROM sequences WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) is null) + return false; + await ReplaceTagsAsync(session, id, tags, cancellationToken).ConfigureAwait(false); + await session.ExecuteAsync("UPDATE sequences SET settings_version = settings_version + 1, updated_at = $2 WHERE id = $1", + [id, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false); + return true; + }, cancellationToken).ConfigureAwait(false); + + /// Sets and clears status flags in one statement. + public async Task UpdateStatusAsync(Guid id, SequenceStatus set, SequenceStatus clear, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync("UPDATE sequences SET status = (status | $2) & ~$3, updated_at = $4 WHERE id = $1", + [id, (int)set, (int)clear, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0; + + /// Recomputes frame count, total bytes and first/last capture from the frames table. + public Task RefreshStatsAsync(Guid id, CancellationToken cancellationToken = default) => + _db.InTransactionAsync(async session => + { + var current = await session.QuerySingleOrDefaultAsync($"{_select} WHERE s.id = $1", [id], Map, cancellationToken).ConfigureAwait(false); + if (current is null) return null; + var stats = await session.QuerySingleAsync( + "SELECT count(*), COALESCE(SUM(size), 0), MIN(captured_at), MAX(captured_at) FROM frames WHERE sequence_id = $1", + [id], cancellationToken).ConfigureAwait(false); + var updated = current with + { + FrameCount = (int)stats.GetInt64(0), + TotalBytes = stats.GetInt64(1), + FirstCapture = stats.GetDateTimeOrNull(2), + LastCapture = stats.GetDateTimeOrNull(3), + FramesVersion = current.FramesVersion + 1, + UpdatedAt = DateTimeOffset.UtcNow, + }; + await UpsertRowAsync(session, updated, cancellationToken).ConfigureAwait(false); + return updated; + }, cancellationToken); + + /// One page of sequences matching . + public Task> ListAsync(SequenceQuery query, CancellationToken cancellationToken = default) + { + var p = new SqlParameters(); + var where = Filter(query, p); + var (column, descending) = SortKey(query.Sort); + var scope = "seq:" + query.Sort; + if (query.Cursor is not null) + { + var (value, id) = Cursor.Decode(scope, query.Cursor); + where.Add(Cursor.After(column, "s.id", descending, value, id, p.Add)); + } + var limit = Cursor.ClampLimit(query.Limit); + var direction = descending ? "DESC" : "ASC"; + var sql = $"{SelectSql($", {column} AS sort_key")}{Where(where)} " + + $"ORDER BY {column} {direction} NULLS LAST, s.id {direction} LIMIT {p.Add(limit + 1)}"; + return _db.RunAsync(async session => + { + var rows = await session.QueryListAsync(sql, p.ToArray(), r => (Item: Map(r), Key: SortValue(query.Sort, r)), cancellationToken).ConfigureAwait(false); + string? next = null; + if (rows.Count > limit) + { + rows.RemoveAt(limit); + next = Cursor.Encode(scope, rows[^1].Key, rows[^1].Item.Id); + } + return new Page(rows.ConvertAll(r => r.Item), next); + }, cancellationToken); + } + + /// Number of sequences matching the filters of (sort and page ignored). + public async Task CountAsync(SequenceQuery query, CancellationToken cancellationToken = default) + { + var p = new SqlParameters(); + var where = Filter(query, p); + return (int)await _db.ScalarAsync($"SELECT count(*) FROM sequences s{Where(where)}", p.ToArray(), cancellationToken).ConfigureAwait(false); + } + + /// Totals, distributions and health counters of one archive, or of all archives. + public Task GetStatsAsync(Guid? archiveId = null, CancellationToken cancellationToken = default) + { + var sql = _db.Sql; + DbParam[] args = archiveId is { } a ? [a] : []; + var seqWhere = archiveId is null ? string.Empty : " WHERE s.archive_id = $1"; + var seqAnd = archiveId is null ? " WHERE" : " WHERE s.archive_id = $1 AND"; + var frameFrom = archiveId is null ? "frames f" : "frames f JOIN sequences s ON s.id = f.sequence_id WHERE s.archive_id = $1"; + var year = sql.Year("s.first_capture"); + return _db.RunAsync(async session => + { + var totals = await session.QuerySingleAsync( + $"SELECT count(*), COALESCE(SUM(s.frame_count), 0), COALESCE(SUM(s.total_bytes), 0) FROM sequences s{seqWhere}", + args, cancellationToken).ConfigureAwait(false); + var byYear = await session.QueryListAsync( + $"SELECT {year}, count(*), COALESCE(SUM(s.frame_count), 0), COALESCE(SUM(s.total_bytes), 0) FROM sequences s" + + $"{seqAnd} s.first_capture IS NOT NULL GROUP BY 1 ORDER BY 1", args, + r => new YearBucket(r.GetInt32(0), r.GetInt32(1), r.GetInt64(2), r.GetInt64(3)), cancellationToken).ConfigureAwait(false); + var byMonth = await session.QueryListAsync( + $"SELECT {year}, {sql.Month("s.first_capture")}, count(*), COALESCE(SUM(s.frame_count), 0) FROM sequences s" + + $"{seqAnd} s.first_capture IS NOT NULL GROUP BY 1, 2 ORDER BY 1, 2", args, + r => new MonthBucket(r.GetInt32(0), r.GetInt32(1), r.GetInt32(2), r.GetInt64(3)), cancellationToken).ConfigureAwait(false); + var byCamera = await session.QueryListAsync( + $"SELECT s.camera_make, s.camera_model, count(*), COALESCE(SUM(s.frame_count), 0), COALESCE(SUM(s.total_bytes), 0) " + + $"FROM sequences s{seqWhere} GROUP BY s.camera_make, s.camera_model ORDER BY 4 DESC, 2 NULLS LAST, 1 NULLS LAST", args, + r => new CameraBucket(r.GetStringOrNull(0), r.GetStringOrNull(1), r.GetInt32(2), r.GetInt64(3), r.GetInt64(4)), cancellationToken).ConfigureAwait(false); + var byFormat = await session.QueryListAsync( + $"SELECT f.format, count(*), COALESCE(SUM(f.size), 0) FROM {frameFrom} GROUP BY f.format ORDER BY 2 DESC, 1", args, + r => new FormatBucket((ImageFileFormat)r.GetInt32(0), r.GetInt64(1), r.GetInt64(2)), cancellationToken).ConfigureAwait(false); + var sequenceHealth = await session.QuerySingleAsync( + "SELECT " + + "COALESCE(SUM(CASE WHEN s.verified_at IS NULL THEN 1 ELSE 0 END), 0), " + + $"COALESCE(SUM(CASE WHEN (s.status & {(int)SequenceStatus.HasWarnings}) <> 0 OR s.warning_count > 0 THEN 1 ELSE 0 END), 0), " + + $"COALESCE(SUM(CASE WHEN (s.status & {(int)SequenceStatus.HasErrors}) <> 0 THEN 1 ELSE 0 END), 0), " + + $"COALESCE(SUM(CASE WHEN (s.status & {(int)SequenceStatus.AnalyzedLight}) = 0 THEN 1 ELSE 0 END), 0) " + + $"FROM sequences s{seqWhere}", args, cancellationToken).ConfigureAwait(false); + var frameHealth = await session.QuerySingleAsync( + "SELECT " + + $"COALESCE(SUM(CASE WHEN (f.flags & {(int)FrameFlags.Missing}) <> 0 THEN 1 ELSE 0 END), 0), " + + $"COALESCE(SUM(CASE WHEN (f.flags & {(int)FrameFlags.Corrupt}) <> 0 THEN 1 ELSE 0 END), 0), " + + $"COALESCE(SUM(CASE WHEN (f.flags & {(int)FrameFlags.Duplicate}) <> 0 THEN 1 ELSE 0 END), 0) " + + $"FROM {frameFrom}", args, cancellationToken).ConfigureAwait(false); + return new ArchiveStats( + totals.GetInt32(0), totals.GetInt64(1), totals.GetInt64(2), byYear, byMonth, byCamera, byFormat, + new ArchiveHealth( + sequenceHealth.GetInt32(0), sequenceHealth.GetInt32(1), sequenceHealth.GetInt32(2), sequenceHealth.GetInt32(3), + frameHealth.GetInt64(0), frameHealth.GetInt64(1), frameHealth.GetInt64(2))); + }, cancellationToken); + } + + private static List Filter(SequenceQuery query, SqlParameters p) + { + var where = new List(); + if (query.ArchiveId is { } archive) where.Add($"s.archive_id = {p.Add(archive)}"); + if (query.Year is { } year) + where.Add($"s.first_capture >= {p.Add(new DateTime(year, 1, 1))} AND s.first_capture < {p.Add(new DateTime(year + 1, 1, 1))}"); + if (query.Camera is { } camera) where.Add($"s.camera_model = {p.Add(camera)}"); + if (query.WithStatus != SequenceStatus.None) + { + var mask = p.Add((int)query.WithStatus); + where.Add($"(s.status & {mask}) = {mask}"); + } + if (query.WithoutStatus != SequenceStatus.None) where.Add($"(s.status & {p.Add((int)query.WithoutStatus)}) = 0"); + if (query.Tag is { } tag) + where.Add("EXISTS (SELECT 1 FROM sequence_tags st JOIN tags t ON t.id = st.tag_id " + + $"WHERE st.sequence_id = s.id AND t.name_key = {p.Add(TagRepository.Key(tag))})"); + if (!string.IsNullOrWhiteSpace(query.Text)) + { + var pattern = p.Add(SqlDialect.ContainsPattern(query.Text.Trim().ToLowerInvariant())); + where.Add($"(s.search_text LIKE {pattern} ESCAPE '\\' OR EXISTS (SELECT 1 FROM sequence_tags st JOIN tags t ON t.id = st.tag_id " + + $"WHERE st.sequence_id = s.id AND t.name_key LIKE {pattern} ESCAPE '\\'))"); + } + if (query.OwnerUserId is { } owner) where.Add($"s.owner_user_id = {p.Add(owner)}"); + return where; + } + + private static string Where(List conditions) => + conditions.Count == 0 ? string.Empty : " WHERE " + string.Join(" AND ", conditions); + + private static (string Column, bool Descending) SortKey(SequenceSort sort) => sort switch + { + SequenceSort.Newest => ("s.first_capture", true), + SequenceSort.Oldest => ("s.first_capture", false), + SequenceSort.Title => ("s.title_key", false), + SequenceSort.FrameCount => ("s.frame_count", true), + SequenceSort.Size => ("s.total_bytes", true), + _ => throw new ArgumentOutOfRangeException(nameof(sort), sort, null), + }; + + private static object? SortValue(SequenceSort sort, DbRow row) => sort switch + { + SequenceSort.Newest or SequenceSort.Oldest => row.GetDateTimeOrNull(SortKeyOrdinal), + SequenceSort.Title => row.GetString(SortKeyOrdinal), + _ => row.GetInt64(SortKeyOrdinal), + }; + + private static Task UpsertRowAsync(IDbSession session, Sequence s, CancellationToken cancellationToken) + { + var search = string.Join(' ', new[] { s.Title, s.CameraMake, s.CameraModel }.Where(v => !string.IsNullOrWhiteSpace(v))).ToLowerInvariant(); + return session.ExecuteAsync(UpsertSql, + [ + s.Id, s.ArchiveId, s.Title, s.DisplayTitle.ToLowerInvariant(), search, s.Folder, s.CameraMake, s.CameraModel, s.CameraSerial, s.Lens, + Wall(s.FirstCapture), Wall(s.LastCapture), s.FrameCount, s.MedianInterval, s.TotalBytes, (int)s.Status, s.RenderedCount, s.VerifiedAt, + s.HeroFrameId, s.Rating, s.Notes, DbParam.Json(s.SettingsJson), DbParam.Json(s.AnalysisJson), (int)s.Origin, (int)s.SyncState, + s.SettingsVersion, s.FramesVersion, s.OwnerUserId, s.WarningCount, s.UpdatedAt, + ], cancellationToken); + } + + private static async Task ReplaceTagsAsync(IDbSession session, Guid sequenceId, IEnumerable tags, CancellationToken cancellationToken) + { + await session.ExecuteAsync("DELETE FROM sequence_tags WHERE sequence_id = $1", [sequenceId], cancellationToken).ConfigureAwait(false); + foreach (var tag in TagRepository.NormalizeSet(tags)) + { + var key = TagRepository.Key(tag); + await session.ExecuteAsync("INSERT INTO tags (id, name, name_key) VALUES ($1, $2, $3) ON CONFLICT (name_key) DO NOTHING", + [Guid.CreateVersion7(), tag, key], cancellationToken).ConfigureAwait(false); + await session.ExecuteAsync("INSERT INTO sequence_tags (sequence_id, tag_id) SELECT $1, id FROM tags WHERE name_key = $2", + [sequenceId, key], cancellationToken).ConfigureAwait(false); + } + } + + /// Camera wall-clock times are stored without zone, whatever kind the caller's value has. + internal static DateTime? Wall(DateTime? value) => + value is { } v ? DateTime.SpecifyKind(v, DateTimeKind.Unspecified) : null; + + private static Sequence Map(DbRow r) + { + var tags = r.GetStringOrNull(TagsOrdinal) is { Length: > 0 } list ? list.Split(SqlDialect.ListSeparator) : []; + Array.Sort(tags, StringComparer.OrdinalIgnoreCase); + return new Sequence + { + Id = r.GetGuid(0), + ArchiveId = r.GetGuid(1), + Title = r.GetStringOrNull(2), + Folder = r.GetString(3), + CameraMake = r.GetStringOrNull(4), + CameraModel = r.GetStringOrNull(5), + CameraSerial = r.GetStringOrNull(6), + Lens = r.GetStringOrNull(7), + FirstCapture = r.GetDateTimeOrNull(8), + LastCapture = r.GetDateTimeOrNull(9), + FrameCount = r.GetInt32(10), + MedianInterval = r.GetDoubleOrNull(11), + TotalBytes = r.GetInt64(12), + Status = (SequenceStatus)r.GetInt32(13), + RenderedCount = r.GetInt32(14), + VerifiedAt = r.GetDateTimeOffsetOrNull(15), + HeroFrameId = r.GetGuidOrNull(16), + Rating = r.GetInt32(17), + Notes = r.GetStringOrNull(18), + SettingsJson = r.GetStringOrNull(19), + AnalysisJson = r.GetStringOrNull(20), + Origin = (SequenceOrigin)r.GetInt32(21), + SyncState = (SyncState)r.GetInt32(22), + SettingsVersion = r.GetInt32(23), + FramesVersion = r.GetInt32(24), + OwnerUserId = r.GetGuidOrNull(25), + WarningCount = r.GetInt32(26), + UpdatedAt = r.GetDateTimeOffset(27), + Tags = tags, + }; + } +} diff --git a/src/Titano.Data/Repositories/SettingsRepository.cs b/src/Titano.Data/Repositories/SettingsRepository.cs new file mode 100644 index 00000000..73c56ca3 --- /dev/null +++ b/src/Titano.Data/Repositories/SettingsRepository.cs @@ -0,0 +1,93 @@ +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Text.Json.Serialization.Metadata; + +namespace Titano.Data.Repositories; + +/// Who a setting belongs to. +public enum SettingScope +{ + /// The installation (client app or server); no scope id. + App, + + /// One archive; the scope id is the archive id. + Archive, + + /// One user; the scope id is the user id. + User, +} + +/// Typed key/value settings stored as JSON (BRIEF §5 Setting). +public sealed class SettingsRepository +{ + private static readonly string UpsertSql = SqlDialect.Upsert("settings", ["scope", "scope_id", "key", "value_json", "updated_at"], 3); + + private readonly DbAccess _db; + + internal SettingsRepository(DbAccess db) => _db = db; + + /// The raw JSON of a setting, or null when unset. + public Task GetJsonAsync(SettingScope scope, Guid? scopeId, string key, CancellationToken cancellationToken = default) => + _db.ScalarAsync("SELECT value_json FROM settings WHERE scope = $1 AND scope_id = $2 AND key = $3", + [ScopeName(scope), ScopeId(scope, scopeId), key], cancellationToken); + + /// Stores a JSON value; malformed JSON is rejected on both dialects. + public async Task SetJsonAsync(SettingScope scope, Guid? scopeId, string key, string json, CancellationToken cancellationToken = default) + { + using (JsonDocument.Parse(json)) + { + } + await _db.ExecuteAsync(UpsertSql, [ScopeName(scope), ScopeId(scope, scopeId), key, DbParam.Json(json), DateTimeOffset.UtcNow], cancellationToken) + .ConfigureAwait(false); + } + + /// Reads a setting with source-generated metadata (trim/AOT safe); default when unset. + public async Task GetAsync(SettingScope scope, Guid? scopeId, string key, JsonTypeInfo typeInfo, CancellationToken cancellationToken = default) => + await GetJsonAsync(scope, scopeId, key, cancellationToken).ConfigureAwait(false) is { } json + ? JsonSerializer.Deserialize(json, typeInfo) + : default; + + public Task SetAsync(SettingScope scope, Guid? scopeId, string key, T value, JsonTypeInfo typeInfo, CancellationToken cancellationToken = default) => + SetJsonAsync(scope, scopeId, key, JsonSerializer.Serialize(value, typeInfo), cancellationToken); + + /// Reads a setting through reflection-based serialization; default when unset. + [RequiresUnreferencedCode("Reflection-based JSON; use the JsonTypeInfo overload in trimmed builds.")] + [RequiresDynamicCode("Reflection-based JSON; use the JsonTypeInfo overload in AOT builds.")] + public async Task GetAsync(SettingScope scope, Guid? scopeId, string key, JsonSerializerOptions? options = null, CancellationToken cancellationToken = default) => + await GetJsonAsync(scope, scopeId, key, cancellationToken).ConfigureAwait(false) is { } json + ? JsonSerializer.Deserialize(json, options) + : default; + + [RequiresUnreferencedCode("Reflection-based JSON; use the JsonTypeInfo overload in trimmed builds.")] + [RequiresDynamicCode("Reflection-based JSON; use the JsonTypeInfo overload in AOT builds.")] + public Task SetAsync(SettingScope scope, Guid? scopeId, string key, T value, JsonSerializerOptions? options = null, CancellationToken cancellationToken = default) => + SetJsonAsync(scope, scopeId, key, JsonSerializer.Serialize(value, options), cancellationToken); + + public async Task DeleteAsync(SettingScope scope, Guid? scopeId, string key, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync("DELETE FROM settings WHERE scope = $1 AND scope_id = $2 AND key = $3", + [ScopeName(scope), ScopeId(scope, scopeId), key], cancellationToken).ConfigureAwait(false) > 0; + + /// Every setting of a scope as key → JSON. + public async Task> ListAsync(SettingScope scope, Guid? scopeId, CancellationToken cancellationToken = default) + { + var rows = await _db.ListAsync("SELECT key, value_json FROM settings WHERE scope = $1 AND scope_id = $2 ORDER BY key", + [ScopeName(scope), ScopeId(scope, scopeId)], r => KeyValuePair.Create(r.GetString(0), r.GetString(1)), cancellationToken).ConfigureAwait(false); + return new Dictionary(rows, StringComparer.Ordinal); + } + + private static string ScopeName(SettingScope scope) => scope switch + { + SettingScope.App => "app", + SettingScope.Archive => "archive", + SettingScope.User => "user", + _ => throw new ArgumentOutOfRangeException(nameof(scope), scope, null), + }; + + private static string ScopeId(SettingScope scope, Guid? scopeId) => (scope, scopeId) switch + { + (SettingScope.App, null) => string.Empty, + (SettingScope.App, _) => throw new ArgumentException("App settings have no scope id.", nameof(scopeId)), + (_, { } id) => TextValues.FormatGuid(id), + _ => throw new ArgumentException($"{scope} settings need a scope id.", nameof(scopeId)), + }; +} diff --git a/src/Titano.Data/Repositories/TagRepository.cs b/src/Titano.Data/Repositories/TagRepository.cs new file mode 100644 index 00000000..07de96ce --- /dev/null +++ b/src/Titano.Data/Repositories/TagRepository.cs @@ -0,0 +1,79 @@ +using System.Text; + +namespace Titano.Data.Repositories; + +/// A tag and how many sequences carry it. +public sealed record TagInfo(Guid Id, string Name, int SequenceCount); + +/// +/// Normalised tags (BRIEF §5 Tag/SequenceTag): one row per case-insensitive name, linked to sequences through +/// sequence_tags. The first spelling of a name is kept. +/// +public sealed class TagRepository +{ + /// Longest tag name accepted, in characters. + public const int MaxLength = 64; + + private readonly DbAccess _db; + + internal TagRepository(DbAccess db) => _db = db; + + /// All tags by name with their usage counts. + public Task> ListAsync(CancellationToken cancellationToken = default) => + _db.ListAsync( + "SELECT t.id, t.name, (SELECT count(*) FROM sequence_tags st WHERE st.tag_id = t.id) FROM tags t ORDER BY t.name_key", + [], r => new TagInfo(r.GetGuid(0), r.GetString(1), r.GetInt32(2)), cancellationToken); + + /// Renames a tag; a name already used by another tag violates the unique key (). + public async Task RenameAsync(Guid id, string name, CancellationToken cancellationToken = default) + { + var normalized = Normalize(name); + return await _db.ExecuteAsync("UPDATE tags SET name = $2, name_key = $3 WHERE id = $1", + [id, normalized, Key(normalized)], cancellationToken).ConfigureAwait(false) > 0; + } + + /// Deletes a tag and removes it from every sequence. + public async Task DeleteAsync(Guid id, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync("DELETE FROM tags WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) > 0; + + /// Deletes tags no sequence uses any more; returns how many. + public Task DeleteUnusedAsync(CancellationToken cancellationToken = default) => + _db.ExecuteAsync("DELETE FROM tags WHERE NOT EXISTS (SELECT 1 FROM sequence_tags st WHERE st.tag_id = tags.id)", [], cancellationToken); + + /// Trims, collapses inner whitespace and validates a tag name. + public static string Normalize(string name) + { + var sb = new StringBuilder(name.Length); + foreach (var c in name.Trim()) + { + if (char.IsControl(c)) throw new ArgumentException("Tags cannot contain control characters.", nameof(name)); + if (char.IsWhiteSpace(c)) + { + if (sb.Length > 0 && sb[^1] != ' ') sb.Append(' '); + } + else + { + sb.Append(c); + } + } + if (sb.Length == 0) throw new ArgumentException("A tag cannot be empty.", nameof(name)); + if (sb.Length > MaxLength) throw new ArgumentException($"A tag is at most {MaxLength} characters.", nameof(name)); + return sb.ToString(); + } + + /// The case-insensitive identity of a tag name. + public static string Key(string name) => Normalize(name).ToLowerInvariant(); + + /// Normalised tags without case-insensitive duplicates, first spelling kept. + internal static List NormalizeSet(IEnumerable tags) + { + var seen = new HashSet(StringComparer.Ordinal); + var result = new List(); + foreach (var tag in tags) + { + var normalized = Normalize(tag); + if (seen.Add(normalized.ToLowerInvariant())) result.Add(normalized); + } + return result; + } +} diff --git a/src/Titano.Data/Repositories/TokenRepository.cs b/src/Titano.Data/Repositories/TokenRepository.cs new file mode 100644 index 00000000..8b88c0fc --- /dev/null +++ b/src/Titano.Data/Repositories/TokenRepository.cs @@ -0,0 +1,57 @@ +using Titano.Core.Model; + +namespace Titano.Data.Repositories; + +/// API session tokens, stored by SHA-256 hash, with sliding expiry and revocation (BRIEF §14.2). +public sealed class TokenRepository +{ + private const string Columns = "token_hash, user_id, client_name, created, expires, last_used, revoked"; + + private readonly DbAccess _db; + + internal TokenRepository(DbAccess db) => _db = db; + + public async Task InsertAsync(SessionToken token, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync($"INSERT INTO tokens ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7)", + [token.TokenHash, token.UserId, token.ClientName, token.Created, token.Expires, token.LastUsed, token.Revoked], + cancellationToken).ConfigureAwait(false); + + /// The token with this hash, revoked or expired ones included (the caller decides). + public Task GetAsync(string tokenHash, CancellationToken cancellationToken = default) => + _db.SingleAsync($"SELECT {Columns} FROM tokens WHERE token_hash = $1", [tokenHash], Map, cancellationToken); + + /// + /// Sliding expiry: records use at and moves the expiry to . + /// Returns false for unknown, revoked or already expired tokens. + /// + public async Task TouchAsync(string tokenHash, DateTimeOffset now, DateTimeOffset expires, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync( + "UPDATE tokens SET last_used = $2, expires = $3 WHERE token_hash = $1 AND revoked = FALSE AND expires > $2", + [tokenHash, now, expires], cancellationToken).ConfigureAwait(false) > 0; + + public async Task RevokeAsync(string tokenHash, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync("UPDATE tokens SET revoked = TRUE WHERE token_hash = $1 AND revoked = FALSE", [tokenHash], cancellationToken) + .ConfigureAwait(false) > 0; + + /// Revokes every live token of a user (password change, account disabled). + public Task RevokeAllAsync(Guid userId, CancellationToken cancellationToken = default) => + _db.ExecuteAsync("UPDATE tokens SET revoked = TRUE WHERE user_id = $1 AND revoked = FALSE", [userId], cancellationToken); + + public Task> ListByUserAsync(Guid userId, CancellationToken cancellationToken = default) => + _db.ListAsync($"SELECT {Columns} FROM tokens WHERE user_id = $1 ORDER BY last_used DESC", [userId], Map, cancellationToken); + + /// Deletes tokens that expired by or were revoked. + public Task PurgeAsync(DateTimeOffset now, CancellationToken cancellationToken = default) => + _db.ExecuteAsync("DELETE FROM tokens WHERE expires <= $1 OR revoked = TRUE", [now], cancellationToken); + + private static SessionToken Map(DbRow r) => new() + { + TokenHash = r.GetString(0), + UserId = r.GetGuid(1), + ClientName = r.GetStringOrNull(2), + Created = r.GetDateTimeOffset(3), + Expires = r.GetDateTimeOffset(4), + LastUsed = r.GetDateTimeOffset(5), + Revoked = r.GetBool(6), + }; +} diff --git a/src/Titano.Data/Repositories/UploadRepository.cs b/src/Titano.Data/Repositories/UploadRepository.cs new file mode 100644 index 00000000..355511ec --- /dev/null +++ b/src/Titano.Data/Repositories/UploadRepository.cs @@ -0,0 +1,135 @@ +namespace Titano.Data.Repositories; + +/// Life cycle of a resumable upload (BRIEF §14.4). +public enum UploadState +{ + /// Chunks are arriving. + Receiving, + + /// All chunks are in; the server is verifying the hash and moving the file into the archive. + Committing, + + /// The file is a frame of the archive (). + Committed, + Failed, + Cancelled, +} + +/// A chunked upload and its received-chunk bitmap (bit n of byte n/8, least significant first). +public sealed record Upload +{ + public required Guid Id { get; init; } + public required Guid UserId { get; init; } + public Guid? SequenceId { get; init; } + public required string FileName { get; init; } + public long Size { get; init; } + + /// Lower-case hex SHA-256 announced by the client. + public required string Sha256 { get; init; } + + public DateTime? CapturedAt { get; init; } + public required int ChunkSize { get; init; } + public byte[] Received { get; init; } = []; + public int ReceivedCount { get; init; } + public UploadState State { get; init; } + public Guid? FrameId { get; init; } + public DateTimeOffset Created { get; init; } + public DateTimeOffset UpdatedAt { get; init; } + + public int ChunkCount => (int)((Size + ChunkSize - 1) / ChunkSize); + + public bool IsComplete => ReceivedCount >= ChunkCount; + + public bool HasChunk(int index) => + index >= 0 && index / 8 < Received.Length && (Received[index / 8] & (1 << (index % 8))) != 0; +} + +/// Resumable uploads: creation, per-chunk bookkeeping, state changes and cleanup. +public sealed class UploadRepository +{ + private const string Columns = + "id, user_id, sequence_id, file_name, size, sha256, captured_at, chunk_size, received, received_count, state, frame_id, created, updated_at"; + + private readonly DbAccess _db; + + internal UploadRepository(DbAccess db) => _db = db; + + /// Stores a new upload with an empty bitmap sized for its chunk count. + public async Task CreateAsync(Upload upload, CancellationToken cancellationToken = default) + { + ArgumentOutOfRangeException.ThrowIfLessThan(upload.ChunkSize, 1); + ArgumentOutOfRangeException.ThrowIfNegative(upload.Size); + var stored = upload with { Received = new byte[(upload.ChunkCount + 7) / 8], ReceivedCount = 0, Sha256 = upload.Sha256.ToLowerInvariant() }; + await _db.ExecuteAsync( + $"INSERT INTO uploads ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)", + [ + stored.Id, stored.UserId, stored.SequenceId, stored.FileName, stored.Size, stored.Sha256, SequenceRepository.Wall(stored.CapturedAt), + stored.ChunkSize, stored.Received, 0, (int)stored.State, stored.FrameId, stored.Created, stored.UpdatedAt, + ], cancellationToken).ConfigureAwait(false); + return stored; + } + + public Task GetAsync(Guid id, CancellationToken cancellationToken = default) => + _db.SingleAsync($"SELECT {Columns} FROM uploads WHERE id = $1", [id], Map, cancellationToken); + + /// An upload of the same content by the same user that is still receiving, so a client can resume it. + public Task FindReceivingAsync(Guid userId, string sha256, CancellationToken cancellationToken = default) => + _db.SingleAsync( + $"SELECT {Columns} FROM uploads WHERE user_id = $1 AND sha256 = $2 AND state = {(int)UploadState.Receiving} ORDER BY created DESC LIMIT 1", + [userId, sha256.ToLowerInvariant()], Map, cancellationToken); + + /// + /// Marks chunk as received (idempotent; concurrent chunks of one upload serialise on + /// its row) and returns the updated upload, or null when it does not exist. + /// + public Task MarkChunkReceivedAsync(Guid id, int index, CancellationToken cancellationToken = default) => + _db.InTransactionAsync(async session => + { + var upload = await session.QuerySingleOrDefaultAsync($"SELECT {Columns} FROM uploads WHERE id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken) + .ConfigureAwait(false); + if (upload is null) return null; + ArgumentOutOfRangeException.ThrowIfNegative(index); + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(index, upload.ChunkCount); + if (upload.HasChunk(index)) return upload; + var bitmap = (byte[])upload.Received.Clone(); + bitmap[index / 8] |= (byte)(1 << (index % 8)); + var updated = upload with { Received = bitmap, ReceivedCount = upload.ReceivedCount + 1, UpdatedAt = DateTimeOffset.UtcNow }; + await session.ExecuteAsync("UPDATE uploads SET received = $2, received_count = $3, updated_at = $4 WHERE id = $1", + [id, bitmap, updated.ReceivedCount, updated.UpdatedAt], cancellationToken).ConfigureAwait(false); + return updated; + }, cancellationToken); + + /// Moves an upload to , recording the frame it became when committed. + public async Task SetStateAsync(Guid id, UploadState state, Guid? frameId = null, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync("UPDATE uploads SET state = $2, frame_id = COALESCE($3, frame_id), updated_at = $4 WHERE id = $1", + [id, (int)state, frameId, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0; + + public Task> ListByUserAsync(Guid userId, CancellationToken cancellationToken = default) => + _db.ListAsync($"SELECT {Columns} FROM uploads WHERE user_id = $1 ORDER BY created DESC", [userId], Map, cancellationToken); + + /// Uploads still receiving but untouched since (abandoned; their part files can go). + public Task> ListStaleAsync(DateTimeOffset updatedBefore, CancellationToken cancellationToken = default) => + _db.ListAsync($"SELECT {Columns} FROM uploads WHERE state = {(int)UploadState.Receiving} AND updated_at < $1 ORDER BY updated_at", + [updatedBefore], Map, cancellationToken); + + public async Task DeleteAsync(Guid id, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync("DELETE FROM uploads WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) > 0; + + private static Upload Map(DbRow r) => new() + { + Id = r.GetGuid(0), + UserId = r.GetGuid(1), + SequenceId = r.GetGuidOrNull(2), + FileName = r.GetString(3), + Size = r.GetInt64(4), + Sha256 = r.GetString(5), + CapturedAt = r.GetDateTimeOrNull(6), + ChunkSize = r.GetInt32(7), + Received = r.GetBytes(8), + ReceivedCount = r.GetInt32(9), + State = (UploadState)r.GetInt32(10), + FrameId = r.GetGuidOrNull(11), + Created = r.GetDateTimeOffset(12), + UpdatedAt = r.GetDateTimeOffset(13), + }; +} diff --git a/src/Titano.Data/Repositories/UserRepository.cs b/src/Titano.Data/Repositories/UserRepository.cs new file mode 100644 index 00000000..4fe75107 --- /dev/null +++ b/src/Titano.Data/Repositories/UserRepository.cs @@ -0,0 +1,75 @@ +using Titano.Core.Model; + +namespace Titano.Data.Repositories; + +/// Server accounts. User names are unique case-insensitively. +public sealed class UserRepository +{ + private const string Columns = + "id, username, display_name, password_hash, role, permissions, quota_bytes, used_bytes, disabled, must_change_password, created, last_login"; + + private readonly DbAccess _db; + + internal UserRepository(DbAccess db) => _db = db; + + /// Adds a user; a taken name raises a with . + public async Task CreateAsync(User user, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync( + "INSERT INTO users (id, username, username_key, display_name, password_hash, role, permissions, quota_bytes, used_bytes, " + + "disabled, must_change_password, created, last_login) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)", + [ + user.Id, user.Username, Key(user.Username), user.DisplayName, user.PasswordHash, (int)user.Role, (long)user.Permissions, + user.QuotaBytes, user.UsedBytes, user.Disabled, user.MustChangePassword, user.Created, user.LastLogin, + ], cancellationToken).ConfigureAwait(false); + + /// Saves every field except id and creation time. + public async Task UpdateAsync(User user, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync( + "UPDATE users SET username = $2, username_key = $3, display_name = $4, password_hash = $5, role = $6, permissions = $7, " + + "quota_bytes = $8, used_bytes = $9, disabled = $10, must_change_password = $11, last_login = $12 WHERE id = $1", + [ + user.Id, user.Username, Key(user.Username), user.DisplayName, user.PasswordHash, (int)user.Role, (long)user.Permissions, + user.QuotaBytes, user.UsedBytes, user.Disabled, user.MustChangePassword, user.LastLogin, + ], cancellationToken).ConfigureAwait(false) > 0; + + public Task GetAsync(Guid id, CancellationToken cancellationToken = default) => + _db.SingleAsync($"SELECT {Columns} FROM users WHERE id = $1", [id], Map, cancellationToken); + + public Task GetByUsernameAsync(string username, CancellationToken cancellationToken = default) => + _db.SingleAsync($"SELECT {Columns} FROM users WHERE username_key = $1", [Key(username)], Map, cancellationToken); + + public Task> ListAsync(CancellationToken cancellationToken = default) => + _db.ListAsync($"SELECT {Columns} FROM users ORDER BY username_key", [], Map, cancellationToken); + + public async Task CountAsync(CancellationToken cancellationToken = default) => + (int)await _db.ScalarAsync("SELECT count(*) FROM users", [], cancellationToken).ConfigureAwait(false); + + /// Deletes the user with their tokens, render nodes and uploads. + public async Task DeleteAsync(Guid id, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync("DELETE FROM users WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) > 0; + + public async Task RecordLoginAsync(Guid id, DateTimeOffset at, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync("UPDATE users SET last_login = $2 WHERE id = $1", [id, at], cancellationToken).ConfigureAwait(false) > 0; + + /// Adds (negative to release) to the user's quota usage atomically. + public async Task AddUsedBytesAsync(Guid id, long delta, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync("UPDATE users SET used_bytes = used_bytes + $2 WHERE id = $1", [id, delta], cancellationToken).ConfigureAwait(false) > 0; + + private static string Key(string username) => username.Trim().ToLowerInvariant(); + + private static User Map(DbRow r) => new() + { + Id = r.GetGuid(0), + Username = r.GetString(1), + DisplayName = r.GetStringOrNull(2), + PasswordHash = r.GetString(3), + Role = (UserRole)r.GetInt32(4), + Permissions = (Permissions)r.GetInt64(5), + QuotaBytes = r.GetInt64(6), + UsedBytes = r.GetInt64(7), + Disabled = r.GetBool(8), + MustChangePassword = r.GetBool(9), + Created = r.GetDateTimeOffset(10), + LastLogin = r.GetDateTimeOffsetOrNull(11), + }; +} diff --git a/src/Titano.Data/TitanoDatabase.cs b/src/Titano.Data/TitanoDatabase.cs new file mode 100644 index 00000000..7913e13a --- /dev/null +++ b/src/Titano.Data/TitanoDatabase.cs @@ -0,0 +1,155 @@ +using Titano.Data.Migrations; +using Titano.Data.Repositories; +using Titano.Data.Sqlite; + +namespace Titano.Data; + +/// The repositories of the Titano schema, bound either to the database (one session per call) or to one transaction. +public abstract class TitanoRepositories +{ + private protected TitanoRepositories(DbAccess access) + { + Archives = new ArchiveRepository(access); + Sequences = new SequenceRepository(access); + Frames = new FrameRepository(access); + Renders = new RenderRepository(access); + Jobs = new JobRepository(access); + MediaCards = new MediaCardRepository(access); + Users = new UserRepository(access); + Tokens = new TokenRepository(access); + RenderNodes = new RenderNodeRepository(access); + Settings = new SettingsRepository(access); + Tags = new TagRepository(access); + Audit = new AuditRepository(access); + Uploads = new UploadRepository(access); + } + + public ArchiveRepository Archives { get; } + public SequenceRepository Sequences { get; } + public FrameRepository Frames { get; } + public RenderRepository Renders { get; } + public JobRepository Jobs { get; } + public MediaCardRepository MediaCards { get; } + public UserRepository Users { get; } + public TokenRepository Tokens { get; } + public RenderNodeRepository RenderNodes { get; } + public SettingsRepository Settings { get; } + public TagRepository Tags { get; } + public AuditRepository Audit { get; } + public UploadRepository Uploads { get; } +} + +/// +/// Entry point of the data layer: opens SQLite or PostgreSQL, migrates the schema and exposes the repositories. +/// Each repository call borrows a pooled session; use to group calls atomically. +/// +/// +/// +/// await using var db = await TitanoDatabase.OpenAsync(DatabaseOptions.Parse(connectionString), ct); +/// var page = await db.Sequences.ListAsync(new SequenceQuery { Year = 2026 }, ct); +/// await using var tx = await db.BeginAsync(ct); +/// await tx.Frames.InsertAsync(frame, ct); +/// await tx.Sequences.RefreshStatsAsync(frame.SequenceId, ct); +/// await tx.CommitAsync(ct); +/// +/// +public sealed class TitanoDatabase : TitanoRepositories, IAsyncDisposable +{ + private TitanoDatabase(IDatabase database, int schemaVersion) + : base(new DbAccess(database)) + { + Database = database; + SchemaVersion = schemaVersion; + } + + /// The underlying engine, for raw sessions and engine-specific features (backups, LISTEN). + public IDatabase Database { get; } + + public Dialect Dialect => Database.Dialect; + + /// Schema version after migration. + public int SchemaVersion { get; private set; } + + /// + /// Opens the database and applies pending migrations. Throws when a newer + /// build already migrated it. + /// + public static async Task OpenAsync(DatabaseOptions options, CancellationToken cancellationToken = default) + { + var database = DatabaseFactory.Open(options); + try + { + var version = await MigrationRunner.MigrateAsync(database, cancellationToken).ConfigureAwait(false); + return new TitanoDatabase(database, version); + } + catch + { + await database.DisposeAsync().ConfigureAwait(false); + throw; + } + } + + /// Starts a transaction whose repositories all run on it; dispose without committing to roll back. + public async Task BeginAsync(CancellationToken cancellationToken = default) + { + var session = await Database.OpenAsync(cancellationToken).ConfigureAwait(false); + try + { + return new TitanoTransaction(session, await session.BeginAsync(cancellationToken).ConfigureAwait(false)); + } + catch + { + await session.DisposeAsync().ConfigureAwait(false); + throw; + } + } + + /// SQLite only: writes a VACUUM INTO copy into , keeping the newest . + public Task BackupAsync(string directory, int keep = 3, CancellationToken cancellationToken = default) => + AsSqlite().BackupAsync(directory, keep, cancellationToken); + + /// SQLite only: replaces the database with a backup, then migrates it to this build's schema. + public async Task RestoreAsync(string backupPath, CancellationToken cancellationToken = default) + { + await AsSqlite().RestoreAsync(backupPath, cancellationToken).ConfigureAwait(false); + SchemaVersion = await MigrationRunner.MigrateAsync(Database, cancellationToken).ConfigureAwait(false); + } + + public ValueTask DisposeAsync() => Database.DisposeAsync(); + + private SqliteDatabase AsSqlite() => + Database as SqliteDatabase ?? throw new NotSupportedException("Backups of a PostgreSQL database are made with the server's own tools."); +} + +/// Repositories bound to one open transaction. +public sealed class TitanoTransaction : TitanoRepositories, IAsyncDisposable +{ + private readonly IDbTransaction _transaction; + + internal TitanoTransaction(IDbSession session, IDbTransaction transaction) + : base(new DbAccess(session)) + { + Session = session; + _transaction = transaction; + } + + /// The session the transaction runs on, for statements of your own inside it. + public IDbSession Session { get; } + + public Task CommitAsync(CancellationToken cancellationToken = default) => _transaction.CommitAsync(cancellationToken); + + public Task RollbackAsync(CancellationToken cancellationToken = default) => _transaction.RollbackAsync(cancellationToken); + + /// Rolls back unless committed, and returns the session to the pool. + public async ValueTask DisposeAsync() + { + try + { + await _transaction.DisposeAsync().ConfigureAwait(false); + } + finally + { + await Session.DisposeAsync().ConfigureAwait(false); + } + } +}