From 2fbb7bc17979c5f0a7dad7d8f218e34f4603c002 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 10:05:43 +0000 Subject: [PATCH] test(data): cover migrations and every repository on both dialects Shared abstract suites run against in-memory/temp-file SQLite and, with TITANO_TEST_PG, a private PostgreSQL schema per test: migration idempotency and version refusal, keyset paging with ties for every sort, statistics, exclusive job claiming under 8 concurrent claimers, and the 100k-frame bulk insert under 15 s. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- .../Infrastructure/TestData.cs | 80 ++++ .../Migrations/MigrationTests.cs | 95 +++++ tests/Titano.Data.Tests/Program.cs | 8 +- .../Repositories/BulkInsertTests.cs | 42 ++ .../Repositories/CatalogRepositoryTests.cs | 387 ++++++++++++++++++ .../Repositories/JobRepositoryTests.cs | 168 ++++++++ .../Repositories/MediaCardRepositoryTests.cs | 100 +++++ .../Repositories/ServerRepositoryTests.cs | 246 +++++++++++ 8 files changed, 1125 insertions(+), 1 deletion(-) create mode 100644 tests/Titano.Data.Tests/Infrastructure/TestData.cs create mode 100644 tests/Titano.Data.Tests/Migrations/MigrationTests.cs create mode 100644 tests/Titano.Data.Tests/Repositories/BulkInsertTests.cs create mode 100644 tests/Titano.Data.Tests/Repositories/CatalogRepositoryTests.cs create mode 100644 tests/Titano.Data.Tests/Repositories/JobRepositoryTests.cs create mode 100644 tests/Titano.Data.Tests/Repositories/MediaCardRepositoryTests.cs create mode 100644 tests/Titano.Data.Tests/Repositories/ServerRepositoryTests.cs diff --git a/tests/Titano.Data.Tests/Infrastructure/TestData.cs b/tests/Titano.Data.Tests/Infrastructure/TestData.cs new file mode 100644 index 00000000..b0bc9814 --- /dev/null +++ b/tests/Titano.Data.Tests/Infrastructure/TestData.cs @@ -0,0 +1,80 @@ +using Titano.Core.Imaging; +using Titano.Core.Model; +using Titano.Testing; + +namespace Titano.Data.Tests.Infrastructure; + +/// Base of repository tests: a migrated per test. +public abstract class RepositoryTest(TestContext context) : DatabaseTest(context) +{ + /// Opens a fresh migrated database; forces a SQLite file. + protected async Task OpenTitanoAsync(bool multiConnection = false) => + Own(await TitanoDatabase.OpenAsync(await NewOptionsAsync(multiConnection), Ct)); +} + +/// Builders of valid entities with distinct ids; override fields with with. +internal static class TestData +{ + /// A fixed reference instant (microsecond precision, like the databases). + public static readonly DateTimeOffset Now = new(2026, 10, 2, 9, 0, 0, TimeSpan.Zero); + + public static Archive Archive(string name = "Main") => new() + { + Id = Guid.CreateVersion7(), + Name = name, + RootPath = "/archive/" + name, + Created = Now, + }; + + public static Sequence Sequence(Guid archiveId, DateTime? firstCapture = null, string? title = null, string camera = "EOS R5") => new() + { + Id = Guid.CreateVersion7(), + ArchiveId = archiveId, + Title = title, + Folder = $"2026/{Guid.NewGuid():N}", + CameraMake = "Canon", + CameraModel = camera, + FirstCapture = firstCapture, + LastCapture = firstCapture?.AddHours(1), + UpdatedAt = Now, + }; + + public static Frame Frame(Guid sequenceId, int index, DateTime? capturedAt = null) => new() + { + Id = Guid.CreateVersion7(), + SequenceId = sequenceId, + Index = index, + RelativePath = $"2026/seq/originals/IMG_{index:0000}.CR3", + FileName = $"IMG_{index:0000}.CR3", + Size = 25_000_000 + index, + Sha256 = Convert.ToHexStringLower(BitConverter.GetBytes((long)index)).PadLeft(64, '0'), + CapturedAt = capturedAt ?? new DateTime(2026, 9, 28, 18, 0, 0).AddSeconds(5 * index), + TimestampSource = TimestampSource.ExifOriginal, + ExposureTime = 1 / 250.0, + FNumber = 8, + Iso = 100, + FocalLength = 24, + Width = 8192, + Height = 5464, + Format = ImageFileFormat.Cr3, + }; + + public static Job Job(JobKind kind = JobKind.Scan, JobPriority priority = JobPriority.Normal, DateTimeOffset? created = null) => new() + { + Id = Guid.CreateVersion7(), + Kind = kind, + Priority = priority, + State = JobState.Queued, + Created = created ?? Now, + }; + + public static User User(string username = "alice") => new() + { + Id = Guid.CreateVersion7(), + Username = username, + PasswordHash = "pbkdf2-sha512$210000$salt$hash", + Role = UserRole.Editor, + Permissions = RolePermissions.Default(UserRole.Editor), + Created = Now, + }; +} diff --git a/tests/Titano.Data.Tests/Migrations/MigrationTests.cs b/tests/Titano.Data.Tests/Migrations/MigrationTests.cs new file mode 100644 index 00000000..755f9c20 --- /dev/null +++ b/tests/Titano.Data.Tests/Migrations/MigrationTests.cs @@ -0,0 +1,95 @@ +using Titano.Data.Migrations; +using Titano.Data.Tests.Infrastructure; +using Titano.Testing; + +namespace Titano.Data.Tests.Migrations; + +public abstract class MigrationTests(TestContext context) : RepositoryTest(context) +{ + private static readonly string[] Tables = + [ + "archives", "sequences", "frames", "renders", "jobs", "media_cards", "import_history", "users", "tokens", + "render_nodes", "settings", "tags", "sequence_tags", "audit_log", "uploads", + ]; + + [Test] + public void EmbeddedScriptsAreFoundForBothDialects() + { + foreach (var dialect in new[] { Dialect.Sqlite, Dialect.Postgres }) + { + var migrations = MigrationRunner.Load(dialect); + Assert.NotEmpty(migrations); + Assert.Equal(1, migrations[0].Version); + Assert.Equal("initial", migrations[0].Name); + Assert.Contains("CREATE TABLE frames", migrations[0].Sql); + } + } + + [Test] + public async Task CreatesEveryTableAndIsIdempotent() + { + var titano = await OpenTitanoAsync(); + var latest = MigrationRunner.Load(Dialect)[^1].Version; + Assert.Equal(latest, titano.SchemaVersion); + Assert.Equal(latest, await MigrationRunner.MigrateAsync(titano.Database, Ct)); + + await using var session = await titano.Database.OpenAsync(Ct); + foreach (var table in Tables) Assert.Equal(0L, await session.ScalarAsync($"SELECT count(*) FROM {table}", [], Ct)); + var recorded = await session.QueryListAsync("SELECT version, name, applied_at FROM schema_migrations ORDER BY version", [], r => r, Ct); + Assert.Equal(MigrationRunner.Load(Dialect).Count, recorded.Count); + Assert.Equal("initial", recorded[0].GetString(1)); + Assert.True(recorded[0].GetDateTimeOffset(2) > DateTimeOffset.UtcNow.AddMinutes(-5)); + } + + [Test] + public async Task AppliesOnlyPendingMigrations() + { + var titano = await OpenTitanoAsync(); + var next = MigrationRunner.Load(Dialect).Append(new Migration(9000, "extra", "CREATE TABLE extra (id integer PRIMARY KEY)")).ToList(); + Assert.Equal(9000, await MigrationRunner.MigrateAsync(titano.Database, next, Ct)); + Assert.Equal(9000, await MigrationRunner.MigrateAsync(titano.Database, next, Ct)); + await using var session = await titano.Database.OpenAsync(Ct); + Assert.Equal(1L, await session.ScalarAsync("SELECT count(*) FROM schema_migrations WHERE version = $1", [9000], Ct)); + Assert.Equal(0L, await session.ScalarAsync("SELECT count(*) FROM extra", [], Ct)); + } + + [Test] + public async Task RefusesADatabaseFromANewerBuild() + { + var titano = await OpenTitanoAsync(); + await using (var session = await titano.Database.OpenAsync(Ct)) + await session.ExecuteAsync("INSERT INTO schema_migrations (version, name, applied_at) VALUES ($1, $2, $3)", [9999, "future", DateTimeOffset.UtcNow], Ct); + var error = await Assert.ThrowsAsync(() => MigrationRunner.MigrateAsync(titano.Database, Ct)); + Assert.Equal(9999, error.DatabaseVersion); + Assert.Equal(MigrationRunner.Load(Dialect)[^1].Version, error.SupportedVersion); + } + + [Test] + public async Task AFailingMigrationLeavesNothingBehind() + { + var database = await OpenDatabaseAsync(); + List broken = + [ + new(1, "good", "CREATE TABLE good_one (id integer PRIMARY KEY)"), + new(2, "bad", "CREATE TABLE good_two (id integer PRIMARY KEY); THIS IS NOT SQL"), + ]; + await Assert.ThrowsAsync(() => MigrationRunner.MigrateAsync(database, broken, Ct)); + await using var session = await database.OpenAsync(Ct); + var leftovers = await session.ScalarAsync(ForDialect( + "SELECT count(*) FROM sqlite_master WHERE name IN ('good_one', 'good_two', 'schema_migrations')", + "SELECT count(*) FROM information_schema.tables WHERE table_schema = current_schema() AND table_name IN ('good_one', 'good_two', 'schema_migrations')"), + [], Ct); + Assert.Equal(0L, leftovers); + } +} + +public sealed class SqliteMigrationTests(TestContext context) : MigrationTests(context) +{ + protected override Backend Backend => Backend.SqliteMemory; +} + +[RequiresEnvironment(PgTestServer.Variable)] +public sealed class PostgresMigrationTests(TestContext context) : MigrationTests(context) +{ + protected override Backend Backend => Backend.Postgres; +} diff --git a/tests/Titano.Data.Tests/Program.cs b/tests/Titano.Data.Tests/Program.cs index 92339124..ffd81b63 100644 --- a/tests/Titano.Data.Tests/Program.cs +++ b/tests/Titano.Data.Tests/Program.cs @@ -1,8 +1,14 @@ +using Titano.Core.Diagnostics; using Titano.Testing; namespace Titano.Data.Tests; internal static class TestMain { - public static Task Main(string[] args) => TestRunner.RunAsync(args, typeof(TestMain).Assembly); + public static Task Main(string[] args) + { + // Migrations and schema drops log at Info for every test database; keep the console for warnings. + Log.MinimumLevel = LogLevel.Warning; + return TestRunner.RunAsync(args, typeof(TestMain).Assembly); + } } diff --git a/tests/Titano.Data.Tests/Repositories/BulkInsertTests.cs b/tests/Titano.Data.Tests/Repositories/BulkInsertTests.cs new file mode 100644 index 00000000..879f4972 --- /dev/null +++ b/tests/Titano.Data.Tests/Repositories/BulkInsertTests.cs @@ -0,0 +1,42 @@ +using System.Diagnostics; +using Titano.Data.Tests.Infrastructure; +using Titano.Testing; + +namespace Titano.Data.Tests.Repositories; + +/// The 100k-frame import of the Phase 2 gate must stay well under 15 s. +public abstract class BulkInsertTests(TestContext context) : RepositoryTest(context) +{ + [Test] + [Timeout(60_000)] + public async Task HundredThousandFramesInsertQuickly() + { + var db = await OpenTitanoAsync(multiConnection: true); + var archive = TestData.Archive(); + await db.Archives.UpsertAsync(archive, Ct); + var sequence = TestData.Sequence(archive.Id); + await db.Sequences.UpsertAsync(sequence, Ct); + var frames = Enumerable.Range(0, 100_000).Select(i => TestData.Frame(sequence.Id, i)).ToList(); + + var watch = Stopwatch.StartNew(); + Assert.Equal(100_000L, await db.Frames.BulkInsertAsync(frames, Ct)); + watch.Stop(); + Context.Log($"{Dialect}: 100k frames in {watch.Elapsed.TotalSeconds:F2} s"); + Assert.Less(watch.Elapsed.TotalSeconds, 15.0, "bulk insert time"); + + var refreshed = await db.Sequences.RefreshStatsAsync(sequence.Id, Ct); + Assert.Equal(100_000, refreshed!.FrameCount); + Assert.Equal(frames[^1], (await db.Frames.ListBySequenceAsync(sequence.Id, afterIndex: 99_998, cancellationToken: Ct)).Single()); + } +} + +public sealed class SqliteBulkInsertTests(TestContext context) : BulkInsertTests(context) +{ + protected override Backend Backend => Backend.SqliteFile; +} + +[RequiresEnvironment(PgTestServer.Variable)] +public sealed class PostgresBulkInsertTests(TestContext context) : BulkInsertTests(context) +{ + protected override Backend Backend => Backend.Postgres; +} diff --git a/tests/Titano.Data.Tests/Repositories/CatalogRepositoryTests.cs b/tests/Titano.Data.Tests/Repositories/CatalogRepositoryTests.cs new file mode 100644 index 00000000..340fe506 --- /dev/null +++ b/tests/Titano.Data.Tests/Repositories/CatalogRepositoryTests.cs @@ -0,0 +1,387 @@ +using System.Text.Json.Nodes; +using Titano.Core.Imaging; +using Titano.Core.Model; +using Titano.Data.Repositories; +using Titano.Data.Tests.Infrastructure; +using Titano.Testing; + +namespace Titano.Data.Tests.Repositories; + +/// Archives, sequences, frames, renders and tags on both dialects. +public abstract class CatalogRepositoryTests(TestContext context) : RepositoryTest(context) +{ + [Test] + public async Task ArchivesRoundTripAndDelete() + { + var db = await OpenTitanoAsync(); + var archive = TestData.Archive() with { ProxyLongEdge = -1, FolderTemplate = "{year}/{title}" }; + await db.Archives.UpsertAsync(archive, Ct); + await db.Archives.UpsertAsync(archive with { Name = "Renamed" }, Ct); + Assert.Equal(archive with { Name = "Renamed" }, await db.Archives.GetAsync(archive.Id, Ct)); + Assert.Count(1, await db.Archives.ListAsync(Ct)); + Assert.True(await db.Archives.DeleteAsync(archive.Id, Ct)); + Assert.Null(await db.Archives.GetAsync(archive.Id, Ct)); + Assert.False(await db.Archives.DeleteAsync(archive.Id, Ct)); + } + + [Test] + public async Task SequencesRoundTripEveryField() + { + var db = await OpenTitanoAsync(); + var archive = TestData.Archive(); + await db.Archives.UpsertAsync(archive, Ct); + var sequence = TestData.Sequence(archive.Id, new DateTime(2026, 9, 28, 18, 30, 15, 250), "Harbour sunset") with + { + CameraSerial = "0123", Lens = "RF 24-70", FrameCount = 1200, MedianInterval = 5.0, TotalBytes = 30_000_000_000, + Status = SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt, RenderedCount = 0, VerifiedAt = TestData.Now, + HeroFrameId = Guid.CreateVersion7(), Rating = 4, Notes = "windy", SettingsJson = "{\"fps\": 25, \"deflicker\": true}", + AnalysisJson = "{\"ev\": [12.5, 13]}", Origin = SequenceOrigin.Server, SyncState = SyncState.LocalAhead, + SettingsVersion = 3, FramesVersion = 7, OwnerUserId = Guid.CreateVersion7(), WarningCount = 2, Tags = ["Sunset", "harbour", "SUNSET"], + }; + await db.Sequences.UpsertAsync(sequence, Ct); + + var read = Assert.NotNull(await db.Sequences.GetAsync(sequence.Id, Ct)); + Assert.True(JsonNode.DeepEquals(JsonNode.Parse(sequence.SettingsJson!), JsonNode.Parse(read.SettingsJson!))); + Assert.True(JsonNode.DeepEquals(JsonNode.Parse(sequence.AnalysisJson!), JsonNode.Parse(read.AnalysisJson!))); + Assert.SequenceEqual(["harbour", "Sunset"], read.Tags); + Assert.Equal(sequence with { Tags = read.Tags, SettingsJson = read.SettingsJson, AnalysisJson = read.AnalysisJson }, read); + + await db.Sequences.UpsertAsync(sequence with { Title = null, Tags = ["night"], Rating = 1 }, Ct); + read = Assert.NotNull(await db.Sequences.GetAsync(sequence.Id, Ct)); + Assert.Null(read.Title); + Assert.Equal("2026-09-28 EOS R5", read.DisplayTitle); + Assert.SequenceEqual(["night"], read.Tags); + Assert.Equal(1, read.Rating); + } + + [Test] + public async Task DeletingASequenceRemovesItsFramesRendersAndTagLinks() + { + var db = await OpenTitanoAsync(); + var archive = TestData.Archive(); + await db.Archives.UpsertAsync(archive, Ct); + var sequence = TestData.Sequence(archive.Id) with { Tags = ["a"] }; + await db.Sequences.UpsertAsync(sequence, Ct); + await db.Frames.BulkInsertAsync(Enumerable.Range(0, 10).Select(i => TestData.Frame(sequence.Id, i)), Ct); + await db.Renders.InsertAsync(Render(sequence.Id), Ct); + + Assert.True(await db.Sequences.DeleteAsync(sequence.Id, Ct)); + Assert.Empty(await db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: Ct)); + Assert.Empty(await db.Renders.ListBySequenceAsync(sequence.Id, Ct)); + Assert.Equal(0, (await db.Tags.ListAsync(Ct)).Single().SequenceCount); + Assert.Equal(1, await db.Tags.DeleteUnusedAsync(Ct)); + } + + [Test] + public async Task ListingFiltersCombine() + { + var db = await OpenTitanoAsync(); + var main = TestData.Archive(); + var other = TestData.Archive("Other"); + await db.Archives.UpsertAsync(main, Ct); + await db.Archives.UpsertAsync(other, Ct); + var owner = Guid.CreateVersion7(); + var harbour = TestData.Sequence(main.Id, new DateTime(2024, 6, 1), "Porto di CITTÀ vecchia") with { Status = SequenceStatus.AnalyzedLight, Tags = ["sea"] }; + var stars = TestData.Sequence(main.Id, new DateTime(2025, 1, 2), "Milky way", "Z8") with { CameraMake = "Nikon", Tags = ["Night Sky"], OwnerUserId = owner }; + var clouds = TestData.Sequence(main.Id, new DateTime(2024, 12, 31, 23, 59, 59), "50%_clouds") with { Status = SequenceStatus.AnalyzedLight | SequenceStatus.Verified }; + var elsewhere = TestData.Sequence(other.Id, new DateTime(2024, 3, 3), "Elsewhere"); + var undated = TestData.Sequence(main.Id, null, "Undated"); + foreach (var s in new[] { harbour, stars, clouds, elsewhere, undated }) await db.Sequences.UpsertAsync(s, Ct); + + async Task Titles(SequenceQuery q) => + [.. (await db.Sequences.ListAsync(q with { Sort = SequenceSort.Title }, Ct)).Items.Select(s => s.Title!)]; + + Assert.SequenceEqual(["50%_clouds", "Porto di CITTÀ vecchia"], await Titles(new() { ArchiveId = main.Id, Year = 2024 })); + Assert.SequenceEqual(["Milky way"], await Titles(new() { Camera = "Z8" })); + Assert.SequenceEqual(["50%_clouds", "Porto di CITTÀ vecchia"], await Titles(new() { WithStatus = SequenceStatus.AnalyzedLight })); + Assert.SequenceEqual(["Porto di CITTÀ vecchia"], await Titles(new() { WithStatus = SequenceStatus.AnalyzedLight, WithoutStatus = SequenceStatus.Verified })); + Assert.SequenceEqual(["Milky way"], await Titles(new() { Tag = " night SKY " })); + Assert.SequenceEqual(["Porto di CITTÀ vecchia"], await Titles(new() { Text = "città" })); + Assert.SequenceEqual(["Milky way"], await Titles(new() { Text = "NIKON" })); + Assert.SequenceEqual(["Milky way"], await Titles(new() { Text = "ght sk" })); + Assert.SequenceEqual(["50%_clouds"], await Titles(new() { Text = "50%_" })); + Assert.Empty(await Titles(new() { Text = "5_%" })); + Assert.SequenceEqual(["Milky way"], await Titles(new() { OwnerUserId = owner })); + Assert.Equal(4, await db.Sequences.CountAsync(new() { ArchiveId = main.Id }, Ct)); + Assert.Equal(2, await db.Sequences.CountAsync(new() { ArchiveId = main.Id, Year = 2024 }, Ct)); + Assert.Equal(5, await db.Sequences.CountAsync(new(), Ct)); + } + + [Theory] + [Case(SequenceSort.Newest)] + [Case(SequenceSort.Oldest)] + [Case(SequenceSort.Title)] + [Case(SequenceSort.FrameCount)] + [Case(SequenceSort.Size)] + public async Task KeysetPagesVisitEveryRowOnceInOrderDespiteTies(SequenceSort sort) + { + var db = await OpenTitanoAsync(); + var archive = TestData.Archive(); + await db.Archives.UpsertAsync(archive, Ct); + var all = Enumerable.Range(0, 23).Select(i => TestData.Sequence(archive.Id, + i % 5 == 4 ? null : new DateTime(2024 + i % 3, 1, 1, 12, 0, 0), + i % 4 == 0 ? null : "T" + (i % 6)) with { FrameCount = i % 3, TotalBytes = i % 4 * 1000L }) + .ToList(); + foreach (var s in all) await db.Sequences.UpsertAsync(s, Ct); + + var expected = Expected(all, sort).Select(s => s.Id).ToList(); + var seen = new List(); + string? cursor = null; + var pages = 0; + do + { + var page = await db.Sequences.ListAsync(new SequenceQuery { ArchiveId = archive.Id, Sort = sort, Limit = 4, Cursor = cursor }, Ct); + Assert.True(page.Items.Count is > 0 and <= 4); + seen.AddRange(page.Items.Select(s => s.Id)); + cursor = page.NextCursor; + pages++; + } + while (cursor is not null); + + Assert.Equal(6, pages); + Assert.SequenceEqual(expected, seen); + } + + [Test] + public async Task CursorsOnlyWorkForTheirOwnSort() + { + var db = await OpenTitanoAsync(); + var archive = TestData.Archive(); + await db.Archives.UpsertAsync(archive, Ct); + for (var i = 0; i < 3; i++) await db.Sequences.UpsertAsync(TestData.Sequence(archive.Id, new DateTime(2024, 1, 1 + i)), Ct); + var page = await db.Sequences.ListAsync(new SequenceQuery { Limit = 1 }, Ct); + Assert.NotNull(page.NextCursor); + await Assert.ThrowsAsync(() => db.Sequences.ListAsync(new SequenceQuery { Sort = SequenceSort.Size, Cursor = page.NextCursor }, Ct)); + await Assert.ThrowsAsync(() => db.Sequences.ListAsync(new SequenceQuery { Cursor = "not-a-cursor" }, Ct)); + } + + [Test] + public async Task StatisticsAggregateTheArchive() + { + var db = await OpenTitanoAsync(); + var main = TestData.Archive(); + var other = TestData.Archive("Other"); + await db.Archives.UpsertAsync(main, Ct); + await db.Archives.UpsertAsync(other, Ct); + Sequence Seq(Guid archive, DateTime? first, string model, int frames, long bytes, SequenceStatus status) => + TestData.Sequence(archive, first, camera: model) with { FrameCount = frames, TotalBytes = bytes, Status = status }; + var a = Seq(main.Id, new DateTime(2024, 3, 10), "EOS R5", 10, 1000, SequenceStatus.AnalyzedLight) with { VerifiedAt = TestData.Now }; + var b = Seq(main.Id, new DateTime(2024, 3, 20), "Z8", 20, 2000, SequenceStatus.HasWarnings) with { CameraMake = "Nikon" }; + var c = Seq(main.Id, new DateTime(2025, 7, 1), "EOS R5", 30, 3000, SequenceStatus.HasErrors | SequenceStatus.AnalyzedLight); + var d = Seq(main.Id, null, "EOS R5", 0, 0, SequenceStatus.None) with { WarningCount = 1 }; + var e = Seq(other.Id, new DateTime(2023, 1, 1), "X-T5", 5, 500, SequenceStatus.None); + foreach (var s in new[] { a, b, c, d, e }) await db.Sequences.UpsertAsync(s, Ct); + await db.Frames.BulkInsertAsync( + [ + TestData.Frame(a.Id, 0) with { Flags = FrameFlags.Missing, Size = 100 }, + TestData.Frame(a.Id, 1) with { Format = ImageFileFormat.Jpeg, Size = 10, Flags = FrameFlags.Duplicate | FrameFlags.Corrupt }, + TestData.Frame(b.Id, 0) with { Size = 200 }, + TestData.Frame(e.Id, 0) with { Format = ImageFileFormat.Raf, Flags = FrameFlags.Missing }, + ], Ct); + + var stats = await db.Sequences.GetStatsAsync(main.Id, Ct); + Assert.Equal(4, stats.Sequences); + Assert.Equal(60L, stats.Photos); + Assert.Equal(6000L, stats.Bytes); + Assert.SequenceEqual([new YearBucket(2024, 2, 30, 3000), new YearBucket(2025, 1, 30, 3000)], stats.ByYear); + Assert.SequenceEqual([new MonthBucket(2024, 3, 2, 30), new MonthBucket(2025, 7, 1, 30)], stats.ByMonth); + Assert.SequenceEqual( + [new CameraBucket("Canon", "EOS R5", 3, 40, 4000), new CameraBucket("Nikon", "Z8", 1, 20, 2000)], + stats.ByCamera); + Assert.SequenceEqual([new FormatBucket(ImageFileFormat.Cr3, 2, 300), new FormatBucket(ImageFileFormat.Jpeg, 1, 10)], stats.ByFormat); + Assert.Equal(new ArchiveHealth(Unverified: 3, WithWarnings: 2, WithErrors: 1, NotAnalyzed: 2, MissingFrames: 1, CorruptFrames: 1, DuplicateFrames: 1), stats.Health); + + var everything = await db.Sequences.GetStatsAsync(cancellationToken: Ct); + Assert.Equal(5, everything.Sequences); + Assert.Equal(2L, everything.Health.MissingFrames); + Assert.Equal(2023, everything.ByYear[0].Year); + } + + [Test] + public async Task FramesBulkInsertPageAndFindDuplicates() + { + var db = await OpenTitanoAsync(); + var archive = TestData.Archive(); + await db.Archives.UpsertAsync(archive, Ct); + var sequence = TestData.Sequence(archive.Id); + await db.Sequences.UpsertAsync(sequence, Ct); + var frames = Enumerable.Range(0, 250).Select(i => TestData.Frame(sequence.Id, i)).ToList(); + frames[7] = frames[7] with + { + OriginalFileName = "DSC_7.NEF", CapturedAt = new DateTime(2026, 9, 28, 18, 0, 35, 123), UtcOffsetMinutes = 120, ExposureBias = -0.3, + WhiteBalance = 5600, Orientation = 6, PreviewOffset = 1024, PreviewLength = 65536, MeanLogLuminance = -2.5, ClipLow = 0.01, + ClipHigh = 0.02, Flags = FrameFlags.Excluded, Sha256 = "AB" + new string('c', 62), + }; + Assert.Equal(250L, await db.Frames.BulkInsertAsync(frames, Ct)); + var copy = TestData.Frame(sequence.Id, 1000) with { Sha256 = frames[7].Sha256 }; + await db.Frames.InsertAsync(copy, Ct); + + Assert.Equal(frames[7] with { Sha256 = frames[7].Sha256!.ToLowerInvariant() }, await db.Frames.GetAsync(frames[7].Id, Ct)); + var pageOne = await db.Frames.ListBySequenceAsync(sequence.Id, limit: 100, cancellationToken: Ct); + var pageThree = await db.Frames.ListBySequenceAsync(sequence.Id, afterIndex: 199, limit: 100, cancellationToken: Ct); + Assert.SequenceEqual(Enumerable.Range(0, 100), pageOne.Select(f => f.Index)); + Assert.SequenceEqual(Enumerable.Range(200, 50).Append(1000), pageThree.Select(f => f.Index)); + var duplicates = await db.Frames.FindBySha256Async(frames[7].Sha256!.ToUpperInvariant(), Ct); + Assert.SequenceEqual([7, 1000], duplicates.Select(f => f.Index)); + + var stats = Assert.NotNull(await db.Sequences.RefreshStatsAsync(sequence.Id, Ct)); + Assert.Equal(251, stats.FrameCount); + Assert.Equal(frames.Sum(f => f.Size) + copy.Size, stats.TotalBytes); + Assert.Equal(frames[0].CapturedAt, stats.FirstCapture); + Assert.Equal(copy.CapturedAt, stats.LastCapture); + Assert.Equal(1, stats.FramesVersion); + } + + [Test] + public async Task FrameAnalysisAndFlagsUpdateInBulk() + { + var db = await OpenTitanoAsync(); + var archive = TestData.Archive(); + await db.Archives.UpsertAsync(archive, Ct); + var sequence = TestData.Sequence(archive.Id); + await db.Sequences.UpsertAsync(sequence, Ct); + var frames = Enumerable.Range(0, 1200).Select(i => TestData.Frame(sequence.Id, i) with { Flags = FrameFlags.Missing }).ToList(); + await db.Frames.BulkInsertAsync(frames, Ct); + + var stats = frames.Select((f, i) => new FrameLightStats(f.Id, -i / 100.0, i % 2 == 0 ? null : 0.5, 0.001 * i)).ToList(); + Assert.Equal(1200, await db.Frames.UpdateLightStatsAsync(stats, Ct)); + var ids = frames.Where(f => f.Index % 3 == 0).Select(f => f.Id).ToList(); + Assert.Equal(400, await db.Frames.UpdateFlagsAsync(ids, FrameFlags.Duplicate | FrameFlags.Excluded, FrameFlags.Missing, Ct)); + + var read = new List(); + for (var after = -1; ; after = read[^1].Index) + { + var page = await db.Frames.ListBySequenceAsync(sequence.Id, after, 500, Ct); + if (page.Count == 0) break; + read.AddRange(page); + } + Assert.Equal(1200, read.Count); + foreach (var f in read) + { + Assert.Equal(-f.Index / 100.0, f.MeanLogLuminance); + Assert.Equal(f.Index % 2 == 0 ? null : 0.5, f.ClipLow); + Assert.Equal(f.Index % 3 == 0 ? FrameFlags.Duplicate | FrameFlags.Excluded : FrameFlags.Missing, f.Flags); + } + Assert.Equal(1200, await db.Frames.DeleteBySequenceAsync(sequence.Id, Ct)); + } + + [Test] + public async Task RendersKeepTheSequenceCountInStep() + { + var db = await OpenTitanoAsync(); + var archive = TestData.Archive(); + await db.Archives.UpsertAsync(archive, Ct); + var sequence = TestData.Sequence(archive.Id); + await db.Sequences.UpsertAsync(sequence, Ct); + var first = Render(sequence.Id) with { Created = TestData.Now }; + var second = Render(sequence.Id) with { Created = TestData.Now.AddMinutes(5), SettingsJson = null, Codec = null }; + await db.Renders.InsertAsync(first, Ct); + await db.Renders.InsertAsync(second, Ct); + Assert.Equal(2, (await db.Sequences.GetAsync(sequence.Id, Ct))!.RenderedCount); + var list = await db.Renders.ListBySequenceAsync(sequence.Id, Ct); + Assert.SequenceEqual([second.Id, first.Id], list.Select(r => r.Id)); + Assert.Equal(second, list[0]); + Assert.True(await db.Renders.DeleteAsync(second.Id, Ct)); + Assert.False(await db.Renders.DeleteAsync(second.Id, Ct)); + Assert.Equal(1, (await db.Sequences.GetAsync(sequence.Id, Ct))!.RenderedCount); + } + + [Test] + public async Task TagsCanBeListedRenamedAndDeleted() + { + var db = await OpenTitanoAsync(); + var archive = TestData.Archive(); + await db.Archives.UpsertAsync(archive, Ct); + var one = TestData.Sequence(archive.Id) with { Tags = ["Alps", "snow"] }; + var two = TestData.Sequence(archive.Id) with { Tags = ["alps"] }; + await db.Sequences.UpsertAsync(one, Ct); + await db.Sequences.UpsertAsync(two, Ct); + Assert.False(await db.Sequences.SetTagsAsync(Guid.CreateVersion7(), ["x"], Ct)); + + var tags = await db.Tags.ListAsync(Ct); + Assert.SequenceEqual([("Alps", 2), ("snow", 1)], tags.Select(t => (t.Name, t.SequenceCount))); + var snow = tags.Single(t => t.Name == "snow"); + await Assert.ThrowsAsync(() => db.Tags.RenameAsync(snow.Id, "ALPS", Ct)); + Assert.True(await db.Tags.RenameAsync(snow.Id, "Snow fields", Ct)); + Assert.SequenceEqual(["Alps", "Snow fields"], (await db.Sequences.GetAsync(one.Id, Ct))!.Tags); + Assert.True(await db.Tags.DeleteAsync(snow.Id, Ct)); + Assert.SequenceEqual(["Alps"], (await db.Sequences.GetAsync(one.Id, Ct))!.Tags); + Assert.True(await db.Sequences.SetTagsAsync(two.Id, [], Ct)); + Assert.Equal(1, (await db.Tags.ListAsync(Ct)).Single().SequenceCount); + Assert.Throws(() => TagRepository.Normalize(" \t ")); + Assert.Throws(() => TagRepository.Normalize("bad\u0001tag")); + } + + [Test] + public async Task TransactionsGroupRepositoryCalls() + { + var db = await OpenTitanoAsync(); + var archive = TestData.Archive(); + await db.Archives.UpsertAsync(archive, Ct); + var sequence = TestData.Sequence(archive.Id); + + await using (var tx = await db.BeginAsync(Ct)) + { + await tx.Sequences.UpsertAsync(sequence, Ct); + await tx.Frames.InsertAsync(TestData.Frame(sequence.Id, 0), Ct); + Assert.NotNull(await tx.Sequences.GetAsync(sequence.Id, Ct)); + } + Assert.Null(await db.Sequences.GetAsync(sequence.Id, Ct)); + + await using (var tx = await db.BeginAsync(Ct)) + { + await tx.Sequences.UpsertAsync(sequence, Ct); + await tx.Frames.InsertAsync(TestData.Frame(sequence.Id, 0), Ct); + await tx.Sequences.RefreshStatsAsync(sequence.Id, Ct); + await tx.CommitAsync(Ct); + } + Assert.Equal(1, (await db.Sequences.GetAsync(sequence.Id, Ct))!.FrameCount); + } + + private static Render Render(Guid sequenceId) => new() + { + Id = Guid.CreateVersion7(), + SequenceId = sequenceId, + PresetName = "4K HEVC", + SettingsJson = "{\"crf\": 18}", + OutputPath = "exports/out.mp4", + Codec = "hevc", + Width = 3840, + Height = 2160, + Fps = 29.97, + DurationSeconds = 41.5, + Bytes = 123_456_789, + Created = TestData.Now, + ProducedBy = "node-1", + }; + + /// The order the database must produce: sort key with NULLS LAST, then id (as text) in the same direction. + private static IEnumerable Expected(List all, SequenceSort sort) + { + int ById(Sequence a, Sequence b) => string.CompareOrdinal(a.Id.ToString("D"), b.Id.ToString("D")); + int NullsLast(T? a, T? b, Comparison compare) where T : struct => + a is null ? (b is null ? 0 : 1) : b is null ? -1 : compare(a.Value, b.Value); + Comparison comparison = sort switch + { + SequenceSort.Newest => (a, b) => NullsLast(a.FirstCapture, b.FirstCapture, (x, y) => y.CompareTo(x)) is var c and not 0 ? c : -ById(a, b), + SequenceSort.Oldest => (a, b) => NullsLast(a.FirstCapture, b.FirstCapture, (x, y) => x.CompareTo(y)) is var c and not 0 ? c : ById(a, b), + SequenceSort.Title => (a, b) => string.CompareOrdinal(a.DisplayTitle.ToLowerInvariant(), b.DisplayTitle.ToLowerInvariant()) is var c and not 0 ? c : ById(a, b), + SequenceSort.FrameCount => (a, b) => b.FrameCount.CompareTo(a.FrameCount) is var c and not 0 ? c : -ById(a, b), + _ => (a, b) => b.TotalBytes.CompareTo(a.TotalBytes) is var c and not 0 ? c : -ById(a, b), + }; + var sorted = all.ToList(); + sorted.Sort(comparison); + return sorted; + } +} + +public sealed class SqliteCatalogRepositoryTests(TestContext context) : CatalogRepositoryTests(context) +{ + protected override Backend Backend => Backend.SqliteMemory; +} + +[RequiresEnvironment(PgTestServer.Variable)] +public sealed class PostgresCatalogRepositoryTests(TestContext context) : CatalogRepositoryTests(context) +{ + protected override Backend Backend => Backend.Postgres; +} diff --git a/tests/Titano.Data.Tests/Repositories/JobRepositoryTests.cs b/tests/Titano.Data.Tests/Repositories/JobRepositoryTests.cs new file mode 100644 index 00000000..b4c661ec --- /dev/null +++ b/tests/Titano.Data.Tests/Repositories/JobRepositoryTests.cs @@ -0,0 +1,168 @@ +using System.Collections.Concurrent; +using Titano.Core.Model; +using Titano.Data.Tests.Infrastructure; +using Titano.Testing; + +namespace Titano.Data.Tests.Repositories; + +public abstract class JobRepositoryTests(TestContext context) : RepositoryTest(context) +{ + [Test] + public async Task JobsRoundTripEveryField() + { + var db = await OpenTitanoAsync(); + var job = TestData.Job(JobKind.Render, JobPriority.User) with + { + TargetIds = [Guid.CreateVersion7(), Guid.CreateVersion7()], TargetLabel = "Harbour sunset", ParametersJson = "{\"preset\": \"4K\"}", + ProgressDone = 5, ProgressTotal = 10, ItemsPerSecond = 2.5, Eta = TimeSpan.FromSeconds(2), Checkpoint = "frame:5", + LogTail = "line 1\nline 2", CreatedBy = Guid.CreateVersion7(), NodeId = "node-a", Attempts = 1, Error = "boom", + State = JobState.Failed, Started = TestData.Now.AddMinutes(1), Finished = TestData.Now.AddMinutes(2), + }; + await db.Jobs.EnqueueAsync(job, Ct); + var read = Assert.NotNull(await db.Jobs.GetAsync(job.Id, Ct)); + Assert.SequenceEqual(job.TargetIds, read.TargetIds); + Assert.Equal(job with { TargetIds = read.TargetIds }, read); + } + + [Test] + public async Task ClaimsByPriorityThenAgeAndKind() + { + var db = await OpenTitanoAsync(); + var oldBackground = TestData.Job(JobKind.Thumbs, JobPriority.Background, TestData.Now.AddMinutes(-10)); + var newUser = TestData.Job(JobKind.Render, JobPriority.User, TestData.Now); + var oldUser = TestData.Job(JobKind.Import, JobPriority.User, TestData.Now.AddMinutes(-1)); + var normal = TestData.Job(JobKind.Scan, JobPriority.Normal, TestData.Now.AddMinutes(-5)); + foreach (var job in new[] { oldBackground, newUser, oldUser, normal }) await db.Jobs.EnqueueAsync(job, Ct); + + var renderOnly = await db.Jobs.ClaimNextAsync("node-gpu", [JobKind.Render], Ct); + Assert.Equal(newUser.Id, renderOnly?.Id); + Assert.Equal(JobState.Running, renderOnly!.State); + Assert.Equal("node-gpu", renderOnly.NodeId); + Assert.Equal(1, renderOnly.Attempts); + Assert.NotNull(renderOnly.Started); + Assert.Null(await db.Jobs.ClaimNextAsync("node-gpu", [JobKind.Render], Ct)); + + var order = new List(); + while (await db.Jobs.ClaimNextAsync("server", cancellationToken: Ct) is { } claimed) order.Add(claimed.Id); + Assert.SequenceEqual([oldUser.Id, normal.Id, oldBackground.Id], order); + } + + [Test] + public async Task ConcurrentClaimersNeverShareAJob() + { + var db = await OpenTitanoAsync(multiConnection: true); + var jobs = Enumerable.Range(0, 40).Select(i => TestData.Job(created: TestData.Now.AddSeconds(i))).ToList(); + foreach (var job in jobs) await db.Jobs.EnqueueAsync(job, Ct); + + var claims = new ConcurrentBag<(string Worker, Guid Job)>(); + await Task.WhenAll(Enumerable.Range(0, 8).Select(w => Task.Run(async () => + { + var worker = "worker-" + w; + while (await db.Jobs.ClaimNextAsync(worker, cancellationToken: Ct) is { } job) claims.Add((worker, job.Id)); + }))); + + Assert.Equal(40, claims.Count); + Assert.Equal(40, claims.Select(c => c.Job).Distinct().Count()); + foreach (var (worker, id) in claims) + { + var stored = await db.Jobs.GetAsync(id, Ct); + Assert.Equal(worker, stored!.NodeId); + Assert.Equal(1, stored.Attempts); + } + } + + [Test] + public async Task ProgressIsRefusedOnceTheJobStopsRunning() + { + var db = await OpenTitanoAsync(); + var job = TestData.Job(); + await db.Jobs.EnqueueAsync(job, Ct); + Assert.False(await db.Jobs.UpdateProgressAsync(job.Id, 1, 10, null, null, null, null, Ct)); + await db.Jobs.ClaimNextAsync("w", cancellationToken: Ct); + Assert.True(await db.Jobs.UpdateProgressAsync(job.Id, 4, 10, 2.0, TimeSpan.FromSeconds(3), "cursor:4", "tail", Ct)); + var running = await db.Jobs.GetAsync(job.Id, Ct); + Assert.Equal((4L, 10L, "cursor:4", "tail"), (running!.ProgressDone, running.ProgressTotal, running.Checkpoint, running.LogTail)); + Assert.Equal(TimeSpan.FromSeconds(3), running.Eta); + + Assert.True(await db.Jobs.SetStateAsync(job.Id, JobState.Cancelled, cancellationToken: Ct)); + Assert.False(await db.Jobs.UpdateProgressAsync(job.Id, 5, 10, null, null, "cursor:5", null, Ct)); + var cancelled = await db.Jobs.GetAsync(job.Id, Ct); + Assert.Equal(JobState.Cancelled, cancelled!.State); + Assert.NotNull(cancelled.Finished); + + Assert.True(await db.Jobs.SetStateAsync(job.Id, JobState.Queued, cancellationToken: Ct)); + Assert.Null((await db.Jobs.GetAsync(job.Id, Ct))!.Finished); + Assert.Equal(2, (await db.Jobs.ClaimNextAsync("w", cancellationToken: Ct))!.Attempts); + Assert.False(await db.Jobs.SetStateAsync(Guid.CreateVersion7(), JobState.Done, cancellationToken: Ct)); + } + + [Test] + public async Task ListingPagesByStateNewestFirst() + { + var db = await OpenTitanoAsync(); + var jobs = Enumerable.Range(0, 7).Select(i => TestData.Job(created: TestData.Now.AddMinutes(i % 3))).ToList(); + foreach (var job in jobs) await db.Jobs.EnqueueAsync(job, Ct); + await db.Jobs.SetStateAsync(jobs[0].Id, JobState.Done, cancellationToken: Ct); + + var seen = new List(); + string? cursor = null; + do + { + var page = await db.Jobs.ListAsync(JobState.Queued, limit: 2, cursor: cursor, cancellationToken: Ct); + seen.AddRange(page.Items.Select(j => j.Id)); + cursor = page.NextCursor; + } + while (cursor is not null); + + var expected = jobs.Skip(1) + .OrderByDescending(j => j.Created) + .ThenByDescending(j => j.Id.ToString("D"), StringComparer.Ordinal) + .Select(j => j.Id); + Assert.SequenceEqual(expected, seen); + Assert.SequenceEqual([jobs[0].Id], (await db.Jobs.ListAsync(JobState.Done, cancellationToken: Ct)).Items.Select(j => j.Id)); + Assert.Equal(7, (await db.Jobs.ListAsync(cancellationToken: Ct)).Items.Count); + } + + [Test] + public async Task HistoryPurgeKeepsRecentAndActiveJobs() + { + var db = await OpenTitanoAsync(); + var old = TestData.Job() with { State = JobState.Done, Finished = TestData.Now.AddDays(-100) }; + var oldFailed = TestData.Job() with { State = JobState.Failed, Finished = TestData.Now.AddDays(-91) }; + var recent = TestData.Job() with { State = JobState.Done, Finished = TestData.Now.AddDays(-10) }; + var queued = TestData.Job(created: TestData.Now.AddDays(-200)); + foreach (var job in new[] { old, oldFailed, recent, queued }) await db.Jobs.EnqueueAsync(job, Ct); + Assert.Equal(2, await db.Jobs.PurgeHistoryAsync(TestData.Now.AddDays(-90), Ct)); + Assert.Equal(2, (await db.Jobs.ListAsync(cancellationToken: Ct)).Items.Count); + } + + [Test] + public async Task RunningJobsAreRequeuedAfterACrash() + { + var db = await OpenTitanoAsync(); + for (var i = 0; i < 3; i++) await db.Jobs.EnqueueAsync(TestData.Job(created: TestData.Now.AddSeconds(i)), Ct); + var a = await db.Jobs.ClaimNextAsync("node-a", cancellationToken: Ct); + var b = await db.Jobs.ClaimNextAsync("node-b", cancellationToken: Ct); + await db.Jobs.UpdateProgressAsync(a!.Id, 3, 9, null, null, "resume-here", null, Ct); + + Assert.Equal(1, await db.Jobs.RequeueRunningAsync("node-a", Ct)); + var requeued = await db.Jobs.GetAsync(a.Id, Ct); + Assert.Equal(JobState.Queued, requeued!.State); + Assert.Equal("resume-here", requeued.Checkpoint); + Assert.Null(requeued.NodeId); + Assert.Equal(JobState.Running, (await db.Jobs.GetAsync(b!.Id, Ct))!.State); + Assert.Equal(1, await db.Jobs.RequeueRunningAsync(cancellationToken: Ct)); + Assert.Equal(0, await db.Jobs.RequeueRunningAsync(cancellationToken: Ct)); + } +} + +public sealed class SqliteJobRepositoryTests(TestContext context) : JobRepositoryTests(context) +{ + protected override Backend Backend => Backend.SqliteMemory; +} + +[RequiresEnvironment(PgTestServer.Variable)] +public sealed class PostgresJobRepositoryTests(TestContext context) : JobRepositoryTests(context) +{ + protected override Backend Backend => Backend.Postgres; +} diff --git a/tests/Titano.Data.Tests/Repositories/MediaCardRepositoryTests.cs b/tests/Titano.Data.Tests/Repositories/MediaCardRepositoryTests.cs new file mode 100644 index 00000000..efc1352a --- /dev/null +++ b/tests/Titano.Data.Tests/Repositories/MediaCardRepositoryTests.cs @@ -0,0 +1,100 @@ +using Titano.Core.Model; +using Titano.Data.Repositories; +using Titano.Data.Tests.Infrastructure; +using Titano.Testing; + +namespace Titano.Data.Tests.Repositories; + +public abstract class MediaCardRepositoryTests(TestContext context) : RepositoryTest(context) +{ + [Test] + public async Task CardsRoundTripAndAreFoundByIdentity() + { + var db = await OpenTitanoAsync(); + var card = Card(); + await db.MediaCards.UpsertAsync(card, Ct); + Assert.Equal(card, await db.MediaCards.GetAsync(card.Id, Ct)); + Assert.Equal(card, await db.MediaCards.FindByIdentityAsync(card.Identity, Ct)); + Assert.Null(await db.MediaCards.FindByIdentityAsync("unknown", Ct)); + + var seen = TestData.Now.AddDays(2); + Assert.True(await db.MediaCards.TouchAsync(card.Id, seen, Ct)); + await db.MediaCards.UpsertAsync(card with { Nickname = "Card B", LastSeen = seen }, Ct); + var other = Card() with { Identity = "1-OTHER-64", LastSeen = TestData.Now.AddDays(5) }; + await db.MediaCards.UpsertAsync(other, Ct); + Assert.SequenceEqual([other.Id, card.Id], (await db.MediaCards.ListAsync(Ct)).Select(c => c.Id)); + Assert.Equal("Card B", (await db.MediaCards.GetAsync(card.Id, Ct))!.Nickname); + + var clash = await Assert.ThrowsAsync(() => db.MediaCards.UpsertAsync(Card() with { Identity = card.Identity }, Ct)); + Assert.True(clash.IsUniqueViolation); + } + + [Test] + public async Task ImportHistoryLetsReinsertsSkipKnownFiles() + { + var db = await OpenTitanoAsync(); + var card = Card(); + await db.MediaCards.UpsertAsync(card, Ct); + var archive = TestData.Archive(); + await db.Archives.UpsertAsync(archive, Ct); + var sequence = TestData.Sequence(archive.Id); + await db.Sequences.UpsertAsync(sequence, Ct); + var frame = TestData.Frame(sequence.Id, 0); + await db.Frames.InsertAsync(frame, Ct); + + // File-system times carry 100 ns ticks; keys compare at the database's microsecond precision. + var modified = new DateTimeOffset(2026, 9, 28, 18, 0, 0, TimeSpan.FromHours(2)).AddTicks(1234567); + var first = new ImportKey("DCIM/100CANON/IMG_0001.CR3", 25_000_000, modified); + var reused = new ImportKey("DCIM/100CANON/IMG_0001.CR3", 24_000_000, modified.AddDays(30)); + var job = Guid.CreateVersion7(); + await db.MediaCards.RecordImportsAsync( + [ + new ImportRecord(card.Id, first, new string('F', 64), frame.Id, job, TestData.Now), + new ImportRecord(card.Id, reused, null, null, job, TestData.Now.AddDays(30)), + ], Ct); + await db.MediaCards.RecordImportsAsync([new ImportRecord(card.Id, first, new string('F', 64), frame.Id, job, TestData.Now)], Ct); + + var keys = await db.MediaCards.GetImportedKeysAsync(card.Id, Ct); + Assert.Equal(2, keys.Count); + Assert.True(keys.Contains(new ImportKey("DCIM/100CANON/IMG_0001.CR3", 25_000_000, modified.AddTicks(2)))); + Assert.True(await db.MediaCards.IsImportedAsync(card.Id, first, Ct)); + Assert.False(await db.MediaCards.IsImportedAsync(card.Id, new ImportKey(first.Path, first.Size + 1, modified), Ct)); + + var history = await db.MediaCards.ListImportsAsync(card.Id, cancellationToken: Ct); + Assert.Equal(reused, history[0].Key); + Assert.Equal(new string('f', 64), history[1].Sha256); + Assert.Equal(frame.Id, history[1].FrameId); + + await db.Frames.DeleteBySequenceAsync(sequence.Id, Ct); + Assert.Null((await db.MediaCards.ListImportsAsync(card.Id, cancellationToken: Ct))[1].FrameId); + Assert.True(await db.MediaCards.DeleteAsync(card.Id, Ct)); + Assert.Equal(0, (await db.MediaCards.GetImportedKeysAsync(card.Id, Ct)).Count); + } + + private static MediaCard Card() => new() + { + Id = Guid.CreateVersion7(), + Identity = "3735928559-EOS_DIGITAL-128043712512", + VolumeSerial = 3735928559, + Label = "EOS_DIGITAL", + Capacity = 128_043_712_512, + CameraMake = "Canon", + CameraModel = "EOS R5", + CameraSerial = "0123", + ProfileJson = "{\"auto\": false}", + AutoImport = true, + FirstSeen = TestData.Now, + LastSeen = TestData.Now, + }; +} + +public sealed class SqliteMediaCardRepositoryTests(TestContext context) : MediaCardRepositoryTests(context) +{ + protected override Backend Backend => Backend.SqliteMemory; +} + +[RequiresEnvironment(PgTestServer.Variable)] +public sealed class PostgresMediaCardRepositoryTests(TestContext context) : MediaCardRepositoryTests(context) +{ + protected override Backend Backend => Backend.Postgres; +} diff --git a/tests/Titano.Data.Tests/Repositories/ServerRepositoryTests.cs b/tests/Titano.Data.Tests/Repositories/ServerRepositoryTests.cs new file mode 100644 index 00000000..61dd5c2c --- /dev/null +++ b/tests/Titano.Data.Tests/Repositories/ServerRepositoryTests.cs @@ -0,0 +1,246 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using Titano.Core.Model; +using Titano.Data.Repositories; +using Titano.Data.Tests.Infrastructure; +using Titano.Testing; + +namespace Titano.Data.Tests.Repositories; + +/// Users, tokens, render nodes, settings, audit log and uploads on both dialects. +public abstract class ServerRepositoryTests(TestContext context) : RepositoryTest(context) +{ + [Test] + public async Task UsersAreUniqueIgnoringCase() + { + var db = await OpenTitanoAsync(); + var alice = TestData.User("Alice") with { DisplayName = "Alice A.", QuotaBytes = 1L << 40, MustChangePassword = true, Role = UserRole.Admin, Permissions = Permissions.All }; + await db.Users.CreateAsync(alice, Ct); + var duplicate = await Assert.ThrowsAsync(() => db.Users.CreateAsync(TestData.User("ALICE"), Ct)); + Assert.True(duplicate.IsUniqueViolation); + + Assert.Equal(alice, await db.Users.GetByUsernameAsync(" alice ", Ct)); + Assert.True(await db.Users.UpdateAsync(alice with { Username = "alice2", Disabled = true, MustChangePassword = false }, Ct)); + Assert.True(await db.Users.RecordLoginAsync(alice.Id, TestData.Now, Ct)); + Assert.True(await db.Users.AddUsedBytesAsync(alice.Id, 5000, Ct)); + Assert.True(await db.Users.AddUsedBytesAsync(alice.Id, -1000, Ct)); + var read = Assert.NotNull(await db.Users.GetAsync(alice.Id, Ct)); + Assert.Equal(alice with { Username = "alice2", Disabled = true, MustChangePassword = false, LastLogin = TestData.Now, UsedBytes = 4000 }, read); + Assert.Null(await db.Users.GetByUsernameAsync("alice", Ct)); + Assert.Equal(1, await db.Users.CountAsync(Ct)); + Assert.Count(1, await db.Users.ListAsync(Ct)); + } + + [Test] + public async Task DeletingAUserRemovesTheirTokensNodesAndUploads() + { + var db = await OpenTitanoAsync(); + var user = TestData.User(); + await db.Users.CreateAsync(user, Ct); + await db.Tokens.InsertAsync(Token(user.Id, "h1"), Ct); + await db.RenderNodes.RegisterAsync(Node(user.Id), Ct); + await db.Uploads.CreateAsync(Upload(user.Id, 10), Ct); + Assert.True(await db.Users.DeleteAsync(user.Id, Ct)); + Assert.Null(await db.Tokens.GetAsync("h1", Ct)); + Assert.Empty(await db.RenderNodes.ListAsync(Ct)); + Assert.Empty(await db.Uploads.ListByUserAsync(user.Id, Ct)); + } + + [Test] + public async Task TokensSlideRevokeAndPurge() + { + var db = await OpenTitanoAsync(); + var user = TestData.User(); + await db.Users.CreateAsync(user, Ct); + var token = Token(user.Id, "hash-1"); + await db.Tokens.InsertAsync(token, Ct); + await db.Tokens.InsertAsync(Token(user.Id, "hash-2"), Ct); + await db.Tokens.InsertAsync(Token(user.Id, "expired") with { Expires = TestData.Now.AddDays(-1) }, Ct); + Assert.Equal(token, await db.Tokens.GetAsync("hash-1", Ct)); + + var later = TestData.Now.AddDays(3); + Assert.True(await db.Tokens.TouchAsync("hash-1", later, later.AddDays(30), Ct)); + var touched = await db.Tokens.GetAsync("hash-1", Ct); + Assert.Equal((later, later.AddDays(30)), (touched!.LastUsed, touched.Expires)); + Assert.False(await db.Tokens.TouchAsync("expired", TestData.Now, later, Ct)); + Assert.False(await db.Tokens.TouchAsync("missing", TestData.Now, later, Ct)); + + Assert.True(await db.Tokens.RevokeAsync("hash-1", Ct)); + Assert.False(await db.Tokens.RevokeAsync("hash-1", Ct)); + Assert.False(await db.Tokens.TouchAsync("hash-1", later, later.AddDays(30), Ct)); + Assert.True((await db.Tokens.GetAsync("hash-1", Ct))!.Revoked); + Assert.Equal(2, await db.Tokens.RevokeAllAsync(user.Id, Ct)); + Assert.Count(3, await db.Tokens.ListByUserAsync(user.Id, Ct)); + Assert.Equal(3, await db.Tokens.PurgeAsync(TestData.Now, Ct)); + } + + [Test] + public async Task RenderNodesReportHeartbeats() + { + var db = await OpenTitanoAsync(); + var user = TestData.User(); + await db.Users.CreateAsync(user, Ct); + var node = Node(user.Id); + await db.RenderNodes.RegisterAsync(node, Ct); + var stale = Node(user.Id) with { ClientName = "old-pc", LastHeartbeat = TestData.Now.AddHours(-1) }; + await db.RenderNodes.RegisterAsync(stale, Ct); + await db.RenderNodes.RegisterAsync(node with { CapabilitiesJson = "{\"encoders\": [\"nvenc\"]}" }, Ct); + Assert.Equal("{\"encoders\": [\"nvenc\"]}", (await db.RenderNodes.GetAsync(node.Id, Ct))!.CapabilitiesJson); + + Assert.SequenceEqual([node.Id], (await db.RenderNodes.ListOnlineAsync(TestData.Now.AddMinutes(-2), Ct)).Select(n => n.Id)); + Assert.True(await db.RenderNodes.HeartbeatAsync(stale.Id, "rendering", TestData.Now.AddMinutes(1), Ct)); + var online = await db.RenderNodes.ListOnlineAsync(TestData.Now.AddMinutes(-2), Ct); + Assert.SequenceEqual([stale.Id, node.Id], online.Select(n => n.Id)); + Assert.Equal("rendering", online[0].State); + Assert.True(await db.RenderNodes.DeleteAsync(stale.Id, Ct)); + Assert.False(await db.RenderNodes.HeartbeatAsync(stale.Id, "idle", TestData.Now, Ct)); + } + + [Test] + public async Task SettingsAreTypedAndScoped() + { + var db = await OpenTitanoAsync(); + var archive = Guid.CreateVersion7(); + var user = Guid.CreateVersion7(); + var preferences = new Preferences("it", 4, [1.5, 2]); + await db.Settings.SetAsync(SettingScope.App, null, "ui", preferences, TestJson.Default.Preferences, Ct); + await db.Settings.SetAsync(SettingScope.User, user, "ui", preferences with { Language = "en" }, TestJson.Default.Preferences, Ct); + await db.Settings.SetJsonAsync(SettingScope.Archive, archive, "proxy", "{\"longEdge\": 1920}", Ct); + + Assert.Equal(preferences, await db.Settings.GetAsync(SettingScope.App, null, "ui", TestJson.Default.Preferences, Ct), "app scope"); + Assert.Equal("en", (await db.Settings.GetAsync(SettingScope.User, user, "ui", TestJson.Default.Preferences, Ct))!.Language); + Assert.Null(await db.Settings.GetAsync(SettingScope.User, Guid.CreateVersion7(), "ui", TestJson.Default.Preferences, Ct)); + Assert.Equal(1920, JsonDocument.Parse((await db.Settings.GetJsonAsync(SettingScope.Archive, archive, "proxy", Ct))!).RootElement.GetProperty("longEdge").GetInt32()); + + await db.Settings.SetAsync(SettingScope.App, null, "ui", preferences with { Columns = 6 }, TestJson.Default.Preferences, Ct); + Assert.Equal(6, (await db.Settings.GetAsync(SettingScope.App, null, "ui", TestJson.Default.Preferences, Ct))!.Columns); + await db.Settings.SetAsync(SettingScope.App, null, "count", 42, TestJson.Default.Int32, Ct); + Assert.SequenceEqual(["count", "ui"], (await db.Settings.ListAsync(SettingScope.App, null, Ct)).Keys.Order(StringComparer.Ordinal)); + Assert.True(await db.Settings.DeleteAsync(SettingScope.App, null, "count", Ct)); + Assert.False(await db.Settings.DeleteAsync(SettingScope.App, null, "count", Ct)); + + await Assert.ThrowsAsync(() => db.Settings.GetJsonAsync(SettingScope.User, null, "ui", Ct)); + await Assert.ThrowsAsync(() => db.Settings.GetJsonAsync(SettingScope.App, user, "ui", Ct)); + await Assert.ThrowsAsync(() => db.Settings.SetJsonAsync(SettingScope.App, null, "bad", "{not json", Ct)); + } + + [Test] + public async Task AuditLogPagesNewestFirst() + { + var db = await OpenTitanoAsync(); + var user = Guid.CreateVersion7(); + var entries = Enumerable.Range(0, 9).Select(i => new AuditEntry + { + Id = Guid.CreateVersion7(), At = TestData.Now.AddSeconds(i / 2), Action = "sequence.delete", UserId = i % 3 == 0 ? user : null, + Username = "alice", TargetKind = "sequence", TargetId = Guid.CreateVersion7().ToString(), Details = "{\"frames\": 10}", RemoteAddress = "10.0.0.1", + }).ToList(); + foreach (var e in entries) await db.Audit.AppendAsync(e, Ct); + + var seen = new List(); + string? cursor = null; + do + { + var page = await db.Audit.ListAsync(limit: 4, cursor: cursor, cancellationToken: Ct); + seen.AddRange(page.Items); + cursor = page.NextCursor; + } + while (cursor is not null); + Assert.SequenceEqual( + entries.OrderByDescending(e => e.At).ThenByDescending(e => e.Id.ToString("D"), StringComparer.Ordinal), + seen); + Assert.Equal(3, (await db.Audit.ListAsync(user, cancellationToken: Ct)).Items.Count); + } + + [Test] + public async Task UploadsTrackChunksAndStates() + { + var db = await OpenTitanoAsync(); + var user = TestData.User(); + await db.Users.CreateAsync(user, Ct); + var created = await db.Uploads.CreateAsync(Upload(user.Id, 8 * 1024 * 1024 * 9 + 1), Ct); + Assert.Equal(10, created.ChunkCount); + Assert.Equal(2, created.Received.Length); + Assert.Equal(created.Id, (await db.Uploads.FindReceivingAsync(user.Id, created.Sha256.ToUpperInvariant(), Ct))!.Id); + + foreach (var chunk in new[] { 0, 9, 3, 9 }) + { + var updated = await db.Uploads.MarkChunkReceivedAsync(created.Id, chunk, Ct); + Assert.True(updated!.HasChunk(chunk)); + } + await Assert.ThrowsAsync(() => db.Uploads.MarkChunkReceivedAsync(created.Id, 10, Ct)); + Assert.Null(await db.Uploads.MarkChunkReceivedAsync(Guid.CreateVersion7(), 0, Ct)); + + var stored = Assert.NotNull(await db.Uploads.GetAsync(created.Id, Ct)); + Assert.Equal(3, stored.ReceivedCount); + Assert.SequenceEqual([0, 3, 9], Enumerable.Range(0, 10).Where(stored.HasChunk)); + Assert.False(stored.IsComplete); + + await Task.WhenAll(Enumerable.Range(0, 10).Select(i => db.Uploads.MarkChunkReceivedAsync(created.Id, i, Ct))); + Assert.True((await db.Uploads.GetAsync(created.Id, Ct))!.IsComplete); + + Assert.Count(1, await db.Uploads.ListStaleAsync(TestData.Now.AddYears(10), Ct)); + var frameId = Guid.CreateVersion7(); + Assert.True(await db.Uploads.SetStateAsync(created.Id, UploadState.Committed, frameId, Ct)); + Assert.True(await db.Uploads.SetStateAsync(created.Id, UploadState.Committed, cancellationToken: Ct)); + var committed = await db.Uploads.GetAsync(created.Id, Ct); + Assert.Equal((UploadState.Committed, (Guid?)frameId), (committed!.State, committed.FrameId)); + Assert.Empty(await db.Uploads.ListStaleAsync(TestData.Now.AddYears(10), Ct)); + Assert.Null(await db.Uploads.FindReceivingAsync(user.Id, created.Sha256, Ct)); + Assert.True(await db.Uploads.DeleteAsync(created.Id, Ct)); + } + + private static SessionToken Token(Guid userId, string hash) => new() + { + TokenHash = hash, + UserId = userId, + ClientName = "Titano desktop", + Created = TestData.Now, + Expires = TestData.Now.AddDays(30), + LastUsed = TestData.Now, + }; + + private static RenderNode Node(Guid userId) => new() + { + Id = Guid.CreateVersion7(), + UserId = userId, + ClientName = "studio-pc", + CapabilitiesJson = "{\"encoders\": []}", + LastHeartbeat = TestData.Now, + }; + + private static Upload Upload(Guid userId, long size) => new() + { + Id = Guid.CreateVersion7(), + UserId = userId, + FileName = "IMG_0001.CR3", + Size = size, + Sha256 = new string('A', 64), + CapturedAt = new DateTime(2026, 9, 28, 18, 0, 0), + ChunkSize = 8 * 1024 * 1024, + Created = TestData.Now, + UpdatedAt = TestData.Now, + }; +} + +public sealed record Preferences(string Language, int Columns, double[] Zoom) +{ + public bool Equals(Preferences? other) => + other is not null && Language == other.Language && Columns == other.Columns && Zoom.SequenceEqual(other.Zoom); + + public override int GetHashCode() => HashCode.Combine(Language, Columns); +} + +[JsonSerializable(typeof(Preferences))] +[JsonSerializable(typeof(int))] +internal sealed partial class TestJson : JsonSerializerContext; + +public sealed class SqliteServerRepositoryTests(TestContext context) : ServerRepositoryTests(context) +{ + protected override Backend Backend => Backend.SqliteMemory; +} + +[RequiresEnvironment(PgTestServer.Variable)] +public sealed class PostgresServerRepositoryTests(TestContext context) : ServerRepositoryTests(context) +{ + protected override Backend Backend => Backend.Postgres; +}