diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml new file mode 100644 index 00000000..5e43f70b --- /dev/null +++ b/.github/workflows/perf.yml @@ -0,0 +1,73 @@ +name: Performance gates + +# BRIEF §1.1 and the Phase 2/3 gates on a synthetic 500,000-photo library (SQLite and PostgreSQL 17) and a +# 100,000-frame archive rebuild from sidecars. Too slow for every push: weekly and on demand. +on: + schedule: + - cron: "41 3 * * 1" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: perf-${{ github.ref }} + cancel-in-progress: true + +env: + DOTNET_NOLOGO: "1" + DOTNET_CLI_TELEMETRY_OPTOUT: "1" + +jobs: + perf: + name: Library and rebuild gates + runs-on: ubuntu-latest + timeout-minutes: 90 + services: + postgres: + image: postgres:17 + env: + POSTGRES_USER: titano + POSTGRES_PASSWORD: titano-ci + POSTGRES_DB: titano_perf + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U titano -d titano_perf" + --health-interval 5s --health-timeout 5s --health-retries 20 + env: + TITANO_PERF: "1" + TITANO_TEST_PG: Host=127.0.0.1;Port=5432;Database=titano_perf;Username=titano;Password=titano-ci;SslMode=disable + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-dotnet@v4 + with: + global-json-file: global.json + + - name: Build + run: dotnet build tests/Titano.Data.Tests -c Release + + - name: Perf tests (SQLite, PostgreSQL 17) + run: | + set -o pipefail + dotnet run --project tests/Titano.Data.Tests -c Release --no-build -- --filter category:perf --verbose --junit out/junit/perf.xml | tee out-perf.log + + - name: Timings + if: always() + run: | + { + echo "## Performance gates" + echo '```' + grep -h "perf:" out-perf.log 2>/dev/null | sed 's/^ *//' || echo "no timings recorded" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload results + if: always() + uses: actions/upload-artifact@v4 + with: + name: perf-results + path: | + out/junit/ + out-perf.log + if-no-files-found: ignore diff --git a/tests/Titano.Data.Tests/Perf/LibraryPerfTests.cs b/tests/Titano.Data.Tests/Perf/LibraryPerfTests.cs new file mode 100644 index 00000000..ffbcc0ed --- /dev/null +++ b/tests/Titano.Data.Tests/Perf/LibraryPerfTests.cs @@ -0,0 +1,84 @@ +using Titano.Core.Model; +using Titano.Data.Repositories; +using Titano.Data.Tests.Infrastructure; +using Titano.Testing; + +namespace Titano.Data.Tests.Perf; + +/// +/// §1.1 / Phase 3 gate on the 500,000-photo library: the Library opens (archive totals, filter lists and the first +/// page, exactly what the Library screen asks at start) in under 1 s on a cold index, and every filter, search and +/// sort answers its first page plus the total in under 200 ms. Opt-in: TITANO_PERF=1. +/// +[Category("perf")] +[RequiresEnvironment(PerfGate.Variable)] +[Serial] +[Timeout(3_600_000)] +public abstract class LibraryPerfTests(TestContext context) : RepositoryTest(context) +{ + public const double OpenBudgetMs = 1_000; + public const double FilterBudgetMs = 200; + + [Test] + public async Task TheLibraryOpensAndFiltersWithinTheGates() + { + var options = await NewOptionsAsync(multiConnection: true); + await using (var build = await TitanoDatabase.OpenAsync(options, Ct)) await SyntheticLibrary.BuildAsync(build, Context); + + // A new connection pool and page cache, as at application start. + var db = Own(await TitanoDatabase.OpenAsync(options, Ct)); + var cold = await PerfGate.TimeAsync(() => OpenLibraryAsync(db)); + var warm = await PerfGate.MedianAsync(() => OpenLibraryAsync(db)); + Context.Log($"perf: {Dialect} library open {cold:F0} ms cold, {warm:F0} ms warm (budget {OpenBudgetMs:F0} ms)"); + + var failures = new List(); + if (cold > OpenBudgetMs) failures.Add($"open {cold:F0} ms"); + var firstPage = await db.Sequences.ListAsync(new SequenceQuery { Limit = 200 }, Ct); + foreach (var (name, query) in Filters(firstPage.NextCursor)) + { + var ms = await PerfGate.MedianAsync(async () => + { + await db.Sequences.ListAsync(query, Ct); + if (query.Cursor is null) await db.Sequences.CountAsync(query, Ct); + }); + Context.Log($"perf: {Dialect} {name} {ms:F1} ms (budget {FilterBudgetMs:F0} ms)"); + if (ms > FilterBudgetMs) failures.Add($"{name} {ms:F0} ms"); + } + Assert.True(failures.Count == 0, "over budget: " + string.Join(", ", failures)); + } + + /// What the Library screen loads when it opens: totals and filter lists, then the first page. + private async Task OpenLibraryAsync(TitanoDatabase db) + { + await db.Sequences.GetStatsAsync(null, Ct); + await db.Tags.ListAsync(Ct); + await db.Archives.ListAsync(Ct); + await db.Sequences.ListAsync(new SequenceQuery { Limit = 200 }, Ct); + } + + private static IEnumerable<(string Name, SequenceQuery Query)> Filters(string? secondPage) + { + yield return ("year", new SequenceQuery { Year = 2021, Limit = 200 }); + yield return ("camera", new SequenceQuery { Camera = "Canon EOS R5", Limit = 200 }); + yield return ("tag", new SequenceQuery { Tag = SyntheticLibrary.Tag(0), Limit = 200 }); + yield return ("status", new SequenceQuery { WithoutStatus = SequenceStatus.AnalyzedLight, Limit = 200 }); + yield return ("search", new SequenceQuery { Text = "harbour", Limit = 200 }); + yield return ("search-miss", new SequenceQuery { Text = "no such words", Limit = 200 }); + yield return ("combined", new SequenceQuery { Year = 2024, Camera = "ILCE-7M4", WithStatus = SequenceStatus.Verified, Limit = 200 }); + yield return ("sort-title", new SequenceQuery { Sort = SequenceSort.Title, Limit = 200 }); + yield return ("sort-size", new SequenceQuery { Sort = SequenceSort.Size, Limit = 200 }); + yield return ("sort-frames", new SequenceQuery { Sort = SequenceSort.FrameCount, Limit = 200 }); + yield return ("next-page", new SequenceQuery { Limit = 200, Cursor = secondPage }); + } +} + +public sealed class SqlitePerfLibraryTests(TestContext context) : LibraryPerfTests(context) +{ + protected override Backend Backend => Backend.SqliteFile; +} + +[RequiresEnvironment(PgTestServer.Variable)] +public sealed class PostgresPerfLibraryTests(TestContext context) : LibraryPerfTests(context) +{ + protected override Backend Backend => Backend.Postgres; +} diff --git a/tests/Titano.Data.Tests/Perf/PerfGate.cs b/tests/Titano.Data.Tests/Perf/PerfGate.cs new file mode 100644 index 00000000..2283b14f --- /dev/null +++ b/tests/Titano.Data.Tests/Perf/PerfGate.cs @@ -0,0 +1,26 @@ +using System.Diagnostics; + +namespace Titano.Data.Tests.Perf; + +/// Timing helpers of the opt-in performance gates (TITANO_PERF=1, category perf). +internal static class PerfGate +{ + public const string Variable = "TITANO_PERF"; + + public static async Task TimeAsync(Func action) + { + var watch = Stopwatch.StartNew(); + await action(); + return watch.Elapsed.TotalMilliseconds; + } + + /// Median of timed runs after one untimed warm-up. + public static async Task MedianAsync(Func action, int runs = 5) + { + await action(); + var times = new double[runs]; + for (var i = 0; i < runs; i++) times[i] = await TimeAsync(action); + Array.Sort(times); + return times[runs / 2]; + } +} diff --git a/tests/Titano.Data.Tests/Perf/RebuildPerfTests.cs b/tests/Titano.Data.Tests/Perf/RebuildPerfTests.cs new file mode 100644 index 00000000..328d9b64 --- /dev/null +++ b/tests/Titano.Data.Tests/Perf/RebuildPerfTests.cs @@ -0,0 +1,85 @@ +using System.Diagnostics; +using System.Text; +using Titano.Data.Archiving; +using Titano.Data.Tests.Archiving; +using Titano.Testing; + +namespace Titano.Data.Tests.Perf; + +/// +/// Phase 2 gate, §1.1 scaled: a 100,000-frame archive adopted from real files, then its index thrown away and rebuilt +/// from the sidecars alone. The rebuilt index must equal the original row for row, within a fifth of the 10-minute +/// budget §1.1 gives 500,000 photos. Opt-in: TITANO_PERF=1. +/// +[Category("perf")] +[RequiresEnvironment(PerfGate.Variable)] +[Serial] +[Timeout(3_600_000)] +public sealed class RebuildPerfTests(TestContext context) +{ + public const int Sequences = 40; + public const int FramesPerSequence = 2_500; + public const double RebuildBudgetSeconds = 600.0 * Sequences * FramesPerSequence / 500_000; + + [Test] + public async Task AHundredThousandFrameArchiveRebuildsFromSidecarsAlone() + { + await using var fx = await ArchiveFixture.CreateAsync(context); + var watch = Stopwatch.StartNew(); + Parallel.For(0, Sequences, new ParallelOptions { CancellationToken = fx.Ct }, s => + { + var start = ArchiveFixture.Start.AddDays(3 * s); + var folder = Path.Combine(fx.Paths.Root, start.Year.ToString(System.Globalization.CultureInfo.InvariantCulture), $"{start:yyyy-MM-dd} Roof {s:00}", "originals"); + Photos.WriteSeries(folder, start, FramesPerSequence, firstSeed: 1 + s * FramesPerSequence); + }); + context.Log($"perf: wrote {Sequences * FramesPerSequence:N0} photos in {watch.Elapsed.TotalSeconds:F1} s"); + + watch.Restart(); + var adopted = await fx.ScanAsync(ScanMode.Full); + context.Log($"perf: adopted {adopted.Adopted} folders in {watch.Elapsed.TotalSeconds:F1} s"); + Assert.Equal(Sequences, adopted.Adopted); + Assert.Empty(adopted.Failed); + var original = await DumpAsync(fx); + + // With the index snapshot the full scan left behind, then from the sidecar files alone (snapshot lost). + foreach (var snapshot in new[] { true, false }) + { + if (!snapshot) File.Delete(fx.Paths.SnapshotPath); + await fx.ReopenWithEmptyIndexAsync(); + watch.Restart(); + var rebuilt = await fx.ScanAsync(ScanMode.Rebuild); + var seconds = watch.Elapsed.TotalSeconds; + context.Log($"perf: rebuilt {Sequences * FramesPerSequence:N0} frames {(snapshot ? "with" : "without")} the snapshot in {seconds:F1} s (budget {RebuildBudgetSeconds:F0} s)"); + Assert.Equal((Sequences, 0), (rebuilt.Indexed, rebuilt.Adopted)); + Assert.Equal(original, await DumpAsync(fx)); + Assert.Less(seconds, RebuildBudgetSeconds, "rebuild from sidecars"); + } + } + + /// Every sequence row and every frame row (paged), in folder and index order. + private static async Task DumpAsync(ArchiveFixture fx) + { + var sb = new StringBuilder(); + var frames = 0; + foreach (var location in await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)) + { + var s = (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!; + sb.Append($"S {s.Id} {s.Folder} '{s.Title}' {s.CameraModel} {s.FirstCapture:O} {s.FrameCount} {s.TotalBytes} {s.Status} ") + .AppendLine($"{s.VerifiedAt:O} {string.Join('|', s.Tags)} {s.SettingsVersion}/{s.FramesVersion} {s.MedianInterval}"); + var after = -1; + while (true) + { + var page = await fx.Db.Frames.ListBySequenceAsync(s.Id, after, 1000, fx.Ct); + if (page.Count == 0) break; + foreach (var f in page) + { + sb.AppendLine($" F {f.Id} {f.Index} {f.RelativePath} {f.OriginalFileName} {f.Size} {f.Sha256} {f.CapturedAt:O} {f.Width}x{f.Height} {f.Format} {f.Flags}"); + frames++; + } + after = page[^1].Index; + } + } + Assert.Equal(Sequences * FramesPerSequence, frames); + return sb.ToString(); + } +} diff --git a/tests/Titano.Data.Tests/Perf/SyntheticLibrary.cs b/tests/Titano.Data.Tests/Perf/SyntheticLibrary.cs new file mode 100644 index 00000000..07bf7b23 --- /dev/null +++ b/tests/Titano.Data.Tests/Perf/SyntheticLibrary.cs @@ -0,0 +1,117 @@ +using System.Diagnostics; +using Titano.Core.Imaging; +using Titano.Core.Model; +using Titano.Testing; + +namespace Titano.Data.Tests.Perf; + +/// +/// The §1.1 library: 2,000 sequences of 250 photos (500,000 frames) over twelve years, a dozen cameras, sixty tags and +/// mixed status flags, written straight into the index (no files). Deterministic for a given seed. +/// +internal static class SyntheticLibrary +{ + public const int Sequences = 2_000; + public const int FramesPerSequence = 250; + public const int Photos = Sequences * FramesPerSequence; + + public static readonly (string Make, string Model)[] Cameras = + [ + ("Canon", "Canon EOS R5"), ("Canon", "Canon EOS 5D Mark IV"), ("Canon", "Canon EOS R6"), ("NIKON CORPORATION", "NIKON Z 7_2"), + ("NIKON CORPORATION", "NIKON D850"), ("SONY", "ILCE-7M4"), ("SONY", "ILCE-7RM5"), ("FUJIFILM", "X-T5"), ("FUJIFILM", "GFX100S"), + ("Panasonic", "DC-S5M2"), ("OM Digital Solutions", "OM-1"), ("GoPro", "HERO12 Black"), + ]; + + private static readonly string[] Subjects = + ["Harbour", "Milky Way", "Storm front", "Construction site", "City lights", "Glacier", "Sunrise", "Aurora", "Harvest", "Tide", "Fog", "Market"]; + + private static readonly string[] Places = + ["Dolomites", "Trieste", "Lofoten", "Atacama", "Iceland", "Rome", "Garda", "Etna", "Patagonia", "Namib", "Kyoto", "Sardinia"]; + + private static readonly ImageFileFormat[] Formats = [ImageFileFormat.Cr3, ImageFileFormat.Nef, ImageFileFormat.Arw, ImageFileFormat.Jpeg, ImageFileFormat.Raf, ImageFileFormat.Dng]; + + public static string Tag(int i) => i switch + { + 0 => "milky way", + 1 => "harbour", + 2 => "client", + _ => $"tag {i:00}", + }; + + /// Writes the library into and returns its archive. + public static async Task BuildAsync(TitanoDatabase db, TestContext context, int seed = 7) + { + var watch = Stopwatch.StartNew(); + var archive = new Archive { Id = Guid.CreateVersion7(), Name = "Perf", RootPath = "/archive/perf", Created = DateTimeOffset.UtcNow }; + await db.Archives.UpsertAsync(archive, context.Cancellation); + var random = new Random(seed); + var sequences = new List(Sequences); + for (var i = 0; i < Sequences; i++) + { + var (make, model) = Cameras[random.Next(Cameras.Length)]; + var first = new DateTime(2015, 1, 1).AddDays(random.Next(12 * 365)).AddMinutes(random.Next(24 * 60)); + var interval = 2 + random.Next(30); + var status = (random.Next(4) > 0 ? SequenceStatus.AnalyzedLight : 0) | (random.Next(3) == 0 ? SequenceStatus.ProxiesBuilt : 0) + | (random.Next(2) == 0 ? SequenceStatus.Verified : 0) | (random.Next(10) == 0 ? SequenceStatus.HasWarnings : 0); + var tags = Enumerable.Range(0, random.Next(5)).Select(_ => Tag(random.Next(60))).Distinct().ToArray(); + sequences.Add(new Sequence + { + Id = Guid.CreateVersion7(), + ArchiveId = archive.Id, + Title = random.Next(8) == 0 ? null : $"{Subjects[random.Next(Subjects.Length)]} {Places[random.Next(Places.Length)]} {i}", + Folder = $"{first.Year}/{first:yyyy-MM-dd}_{i:0000}", + CameraMake = make, + CameraModel = model, + FirstCapture = first, + LastCapture = first.AddSeconds(interval * (FramesPerSequence - 1)), + FrameCount = FramesPerSequence, + MedianInterval = interval, + TotalBytes = FramesPerSequence * 25_000_000L, + Status = status, + Rating = random.Next(6), + Tags = tags, + UpdatedAt = DateTimeOffset.UtcNow, + }); + } + foreach (var s in sequences) await db.Sequences.UpsertAsync(s, context.Cancellation); + await db.Frames.BulkInsertAsync(sequences.SelectMany(s => Frames(s, seed)), context.Cancellation); + context.Log($"perf: built {Sequences:N0} sequences / {Photos:N0} photos in {watch.Elapsed.TotalSeconds:F1} s"); + return archive; + } + + private static IEnumerable Frames(Sequence sequence, int seed) + { + var random = new Random(HashCode.Combine(seed, sequence.Id)); + var format = Formats[random.Next(Formats.Length)]; + for (var i = 0; i < FramesPerSequence; i++) + { + var flags = random.Next(2_000) switch + { + 0 => FrameFlags.Missing, + 1 => FrameFlags.Corrupt, + 2 or 3 => FrameFlags.Duplicate, + _ => FrameFlags.None, + }; + yield return new Frame + { + Id = Guid.CreateVersion7(), + SequenceId = sequence.Id, + Index = i, + RelativePath = $"{sequence.Folder}/originals/IMG_{i:0000}.{format.ToString().ToUpperInvariant()}", + FileName = $"IMG_{i:0000}.{format.ToString().ToUpperInvariant()}", + Size = 20_000_000 + random.Next(10_000_000), + Sha256 = Convert.ToHexStringLower(Guid.NewGuid().ToByteArray()) + Convert.ToHexStringLower(Guid.NewGuid().ToByteArray()), + CapturedAt = sequence.FirstCapture!.Value.AddSeconds(sequence.MedianInterval!.Value * i), + TimestampSource = TimestampSource.ExifOriginal, + ExposureTime = 1 / 250.0, + FNumber = 8, + Iso = 100, + FocalLength = 24, + Width = 6000, + Height = 4000, + Format = format, + Flags = flags, + }; + } + } +}