diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml
new file mode 100644
index 00000000..5e43f70b
--- /dev/null
+++ b/.github/workflows/perf.yml
@@ -0,0 +1,73 @@
+name: Performance gates
+
+# BRIEF §1.1 and the Phase 2/3 gates on a synthetic 500,000-photo library (SQLite and PostgreSQL 17) and a
+# 100,000-frame archive rebuild from sidecars. Too slow for every push: weekly and on demand.
+on:
+ schedule:
+ - cron: "41 3 * * 1"
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+concurrency:
+ group: perf-${{ github.ref }}
+ cancel-in-progress: true
+
+env:
+ DOTNET_NOLOGO: "1"
+ DOTNET_CLI_TELEMETRY_OPTOUT: "1"
+
+jobs:
+ perf:
+ name: Library and rebuild gates
+ runs-on: ubuntu-latest
+ timeout-minutes: 90
+ services:
+ postgres:
+ image: postgres:17
+ env:
+ POSTGRES_USER: titano
+ POSTGRES_PASSWORD: titano-ci
+ POSTGRES_DB: titano_perf
+ ports:
+ - 5432:5432
+ options: >-
+ --health-cmd "pg_isready -U titano -d titano_perf"
+ --health-interval 5s --health-timeout 5s --health-retries 20
+ env:
+ TITANO_PERF: "1"
+ TITANO_TEST_PG: Host=127.0.0.1;Port=5432;Database=titano_perf;Username=titano;Password=titano-ci;SslMode=disable
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-dotnet@v4
+ with:
+ global-json-file: global.json
+
+ - name: Build
+ run: dotnet build tests/Titano.Data.Tests -c Release
+
+ - name: Perf tests (SQLite, PostgreSQL 17)
+ run: |
+ set -o pipefail
+ dotnet run --project tests/Titano.Data.Tests -c Release --no-build -- --filter category:perf --verbose --junit out/junit/perf.xml | tee out-perf.log
+
+ - name: Timings
+ if: always()
+ run: |
+ {
+ echo "## Performance gates"
+ echo '```'
+ grep -h "perf:" out-perf.log 2>/dev/null | sed 's/^ *//' || echo "no timings recorded"
+ echo '```'
+ } >> "$GITHUB_STEP_SUMMARY"
+
+ - name: Upload results
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: perf-results
+ path: |
+ out/junit/
+ out-perf.log
+ if-no-files-found: ignore
diff --git a/tests/Titano.Data.Tests/Perf/LibraryPerfTests.cs b/tests/Titano.Data.Tests/Perf/LibraryPerfTests.cs
new file mode 100644
index 00000000..ffbcc0ed
--- /dev/null
+++ b/tests/Titano.Data.Tests/Perf/LibraryPerfTests.cs
@@ -0,0 +1,84 @@
+using Titano.Core.Model;
+using Titano.Data.Repositories;
+using Titano.Data.Tests.Infrastructure;
+using Titano.Testing;
+
+namespace Titano.Data.Tests.Perf;
+
+///
+/// §1.1 / Phase 3 gate on the 500,000-photo library: the Library opens (archive totals, filter lists and the first
+/// page, exactly what the Library screen asks at start) in under 1 s on a cold index, and every filter, search and
+/// sort answers its first page plus the total in under 200 ms. Opt-in: TITANO_PERF=1.
+///
+[Category("perf")]
+[RequiresEnvironment(PerfGate.Variable)]
+[Serial]
+[Timeout(3_600_000)]
+public abstract class LibraryPerfTests(TestContext context) : RepositoryTest(context)
+{
+ public const double OpenBudgetMs = 1_000;
+ public const double FilterBudgetMs = 200;
+
+ [Test]
+ public async Task TheLibraryOpensAndFiltersWithinTheGates()
+ {
+ var options = await NewOptionsAsync(multiConnection: true);
+ await using (var build = await TitanoDatabase.OpenAsync(options, Ct)) await SyntheticLibrary.BuildAsync(build, Context);
+
+ // A new connection pool and page cache, as at application start.
+ var db = Own(await TitanoDatabase.OpenAsync(options, Ct));
+ var cold = await PerfGate.TimeAsync(() => OpenLibraryAsync(db));
+ var warm = await PerfGate.MedianAsync(() => OpenLibraryAsync(db));
+ Context.Log($"perf: {Dialect} library open {cold:F0} ms cold, {warm:F0} ms warm (budget {OpenBudgetMs:F0} ms)");
+
+ var failures = new List();
+ if (cold > OpenBudgetMs) failures.Add($"open {cold:F0} ms");
+ var firstPage = await db.Sequences.ListAsync(new SequenceQuery { Limit = 200 }, Ct);
+ foreach (var (name, query) in Filters(firstPage.NextCursor))
+ {
+ var ms = await PerfGate.MedianAsync(async () =>
+ {
+ await db.Sequences.ListAsync(query, Ct);
+ if (query.Cursor is null) await db.Sequences.CountAsync(query, Ct);
+ });
+ Context.Log($"perf: {Dialect} {name} {ms:F1} ms (budget {FilterBudgetMs:F0} ms)");
+ if (ms > FilterBudgetMs) failures.Add($"{name} {ms:F0} ms");
+ }
+ Assert.True(failures.Count == 0, "over budget: " + string.Join(", ", failures));
+ }
+
+ /// What the Library screen loads when it opens: totals and filter lists, then the first page.
+ private async Task OpenLibraryAsync(TitanoDatabase db)
+ {
+ await db.Sequences.GetStatsAsync(null, Ct);
+ await db.Tags.ListAsync(Ct);
+ await db.Archives.ListAsync(Ct);
+ await db.Sequences.ListAsync(new SequenceQuery { Limit = 200 }, Ct);
+ }
+
+ private static IEnumerable<(string Name, SequenceQuery Query)> Filters(string? secondPage)
+ {
+ yield return ("year", new SequenceQuery { Year = 2021, Limit = 200 });
+ yield return ("camera", new SequenceQuery { Camera = "Canon EOS R5", Limit = 200 });
+ yield return ("tag", new SequenceQuery { Tag = SyntheticLibrary.Tag(0), Limit = 200 });
+ yield return ("status", new SequenceQuery { WithoutStatus = SequenceStatus.AnalyzedLight, Limit = 200 });
+ yield return ("search", new SequenceQuery { Text = "harbour", Limit = 200 });
+ yield return ("search-miss", new SequenceQuery { Text = "no such words", Limit = 200 });
+ yield return ("combined", new SequenceQuery { Year = 2024, Camera = "ILCE-7M4", WithStatus = SequenceStatus.Verified, Limit = 200 });
+ yield return ("sort-title", new SequenceQuery { Sort = SequenceSort.Title, Limit = 200 });
+ yield return ("sort-size", new SequenceQuery { Sort = SequenceSort.Size, Limit = 200 });
+ yield return ("sort-frames", new SequenceQuery { Sort = SequenceSort.FrameCount, Limit = 200 });
+ yield return ("next-page", new SequenceQuery { Limit = 200, Cursor = secondPage });
+ }
+}
+
+public sealed class SqlitePerfLibraryTests(TestContext context) : LibraryPerfTests(context)
+{
+ protected override Backend Backend => Backend.SqliteFile;
+}
+
+[RequiresEnvironment(PgTestServer.Variable)]
+public sealed class PostgresPerfLibraryTests(TestContext context) : LibraryPerfTests(context)
+{
+ protected override Backend Backend => Backend.Postgres;
+}
diff --git a/tests/Titano.Data.Tests/Perf/PerfGate.cs b/tests/Titano.Data.Tests/Perf/PerfGate.cs
new file mode 100644
index 00000000..2283b14f
--- /dev/null
+++ b/tests/Titano.Data.Tests/Perf/PerfGate.cs
@@ -0,0 +1,26 @@
+using System.Diagnostics;
+
+namespace Titano.Data.Tests.Perf;
+
+/// Timing helpers of the opt-in performance gates (TITANO_PERF=1, category perf).
+internal static class PerfGate
+{
+ public const string Variable = "TITANO_PERF";
+
+ public static async Task TimeAsync(Func action)
+ {
+ var watch = Stopwatch.StartNew();
+ await action();
+ return watch.Elapsed.TotalMilliseconds;
+ }
+
+ /// Median of timed runs after one untimed warm-up.
+ public static async Task MedianAsync(Func action, int runs = 5)
+ {
+ await action();
+ var times = new double[runs];
+ for (var i = 0; i < runs; i++) times[i] = await TimeAsync(action);
+ Array.Sort(times);
+ return times[runs / 2];
+ }
+}
diff --git a/tests/Titano.Data.Tests/Perf/RebuildPerfTests.cs b/tests/Titano.Data.Tests/Perf/RebuildPerfTests.cs
new file mode 100644
index 00000000..328d9b64
--- /dev/null
+++ b/tests/Titano.Data.Tests/Perf/RebuildPerfTests.cs
@@ -0,0 +1,85 @@
+using System.Diagnostics;
+using System.Text;
+using Titano.Data.Archiving;
+using Titano.Data.Tests.Archiving;
+using Titano.Testing;
+
+namespace Titano.Data.Tests.Perf;
+
+///
+/// Phase 2 gate, §1.1 scaled: a 100,000-frame archive adopted from real files, then its index thrown away and rebuilt
+/// from the sidecars alone. The rebuilt index must equal the original row for row, within a fifth of the 10-minute
+/// budget §1.1 gives 500,000 photos. Opt-in: TITANO_PERF=1.
+///
+[Category("perf")]
+[RequiresEnvironment(PerfGate.Variable)]
+[Serial]
+[Timeout(3_600_000)]
+public sealed class RebuildPerfTests(TestContext context)
+{
+ public const int Sequences = 40;
+ public const int FramesPerSequence = 2_500;
+ public const double RebuildBudgetSeconds = 600.0 * Sequences * FramesPerSequence / 500_000;
+
+ [Test]
+ public async Task AHundredThousandFrameArchiveRebuildsFromSidecarsAlone()
+ {
+ await using var fx = await ArchiveFixture.CreateAsync(context);
+ var watch = Stopwatch.StartNew();
+ Parallel.For(0, Sequences, new ParallelOptions { CancellationToken = fx.Ct }, s =>
+ {
+ var start = ArchiveFixture.Start.AddDays(3 * s);
+ var folder = Path.Combine(fx.Paths.Root, start.Year.ToString(System.Globalization.CultureInfo.InvariantCulture), $"{start:yyyy-MM-dd} Roof {s:00}", "originals");
+ Photos.WriteSeries(folder, start, FramesPerSequence, firstSeed: 1 + s * FramesPerSequence);
+ });
+ context.Log($"perf: wrote {Sequences * FramesPerSequence:N0} photos in {watch.Elapsed.TotalSeconds:F1} s");
+
+ watch.Restart();
+ var adopted = await fx.ScanAsync(ScanMode.Full);
+ context.Log($"perf: adopted {adopted.Adopted} folders in {watch.Elapsed.TotalSeconds:F1} s");
+ Assert.Equal(Sequences, adopted.Adopted);
+ Assert.Empty(adopted.Failed);
+ var original = await DumpAsync(fx);
+
+ // With the index snapshot the full scan left behind, then from the sidecar files alone (snapshot lost).
+ foreach (var snapshot in new[] { true, false })
+ {
+ if (!snapshot) File.Delete(fx.Paths.SnapshotPath);
+ await fx.ReopenWithEmptyIndexAsync();
+ watch.Restart();
+ var rebuilt = await fx.ScanAsync(ScanMode.Rebuild);
+ var seconds = watch.Elapsed.TotalSeconds;
+ context.Log($"perf: rebuilt {Sequences * FramesPerSequence:N0} frames {(snapshot ? "with" : "without")} the snapshot in {seconds:F1} s (budget {RebuildBudgetSeconds:F0} s)");
+ Assert.Equal((Sequences, 0), (rebuilt.Indexed, rebuilt.Adopted));
+ Assert.Equal(original, await DumpAsync(fx));
+ Assert.Less(seconds, RebuildBudgetSeconds, "rebuild from sidecars");
+ }
+ }
+
+ /// Every sequence row and every frame row (paged), in folder and index order.
+ private static async Task DumpAsync(ArchiveFixture fx)
+ {
+ var sb = new StringBuilder();
+ var frames = 0;
+ foreach (var location in await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct))
+ {
+ var s = (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!;
+ sb.Append($"S {s.Id} {s.Folder} '{s.Title}' {s.CameraModel} {s.FirstCapture:O} {s.FrameCount} {s.TotalBytes} {s.Status} ")
+ .AppendLine($"{s.VerifiedAt:O} {string.Join('|', s.Tags)} {s.SettingsVersion}/{s.FramesVersion} {s.MedianInterval}");
+ var after = -1;
+ while (true)
+ {
+ var page = await fx.Db.Frames.ListBySequenceAsync(s.Id, after, 1000, fx.Ct);
+ if (page.Count == 0) break;
+ foreach (var f in page)
+ {
+ sb.AppendLine($" F {f.Id} {f.Index} {f.RelativePath} {f.OriginalFileName} {f.Size} {f.Sha256} {f.CapturedAt:O} {f.Width}x{f.Height} {f.Format} {f.Flags}");
+ frames++;
+ }
+ after = page[^1].Index;
+ }
+ }
+ Assert.Equal(Sequences * FramesPerSequence, frames);
+ return sb.ToString();
+ }
+}
diff --git a/tests/Titano.Data.Tests/Perf/SyntheticLibrary.cs b/tests/Titano.Data.Tests/Perf/SyntheticLibrary.cs
new file mode 100644
index 00000000..07bf7b23
--- /dev/null
+++ b/tests/Titano.Data.Tests/Perf/SyntheticLibrary.cs
@@ -0,0 +1,117 @@
+using System.Diagnostics;
+using Titano.Core.Imaging;
+using Titano.Core.Model;
+using Titano.Testing;
+
+namespace Titano.Data.Tests.Perf;
+
+///
+/// The §1.1 library: 2,000 sequences of 250 photos (500,000 frames) over twelve years, a dozen cameras, sixty tags and
+/// mixed status flags, written straight into the index (no files). Deterministic for a given seed.
+///
+internal static class SyntheticLibrary
+{
+ public const int Sequences = 2_000;
+ public const int FramesPerSequence = 250;
+ public const int Photos = Sequences * FramesPerSequence;
+
+ public static readonly (string Make, string Model)[] Cameras =
+ [
+ ("Canon", "Canon EOS R5"), ("Canon", "Canon EOS 5D Mark IV"), ("Canon", "Canon EOS R6"), ("NIKON CORPORATION", "NIKON Z 7_2"),
+ ("NIKON CORPORATION", "NIKON D850"), ("SONY", "ILCE-7M4"), ("SONY", "ILCE-7RM5"), ("FUJIFILM", "X-T5"), ("FUJIFILM", "GFX100S"),
+ ("Panasonic", "DC-S5M2"), ("OM Digital Solutions", "OM-1"), ("GoPro", "HERO12 Black"),
+ ];
+
+ private static readonly string[] Subjects =
+ ["Harbour", "Milky Way", "Storm front", "Construction site", "City lights", "Glacier", "Sunrise", "Aurora", "Harvest", "Tide", "Fog", "Market"];
+
+ private static readonly string[] Places =
+ ["Dolomites", "Trieste", "Lofoten", "Atacama", "Iceland", "Rome", "Garda", "Etna", "Patagonia", "Namib", "Kyoto", "Sardinia"];
+
+ private static readonly ImageFileFormat[] Formats = [ImageFileFormat.Cr3, ImageFileFormat.Nef, ImageFileFormat.Arw, ImageFileFormat.Jpeg, ImageFileFormat.Raf, ImageFileFormat.Dng];
+
+ public static string Tag(int i) => i switch
+ {
+ 0 => "milky way",
+ 1 => "harbour",
+ 2 => "client",
+ _ => $"tag {i:00}",
+ };
+
+ /// Writes the library into and returns its archive.
+ public static async Task BuildAsync(TitanoDatabase db, TestContext context, int seed = 7)
+ {
+ var watch = Stopwatch.StartNew();
+ var archive = new Archive { Id = Guid.CreateVersion7(), Name = "Perf", RootPath = "/archive/perf", Created = DateTimeOffset.UtcNow };
+ await db.Archives.UpsertAsync(archive, context.Cancellation);
+ var random = new Random(seed);
+ var sequences = new List(Sequences);
+ for (var i = 0; i < Sequences; i++)
+ {
+ var (make, model) = Cameras[random.Next(Cameras.Length)];
+ var first = new DateTime(2015, 1, 1).AddDays(random.Next(12 * 365)).AddMinutes(random.Next(24 * 60));
+ var interval = 2 + random.Next(30);
+ var status = (random.Next(4) > 0 ? SequenceStatus.AnalyzedLight : 0) | (random.Next(3) == 0 ? SequenceStatus.ProxiesBuilt : 0)
+ | (random.Next(2) == 0 ? SequenceStatus.Verified : 0) | (random.Next(10) == 0 ? SequenceStatus.HasWarnings : 0);
+ var tags = Enumerable.Range(0, random.Next(5)).Select(_ => Tag(random.Next(60))).Distinct().ToArray();
+ sequences.Add(new Sequence
+ {
+ Id = Guid.CreateVersion7(),
+ ArchiveId = archive.Id,
+ Title = random.Next(8) == 0 ? null : $"{Subjects[random.Next(Subjects.Length)]} {Places[random.Next(Places.Length)]} {i}",
+ Folder = $"{first.Year}/{first:yyyy-MM-dd}_{i:0000}",
+ CameraMake = make,
+ CameraModel = model,
+ FirstCapture = first,
+ LastCapture = first.AddSeconds(interval * (FramesPerSequence - 1)),
+ FrameCount = FramesPerSequence,
+ MedianInterval = interval,
+ TotalBytes = FramesPerSequence * 25_000_000L,
+ Status = status,
+ Rating = random.Next(6),
+ Tags = tags,
+ UpdatedAt = DateTimeOffset.UtcNow,
+ });
+ }
+ foreach (var s in sequences) await db.Sequences.UpsertAsync(s, context.Cancellation);
+ await db.Frames.BulkInsertAsync(sequences.SelectMany(s => Frames(s, seed)), context.Cancellation);
+ context.Log($"perf: built {Sequences:N0} sequences / {Photos:N0} photos in {watch.Elapsed.TotalSeconds:F1} s");
+ return archive;
+ }
+
+ private static IEnumerable Frames(Sequence sequence, int seed)
+ {
+ var random = new Random(HashCode.Combine(seed, sequence.Id));
+ var format = Formats[random.Next(Formats.Length)];
+ for (var i = 0; i < FramesPerSequence; i++)
+ {
+ var flags = random.Next(2_000) switch
+ {
+ 0 => FrameFlags.Missing,
+ 1 => FrameFlags.Corrupt,
+ 2 or 3 => FrameFlags.Duplicate,
+ _ => FrameFlags.None,
+ };
+ yield return new Frame
+ {
+ Id = Guid.CreateVersion7(),
+ SequenceId = sequence.Id,
+ Index = i,
+ RelativePath = $"{sequence.Folder}/originals/IMG_{i:0000}.{format.ToString().ToUpperInvariant()}",
+ FileName = $"IMG_{i:0000}.{format.ToString().ToUpperInvariant()}",
+ Size = 20_000_000 + random.Next(10_000_000),
+ Sha256 = Convert.ToHexStringLower(Guid.NewGuid().ToByteArray()) + Convert.ToHexStringLower(Guid.NewGuid().ToByteArray()),
+ CapturedAt = sequence.FirstCapture!.Value.AddSeconds(sequence.MedianInterval!.Value * i),
+ TimestampSource = TimestampSource.ExifOriginal,
+ ExposureTime = 1 / 250.0,
+ FNumber = 8,
+ Iso = 100,
+ FocalLength = 24,
+ Width = 6000,
+ Height = 4000,
+ Format = format,
+ Flags = flags,
+ };
+ }
+ }
+}