test(perf): opt-in gates for the 500k-photo library and the 100k-frame rebuild
Category perf, skipped unless TITANO_PERF=1. A synthetic library of 2,000 sequences and 500,000 frames (SQLite, and PostgreSQL when TITANO_TEST_PG is set) must open the Library in under 1 s cold and answer every filter, search, sort and next page in under 200 ms; a 100,000-frame archive adopted from real files must rebuild from its sidecars, with and without the index snapshot, into the identical index within the scaled 2-minute budget. perf.yml runs them weekly and on demand against PostgreSQL 17 and writes the timings to the job summary. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
5 files changed
+385
No files matched your search
@@ -0,0 +1,73 @@
|
||||
name: Performance gates
|
||||
|
||||
# BRIEF §1.1 and the Phase 2/3 gates on a synthetic 500,000-photo library (SQLite and PostgreSQL 17) and a
|
||||
# 100,000-frame archive rebuild from sidecars. Too slow for every push: weekly and on demand.
|
||||
on:
|
||||
schedule:
|
||||
- cron: "41 3 * * 1"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: perf-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
DOTNET_NOLOGO: "1"
|
||||
DOTNET_CLI_TELEMETRY_OPTOUT: "1"
|
||||
|
||||
jobs:
|
||||
perf:
|
||||
name: Library and rebuild gates
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17
|
||||
env:
|
||||
POSTGRES_USER: titano
|
||||
POSTGRES_PASSWORD: titano-ci
|
||||
POSTGRES_DB: titano_perf
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U titano -d titano_perf"
|
||||
--health-interval 5s --health-timeout 5s --health-retries 20
|
||||
env:
|
||||
TITANO_PERF: "1"
|
||||
TITANO_TEST_PG: Host=127.0.0.1;Port=5432;Database=titano_perf;Username=titano;Password=titano-ci;SslMode=disable
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-dotnet@v4
|
||||
with:
|
||||
global-json-file: global.json
|
||||
|
||||
- name: Build
|
||||
run: dotnet build tests/Titano.Data.Tests -c Release
|
||||
|
||||
- name: Perf tests (SQLite, PostgreSQL 17)
|
||||
run: |
|
||||
set -o pipefail
|
||||
dotnet run --project tests/Titano.Data.Tests -c Release --no-build -- --filter category:perf --verbose --junit out/junit/perf.xml | tee out-perf.log
|
||||
|
||||
- name: Timings
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "## Performance gates"
|
||||
echo '```'
|
||||
grep -h "perf:" out-perf.log 2>/dev/null | sed 's/^ *//' || echo "no timings recorded"
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Upload results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: perf-results
|
||||
path: |
|
||||
out/junit/
|
||||
out-perf.log
|
||||
if-no-files-found: ignore
|
||||
@@ -0,0 +1,84 @@
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Repositories;
|
||||
using Titano.Data.Tests.Infrastructure;
|
||||
using Titano.Testing;
|
||||
|
||||
namespace Titano.Data.Tests.Perf;
|
||||
|
||||
/// <summary>
|
||||
/// §1.1 / Phase 3 gate on the 500,000-photo library: the Library opens (archive totals, filter lists and the first
|
||||
/// page, exactly what the Library screen asks at start) in under 1 s on a cold index, and every filter, search and
|
||||
/// sort answers its first page plus the total in under 200 ms. Opt-in: <c>TITANO_PERF=1</c>.
|
||||
/// </summary>
|
||||
[Category("perf")]
|
||||
[RequiresEnvironment(PerfGate.Variable)]
|
||||
[Serial]
|
||||
[Timeout(3_600_000)]
|
||||
public abstract class LibraryPerfTests(TestContext context) : RepositoryTest(context)
|
||||
{
|
||||
public const double OpenBudgetMs = 1_000;
|
||||
public const double FilterBudgetMs = 200;
|
||||
|
||||
[Test]
|
||||
public async Task TheLibraryOpensAndFiltersWithinTheGates()
|
||||
{
|
||||
var options = await NewOptionsAsync(multiConnection: true);
|
||||
await using (var build = await TitanoDatabase.OpenAsync(options, Ct)) await SyntheticLibrary.BuildAsync(build, Context);
|
||||
|
||||
// A new connection pool and page cache, as at application start.
|
||||
var db = Own(await TitanoDatabase.OpenAsync(options, Ct));
|
||||
var cold = await PerfGate.TimeAsync(() => OpenLibraryAsync(db));
|
||||
var warm = await PerfGate.MedianAsync(() => OpenLibraryAsync(db));
|
||||
Context.Log($"perf: {Dialect} library open {cold:F0} ms cold, {warm:F0} ms warm (budget {OpenBudgetMs:F0} ms)");
|
||||
|
||||
var failures = new List<string>();
|
||||
if (cold > OpenBudgetMs) failures.Add($"open {cold:F0} ms");
|
||||
var firstPage = await db.Sequences.ListAsync(new SequenceQuery { Limit = 200 }, Ct);
|
||||
foreach (var (name, query) in Filters(firstPage.NextCursor))
|
||||
{
|
||||
var ms = await PerfGate.MedianAsync(async () =>
|
||||
{
|
||||
await db.Sequences.ListAsync(query, Ct);
|
||||
if (query.Cursor is null) await db.Sequences.CountAsync(query, Ct);
|
||||
});
|
||||
Context.Log($"perf: {Dialect} {name} {ms:F1} ms (budget {FilterBudgetMs:F0} ms)");
|
||||
if (ms > FilterBudgetMs) failures.Add($"{name} {ms:F0} ms");
|
||||
}
|
||||
Assert.True(failures.Count == 0, "over budget: " + string.Join(", ", failures));
|
||||
}
|
||||
|
||||
/// <summary>What the Library screen loads when it opens: totals and filter lists, then the first page.</summary>
|
||||
private async Task OpenLibraryAsync(TitanoDatabase db)
|
||||
{
|
||||
await db.Sequences.GetStatsAsync(null, Ct);
|
||||
await db.Tags.ListAsync(Ct);
|
||||
await db.Archives.ListAsync(Ct);
|
||||
await db.Sequences.ListAsync(new SequenceQuery { Limit = 200 }, Ct);
|
||||
}
|
||||
|
||||
private static IEnumerable<(string Name, SequenceQuery Query)> Filters(string? secondPage)
|
||||
{
|
||||
yield return ("year", new SequenceQuery { Year = 2021, Limit = 200 });
|
||||
yield return ("camera", new SequenceQuery { Camera = "Canon EOS R5", Limit = 200 });
|
||||
yield return ("tag", new SequenceQuery { Tag = SyntheticLibrary.Tag(0), Limit = 200 });
|
||||
yield return ("status", new SequenceQuery { WithoutStatus = SequenceStatus.AnalyzedLight, Limit = 200 });
|
||||
yield return ("search", new SequenceQuery { Text = "harbour", Limit = 200 });
|
||||
yield return ("search-miss", new SequenceQuery { Text = "no such words", Limit = 200 });
|
||||
yield return ("combined", new SequenceQuery { Year = 2024, Camera = "ILCE-7M4", WithStatus = SequenceStatus.Verified, Limit = 200 });
|
||||
yield return ("sort-title", new SequenceQuery { Sort = SequenceSort.Title, Limit = 200 });
|
||||
yield return ("sort-size", new SequenceQuery { Sort = SequenceSort.Size, Limit = 200 });
|
||||
yield return ("sort-frames", new SequenceQuery { Sort = SequenceSort.FrameCount, Limit = 200 });
|
||||
yield return ("next-page", new SequenceQuery { Limit = 200, Cursor = secondPage });
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class SqlitePerfLibraryTests(TestContext context) : LibraryPerfTests(context)
|
||||
{
|
||||
protected override Backend Backend => Backend.SqliteFile;
|
||||
}
|
||||
|
||||
[RequiresEnvironment(PgTestServer.Variable)]
|
||||
public sealed class PostgresPerfLibraryTests(TestContext context) : LibraryPerfTests(context)
|
||||
{
|
||||
protected override Backend Backend => Backend.Postgres;
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
using System.Diagnostics;
|
||||
|
||||
namespace Titano.Data.Tests.Perf;
|
||||
|
||||
/// <summary>Timing helpers of the opt-in performance gates (<c>TITANO_PERF=1</c>, category <c>perf</c>).</summary>
|
||||
internal static class PerfGate
|
||||
{
|
||||
public const string Variable = "TITANO_PERF";
|
||||
|
||||
public static async Task<double> TimeAsync(Func<Task> action)
|
||||
{
|
||||
var watch = Stopwatch.StartNew();
|
||||
await action();
|
||||
return watch.Elapsed.TotalMilliseconds;
|
||||
}
|
||||
|
||||
/// <summary>Median of <paramref name="runs"/> timed runs after one untimed warm-up.</summary>
|
||||
public static async Task<double> MedianAsync(Func<Task> action, int runs = 5)
|
||||
{
|
||||
await action();
|
||||
var times = new double[runs];
|
||||
for (var i = 0; i < runs; i++) times[i] = await TimeAsync(action);
|
||||
Array.Sort(times);
|
||||
return times[runs / 2];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
using System.Diagnostics;
|
||||
using System.Text;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Data.Tests.Archiving;
|
||||
using Titano.Testing;
|
||||
|
||||
namespace Titano.Data.Tests.Perf;
|
||||
|
||||
/// <summary>
|
||||
/// Phase 2 gate, §1.1 scaled: a 100,000-frame archive adopted from real files, then its index thrown away and rebuilt
|
||||
/// from the sidecars alone. The rebuilt index must equal the original row for row, within a fifth of the 10-minute
|
||||
/// budget §1.1 gives 500,000 photos. Opt-in: <c>TITANO_PERF=1</c>.
|
||||
/// </summary>
|
||||
[Category("perf")]
|
||||
[RequiresEnvironment(PerfGate.Variable)]
|
||||
[Serial]
|
||||
[Timeout(3_600_000)]
|
||||
public sealed class RebuildPerfTests(TestContext context)
|
||||
{
|
||||
public const int Sequences = 40;
|
||||
public const int FramesPerSequence = 2_500;
|
||||
public const double RebuildBudgetSeconds = 600.0 * Sequences * FramesPerSequence / 500_000;
|
||||
|
||||
[Test]
|
||||
public async Task AHundredThousandFrameArchiveRebuildsFromSidecarsAlone()
|
||||
{
|
||||
await using var fx = await ArchiveFixture.CreateAsync(context);
|
||||
var watch = Stopwatch.StartNew();
|
||||
Parallel.For(0, Sequences, new ParallelOptions { CancellationToken = fx.Ct }, s =>
|
||||
{
|
||||
var start = ArchiveFixture.Start.AddDays(3 * s);
|
||||
var folder = Path.Combine(fx.Paths.Root, start.Year.ToString(System.Globalization.CultureInfo.InvariantCulture), $"{start:yyyy-MM-dd} Roof {s:00}", "originals");
|
||||
Photos.WriteSeries(folder, start, FramesPerSequence, firstSeed: 1 + s * FramesPerSequence);
|
||||
});
|
||||
context.Log($"perf: wrote {Sequences * FramesPerSequence:N0} photos in {watch.Elapsed.TotalSeconds:F1} s");
|
||||
|
||||
watch.Restart();
|
||||
var adopted = await fx.ScanAsync(ScanMode.Full);
|
||||
context.Log($"perf: adopted {adopted.Adopted} folders in {watch.Elapsed.TotalSeconds:F1} s");
|
||||
Assert.Equal(Sequences, adopted.Adopted);
|
||||
Assert.Empty(adopted.Failed);
|
||||
var original = await DumpAsync(fx);
|
||||
|
||||
// With the index snapshot the full scan left behind, then from the sidecar files alone (snapshot lost).
|
||||
foreach (var snapshot in new[] { true, false })
|
||||
{
|
||||
if (!snapshot) File.Delete(fx.Paths.SnapshotPath);
|
||||
await fx.ReopenWithEmptyIndexAsync();
|
||||
watch.Restart();
|
||||
var rebuilt = await fx.ScanAsync(ScanMode.Rebuild);
|
||||
var seconds = watch.Elapsed.TotalSeconds;
|
||||
context.Log($"perf: rebuilt {Sequences * FramesPerSequence:N0} frames {(snapshot ? "with" : "without")} the snapshot in {seconds:F1} s (budget {RebuildBudgetSeconds:F0} s)");
|
||||
Assert.Equal((Sequences, 0), (rebuilt.Indexed, rebuilt.Adopted));
|
||||
Assert.Equal(original, await DumpAsync(fx));
|
||||
Assert.Less(seconds, RebuildBudgetSeconds, "rebuild from sidecars");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Every sequence row and every frame row (paged), in folder and index order.</summary>
|
||||
private static async Task<string> DumpAsync(ArchiveFixture fx)
|
||||
{
|
||||
var sb = new StringBuilder();
|
||||
var frames = 0;
|
||||
foreach (var location in await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct))
|
||||
{
|
||||
var s = (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!;
|
||||
sb.Append($"S {s.Id} {s.Folder} '{s.Title}' {s.CameraModel} {s.FirstCapture:O} {s.FrameCount} {s.TotalBytes} {s.Status} ")
|
||||
.AppendLine($"{s.VerifiedAt:O} {string.Join('|', s.Tags)} {s.SettingsVersion}/{s.FramesVersion} {s.MedianInterval}");
|
||||
var after = -1;
|
||||
while (true)
|
||||
{
|
||||
var page = await fx.Db.Frames.ListBySequenceAsync(s.Id, after, 1000, fx.Ct);
|
||||
if (page.Count == 0) break;
|
||||
foreach (var f in page)
|
||||
{
|
||||
sb.AppendLine($" F {f.Id} {f.Index} {f.RelativePath} {f.OriginalFileName} {f.Size} {f.Sha256} {f.CapturedAt:O} {f.Width}x{f.Height} {f.Format} {f.Flags}");
|
||||
frames++;
|
||||
}
|
||||
after = page[^1].Index;
|
||||
}
|
||||
}
|
||||
Assert.Equal(Sequences * FramesPerSequence, frames);
|
||||
return sb.ToString();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
using System.Diagnostics;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Testing;
|
||||
|
||||
namespace Titano.Data.Tests.Perf;
|
||||
|
||||
/// <summary>
|
||||
/// The §1.1 library: 2,000 sequences of 250 photos (500,000 frames) over twelve years, a dozen cameras, sixty tags and
|
||||
/// mixed status flags, written straight into the index (no files). Deterministic for a given seed.
|
||||
/// </summary>
|
||||
internal static class SyntheticLibrary
|
||||
{
|
||||
public const int Sequences = 2_000;
|
||||
public const int FramesPerSequence = 250;
|
||||
public const int Photos = Sequences * FramesPerSequence;
|
||||
|
||||
public static readonly (string Make, string Model)[] Cameras =
|
||||
[
|
||||
("Canon", "Canon EOS R5"), ("Canon", "Canon EOS 5D Mark IV"), ("Canon", "Canon EOS R6"), ("NIKON CORPORATION", "NIKON Z 7_2"),
|
||||
("NIKON CORPORATION", "NIKON D850"), ("SONY", "ILCE-7M4"), ("SONY", "ILCE-7RM5"), ("FUJIFILM", "X-T5"), ("FUJIFILM", "GFX100S"),
|
||||
("Panasonic", "DC-S5M2"), ("OM Digital Solutions", "OM-1"), ("GoPro", "HERO12 Black"),
|
||||
];
|
||||
|
||||
private static readonly string[] Subjects =
|
||||
["Harbour", "Milky Way", "Storm front", "Construction site", "City lights", "Glacier", "Sunrise", "Aurora", "Harvest", "Tide", "Fog", "Market"];
|
||||
|
||||
private static readonly string[] Places =
|
||||
["Dolomites", "Trieste", "Lofoten", "Atacama", "Iceland", "Rome", "Garda", "Etna", "Patagonia", "Namib", "Kyoto", "Sardinia"];
|
||||
|
||||
private static readonly ImageFileFormat[] Formats = [ImageFileFormat.Cr3, ImageFileFormat.Nef, ImageFileFormat.Arw, ImageFileFormat.Jpeg, ImageFileFormat.Raf, ImageFileFormat.Dng];
|
||||
|
||||
public static string Tag(int i) => i switch
|
||||
{
|
||||
0 => "milky way",
|
||||
1 => "harbour",
|
||||
2 => "client",
|
||||
_ => $"tag {i:00}",
|
||||
};
|
||||
|
||||
/// <summary>Writes the library into <paramref name="db"/> and returns its archive.</summary>
|
||||
public static async Task<Archive> BuildAsync(TitanoDatabase db, TestContext context, int seed = 7)
|
||||
{
|
||||
var watch = Stopwatch.StartNew();
|
||||
var archive = new Archive { Id = Guid.CreateVersion7(), Name = "Perf", RootPath = "/archive/perf", Created = DateTimeOffset.UtcNow };
|
||||
await db.Archives.UpsertAsync(archive, context.Cancellation);
|
||||
var random = new Random(seed);
|
||||
var sequences = new List<Sequence>(Sequences);
|
||||
for (var i = 0; i < Sequences; i++)
|
||||
{
|
||||
var (make, model) = Cameras[random.Next(Cameras.Length)];
|
||||
var first = new DateTime(2015, 1, 1).AddDays(random.Next(12 * 365)).AddMinutes(random.Next(24 * 60));
|
||||
var interval = 2 + random.Next(30);
|
||||
var status = (random.Next(4) > 0 ? SequenceStatus.AnalyzedLight : 0) | (random.Next(3) == 0 ? SequenceStatus.ProxiesBuilt : 0)
|
||||
| (random.Next(2) == 0 ? SequenceStatus.Verified : 0) | (random.Next(10) == 0 ? SequenceStatus.HasWarnings : 0);
|
||||
var tags = Enumerable.Range(0, random.Next(5)).Select(_ => Tag(random.Next(60))).Distinct().ToArray();
|
||||
sequences.Add(new Sequence
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
ArchiveId = archive.Id,
|
||||
Title = random.Next(8) == 0 ? null : $"{Subjects[random.Next(Subjects.Length)]} {Places[random.Next(Places.Length)]} {i}",
|
||||
Folder = $"{first.Year}/{first:yyyy-MM-dd}_{i:0000}",
|
||||
CameraMake = make,
|
||||
CameraModel = model,
|
||||
FirstCapture = first,
|
||||
LastCapture = first.AddSeconds(interval * (FramesPerSequence - 1)),
|
||||
FrameCount = FramesPerSequence,
|
||||
MedianInterval = interval,
|
||||
TotalBytes = FramesPerSequence * 25_000_000L,
|
||||
Status = status,
|
||||
Rating = random.Next(6),
|
||||
Tags = tags,
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
});
|
||||
}
|
||||
foreach (var s in sequences) await db.Sequences.UpsertAsync(s, context.Cancellation);
|
||||
await db.Frames.BulkInsertAsync(sequences.SelectMany(s => Frames(s, seed)), context.Cancellation);
|
||||
context.Log($"perf: built {Sequences:N0} sequences / {Photos:N0} photos in {watch.Elapsed.TotalSeconds:F1} s");
|
||||
return archive;
|
||||
}
|
||||
|
||||
private static IEnumerable<Frame> Frames(Sequence sequence, int seed)
|
||||
{
|
||||
var random = new Random(HashCode.Combine(seed, sequence.Id));
|
||||
var format = Formats[random.Next(Formats.Length)];
|
||||
for (var i = 0; i < FramesPerSequence; i++)
|
||||
{
|
||||
var flags = random.Next(2_000) switch
|
||||
{
|
||||
0 => FrameFlags.Missing,
|
||||
1 => FrameFlags.Corrupt,
|
||||
2 or 3 => FrameFlags.Duplicate,
|
||||
_ => FrameFlags.None,
|
||||
};
|
||||
yield return new Frame
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
SequenceId = sequence.Id,
|
||||
Index = i,
|
||||
RelativePath = $"{sequence.Folder}/originals/IMG_{i:0000}.{format.ToString().ToUpperInvariant()}",
|
||||
FileName = $"IMG_{i:0000}.{format.ToString().ToUpperInvariant()}",
|
||||
Size = 20_000_000 + random.Next(10_000_000),
|
||||
Sha256 = Convert.ToHexStringLower(Guid.NewGuid().ToByteArray()) + Convert.ToHexStringLower(Guid.NewGuid().ToByteArray()),
|
||||
CapturedAt = sequence.FirstCapture!.Value.AddSeconds(sequence.MedianInterval!.Value * i),
|
||||
TimestampSource = TimestampSource.ExifOriginal,
|
||||
ExposureTime = 1 / 250.0,
|
||||
FNumber = 8,
|
||||
Iso = 100,
|
||||
FocalLength = 24,
|
||||
Width = 6000,
|
||||
Height = 4000,
|
||||
Format = format,
|
||||
Flags = flags,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user