Files
Chronos/tests/Titano.Data.Tests/Perf/RebuildPerfTests.cs
T
Claude 54fa588ecc test(perf): rebuild gate in the §1.1 shape on both dialects, every column compared
- The rebuild gate adopts 400 sequences of 250 real photos (one fifth of the
  §1.1 library, same frames per sequence) and runs on SQLite and PostgreSQL.
- The rebuilt index is compared with the original in every column of every
  sequence and frame (only UpdatedAt, the rebuild time, is left out), and the
  first differing row is named.
- Synthetic frames are seeded by the sequence's position, so every run builds
  the same library; the Library open gate says it runs on a new connection pool
  with a warm OS cache instead of calling it cold.

Measured locally (PostgreSQL 16): Library open 259 ms SQLite / 181 ms PostgreSQL,
filters <= 6.4 ms; rebuild of 100,000 frames 1.8 s SQLite / 4.7 s PostgreSQL
(budget 120 s).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
2026-10-03 10:13:06 +00:00

118 lines
5.9 KiB
C#

using System.Diagnostics;
using System.Globalization;
using System.Text.Json;
using Titano.Core.Archiving;
using Titano.Core.Model;
using Titano.Data.Archiving;
using Titano.Data.Tests.Archiving;
using Titano.Data.Tests.Infrastructure;
using Titano.Testing;
namespace Titano.Data.Tests.Perf;
/// <summary>
/// Phase 2 gate, §1.1 scaled to one fifth: 400 sequences of 250 real photos (100,000 frames; the §1.1 shape has 2,000
/// such sequences) adopted from files, then the index thrown away and rebuilt from the archive — once with the index
/// snapshot the scan left behind and once from the sidecars alone. The rebuilt index must equal the original in every
/// column of every sequence and frame (only <see cref="Sequence.UpdatedAt"/>, the time of the rebuild, may differ),
/// within a fifth of the 10 minutes §1.1 allows for 500,000 photos. Opt-in: <c>TITANO_PERF=1</c>.
/// </summary>
[Category("perf")]
[RequiresEnvironment(PerfGate.Variable)]
[Serial]
[Timeout(3_600_000)]
public abstract class RebuildPerfTests(TestContext context) : DatabaseTest(context)
{
public const int Scale = 5;
public const int Sequences = SyntheticLibrary.Sequences / Scale;
public const int FramesPerSequence = SyntheticLibrary.FramesPerSequence;
public const double RebuildBudgetSeconds = 600.0 / Scale;
private static readonly JsonSerializerOptions Json = new() { WriteIndented = false };
[Test]
public async Task AHundredThousandFrameArchiveRebuildsFromSidecarsAlone()
{
var root = Context.CreateTempDirectory("archive");
var watch = Stopwatch.StartNew();
Parallel.For(0, Sequences, new ParallelOptions { CancellationToken = Ct }, s =>
{
var start = ArchiveFixture.Start.AddDays(3 * s);
var folder = Path.Combine(root, start.Year.ToString(CultureInfo.InvariantCulture), $"{start:yyyy-MM-dd} Roof {s:000}", "originals");
Photos.WriteSeries(folder, start, FramesPerSequence, firstSeed: 1 + s * FramesPerSequence);
});
Context.Log($"perf: wrote {Sequences * FramesPerSequence:N0} photos in {Sequences:N0} folders in {watch.Elapsed.TotalSeconds:F1} s");
var (db, service, archive) = await OpenIndexAsync(root, adopt: true);
watch.Restart();
var adopted = await service.Indexer.ScanAsync(archive, ScanMode.Full, null, _ => ValueTask.CompletedTask, Ct);
Context.Log($"perf: {Dialect} adopted {adopted.Adopted} folders in {watch.Elapsed.TotalSeconds:F1} s");
Assert.Equal(Sequences, adopted.Adopted);
Assert.Empty(adopted.Failed);
var original = await DumpAsync(db, archive.Id);
Assert.Equal(Sequences * (FramesPerSequence + 1), original.Count);
foreach (var snapshot in new[] { true, false })
{
if (!snapshot) File.Delete(new ArchivePaths(root).SnapshotPath);
var (freshDb, fresh, reopened) = await OpenIndexAsync(root, adopt: false);
watch.Restart();
var rebuilt = await fresh.Indexer.ScanAsync(reopened, ScanMode.Rebuild, null, _ => ValueTask.CompletedTask, Ct);
var seconds = watch.Elapsed.TotalSeconds;
Context.Log($"perf: {Dialect} rebuilt {Sequences * FramesPerSequence:N0} frames {(snapshot ? "with" : "without")} the snapshot in {seconds:F1} s (budget {RebuildBudgetSeconds:F0} s)");
Assert.Equal((Sequences, 0), (rebuilt.Indexed, rebuilt.Adopted));
AssertSameIndex(original, await DumpAsync(freshDb, reopened.Id));
Assert.Less(seconds, RebuildBudgetSeconds, "rebuild from sidecars");
}
}
/// <summary>A new empty index of this test's dialect with the archive at <paramref name="root"/> adopted or opened in it.</summary>
private async Task<(TitanoDatabase Db, ArchiveService Service, Archive Archive)> OpenIndexAsync(string root, bool adopt)
{
var db = Own(await TitanoDatabase.OpenAsync(await NewOptionsAsync(multiConnection: true), Ct));
var service = new ArchiveService(db, new ArchiveServiceOptions { Owner = "perf", LeaseWait = TimeSpan.FromSeconds(20) });
var archive = adopt ? await service.AdoptAsync(root, "Main", Ct) : await service.OpenAsync(root, Ct);
return (db, service, archive);
}
/// <summary>Every sequence and frame row as JSON with all its columns, in folder and index order.</summary>
private async Task<List<string>> DumpAsync(TitanoDatabase db, Guid archiveId)
{
var rows = new List<string>();
foreach (var location in await db.Sequences.ListLocationsAsync(archiveId, Ct))
{
var sequence = JsonSerializer.SerializeToNode((await db.Sequences.GetAsync(location.Id, Ct))!, Json)!.AsObject();
sequence.Remove(nameof(Sequence.UpdatedAt));
rows.Add("S " + sequence.ToJsonString(Json));
var after = -1;
while (await db.Frames.ListBySequenceAsync(location.Id, after, 1000, Ct) is { Count: > 0 } page)
{
rows.AddRange(page.Select(f => " F " + JsonSerializer.Serialize(f, Json)));
after = page[^1].Index;
}
}
return rows;
}
/// <summary>Equal row for row; a difference names its first row instead of printing two 100,000-line dumps.</summary>
private static void AssertSameIndex(List<string> expected, List<string> actual)
{
for (var i = 0; i < System.Math.Min(expected.Count, actual.Count); i++)
{
if (expected[i] != actual[i]) Assert.Fail($"row {i} differs after the rebuild:\n before {expected[i]}\n after {actual[i]}");
}
Assert.Equal(expected.Count, actual.Count, "rows after the rebuild");
}
}
public sealed class SqliteRebuildPerfTests(TestContext context) : RebuildPerfTests(context)
{
protected override Backend Backend => Backend.SqliteFile;
}
[RequiresEnvironment(PgTestServer.Variable)]
public sealed class PostgresRebuildPerfTests(TestContext context) : RebuildPerfTests(context)
{
protected override Backend Backend => Backend.Postgres;
}