using System.Diagnostics; using System.Globalization; using System.Text.Json; using Titano.Core.Archiving; using Titano.Core.Model; using Titano.Data.Archiving; using Titano.Data.Tests.Archiving; using Titano.Data.Tests.Infrastructure; using Titano.Testing; namespace Titano.Data.Tests.Perf; /// /// Phase 2 gate, §1.1 scaled to one fifth: 400 sequences of 250 real photos (100,000 frames; the §1.1 shape has 2,000 /// such sequences) adopted from files, then the index thrown away and rebuilt from the archive — once with the index /// snapshot the scan left behind and once from the sidecars alone. The rebuilt index must equal the original in every /// column of every sequence and frame (only , the time of the rebuild, may differ), /// within a fifth of the 10 minutes §1.1 allows for 500,000 photos. Opt-in: TITANO_PERF=1. /// [Category("perf")] [RequiresEnvironment(PerfGate.Variable)] [Serial] [Timeout(3_600_000)] public abstract class RebuildPerfTests(TestContext context) : DatabaseTest(context) { public const int Scale = 5; public const int Sequences = SyntheticLibrary.Sequences / Scale; public const int FramesPerSequence = SyntheticLibrary.FramesPerSequence; public const double RebuildBudgetSeconds = 600.0 / Scale; private static readonly JsonSerializerOptions Json = new() { WriteIndented = false }; [Test] public async Task AHundredThousandFrameArchiveRebuildsFromSidecarsAlone() { var root = Context.CreateTempDirectory("archive"); var watch = Stopwatch.StartNew(); Parallel.For(0, Sequences, new ParallelOptions { CancellationToken = Ct }, s => { var start = ArchiveFixture.Start.AddDays(3 * s); var folder = Path.Combine(root, start.Year.ToString(CultureInfo.InvariantCulture), $"{start:yyyy-MM-dd} Roof {s:000}", "originals"); Photos.WriteSeries(folder, start, FramesPerSequence, firstSeed: 1 + s * FramesPerSequence); }); Context.Log($"perf: wrote {Sequences * FramesPerSequence:N0} photos in {Sequences:N0} folders in {watch.Elapsed.TotalSeconds:F1} s"); var (db, service, archive) = await OpenIndexAsync(root, adopt: true); watch.Restart(); var adopted = await service.Indexer.ScanAsync(archive, ScanMode.Full, null, _ => ValueTask.CompletedTask, Ct); Context.Log($"perf: {Dialect} adopted {adopted.Adopted} folders in {watch.Elapsed.TotalSeconds:F1} s"); Assert.Equal(Sequences, adopted.Adopted); Assert.Empty(adopted.Failed); var original = await DumpAsync(db, archive.Id); Assert.Equal(Sequences * (FramesPerSequence + 1), original.Count); foreach (var snapshot in new[] { true, false }) { if (!snapshot) File.Delete(new ArchivePaths(root).SnapshotPath); var (freshDb, fresh, reopened) = await OpenIndexAsync(root, adopt: false); watch.Restart(); var rebuilt = await fresh.Indexer.ScanAsync(reopened, ScanMode.Rebuild, null, _ => ValueTask.CompletedTask, Ct); var seconds = watch.Elapsed.TotalSeconds; Context.Log($"perf: {Dialect} rebuilt {Sequences * FramesPerSequence:N0} frames {(snapshot ? "with" : "without")} the snapshot in {seconds:F1} s (budget {RebuildBudgetSeconds:F0} s)"); Assert.Equal((Sequences, 0), (rebuilt.Indexed, rebuilt.Adopted)); AssertSameIndex(original, await DumpAsync(freshDb, reopened.Id)); Assert.Less(seconds, RebuildBudgetSeconds, "rebuild from sidecars"); } } /// A new empty index of this test's dialect with the archive at adopted or opened in it. private async Task<(TitanoDatabase Db, ArchiveService Service, Archive Archive)> OpenIndexAsync(string root, bool adopt) { var db = Own(await TitanoDatabase.OpenAsync(await NewOptionsAsync(multiConnection: true), Ct)); var service = new ArchiveService(db, new ArchiveServiceOptions { Owner = "perf", LeaseWait = TimeSpan.FromSeconds(20) }); var archive = adopt ? await service.AdoptAsync(root, "Main", Ct) : await service.OpenAsync(root, Ct); return (db, service, archive); } /// Every sequence and frame row as JSON with all its columns, in folder and index order. private async Task> DumpAsync(TitanoDatabase db, Guid archiveId) { var rows = new List(); foreach (var location in await db.Sequences.ListLocationsAsync(archiveId, Ct)) { var sequence = JsonSerializer.SerializeToNode((await db.Sequences.GetAsync(location.Id, Ct))!, Json)!.AsObject(); sequence.Remove(nameof(Sequence.UpdatedAt)); rows.Add("S " + sequence.ToJsonString(Json)); var after = -1; while (await db.Frames.ListBySequenceAsync(location.Id, after, 1000, Ct) is { Count: > 0 } page) { rows.AddRange(page.Select(f => " F " + JsonSerializer.Serialize(f, Json))); after = page[^1].Index; } } return rows; } /// Equal row for row; a difference names its first row instead of printing two 100,000-line dumps. private static void AssertSameIndex(List expected, List actual) { for (var i = 0; i < System.Math.Min(expected.Count, actual.Count); i++) { if (expected[i] != actual[i]) Assert.Fail($"row {i} differs after the rebuild:\n before {expected[i]}\n after {actual[i]}"); } Assert.Equal(expected.Count, actual.Count, "rows after the rebuild"); } } public sealed class SqliteRebuildPerfTests(TestContext context) : RebuildPerfTests(context) { protected override Backend Backend => Backend.SqliteFile; } [RequiresEnvironment(PgTestServer.Variable)] public sealed class PostgresRebuildPerfTests(TestContext context) : RebuildPerfTests(context) { protected override Backend Backend => Backend.Postgres; }