using System.Diagnostics; using Febe.Core.Domain; using Febe.Core.Storage; using Febe.Data; using Febe.Tests.Support; namespace Febe.Tests.Data; /// Spec §4.6: the library must stay fluid with 200,000 entries. [TestClass] public sealed class LibraryScaleTests { private const int Entries = 200_000; private const int Accounts = 40; public TestContext TestContext { get; set; } = null!; [TestMethod] public async Task LibraryQueriesStayFastWith200000Entries() { using var temp = new TempFolder(); await using var db = await TestDatabase.OpenAsync(temp); var accountIds = new List(); for (var a = 0; a < Accounts; a++) { accountIds.Add((await db.Accounts.AddAsync(Make.Account($"account{a:00}.demo", $"{1000 + a}"))).Id); } var words = new[] { "mare", "montagna", "città", "tramonto", "cena", "amici", "viaggio", "festa", "libro", "caffè" }; var random = new Random(7); var start = new DateTimeOffset(2016, 1, 1, 0, 0, 0, TimeSpan.Zero); var span = (long)(new DateTimeOffset(2026, 10, 13, 0, 0, 0, TimeSpan.Zero) - start).TotalSeconds; var fill = Stopwatch.StartNew(); for (var batch = 0; batch < Entries / 5000; batch++) { var items = new List(5000); for (var i = 0; i < 5000; i++) { var n = (batch * 5000) + i; var kind = (n % 10) switch { 0 or 1 => EntryKind.Story, 2 => EntryKind.Highlight, 3 => EntryKind.Tagged, _ => EntryKind.Post }; var caption = string.Join(' ', Enumerable.Range(0, 12).Select(_ => words[random.Next(words.Length)])) + (n == 123_456 ? " parolarara" : string.Empty); var media = new List { Make.Stored(n % 7 == 0 ? Make.Video() : Make.Photo() with { PerceptualHash = (ulong)n * 0x9E3779B97F4A7C15UL }, Make.Sha(n + 1), $"x/{n}.jpg") }; if (n % 5 == 0) { media.Add(Make.Stored(Make.Photo(2), Make.Sha(n + 10_000_000), $"x/{n}_2.jpg")); } items.Add(new NewEntry( Make.Entry(accountIds[n % Accounts], $"id{n}", start.AddSeconds(random.NextInt64(span)), kind, caption) with { Provenance = n % 13 == 0 ? Provenance.Imported : Provenance.Downloaded }, media)); } await db.Entries.AddManyAsync(items); } fill.Stop(); TestContext.WriteLine($"Riempimento: {Entries:N0} elementi in {fill.Elapsed.TotalSeconds:F1} s"); // Plans differ with and without ANALYZE statistics: both must be fast. await MeasureAll(db, accountIds, "senza statistiche"); await db.WriteAsync(w => w.Command("ANALYZE").ExecuteNonQuery(), default); await MeasureAll(db, accountIds, "con ANALYZE"); } private async Task MeasureAll(FebeDatabase db, List accountIds, string label) { var times = new Dictionary(); async Task Measure(string name, Func> query) { await query(); // warm the page cache var best = double.MaxValue; T result = default!; for (var run = 0; run < 3; run++) { var watch = Stopwatch.StartNew(); result = await query(); best = Math.Min(best, watch.Elapsed.TotalMilliseconds); } times[name] = best; return result; } var first = await Measure("prima finestra", () => db.Entries.QueryLibraryAsync(new LibraryQuery { Limit = 200 })); Assert.HasCount(200, first.Tiles); var middleCursor = LibraryCursor.At(new DateTimeOffset(2021, 6, 1, 0, 0, 0, TimeSpan.Zero)); var middle = await Measure("finestra a metà (salto a una data)", () => db.Entries.QueryLibraryAsync(new LibraryQuery { Limit = 200, Cursor = middleCursor })); Assert.HasCount(200, middle.Tiles); Assert.IsTrue(middle.Tiles.All(t => t.PublishedAt < middleCursor.PublishedAt)); await Measure("finestra più recente da metà", () => db.Entries.QueryLibraryAsync(new LibraryQuery { Limit = 200, Cursor = middleCursor, Direction = LibraryDirection.Newer })); await Measure("filtro Video", () => db.Entries.QueryLibraryAsync(new LibraryQuery { Limit = 200, Filter = new LibraryFilter { Kind = LibraryKindFilter.Videos } })); await Measure("filtro Storie", () => db.Entries.QueryLibraryAsync(new LibraryQuery { Limit = 200, Filter = new LibraryFilter { Kind = LibraryKindFilter.Stories } })); await Measure("filtro In evidenza", () => db.Entries.QueryLibraryAsync(new LibraryQuery { Limit = 200, Filter = new LibraryFilter { Kind = LibraryKindFilter.Highlights } })); await Measure("filtro un account", () => db.Entries.QueryLibraryAsync(new LibraryQuery { Limit = 200, Filter = new LibraryFilter { AccountIds = [accountIds[3]] } })); await Measure("filtro tre account", () => db.Entries.QueryLibraryAsync(new LibraryQuery { Limit = 200, Filter = new LibraryFilter { AccountIds = [accountIds[3], accountIds[9], accountIds[30]] } })); await Measure("filtro periodo e provenienza", () => db.Entries.QueryLibraryAsync(new LibraryQuery { Limit = 200, Filter = new LibraryFilter { From = new DateTimeOffset(2020, 1, 1, 0, 0, 0, TimeSpan.Zero), To = new DateTimeOffset(2020, 2, 1, 0, 0, 0, TimeSpan.Zero), Provenance = Provenance.Imported }, })); var rare = await Measure("ricerca rara nella didascalia", () => db.Entries.QueryLibraryAsync(new LibraryQuery { Limit = 200, Filter = new LibraryFilter { Search = "PAROLARARA" } })); Assert.HasCount(1, rare.Tiles); await Measure("ricerca comune", () => db.Entries.QueryLibraryAsync(new LibraryQuery { Limit = 200, Filter = new LibraryFilter { Search = "caffe" } })); await Measure("ricerca per account", () => db.Entries.QueryLibraryAsync(new LibraryQuery { Limit = 200, Filter = new LibraryFilter { Search = "account07" } })); var totals = await Measure("totali", () => db.Entries.GetLibraryTotalsAsync(new LibraryFilter())); Assert.AreEqual(new LibraryTotals(Entries, Accounts), totals); var days = await Measure("conteggi per giorno", () => db.Entries.CountLibraryByDayAsync(new LibraryFilter(), TestZones.Rome)); Assert.AreEqual(Entries, days.Sum(d => d.Count)); await Measure("è già noto? (pagina di 12)", async () => await db.Entries.GetKnownInstagramIdsAsync(accountIds[5], EntryKind.Post, Enumerable.Range(0, 12).Select(i => $"id{5 + (i * 40)}").ToList())); await Measure("SHA-256 in tutta la libreria", () => db.Media.FindBySha256Async(Make.Sha(150_001))); var summaries = await Measure("elenco account con conteggi", () => db.Accounts.ListSummariesAsync(new DateTimeOffset(2026, 10, 1, 0, 0, 0, TimeSpan.Zero))); Assert.AreEqual(Entries, summaries.Sum(s => s.EntriesInLibrary)); await Measure("scheda account", () => db.Accounts.GetStatsAsync(accountIds[2])); await Measure("coda degli scaricamenti", () => db.Media.ListPendingDownloadsAsync(50)); var hashes = await Measure("caricamento impronte", () => db.Media.LoadPerceptualHashesAsync()); Assert.IsGreaterThan(150_000, hashes.Count); foreach (var (name, ms) in times) { TestContext.WriteLine($"[{label}] {name,-40} {ms,8:F1} ms"); } // Bounds with room for a slower CI runner and for tests running in parallel; measured values are ~10× lower. foreach (var (name, ms) in times) { var heavy = name.StartsWith("ricerca", StringComparison.Ordinal) || name.StartsWith("conteggi", StringComparison.Ordinal) || name.StartsWith("totali", StringComparison.Ordinal) || name.StartsWith("caricamento", StringComparison.Ordinal); Assert.IsLessThan(heavy ? 1500.0 : 250.0, ms, $"[{label}] {name}"); } } }