From 1fb040afe697d697c714f22a29cfa6858dcc8c0e Mon Sep 17 00:00:00 2001 From: Alberto Balbo Date: Tue, 27 Oct 2026 18:36:45 +0100 Subject: [PATCH] fix(server): a verify that hashes photos plans Reorganize again; keep refusals, bound planning The kept-plan stamp left out whether a planned sequence's photos are hashed. A verify writes frames.sha256 (and only status and verified_at on the row) without another frames version, so after verifying never-hashed photos the same dry run kept answering "verify the sequence first" for up to five minutes, and its confirmation queued a job certain to fail. - SequenceRepository.GetStampAsync adds, for every planned sequence, how many of its photos have their hash in the index (one grouped query over the archive with all, the selected ids in batches of 500 otherwise). - ReorganizePlanCache keeps a refusal (nested_target, not_found) as it keeps a plan: for the same stamp and templates it is the same, and a long plan that ends refused after a cut-off is no longer planned again on every retry. Failures are still not kept. - Planning is bounded: at most two plans are made at once, the others wait for their turn; a plan still being made 15 minutes after it was asked, let go at capacity, or when the server stops is cancelled, and whoever waits for it gets 503 plan_dropped. A read that hangs keeps its turn until it returns but no longer keeps its waiters. - The Capacity summary now says what eviction does (the kept plan asked for first, else the plan being made asked for first); the stamp, protocol and endpoint docs say that a verify hashing photos is a change the plan reads. - Tests: a scan's photos, verified, are planned again and renamed by the job; the stamp follows hashes of planned photos only; refusals kept, failures not; turns, eviction and deadline of plans being made, a hung read included. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 56582df45453fc555e525f19de7de1803204d12c) --- i18n/en/Server.resx | 1 + i18n/it/Server.it.resx | 1 + .../Repositories/SequenceRepository.cs | 40 +++++- src/Chronos.Protocol/Library.cs | 10 +- src/Chronos.Server/Api/ArchiveEndpoints.cs | 32 +++-- src/Chronos.Server/Api/ReorganizePlanCache.cs | 119 ++++++++++++--- .../Repositories/SequenceStampTests.cs | 40 ++++-- .../Api/ReorganizeTests.cs | 135 +++++++++++++++++- 8 files changed, 320 insertions(+), 58 deletions(-) diff --git a/i18n/en/Server.resx b/i18n/en/Server.resx index 9131c2eb..ff1f51e7 100644 --- a/i18n/en/Server.resx +++ b/i18n/en/Server.resx @@ -77,6 +77,7 @@ The photo was deleted from the sequence while it was being uploaded, so it was not added.long {0} cannot be read, and without its hash other copies would not learn it was deleted; close the program using it or check its permissions, then try again.long The archive changed after the dry run; review the plan again.long + The server stopped making this plan: it took too long, or too many were asked at once; try again shortly.long This page did not come through a trusted proxy over HTTPS. Open it through your proxy, or add the proxy's address to CHRONOS_TRUSTED_PROXIES.long This archive makes no proxies.long The proxy was made from another photo.long diff --git a/i18n/it/Server.it.resx b/i18n/it/Server.it.resx index 53defcb3..e49b2c47 100644 --- a/i18n/it/Server.it.resx +++ b/i18n/it/Server.it.resx @@ -75,6 +75,7 @@ La foto è stata eliminata dalla sequenza mentre veniva caricata, quindi non è stata aggiunta.long Impossibile leggere {0}: senza il suo hash le altre copie non saprebbero che è stato eliminato. Chiudi il programma che lo usa o controlla i permessi, poi riprova. L'archivio è cambiato dopo la simulazione; rivedi di nuovo il piano. + Il server ha smesso di preparare questo piano: richiedeva troppo tempo, o ne sono stati chiesti troppi insieme; riprova tra poco. Questa pagina non è arrivata da un proxy attendibile in HTTPS. Aprila tramite il proxy o aggiungi l'indirizzo del proxy a CHRONOS_TRUSTED_PROXIES. Questo archivio non crea proxy. Il proxy è stato creato da un'altra foto. diff --git a/src/Chronos.Data/Repositories/SequenceRepository.cs b/src/Chronos.Data/Repositories/SequenceRepository.cs index 81b9e4b6..699dd68f 100644 --- a/src/Chronos.Data/Repositories/SequenceRepository.cs +++ b/src/Chronos.Data/Repositories/SequenceRepository.cs @@ -431,9 +431,11 @@ public sealed class SequenceRepository /// A digest of what a Reorganize of (null: every sequence) plans from in the index of /// (owner review 2, R7): every sequence of the archive and its folder, which decide the /// folders that get " (2)" and what a moving folder carries; of the sequences planned, their title, first capture, - /// camera, lens, photo count and frames version; of the others inside their folders, the title. Another value once one - /// of these changes. Any other write (an upload into a sequence not planned, a verify, a status, tag or note change) - /// leaves it as it was, and so does a change made on disk alone, which the index does not have. + /// camera, lens, photo count, frames version and how many of their photos are hashed (only those get a new name, and a + /// verify hashes photos without another frames version); of the others inside their folders, the title. Another value + /// once one of these changes. Any other write (an upload into a sequence not planned, a verify that hashes no photo of + /// a planned sequence, a status, tag or note change) leaves it as it was, and so does a change made on disk alone, which + /// the index does not have. /// public async Task GetStampAsync(Guid archiveId, IReadOnlyCollection? sequenceIds, CancellationToken cancellationToken = default) { @@ -443,12 +445,15 @@ public sealed class SequenceRepository $"{r.GetStringOrNull(2)}\u001f{r.GetDateTimeOrNull(3):O}\u001f{r.GetStringOrNull(4)}\u001f{r.GetStringOrNull(5)}\u001f{r.GetStringOrNull(6)}\u001f" + $"{r.GetInt32(7)}\u001f{r.GetInt32(8)}")), cancellationToken).ConfigureAwait(false); var selected = sequenceIds?.ToHashSet(); + var hashed = await CountHashedAsync(archiveId, selected, cancellationToken).ConfigureAwait(false); var planned = new HashSet(rows.Where(r => selected?.Contains(r.Id) != false && r.Folder.Length > 0).Select(r => r.Folder), StringComparer.OrdinalIgnoreCase); using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); foreach (var row in rows) { - var read = selected?.Contains(row.Id) != false ? row.Read : Inside(row.Folder) ? row.Title : null; + var read = selected?.Contains(row.Id) != false + ? string.Create(CultureInfo.InvariantCulture, $"{row.Read}\u001f{hashed.GetValueOrDefault(row.Id)}") + : Inside(row.Folder) ? row.Title : null; hash.AppendData(Encoding.UTF8.GetBytes(string.Create(CultureInfo.InvariantCulture, $"{row.Id:N}\u001f{row.Folder}\u001e{read}\n"))); } return Convert.ToHexStringLower(hash.GetHashAndReset()); @@ -464,6 +469,33 @@ public sealed class SequenceRepository } } + /// + /// How many photos of each of (null: every sequence of ) have + /// their SHA-256 in the index; a sequence without one is not listed. + /// + private async Task> CountHashedAsync(Guid archiveId, IReadOnlyCollection? sequenceIds, CancellationToken cancellationToken) + { + var counts = new Dictionary(); + if (sequenceIds is null) + { + foreach (var (id, count) in await _db.ListAsync( + "SELECT f.sequence_id, count(*) FROM frames f JOIN sequences s ON s.id = f.sequence_id " + + "WHERE s.archive_id = $1 AND f.sha256 IS NOT NULL GROUP BY f.sequence_id", + [archiveId], r => (r.GetGuid(0), r.GetInt64(1)), cancellationToken).ConfigureAwait(false)) counts[id] = count; + return counts; + } + // 500 ids per statement stays below SQLite's historical 999-variable limit. + foreach (var chunk in sequenceIds.Chunk(500)) + { + var p = new SqlParameters(); + var ids = string.Join(", ", chunk.Select(id => p.Add(id))); + foreach (var (id, count) in await _db.ListAsync( + $"SELECT sequence_id, count(*) FROM frames WHERE sequence_id IN ({ids}) AND sha256 IS NOT NULL GROUP BY sequence_id", + p.ToArray(), r => (r.GetGuid(0), r.GetInt64(1)), cancellationToken).ConfigureAwait(false)) counts[id] = count; + } + return counts; + } + /// /// The sequences of a rolling verification checks next (owner review 2, R10): those never /// verified first, then the longest unverified, at most . With diff --git a/src/Chronos.Protocol/Library.cs b/src/Chronos.Protocol/Library.cs index e7eb308e..8b3f863e 100644 --- a/src/Chronos.Protocol/Library.cs +++ b/src/Chronos.Protocol/Library.cs @@ -87,10 +87,12 @@ public sealed record LibraryFacetsDto(IReadOnlyList Years, IReadOnlyLi /// templates applied to sequences already in the archive, or every one with /// (exactly one of the two). A null template keeps that part; at least one is given, each valid as the archive settings' /// templates are. answers the plan () and changes nothing; the server -/// keeps it a few minutes, so the same dry run again (also after one cut off) is answered from it. The change itself sends -/// that plan's back and runs as a move job (202): 409 plan_changed when the index -/// changed since in what the plan reads; a change made on disk alone, which the index does not have yet, is found by the -/// job, which then fails and changes nothing. +/// keeps it a few minutes, a refusal too, so the same dry run again (also after one cut off) is answered from it; 503 +/// plan_dropped when the server stopped making it (it took too long, or too many were asked at once). The change +/// itself sends that plan's back and runs as a move job (202): 409 plan_changed +/// when the index changed since in what the plan reads (the sequences' rows, and which of their photos are hashed, as a +/// verify does); a change made on disk alone, which the index does not have yet, is found by the job, which then fails and +/// changes nothing. /// public sealed record ReorganizeRequest(IReadOnlyList? SequenceIds, bool All, string? FolderTemplate, string? FileTemplate, string? TitleTemplate, bool DryRun, string? Fingerprint = null); diff --git a/src/Chronos.Server/Api/ArchiveEndpoints.cs b/src/Chronos.Server/Api/ArchiveEndpoints.cs index 4303737a..37d1ad71 100644 --- a/src/Chronos.Server/Api/ArchiveEndpoints.cs +++ b/src/Chronos.Server/Api/ArchiveEndpoints.cs @@ -141,18 +141,20 @@ public static class ArchiveEndpoints /// titleTemplate, with the rule it breaks); 404 /// not_found (sequence: not in the archive); 409 nested_target (details: the folder that would lie /// inside a sequence's folder), plan_changed (details: the fingerprint now), job_unavailable; 503 - /// archive_offline. Audited; the job tells each sequence it changes as sequenceChanged (reorganized) - /// once that sequence's step is journaled (a step that failed half-way, once its folder moved and its index rows followed; - /// a plan the server completes as it starts, no one: no client is connected yet). Photos a sequence receives between the - /// dry run and the confirmation under the names the plan leaves them keep the fingerprint as it was. A dry run's plan is made apart from - /// the request, until the server stops, and kept a few minutes while what it reads from the index stays as it was - /// (): the same dry run again (also after the first was cut off), and its confirmation, - /// take it instead of planning again. So plan_changed answers a change the index has; one made on disk alone since - /// the dry run (a folder made by hand, a desktop writing to a shared archive before a scan) is found by the job, which - /// then fails and changes nothing. + /// archive_offline, plan_dropped (the plan was stopped: past its time, let go for newer ones, or the server + /// stopping). Audited; the job tells each sequence it changes as sequenceChanged (reorganized) + /// once that sequence's step is journaled (a step that failed half-way, once its folder moved and its index rows followed; a plan the server + /// completes as it starts, no one: no client is connected yet). Photos a sequence receives between the dry run and the + /// confirmation under the names the plan leaves them keep the fingerprint as it was. A dry run's plan is made apart from + /// the request, a few at once and each for a limited time, and kept a few minutes, a refusal too, while what it reads + /// from the index stays as it was (; the sequences' rows, and how many photos of those + /// planned are hashed): the same dry run again (also after the first was cut off), and its confirmation, take it instead + /// of planning again. So plan_changed answers a change the index has, photos a verify hashed among them; one made + /// on disk alone since the dry run (a folder made by hand, a desktop writing to a shared archive before a scan) is found + /// by the job, which then fails and changes nothing. /// private static async Task ReorganizeAsync(HttpContext context, Caller caller, ReorganizeRequest request, ChronosDatabase db, ServerArchive archive, - [FromServices] ArchiveService? archives, JobEngineHost host, AuditLog audit, ReorganizePlanCache plans, IHostApplicationLifetime lifetime) + [FromServices] ArchiveService? archives, JobEngineHost host, AuditLog audit, ReorganizePlanCache plans) { var ct = context.RequestAborted; if (request.All == (request.SequenceIds is not null) || request.SequenceIds is { Count: 0 or > LibraryEndpoints.MaxBulkIds }) @@ -169,9 +171,8 @@ public static class ArchiveEndpoints RenamePlan plan; try { - // Made until the server stops, whatever becomes of this request: the same request again takes it. - var stopping = lifetime.ApplicationStopping; - plan = await plans.GetOrPlanAsync(key, () => LibraryEndpoints.Engine(archives, db).PlanReorganizeAsync(archive.Id, sequences, templates, stopping), ct); + // Made whatever becomes of this request, until the cache stops it: the same request again takes it. + plan = await plans.GetOrPlanAsync(key, stop => LibraryEndpoints.Engine(archives, db).PlanReorganizeAsync(archive.Id, sequences, templates, stop), ct); } catch (NestedRenameException ex) { @@ -181,6 +182,11 @@ public static class ArchiveEndpoints { return ApiErrors.NotFound(context, "sequence"); } + catch (OperationCanceledException) when (!ct.IsCancellationRequested) + { + // Stopped: past its time (a read that hangs), let go for newer plans, or the server stopping. + return ApiErrors.Result(context, StatusCodes.Status503ServiceUnavailable, "plan_dropped"); + } var fingerprint = plan.Fingerprint(); if (request.DryRun) return Results.Json(Plan(context, plan, fingerprint), ProtocolJson.Default.ReorganizePlanDto); if (!string.Equals(fingerprint, request.Fingerprint, StringComparison.Ordinal)) return ApiErrors.Conflict(context, "plan_changed", fingerprint); diff --git a/src/Chronos.Server/Api/ReorganizePlanCache.cs b/src/Chronos.Server/Api/ReorganizePlanCache.cs index 93b2df09..aee06934 100644 --- a/src/Chronos.Server/Api/ReorganizePlanCache.cs +++ b/src/Chronos.Server/Api/ReorganizePlanCache.cs @@ -7,27 +7,51 @@ namespace Chronos.Server.Api; /// Reorganize's dry runs kept a short while (owner review 2, R7): the same dry run asked again, and the confirmation that /// follows it, take the plan made then instead of reading every sidecar and photo folder of the selection again (with /// all, of the whole archive), while what the plan reads from the index is as it was then -/// () and for at most once made. A plan is made apart -/// from the request that asked for it (until the server stops), once for every request of the same key: a request cut off -/// meanwhile (a proxy's timeout) leaves it to be made, and the same request again waits for it or takes it. A change made -/// on disk alone, which the index does not have, is not seen by a kept plan: the move job plans again and refuses a -/// plan that no longer matches the archive, so nothing changes that the dry run did not show. +/// () and for at most once made. A refusal (a folder +/// inside a sequence's, a sequence not in the archive) follows from the same reads, so it is kept as a plan is; a failure +/// is not. A plan is made apart from the request that asked for it, once for every request of the same key: a request cut +/// off meanwhile (a proxy's timeout) leaves it to be made, and the same request again waits for it or takes it. At most +/// plans are made at once; one still being made past , let go for newer +/// ones (), or when the server stops, is stopped, and whoever waits for it is told it was dropped +/// (). A change made on disk alone, which the index does not have, is not seen by a +/// kept plan: the move job plans again and refuses a plan that no longer matches the archive, so nothing changes +/// that the dry run did not show. /// -public sealed class ReorganizePlanCache(TimeProvider time) : IDisposable +public sealed class ReorganizePlanCache : IDisposable { /// How long a plan is kept once made; then its timer lets it go. public static readonly TimeSpan Lifetime = TimeSpan.FromMinutes(5); /// /// Plans kept or being made at most (each holds a step for every sequence it changes or carries, or whose photos keep - /// their names); the oldest made goes first, then the oldest being made, which is still answered but not kept. + /// their names). One more lets go the kept plan asked for first; with none kept, the plan being made asked for first, + /// which stops. /// public const int Capacity = 8; + /// Plans made at once at most; the others wait for their turn. + public const int Concurrency = 2; + + /// + /// How long a plan may take, from when it was asked (its wait for a turn included): then it stops. A read that hangs + /// still holds its turn until it returns. + /// + public static readonly TimeSpan PlanTimeout = TimeSpan.FromMinutes(15); + + private readonly TimeProvider _time; private readonly Lock _gate = new(); private readonly Dictionary _plans = []; + private readonly SemaphoreSlim _turns = new(Concurrency); + private readonly CancellationTokenRegistration _stopping; private long _order; + /// The host, whose stopping stops the plans being made. + public ReorganizePlanCache(TimeProvider time, IHostApplicationLifetime? lifetime = null) + { + _time = time; + if (lifetime is not null) _stopping = lifetime.ApplicationStopping.Register(Clear); + } + /// /// What a plan is kept by: the archive, the stamp of what it reads from the index, the sequences (all of them, or the ids /// in the order given, which decides the folder that gets " (2)") and the templates. @@ -50,10 +74,12 @@ public sealed class ReorganizePlanCache(TimeProvider time) : IDisposable /// /// The plan kept for , the one being made for it, or the one starts making - /// now, apart from the caller, then kept. A refusal ( throws) is passed to every caller waiting for - /// it and not kept. stops the wait only: the plan is still made, and kept. + /// now, apart from the caller, then kept; stops when its token is cancelled. A refusal + /// (, ) is kept as a plan is; a failure is passed to + /// every caller waiting for it and not kept, and so is a plan stopped (). + /// stops the wait only: the plan is still made, and kept. /// - public Task GetOrPlanAsync(Key key, Func> plan, CancellationToken cancellationToken = default) + public Task GetOrPlanAsync(Key key, Func> plan, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(plan); Entry? entry; @@ -61,13 +87,14 @@ public sealed class ReorganizePlanCache(TimeProvider time) : IDisposable lock (_gate) { // Never past its lifetime, whatever its timer. - if (_plans.TryGetValue(key, out entry) && entry.Made is { } made && time.GetUtcNow() - made >= Lifetime) Remove(key); + if (_plans.TryGetValue(key, out entry) && entry.Made is { } made && _time.GetUtcNow() - made >= Lifetime) Remove(key); start = !_plans.TryGetValue(key, out entry); if (start) { if (_plans.Count >= Capacity) Remove(_plans.MinBy(p => (p.Value.Made is null, p.Value.Order)).Key); - entry = new Entry(++_order); - _plans[key] = entry; + var asked = new Entry(++_order); + asked.Deadline = _time.CreateTimer(_ => TimeOut(key, asked), null, PlanTimeout, Timeout.InfiniteTimeSpan); + _plans[key] = entry = asked; } } if (start) _ = MakeAsync(key, entry!, plan); @@ -81,6 +108,13 @@ public sealed class ReorganizePlanCache(TimeProvider time) : IDisposable } public void Dispose() + { + _stopping.Dispose(); + Clear(); + } + + /// Lets every plan go; those being made stop. + private void Clear() { lock (_gate) { @@ -88,16 +122,31 @@ public sealed class ReorganizePlanCache(TimeProvider time) : IDisposable } } - private async Task MakeAsync(Key key, Entry entry, Func> plan) + private async Task MakeAsync(Key key, Entry entry, Func> plan) { - RenamePlan made; + var stop = entry.Stop.Token; + RenamePlan? made = null; + Exception? refused = null; try { - made = await Task.Run(plan).ConfigureAwait(false); + await _turns.WaitAsync(stop).ConfigureAwait(false); + var making = Task.Run(() => plan(stop), CancellationToken.None); + // The turn ends with the work, also when no one waits for it any more (a read that hangs). + _ = making.ContinueWith(t => + { + _ = t.Exception; + _turns.Release(); + }, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); + made = await making.WaitAsync(stop).ConfigureAwait(false); + } + catch (Exception ex) when (ex is NestedRenameException || ex.GetType() == typeof(ArgumentException)) + { + // A folder inside a sequence's, a sequence not in the archive: what the key holds decides it, as it does a plan. + refused = ex; } catch (Exception ex) { - // A refusal, a failure, or the server stopping: the next request plans again. + // A failure, or the plan stopped (past its time, let go, the server stopping): the next request plans again. lock (_gate) { if (_plans.GetValueOrDefault(key) == entry) Remove(key); @@ -112,11 +161,28 @@ public sealed class ReorganizePlanCache(TimeProvider time) : IDisposable { if (_plans.GetValueOrDefault(key) == entry) { - entry.Made = time.GetUtcNow(); - entry.Expiry = time.CreateTimer(_ => Expire(key, entry), null, Lifetime, Timeout.InfiniteTimeSpan); + entry.Made = _time.GetUtcNow(); + entry.Deadline?.Dispose(); + entry.Expiry = _time.CreateTimer(_ => Expire(key, entry), null, Lifetime, Timeout.InfiniteTimeSpan); } } - entry.Plan.SetResult(made); + if (refused is null) + { + entry.Plan.SetResult(made!); + return; + } + entry.Plan.SetException(refused); + _ = entry.Plan.Task.Exception; + } + + /// A plan still being made at stops. + private void TimeOut(Key key, Entry entry) + { + lock (_gate) + { + // Made as its deadline fired: kept for its lifetime. + if (_plans.GetValueOrDefault(key) == entry && entry.Made is null) Remove(key); + } } private void Expire(Key key, Entry entry) @@ -129,7 +195,11 @@ public sealed class ReorganizePlanCache(TimeProvider time) : IDisposable private void Remove(Key key) { - if (_plans.Remove(key, out var entry)) entry.Expiry?.Dispose(); + if (!_plans.Remove(key, out var entry)) return; + entry.Deadline?.Dispose(); + entry.Expiry?.Dispose(); + // Stopped outside this lock: the wait for it ends on the thread pool. + if (entry.Made is null) _ = entry.Stop.CancelAsync(); } /// A plan being made ( null) or kept, and the order it was asked in. @@ -139,8 +209,15 @@ public sealed class ReorganizePlanCache(TimeProvider time) : IDisposable public TaskCompletionSource Plan { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + /// Stops the plan while it is made; a source with no timer and no parent, so nothing to release. + public CancellationTokenSource Stop { get; } = new(); + public DateTimeOffset? Made { get; set; } + /// Stops the plan at while it is made. + public ITimer? Deadline { get; set; } + + /// Lets the plan go at once made. public ITimer? Expiry { get; set; } } } diff --git a/tests/Chronos.Data.Tests/Repositories/SequenceStampTests.cs b/tests/Chronos.Data.Tests/Repositories/SequenceStampTests.cs index ab35011c..d48cec95 100644 --- a/tests/Chronos.Data.Tests/Repositories/SequenceStampTests.cs +++ b/tests/Chronos.Data.Tests/Repositories/SequenceStampTests.cs @@ -32,8 +32,8 @@ public abstract class SequenceStampTests(TestContext context) : RepositoryTest(c // What the plan does not read: another archive, and the writes of every upload, verify, tag or status change elsewhere. await db.Sequences.UpsertAsync(TestData.Sequence(other.Id), Ct); - await PhotoAsync(db, ridge, 0); - await PhotoAsync(db, inner, 0); + await VerifyAsync(db, await PhotoAsync(db, ridge, 0, hashed: false)); + await VerifyAsync(db, await PhotoAsync(db, inner, 0, hashed: false)); Assert.Equal(SequenceEditOutcome.Updated, (await db.Sequences.EditAsync(ridge.Id, new SequenceEdit { Title = "Ridge at dawn", Notes = "cold" }, null, Ct)).Outcome); foreach (var id in new[] { roof.Id, ridge.Id }) { @@ -48,8 +48,10 @@ public abstract class SequenceStampTests(TestContext context) : RepositoryTest(c async Task ChangedAsync(string what) => Assert.True(seen.Add(await db.Sequences.GetStampAsync(archive.Id, planned, Ct)), what); Assert.Equal(SequenceEditOutcome.Updated, (await db.Sequences.EditAsync(roof.Id, new SequenceEdit { Title = "Roof at noon" }, null, Ct)).Outcome); await ChangedAsync("title"); - await PhotoAsync(db, roof, 0); + var scanned = await PhotoAsync(db, roof, 0, hashed: false); await ChangedAsync("photos"); + await VerifyAsync(db, scanned); + await ChangedAsync("photos hashed by a verify, which a file template renames"); Assert.Equal(SequenceEditOutcome.Updated, (await db.Sequences.EditAsync(inner.Id, new SequenceEdit { Title = "Inner court" }, null, Ct)).Outcome); await ChangedAsync("title of a sequence a moving folder would carry"); Assert.True(await db.Sequences.RelocateAsync(ridge.Id, "2026/ridge", null, Ct)); @@ -62,20 +64,40 @@ public abstract class SequenceStampTests(TestContext context) : RepositoryTest(c Assert.True(await db.Sequences.DeleteAsync(added.Id, Ct)); Assert.Contains(await db.Sequences.GetStampAsync(archive.Id, planned, Ct), seen); - // With every sequence planned, a photo anywhere changes the plan read; a verify still does not. + // With every sequence planned, a photo anywhere changes the plan read, and so does a verify that hashes one; a verify + // that hashes none still does not. var all = await db.Sequences.GetStampAsync(archive.Id, null, Ct); Assert.True(await db.Sequences.RecordVerifiedAsync(ridge.Id, SequenceStatus.None, SequenceStatus.None, TestData.Now.AddDays(1), Ct)); Assert.Equal(all, await db.Sequences.GetStampAsync(archive.Id, null, Ct)); - await PhotoAsync(db, ridge, 1); - Assert.NotEqual(all, await db.Sequences.GetStampAsync(archive.Id, null, Ct)); + var photo = await PhotoAsync(db, ridge, 1, hashed: false); + var uploaded = await db.Sequences.GetStampAsync(archive.Id, null, Ct); + Assert.NotEqual(all, uploaded); + await VerifyAsync(db, photo); + var verified = await db.Sequences.GetStampAsync(archive.Id, null, Ct); + Assert.False(verified == all || verified == uploaded, "a photo hashed"); } - /// A photo indexed into a sequence, as an upload does: the frame, then the row's counts and frames version. - private async Task PhotoAsync(ChronosDatabase db, Sequence sequence, int index) + /// + /// A photo indexed into a sequence, as an upload does, or without its hash, as a scan does: the frame, then the row's + /// counts and frames version. + /// + private async Task PhotoAsync(ChronosDatabase db, Sequence sequence, int index, bool hashed = true) { var frame = TestData.Frame(sequence.Id, index); - await db.Frames.BulkInsertAsync([frame with { RelativePath = $"{sequence.Folder}/originals/{frame.FileName}" }], Ct); + frame = frame with { RelativePath = $"{sequence.Folder}/originals/{frame.FileName}", Sha256 = hashed ? frame.Sha256 : null }; + await db.Frames.BulkInsertAsync([frame], Ct); Assert.NotNull(await db.Sequences.RefreshStatsAsync(sequence.Id, Ct)); + return frame; + } + + /// What a verify writes once it hashed a photo never hashed: the hash, then the sequence's verify time, and not another frames version. + private async Task VerifyAsync(ChronosDatabase db, Frame frame) + { + var version = (await db.Sequences.GetAsync(frame.SequenceId, Ct))!.FramesVersion; + var sha256 = TestData.Frame(frame.SequenceId, frame.Index).Sha256; + Assert.Equal(1, await db.Frames.UpdateIntegrityAsync([new FrameIntegrity(frame.Id, FrameFlags.None, sha256)], Ct)); + Assert.True(await db.Sequences.RecordVerifiedAsync(frame.SequenceId, SequenceStatus.Verified, SequenceStatus.None, TestData.Now, Ct)); + Assert.Equal(version, (await db.Sequences.GetAsync(frame.SequenceId, Ct))!.FramesVersion); } } diff --git a/tests/Chronos.Server.Tests/Api/ReorganizeTests.cs b/tests/Chronos.Server.Tests/Api/ReorganizeTests.cs index 83e86819..e90c99aa 100644 --- a/tests/Chronos.Server.Tests/Api/ReorganizeTests.cs +++ b/tests/Chronos.Server.Tests/Api/ReorganizeTests.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using System.Net; using System.Net.Http.Json; using Microsoft.Extensions.DependencyInjection; @@ -496,7 +497,8 @@ public sealed class ReorganizeTests(TestContext context) Directory.CreateDirectory(target); Assert.Equal(shown.Fingerprint, (await DryRunAsync(admin, request)).Fingerprint); - // Neither are what the plan does not read: a photo uploaded into another sequence (and the jobs it queues), a verify. + // Neither are what the plan does not read: a photo uploaded into another sequence (and the jobs it queues), a verify + // of photos an upload hashed already. var versions = (await server.Database.Sequences.GetAsync(other))!.FramesVersion; await Uploads.UploadAsync(editor, Photo(6), "IMG_0101.JPG", sessionKey: "other"); using (var verify = await admin.PostAsync("/api/v1/archive/verify", null)) await verify.ExpectAsync(HttpStatusCode.Accepted); @@ -534,6 +536,35 @@ public sealed class ReorganizeTests(TestContext context) Assert.Equal("moved/2026-09-28 (2)", (await server.Database.Sequences.GetAsync(id))!.Folder); } + [Test] + public async Task ADryRunPlansAgainOnceAVerifyHashedItsPhotos() + { + await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); + var admin = await server.AdminAsync(); + // Photos copied into the archive's layout by hand and found by a scan: never hashed, they keep their names. + var originals = Path.Combine(server.Options.ArchivePath, "trips", "originals"); + Directory.CreateDirectory(originals); + for (var i = 0; i < 2; i++) await File.WriteAllBytesAsync(Path.Combine(originals, $"IMG_{i:0000}.JPG"), Photo(i)); + using (var scan = await admin.PostAsync("/api/v1/archive/scan", null)) await scan.ExpectAsync(HttpStatusCode.Accepted); + await UntilIdleAsync(server); + var id = (await server.Database.Sequences.ListLocationsAsync((await server.Database.Archives.ListAsync()).Single().Id)).Single().Id; + var request = new ReorganizeRequest([id], false, null, "{n:000}", null, DryRun: true); + var unhashed = await DryRunAsync(admin, request); + Assert.Equal((0, 2), (unhashed.Files, unhashed.Kept)); + Assert.Equal("unhashed", unhashed.Steps.Single().Blocked.Single().Reason); + + // A verify hashes them and leaves the sequence's frames version as it was: the same dry run plans again, and renames them. + var versions = (await server.Database.Sequences.GetAsync(id))!.FramesVersion; + using (var verify = await admin.PostAsync("/api/v1/archive/verify", null)) await verify.ExpectAsync(HttpStatusCode.Accepted); + await UntilIdleAsync(server); + Assert.Equal(versions, (await server.Database.Sequences.GetAsync(id))!.FramesVersion); + var hashed = await DryRunAsync(admin, request); + Assert.Equal((2, 0), (hashed.Files, hashed.Kept)); + await UntilJobAsync(admin, await ConfirmAsync(admin, request with { DryRun = false, Fingerprint = hashed.Fingerprint }), "done"); + Assert.SequenceEqual(["trips/originals/001.JPG", "trips/originals/002.JPG"], (await server.Database.Frames.ListBySequenceAsync(id)).Select(f => f.RelativePath)); + Assert.True(File.Exists(Path.Combine(originals, "001.JPG")), "renamed on disk"); + } + [Test] public async Task KeptPlansAreTheNewestFewWithinTheirLifetime() { @@ -542,7 +573,7 @@ public sealed class ReorganizeTests(TestContext context) var archive = Guid.NewGuid(); var templates = new RenameTemplates(Folder: "moved/{date}"); var made = 0; - Task Plan() + Task Plan(CancellationToken stop) { made++; return Task.FromResult(new RenamePlan(Guid.NewGuid(), archive, [])); @@ -580,9 +611,15 @@ public sealed class ReorganizeTests(TestContext context) await plans.GetOrPlanAsync(Key(0), Plan); Assert.Equal(3 + ReorganizePlanCache.Capacity, made); - // A refusal is not kept, and a plan forgotten is made again. - await Assert.ThrowsAsync(() => plans.GetOrPlanAsync(Key(99), () => throw new NestedRenameException("a/b", "a"))); - await plans.GetOrPlanAsync(Key(99), Plan); + // A refusal follows from the same reads, and is kept as a plan is: a folder inside a sequence's, a sequence not in the + // archive. A failure is not kept, and neither is a plan or refusal forgotten. + await Assert.ThrowsAsync(() => plans.GetOrPlanAsync(Key(99), _ => throw new NestedRenameException("a/b", "a"))); + await Assert.ThrowsAsync(() => plans.GetOrPlanAsync(Key(99), Plan)); + await Assert.ThrowsAsync(() => plans.GetOrPlanAsync(Key(98), _ => throw new ArgumentException("Not in the archive."))); + await Assert.ThrowsAsync(() => plans.GetOrPlanAsync(Key(98), Plan)); + Assert.Equal(3 + ReorganizePlanCache.Capacity, made); + await Assert.ThrowsAsync(() => plans.GetOrPlanAsync(Key(97), _ => throw new IOException("The sidecar cannot be read."))); + await plans.GetOrPlanAsync(Key(97), Plan); Assert.Equal(4 + ReorganizePlanCache.Capacity, made); plans.Forget(Key(99)); await plans.GetOrPlanAsync(Key(99), Plan); @@ -598,7 +635,7 @@ public sealed class ReorganizeTests(TestContext context) var key = ReorganizePlanCache.Key.Of(Guid.NewGuid(), "stamp", null, templates); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var made = 0; - async Task SlowPlan() + async Task SlowPlan(CancellationToken stop) { Interlocked.Increment(ref made); await release.Task; @@ -632,13 +669,97 @@ public sealed class ReorganizeTests(TestContext context) await plans.GetOrPlanAsync(key, SlowPlan); Assert.Equal(2 + ReorganizePlanCache.Capacity, made); - Task PlanAtOnce() + Task PlanAtOnce(CancellationToken stop) { Interlocked.Increment(ref made); return Task.FromResult(new RenamePlan(Guid.NewGuid(), key.ArchiveId, [])); } } + [Test] + public async Task PlansAreMadeAFewAtOnceAndStopWhenLetGoOrPastTheirTime() + { + var clock = new SteppedTimeProvider(); + using var plans = new ReorganizePlanCache(clock); + var archive = Guid.NewGuid(); + var templates = new RenameTemplates(Folder: "moved/{date}"); + ReorganizePlanCache.Key Key(int i) => ReorganizePlanCache.Key.Of(archive, "stamp", [new Guid(i, 0, 0, new byte[8])], templates); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var started = new ConcurrentDictionary(); + var stopped = new ConcurrentDictionary(); + // A plan made once released, which tells when it starts and when it is stopped. + Func> Plan(int i) => async stop => + { + started[i] = true; + try + { + await release.Task.WaitAsync(stop); + } + catch (OperationCanceledException) + { + stopped[i] = true; + throw; + } + return new RenamePlan(Guid.NewGuid(), archive, []); + }; + + // A few are made at once; the others wait for their turn. + var asked = Enumerable.Range(0, ReorganizePlanCache.Capacity).Select(i => plans.GetOrPlanAsync(Key(i), Plan(i))).ToList(); + await EventuallyAsync(() => started.Count == ReorganizePlanCache.Concurrency); + await Task.Delay(200); + Assert.SequenceEqual(Enumerable.Range(0, ReorganizePlanCache.Concurrency), started.Keys.Order()); + + // One more lets go the plan asked first, being made with none kept: it stops, its waiter is told, the next one starts. + var newest = plans.GetOrPlanAsync(Key(ReorganizePlanCache.Capacity), Plan(ReorganizePlanCache.Capacity)); + await Assert.ThrowsAsync(() => asked[0]); + await EventuallyAsync(() => stopped.ContainsKey(0) && started.ContainsKey(ReorganizePlanCache.Concurrency)); + Assert.Equal(ReorganizePlanCache.Capacity, plans.Count); + release.SetResult(); + foreach (var plan in asked.Skip(1).Append(newest)) await plan; + Assert.SequenceEqual([0], stopped.Keys); + + // A plan still being made at its time stops, also when its work never ends (a read that hangs), and so does one still + // waiting for its turn, which never starts. + var hang = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + Func> Hanging(int i) => _ => + { + started[i] = true; + return hang.Task; + }; + var hung = plans.GetOrPlanAsync(Key(20), Hanging(20)); + var stuck = plans.GetOrPlanAsync(Key(21), Hanging(21)); + await EventuallyAsync(() => started.ContainsKey(20) && started.ContainsKey(21)); + var waiting = plans.GetOrPlanAsync(Key(22), Plan(22)); + clock.Advance(ReorganizePlanCache.PlanTimeout - TimeSpan.FromSeconds(1)); + Assert.False(hung.IsCompleted, "still in time"); + clock.Advance(TimeSpan.FromSeconds(1)); + foreach (var plan in new[] { hung, stuck, waiting }) await Assert.ThrowsAsync(() => plan); + Assert.False(started.ContainsKey(22), "stopped before its turn"); + Assert.Equal(0, plans.Count); + + // Asked again, it is made once a turn is free: the work that hangs holds its turn until it ends, and what it makes + // then is not kept. + var again = plans.GetOrPlanAsync(Key(20), Plan(30)); + await Task.Delay(200); + Assert.False(started.ContainsKey(30), "no turn while two works hang"); + hang.SetResult(new RenamePlan(Guid.NewGuid(), archive, [])); + var made = await again; + Assert.True(ReferenceEquals(made, await plans.GetOrPlanAsync(Key(20), Plan(31))), "kept"); + Assert.False(started.ContainsKey(31), "kept, not made again"); + Assert.Equal(1, plans.Count); + } + + /// Waits until holds. + private static async Task EventuallyAsync(Func condition, [System.Runtime.CompilerServices.CallerArgumentExpression(nameof(condition))] string? expression = null) + { + var until = DateTime.UtcNow + Wait; + while (!condition()) + { + Assert.True(DateTime.UtcNow < until, expression); + await Task.Delay(10); + } + } + private static async Task DryRunAsync(HttpClient client, ReorganizeRequest request) { using var response = await client.PostAsJsonAsync(Url, request with { DryRun = true }, ProtocolJson.Default.ReorganizeRequest);