diff --git a/i18n/en/Server.resx b/i18n/en/Server.resx index 9131c2eb..ff1f51e7 100644 --- a/i18n/en/Server.resx +++ b/i18n/en/Server.resx @@ -77,6 +77,7 @@ The photo was deleted from the sequence while it was being uploaded, so it was not added.long {0} cannot be read, and without its hash other copies would not learn it was deleted; close the program using it or check its permissions, then try again.long The archive changed after the dry run; review the plan again.long + The server stopped making this plan: it took too long, or too many were asked at once; try again shortly.long This page did not come through a trusted proxy over HTTPS. Open it through your proxy, or add the proxy's address to CHRONOS_TRUSTED_PROXIES.long This archive makes no proxies.long The proxy was made from another photo.long diff --git a/i18n/it/Server.it.resx b/i18n/it/Server.it.resx index 53defcb3..e49b2c47 100644 --- a/i18n/it/Server.it.resx +++ b/i18n/it/Server.it.resx @@ -75,6 +75,7 @@ La foto è stata eliminata dalla sequenza mentre veniva caricata, quindi non è stata aggiunta.long Impossibile leggere {0}: senza il suo hash le altre copie non saprebbero che è stato eliminato. Chiudi il programma che lo usa o controlla i permessi, poi riprova. L'archivio è cambiato dopo la simulazione; rivedi di nuovo il piano. + Il server ha smesso di preparare questo piano: richiedeva troppo tempo, o ne sono stati chiesti troppi insieme; riprova tra poco. Questa pagina non è arrivata da un proxy attendibile in HTTPS. Aprila tramite il proxy o aggiungi l'indirizzo del proxy a CHRONOS_TRUSTED_PROXIES. Questo archivio non crea proxy. Il proxy è stato creato da un'altra foto. diff --git a/src/Chronos.Data/Repositories/SequenceRepository.cs b/src/Chronos.Data/Repositories/SequenceRepository.cs index 81b9e4b6..699dd68f 100644 --- a/src/Chronos.Data/Repositories/SequenceRepository.cs +++ b/src/Chronos.Data/Repositories/SequenceRepository.cs @@ -431,9 +431,11 @@ public sealed class SequenceRepository /// A digest of what a Reorganize of (null: every sequence) plans from in the index of /// (owner review 2, R7): every sequence of the archive and its folder, which decide the /// folders that get " (2)" and what a moving folder carries; of the sequences planned, their title, first capture, - /// camera, lens, photo count and frames version; of the others inside their folders, the title. Another value once one - /// of these changes. Any other write (an upload into a sequence not planned, a verify, a status, tag or note change) - /// leaves it as it was, and so does a change made on disk alone, which the index does not have. + /// camera, lens, photo count, frames version and how many of their photos are hashed (only those get a new name, and a + /// verify hashes photos without another frames version); of the others inside their folders, the title. Another value + /// once one of these changes. Any other write (an upload into a sequence not planned, a verify that hashes no photo of + /// a planned sequence, a status, tag or note change) leaves it as it was, and so does a change made on disk alone, which + /// the index does not have. /// public async Task GetStampAsync(Guid archiveId, IReadOnlyCollection? sequenceIds, CancellationToken cancellationToken = default) { @@ -443,12 +445,15 @@ public sealed class SequenceRepository $"{r.GetStringOrNull(2)}\u001f{r.GetDateTimeOrNull(3):O}\u001f{r.GetStringOrNull(4)}\u001f{r.GetStringOrNull(5)}\u001f{r.GetStringOrNull(6)}\u001f" + $"{r.GetInt32(7)}\u001f{r.GetInt32(8)}")), cancellationToken).ConfigureAwait(false); var selected = sequenceIds?.ToHashSet(); + var hashed = await CountHashedAsync(archiveId, selected, cancellationToken).ConfigureAwait(false); var planned = new HashSet(rows.Where(r => selected?.Contains(r.Id) != false && r.Folder.Length > 0).Select(r => r.Folder), StringComparer.OrdinalIgnoreCase); using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); foreach (var row in rows) { - var read = selected?.Contains(row.Id) != false ? row.Read : Inside(row.Folder) ? row.Title : null; + var read = selected?.Contains(row.Id) != false + ? string.Create(CultureInfo.InvariantCulture, $"{row.Read}\u001f{hashed.GetValueOrDefault(row.Id)}") + : Inside(row.Folder) ? row.Title : null; hash.AppendData(Encoding.UTF8.GetBytes(string.Create(CultureInfo.InvariantCulture, $"{row.Id:N}\u001f{row.Folder}\u001e{read}\n"))); } return Convert.ToHexStringLower(hash.GetHashAndReset()); @@ -464,6 +469,33 @@ public sealed class SequenceRepository } } + /// + /// How many photos of each of (null: every sequence of ) have + /// their SHA-256 in the index; a sequence without one is not listed. + /// + private async Task> CountHashedAsync(Guid archiveId, IReadOnlyCollection? sequenceIds, CancellationToken cancellationToken) + { + var counts = new Dictionary(); + if (sequenceIds is null) + { + foreach (var (id, count) in await _db.ListAsync( + "SELECT f.sequence_id, count(*) FROM frames f JOIN sequences s ON s.id = f.sequence_id " + + "WHERE s.archive_id = $1 AND f.sha256 IS NOT NULL GROUP BY f.sequence_id", + [archiveId], r => (r.GetGuid(0), r.GetInt64(1)), cancellationToken).ConfigureAwait(false)) counts[id] = count; + return counts; + } + // 500 ids per statement stays below SQLite's historical 999-variable limit. + foreach (var chunk in sequenceIds.Chunk(500)) + { + var p = new SqlParameters(); + var ids = string.Join(", ", chunk.Select(id => p.Add(id))); + foreach (var (id, count) in await _db.ListAsync( + $"SELECT sequence_id, count(*) FROM frames WHERE sequence_id IN ({ids}) AND sha256 IS NOT NULL GROUP BY sequence_id", + p.ToArray(), r => (r.GetGuid(0), r.GetInt64(1)), cancellationToken).ConfigureAwait(false)) counts[id] = count; + } + return counts; + } + /// /// The sequences of a rolling verification checks next (owner review 2, R10): those never /// verified first, then the longest unverified, at most . With diff --git a/src/Chronos.Protocol/Library.cs b/src/Chronos.Protocol/Library.cs index e7eb308e..8b3f863e 100644 --- a/src/Chronos.Protocol/Library.cs +++ b/src/Chronos.Protocol/Library.cs @@ -87,10 +87,12 @@ public sealed record LibraryFacetsDto(IReadOnlyList Years, IReadOnlyLi /// templates applied to sequences already in the archive, or every one with /// (exactly one of the two). A null template keeps that part; at least one is given, each valid as the archive settings' /// templates are. answers the plan () and changes nothing; the server -/// keeps it a few minutes, so the same dry run again (also after one cut off) is answered from it. The change itself sends -/// that plan's back and runs as a move job (202): 409 plan_changed when the index -/// changed since in what the plan reads; a change made on disk alone, which the index does not have yet, is found by the -/// job, which then fails and changes nothing. +/// keeps it a few minutes, a refusal too, so the same dry run again (also after one cut off) is answered from it; 503 +/// plan_dropped when the server stopped making it (it took too long, or too many were asked at once). The change +/// itself sends that plan's back and runs as a move job (202): 409 plan_changed +/// when the index changed since in what the plan reads (the sequences' rows, and which of their photos are hashed, as a +/// verify does); a change made on disk alone, which the index does not have yet, is found by the job, which then fails and +/// changes nothing. /// public sealed record ReorganizeRequest(IReadOnlyList? SequenceIds, bool All, string? FolderTemplate, string? FileTemplate, string? TitleTemplate, bool DryRun, string? Fingerprint = null); diff --git a/src/Chronos.Server/Api/ArchiveEndpoints.cs b/src/Chronos.Server/Api/ArchiveEndpoints.cs index 4303737a..37d1ad71 100644 --- a/src/Chronos.Server/Api/ArchiveEndpoints.cs +++ b/src/Chronos.Server/Api/ArchiveEndpoints.cs @@ -141,18 +141,20 @@ public static class ArchiveEndpoints /// titleTemplate, with the rule it breaks); 404 /// not_found (sequence: not in the archive); 409 nested_target (details: the folder that would lie /// inside a sequence's folder), plan_changed (details: the fingerprint now), job_unavailable; 503 - /// archive_offline. Audited; the job tells each sequence it changes as sequenceChanged (reorganized) - /// once that sequence's step is journaled (a step that failed half-way, once its folder moved and its index rows followed; - /// a plan the server completes as it starts, no one: no client is connected yet). Photos a sequence receives between the - /// dry run and the confirmation under the names the plan leaves them keep the fingerprint as it was. A dry run's plan is made apart from - /// the request, until the server stops, and kept a few minutes while what it reads from the index stays as it was - /// (): the same dry run again (also after the first was cut off), and its confirmation, - /// take it instead of planning again. So plan_changed answers a change the index has; one made on disk alone since - /// the dry run (a folder made by hand, a desktop writing to a shared archive before a scan) is found by the job, which - /// then fails and changes nothing. + /// archive_offline, plan_dropped (the plan was stopped: past its time, let go for newer ones, or the server + /// stopping). Audited; the job tells each sequence it changes as sequenceChanged (reorganized) + /// once that sequence's step is journaled (a step that failed half-way, once its folder moved and its index rows followed; a plan the server + /// completes as it starts, no one: no client is connected yet). Photos a sequence receives between the dry run and the + /// confirmation under the names the plan leaves them keep the fingerprint as it was. A dry run's plan is made apart from + /// the request, a few at once and each for a limited time, and kept a few minutes, a refusal too, while what it reads + /// from the index stays as it was (; the sequences' rows, and how many photos of those + /// planned are hashed): the same dry run again (also after the first was cut off), and its confirmation, take it instead + /// of planning again. So plan_changed answers a change the index has, photos a verify hashed among them; one made + /// on disk alone since the dry run (a folder made by hand, a desktop writing to a shared archive before a scan) is found + /// by the job, which then fails and changes nothing. /// private static async Task ReorganizeAsync(HttpContext context, Caller caller, ReorganizeRequest request, ChronosDatabase db, ServerArchive archive, - [FromServices] ArchiveService? archives, JobEngineHost host, AuditLog audit, ReorganizePlanCache plans, IHostApplicationLifetime lifetime) + [FromServices] ArchiveService? archives, JobEngineHost host, AuditLog audit, ReorganizePlanCache plans) { var ct = context.RequestAborted; if (request.All == (request.SequenceIds is not null) || request.SequenceIds is { Count: 0 or > LibraryEndpoints.MaxBulkIds }) @@ -169,9 +171,8 @@ public static class ArchiveEndpoints RenamePlan plan; try { - // Made until the server stops, whatever becomes of this request: the same request again takes it. - var stopping = lifetime.ApplicationStopping; - plan = await plans.GetOrPlanAsync(key, () => LibraryEndpoints.Engine(archives, db).PlanReorganizeAsync(archive.Id, sequences, templates, stopping), ct); + // Made whatever becomes of this request, until the cache stops it: the same request again takes it. + plan = await plans.GetOrPlanAsync(key, stop => LibraryEndpoints.Engine(archives, db).PlanReorganizeAsync(archive.Id, sequences, templates, stop), ct); } catch (NestedRenameException ex) { @@ -181,6 +182,11 @@ public static class ArchiveEndpoints { return ApiErrors.NotFound(context, "sequence"); } + catch (OperationCanceledException) when (!ct.IsCancellationRequested) + { + // Stopped: past its time (a read that hangs), let go for newer plans, or the server stopping. + return ApiErrors.Result(context, StatusCodes.Status503ServiceUnavailable, "plan_dropped"); + } var fingerprint = plan.Fingerprint(); if (request.DryRun) return Results.Json(Plan(context, plan, fingerprint), ProtocolJson.Default.ReorganizePlanDto); if (!string.Equals(fingerprint, request.Fingerprint, StringComparison.Ordinal)) return ApiErrors.Conflict(context, "plan_changed", fingerprint); diff --git a/src/Chronos.Server/Api/ReorganizePlanCache.cs b/src/Chronos.Server/Api/ReorganizePlanCache.cs index 93b2df09..aee06934 100644 --- a/src/Chronos.Server/Api/ReorganizePlanCache.cs +++ b/src/Chronos.Server/Api/ReorganizePlanCache.cs @@ -7,27 +7,51 @@ namespace Chronos.Server.Api; /// Reorganize's dry runs kept a short while (owner review 2, R7): the same dry run asked again, and the confirmation that /// follows it, take the plan made then instead of reading every sidecar and photo folder of the selection again (with /// all, of the whole archive), while what the plan reads from the index is as it was then -/// () and for at most once made. A plan is made apart -/// from the request that asked for it (until the server stops), once for every request of the same key: a request cut off -/// meanwhile (a proxy's timeout) leaves it to be made, and the same request again waits for it or takes it. A change made -/// on disk alone, which the index does not have, is not seen by a kept plan: the move job plans again and refuses a -/// plan that no longer matches the archive, so nothing changes that the dry run did not show. +/// () and for at most once made. A refusal (a folder +/// inside a sequence's, a sequence not in the archive) follows from the same reads, so it is kept as a plan is; a failure +/// is not. A plan is made apart from the request that asked for it, once for every request of the same key: a request cut +/// off meanwhile (a proxy's timeout) leaves it to be made, and the same request again waits for it or takes it. At most +/// plans are made at once; one still being made past , let go for newer +/// ones (), or when the server stops, is stopped, and whoever waits for it is told it was dropped +/// (). A change made on disk alone, which the index does not have, is not seen by a +/// kept plan: the move job plans again and refuses a plan that no longer matches the archive, so nothing changes +/// that the dry run did not show. /// -public sealed class ReorganizePlanCache(TimeProvider time) : IDisposable +public sealed class ReorganizePlanCache : IDisposable { /// How long a plan is kept once made; then its timer lets it go. public static readonly TimeSpan Lifetime = TimeSpan.FromMinutes(5); /// /// Plans kept or being made at most (each holds a step for every sequence it changes or carries, or whose photos keep - /// their names); the oldest made goes first, then the oldest being made, which is still answered but not kept. + /// their names). One more lets go the kept plan asked for first; with none kept, the plan being made asked for first, + /// which stops. /// public const int Capacity = 8; + /// Plans made at once at most; the others wait for their turn. + public const int Concurrency = 2; + + /// + /// How long a plan may take, from when it was asked (its wait for a turn included): then it stops. A read that hangs + /// still holds its turn until it returns. + /// + public static readonly TimeSpan PlanTimeout = TimeSpan.FromMinutes(15); + + private readonly TimeProvider _time; private readonly Lock _gate = new(); private readonly Dictionary _plans = []; + private readonly SemaphoreSlim _turns = new(Concurrency); + private readonly CancellationTokenRegistration _stopping; private long _order; + /// The host, whose stopping stops the plans being made. + public ReorganizePlanCache(TimeProvider time, IHostApplicationLifetime? lifetime = null) + { + _time = time; + if (lifetime is not null) _stopping = lifetime.ApplicationStopping.Register(Clear); + } + /// /// What a plan is kept by: the archive, the stamp of what it reads from the index, the sequences (all of them, or the ids /// in the order given, which decides the folder that gets " (2)") and the templates. @@ -50,10 +74,12 @@ public sealed class ReorganizePlanCache(TimeProvider time) : IDisposable /// /// The plan kept for , the one being made for it, or the one starts making - /// now, apart from the caller, then kept. A refusal ( throws) is passed to every caller waiting for - /// it and not kept. stops the wait only: the plan is still made, and kept. + /// now, apart from the caller, then kept; stops when its token is cancelled. A refusal + /// (, ) is kept as a plan is; a failure is passed to + /// every caller waiting for it and not kept, and so is a plan stopped (). + /// stops the wait only: the plan is still made, and kept. /// - public Task GetOrPlanAsync(Key key, Func> plan, CancellationToken cancellationToken = default) + public Task GetOrPlanAsync(Key key, Func> plan, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(plan); Entry? entry; @@ -61,13 +87,14 @@ public sealed class ReorganizePlanCache(TimeProvider time) : IDisposable lock (_gate) { // Never past its lifetime, whatever its timer. - if (_plans.TryGetValue(key, out entry) && entry.Made is { } made && time.GetUtcNow() - made >= Lifetime) Remove(key); + if (_plans.TryGetValue(key, out entry) && entry.Made is { } made && _time.GetUtcNow() - made >= Lifetime) Remove(key); start = !_plans.TryGetValue(key, out entry); if (start) { if (_plans.Count >= Capacity) Remove(_plans.MinBy(p => (p.Value.Made is null, p.Value.Order)).Key); - entry = new Entry(++_order); - _plans[key] = entry; + var asked = new Entry(++_order); + asked.Deadline = _time.CreateTimer(_ => TimeOut(key, asked), null, PlanTimeout, Timeout.InfiniteTimeSpan); + _plans[key] = entry = asked; } } if (start) _ = MakeAsync(key, entry!, plan); @@ -81,6 +108,13 @@ public sealed class ReorganizePlanCache(TimeProvider time) : IDisposable } public void Dispose() + { + _stopping.Dispose(); + Clear(); + } + + /// Lets every plan go; those being made stop. + private void Clear() { lock (_gate) { @@ -88,16 +122,31 @@ public sealed class ReorganizePlanCache(TimeProvider time) : IDisposable } } - private async Task MakeAsync(Key key, Entry entry, Func> plan) + private async Task MakeAsync(Key key, Entry entry, Func> plan) { - RenamePlan made; + var stop = entry.Stop.Token; + RenamePlan? made = null; + Exception? refused = null; try { - made = await Task.Run(plan).ConfigureAwait(false); + await _turns.WaitAsync(stop).ConfigureAwait(false); + var making = Task.Run(() => plan(stop), CancellationToken.None); + // The turn ends with the work, also when no one waits for it any more (a read that hangs). + _ = making.ContinueWith(t => + { + _ = t.Exception; + _turns.Release(); + }, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); + made = await making.WaitAsync(stop).ConfigureAwait(false); + } + catch (Exception ex) when (ex is NestedRenameException || ex.GetType() == typeof(ArgumentException)) + { + // A folder inside a sequence's, a sequence not in the archive: what the key holds decides it, as it does a plan. + refused = ex; } catch (Exception ex) { - // A refusal, a failure, or the server stopping: the next request plans again. + // A failure, or the plan stopped (past its time, let go, the server stopping): the next request plans again. lock (_gate) { if (_plans.GetValueOrDefault(key) == entry) Remove(key); @@ -112,11 +161,28 @@ public sealed class ReorganizePlanCache(TimeProvider time) : IDisposable { if (_plans.GetValueOrDefault(key) == entry) { - entry.Made = time.GetUtcNow(); - entry.Expiry = time.CreateTimer(_ => Expire(key, entry), null, Lifetime, Timeout.InfiniteTimeSpan); + entry.Made = _time.GetUtcNow(); + entry.Deadline?.Dispose(); + entry.Expiry = _time.CreateTimer(_ => Expire(key, entry), null, Lifetime, Timeout.InfiniteTimeSpan); } } - entry.Plan.SetResult(made); + if (refused is null) + { + entry.Plan.SetResult(made!); + return; + } + entry.Plan.SetException(refused); + _ = entry.Plan.Task.Exception; + } + + /// A plan still being made at stops. + private void TimeOut(Key key, Entry entry) + { + lock (_gate) + { + // Made as its deadline fired: kept for its lifetime. + if (_plans.GetValueOrDefault(key) == entry && entry.Made is null) Remove(key); + } } private void Expire(Key key, Entry entry) @@ -129,7 +195,11 @@ public sealed class ReorganizePlanCache(TimeProvider time) : IDisposable private void Remove(Key key) { - if (_plans.Remove(key, out var entry)) entry.Expiry?.Dispose(); + if (!_plans.Remove(key, out var entry)) return; + entry.Deadline?.Dispose(); + entry.Expiry?.Dispose(); + // Stopped outside this lock: the wait for it ends on the thread pool. + if (entry.Made is null) _ = entry.Stop.CancelAsync(); } /// A plan being made ( null) or kept, and the order it was asked in. @@ -139,8 +209,15 @@ public sealed class ReorganizePlanCache(TimeProvider time) : IDisposable public TaskCompletionSource Plan { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + /// Stops the plan while it is made; a source with no timer and no parent, so nothing to release. + public CancellationTokenSource Stop { get; } = new(); + public DateTimeOffset? Made { get; set; } + /// Stops the plan at while it is made. + public ITimer? Deadline { get; set; } + + /// Lets the plan go at once made. public ITimer? Expiry { get; set; } } } diff --git a/tests/Chronos.Data.Tests/Repositories/SequenceStampTests.cs b/tests/Chronos.Data.Tests/Repositories/SequenceStampTests.cs index ab35011c..d48cec95 100644 --- a/tests/Chronos.Data.Tests/Repositories/SequenceStampTests.cs +++ b/tests/Chronos.Data.Tests/Repositories/SequenceStampTests.cs @@ -32,8 +32,8 @@ public abstract class SequenceStampTests(TestContext context) : RepositoryTest(c // What the plan does not read: another archive, and the writes of every upload, verify, tag or status change elsewhere. await db.Sequences.UpsertAsync(TestData.Sequence(other.Id), Ct); - await PhotoAsync(db, ridge, 0); - await PhotoAsync(db, inner, 0); + await VerifyAsync(db, await PhotoAsync(db, ridge, 0, hashed: false)); + await VerifyAsync(db, await PhotoAsync(db, inner, 0, hashed: false)); Assert.Equal(SequenceEditOutcome.Updated, (await db.Sequences.EditAsync(ridge.Id, new SequenceEdit { Title = "Ridge at dawn", Notes = "cold" }, null, Ct)).Outcome); foreach (var id in new[] { roof.Id, ridge.Id }) { @@ -48,8 +48,10 @@ public abstract class SequenceStampTests(TestContext context) : RepositoryTest(c async Task ChangedAsync(string what) => Assert.True(seen.Add(await db.Sequences.GetStampAsync(archive.Id, planned, Ct)), what); Assert.Equal(SequenceEditOutcome.Updated, (await db.Sequences.EditAsync(roof.Id, new SequenceEdit { Title = "Roof at noon" }, null, Ct)).Outcome); await ChangedAsync("title"); - await PhotoAsync(db, roof, 0); + var scanned = await PhotoAsync(db, roof, 0, hashed: false); await ChangedAsync("photos"); + await VerifyAsync(db, scanned); + await ChangedAsync("photos hashed by a verify, which a file template renames"); Assert.Equal(SequenceEditOutcome.Updated, (await db.Sequences.EditAsync(inner.Id, new SequenceEdit { Title = "Inner court" }, null, Ct)).Outcome); await ChangedAsync("title of a sequence a moving folder would carry"); Assert.True(await db.Sequences.RelocateAsync(ridge.Id, "2026/ridge", null, Ct)); @@ -62,20 +64,40 @@ public abstract class SequenceStampTests(TestContext context) : RepositoryTest(c Assert.True(await db.Sequences.DeleteAsync(added.Id, Ct)); Assert.Contains(await db.Sequences.GetStampAsync(archive.Id, planned, Ct), seen); - // With every sequence planned, a photo anywhere changes the plan read; a verify still does not. + // With every sequence planned, a photo anywhere changes the plan read, and so does a verify that hashes one; a verify + // that hashes none still does not. var all = await db.Sequences.GetStampAsync(archive.Id, null, Ct); Assert.True(await db.Sequences.RecordVerifiedAsync(ridge.Id, SequenceStatus.None, SequenceStatus.None, TestData.Now.AddDays(1), Ct)); Assert.Equal(all, await db.Sequences.GetStampAsync(archive.Id, null, Ct)); - await PhotoAsync(db, ridge, 1); - Assert.NotEqual(all, await db.Sequences.GetStampAsync(archive.Id, null, Ct)); + var photo = await PhotoAsync(db, ridge, 1, hashed: false); + var uploaded = await db.Sequences.GetStampAsync(archive.Id, null, Ct); + Assert.NotEqual(all, uploaded); + await VerifyAsync(db, photo); + var verified = await db.Sequences.GetStampAsync(archive.Id, null, Ct); + Assert.False(verified == all || verified == uploaded, "a photo hashed"); } - /// A photo indexed into a sequence, as an upload does: the frame, then the row's counts and frames version. - private async Task PhotoAsync(ChronosDatabase db, Sequence sequence, int index) + /// + /// A photo indexed into a sequence, as an upload does, or without its hash, as a scan does: the frame, then the row's + /// counts and frames version. + /// + private async Task PhotoAsync(ChronosDatabase db, Sequence sequence, int index, bool hashed = true) { var frame = TestData.Frame(sequence.Id, index); - await db.Frames.BulkInsertAsync([frame with { RelativePath = $"{sequence.Folder}/originals/{frame.FileName}" }], Ct); + frame = frame with { RelativePath = $"{sequence.Folder}/originals/{frame.FileName}", Sha256 = hashed ? frame.Sha256 : null }; + await db.Frames.BulkInsertAsync([frame], Ct); Assert.NotNull(await db.Sequences.RefreshStatsAsync(sequence.Id, Ct)); + return frame; + } + + /// What a verify writes once it hashed a photo never hashed: the hash, then the sequence's verify time, and not another frames version. + private async Task VerifyAsync(ChronosDatabase db, Frame frame) + { + var version = (await db.Sequences.GetAsync(frame.SequenceId, Ct))!.FramesVersion; + var sha256 = TestData.Frame(frame.SequenceId, frame.Index).Sha256; + Assert.Equal(1, await db.Frames.UpdateIntegrityAsync([new FrameIntegrity(frame.Id, FrameFlags.None, sha256)], Ct)); + Assert.True(await db.Sequences.RecordVerifiedAsync(frame.SequenceId, SequenceStatus.Verified, SequenceStatus.None, TestData.Now, Ct)); + Assert.Equal(version, (await db.Sequences.GetAsync(frame.SequenceId, Ct))!.FramesVersion); } } diff --git a/tests/Chronos.Server.Tests/Api/ReorganizeTests.cs b/tests/Chronos.Server.Tests/Api/ReorganizeTests.cs index 83e86819..e90c99aa 100644 --- a/tests/Chronos.Server.Tests/Api/ReorganizeTests.cs +++ b/tests/Chronos.Server.Tests/Api/ReorganizeTests.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using System.Net; using System.Net.Http.Json; using Microsoft.Extensions.DependencyInjection; @@ -496,7 +497,8 @@ public sealed class ReorganizeTests(TestContext context) Directory.CreateDirectory(target); Assert.Equal(shown.Fingerprint, (await DryRunAsync(admin, request)).Fingerprint); - // Neither are what the plan does not read: a photo uploaded into another sequence (and the jobs it queues), a verify. + // Neither are what the plan does not read: a photo uploaded into another sequence (and the jobs it queues), a verify + // of photos an upload hashed already. var versions = (await server.Database.Sequences.GetAsync(other))!.FramesVersion; await Uploads.UploadAsync(editor, Photo(6), "IMG_0101.JPG", sessionKey: "other"); using (var verify = await admin.PostAsync("/api/v1/archive/verify", null)) await verify.ExpectAsync(HttpStatusCode.Accepted); @@ -534,6 +536,35 @@ public sealed class ReorganizeTests(TestContext context) Assert.Equal("moved/2026-09-28 (2)", (await server.Database.Sequences.GetAsync(id))!.Folder); } + [Test] + public async Task ADryRunPlansAgainOnceAVerifyHashedItsPhotos() + { + await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); + var admin = await server.AdminAsync(); + // Photos copied into the archive's layout by hand and found by a scan: never hashed, they keep their names. + var originals = Path.Combine(server.Options.ArchivePath, "trips", "originals"); + Directory.CreateDirectory(originals); + for (var i = 0; i < 2; i++) await File.WriteAllBytesAsync(Path.Combine(originals, $"IMG_{i:0000}.JPG"), Photo(i)); + using (var scan = await admin.PostAsync("/api/v1/archive/scan", null)) await scan.ExpectAsync(HttpStatusCode.Accepted); + await UntilIdleAsync(server); + var id = (await server.Database.Sequences.ListLocationsAsync((await server.Database.Archives.ListAsync()).Single().Id)).Single().Id; + var request = new ReorganizeRequest([id], false, null, "{n:000}", null, DryRun: true); + var unhashed = await DryRunAsync(admin, request); + Assert.Equal((0, 2), (unhashed.Files, unhashed.Kept)); + Assert.Equal("unhashed", unhashed.Steps.Single().Blocked.Single().Reason); + + // A verify hashes them and leaves the sequence's frames version as it was: the same dry run plans again, and renames them. + var versions = (await server.Database.Sequences.GetAsync(id))!.FramesVersion; + using (var verify = await admin.PostAsync("/api/v1/archive/verify", null)) await verify.ExpectAsync(HttpStatusCode.Accepted); + await UntilIdleAsync(server); + Assert.Equal(versions, (await server.Database.Sequences.GetAsync(id))!.FramesVersion); + var hashed = await DryRunAsync(admin, request); + Assert.Equal((2, 0), (hashed.Files, hashed.Kept)); + await UntilJobAsync(admin, await ConfirmAsync(admin, request with { DryRun = false, Fingerprint = hashed.Fingerprint }), "done"); + Assert.SequenceEqual(["trips/originals/001.JPG", "trips/originals/002.JPG"], (await server.Database.Frames.ListBySequenceAsync(id)).Select(f => f.RelativePath)); + Assert.True(File.Exists(Path.Combine(originals, "001.JPG")), "renamed on disk"); + } + [Test] public async Task KeptPlansAreTheNewestFewWithinTheirLifetime() { @@ -542,7 +573,7 @@ public sealed class ReorganizeTests(TestContext context) var archive = Guid.NewGuid(); var templates = new RenameTemplates(Folder: "moved/{date}"); var made = 0; - Task Plan() + Task Plan(CancellationToken stop) { made++; return Task.FromResult(new RenamePlan(Guid.NewGuid(), archive, [])); @@ -580,9 +611,15 @@ public sealed class ReorganizeTests(TestContext context) await plans.GetOrPlanAsync(Key(0), Plan); Assert.Equal(3 + ReorganizePlanCache.Capacity, made); - // A refusal is not kept, and a plan forgotten is made again. - await Assert.ThrowsAsync(() => plans.GetOrPlanAsync(Key(99), () => throw new NestedRenameException("a/b", "a"))); - await plans.GetOrPlanAsync(Key(99), Plan); + // A refusal follows from the same reads, and is kept as a plan is: a folder inside a sequence's, a sequence not in the + // archive. A failure is not kept, and neither is a plan or refusal forgotten. + await Assert.ThrowsAsync(() => plans.GetOrPlanAsync(Key(99), _ => throw new NestedRenameException("a/b", "a"))); + await Assert.ThrowsAsync(() => plans.GetOrPlanAsync(Key(99), Plan)); + await Assert.ThrowsAsync(() => plans.GetOrPlanAsync(Key(98), _ => throw new ArgumentException("Not in the archive."))); + await Assert.ThrowsAsync(() => plans.GetOrPlanAsync(Key(98), Plan)); + Assert.Equal(3 + ReorganizePlanCache.Capacity, made); + await Assert.ThrowsAsync(() => plans.GetOrPlanAsync(Key(97), _ => throw new IOException("The sidecar cannot be read."))); + await plans.GetOrPlanAsync(Key(97), Plan); Assert.Equal(4 + ReorganizePlanCache.Capacity, made); plans.Forget(Key(99)); await plans.GetOrPlanAsync(Key(99), Plan); @@ -598,7 +635,7 @@ public sealed class ReorganizeTests(TestContext context) var key = ReorganizePlanCache.Key.Of(Guid.NewGuid(), "stamp", null, templates); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var made = 0; - async Task SlowPlan() + async Task SlowPlan(CancellationToken stop) { Interlocked.Increment(ref made); await release.Task; @@ -632,13 +669,97 @@ public sealed class ReorganizeTests(TestContext context) await plans.GetOrPlanAsync(key, SlowPlan); Assert.Equal(2 + ReorganizePlanCache.Capacity, made); - Task PlanAtOnce() + Task PlanAtOnce(CancellationToken stop) { Interlocked.Increment(ref made); return Task.FromResult(new RenamePlan(Guid.NewGuid(), key.ArchiveId, [])); } } + [Test] + public async Task PlansAreMadeAFewAtOnceAndStopWhenLetGoOrPastTheirTime() + { + var clock = new SteppedTimeProvider(); + using var plans = new ReorganizePlanCache(clock); + var archive = Guid.NewGuid(); + var templates = new RenameTemplates(Folder: "moved/{date}"); + ReorganizePlanCache.Key Key(int i) => ReorganizePlanCache.Key.Of(archive, "stamp", [new Guid(i, 0, 0, new byte[8])], templates); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var started = new ConcurrentDictionary(); + var stopped = new ConcurrentDictionary(); + // A plan made once released, which tells when it starts and when it is stopped. + Func> Plan(int i) => async stop => + { + started[i] = true; + try + { + await release.Task.WaitAsync(stop); + } + catch (OperationCanceledException) + { + stopped[i] = true; + throw; + } + return new RenamePlan(Guid.NewGuid(), archive, []); + }; + + // A few are made at once; the others wait for their turn. + var asked = Enumerable.Range(0, ReorganizePlanCache.Capacity).Select(i => plans.GetOrPlanAsync(Key(i), Plan(i))).ToList(); + await EventuallyAsync(() => started.Count == ReorganizePlanCache.Concurrency); + await Task.Delay(200); + Assert.SequenceEqual(Enumerable.Range(0, ReorganizePlanCache.Concurrency), started.Keys.Order()); + + // One more lets go the plan asked first, being made with none kept: it stops, its waiter is told, the next one starts. + var newest = plans.GetOrPlanAsync(Key(ReorganizePlanCache.Capacity), Plan(ReorganizePlanCache.Capacity)); + await Assert.ThrowsAsync(() => asked[0]); + await EventuallyAsync(() => stopped.ContainsKey(0) && started.ContainsKey(ReorganizePlanCache.Concurrency)); + Assert.Equal(ReorganizePlanCache.Capacity, plans.Count); + release.SetResult(); + foreach (var plan in asked.Skip(1).Append(newest)) await plan; + Assert.SequenceEqual([0], stopped.Keys); + + // A plan still being made at its time stops, also when its work never ends (a read that hangs), and so does one still + // waiting for its turn, which never starts. + var hang = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + Func> Hanging(int i) => _ => + { + started[i] = true; + return hang.Task; + }; + var hung = plans.GetOrPlanAsync(Key(20), Hanging(20)); + var stuck = plans.GetOrPlanAsync(Key(21), Hanging(21)); + await EventuallyAsync(() => started.ContainsKey(20) && started.ContainsKey(21)); + var waiting = plans.GetOrPlanAsync(Key(22), Plan(22)); + clock.Advance(ReorganizePlanCache.PlanTimeout - TimeSpan.FromSeconds(1)); + Assert.False(hung.IsCompleted, "still in time"); + clock.Advance(TimeSpan.FromSeconds(1)); + foreach (var plan in new[] { hung, stuck, waiting }) await Assert.ThrowsAsync(() => plan); + Assert.False(started.ContainsKey(22), "stopped before its turn"); + Assert.Equal(0, plans.Count); + + // Asked again, it is made once a turn is free: the work that hangs holds its turn until it ends, and what it makes + // then is not kept. + var again = plans.GetOrPlanAsync(Key(20), Plan(30)); + await Task.Delay(200); + Assert.False(started.ContainsKey(30), "no turn while two works hang"); + hang.SetResult(new RenamePlan(Guid.NewGuid(), archive, [])); + var made = await again; + Assert.True(ReferenceEquals(made, await plans.GetOrPlanAsync(Key(20), Plan(31))), "kept"); + Assert.False(started.ContainsKey(31), "kept, not made again"); + Assert.Equal(1, plans.Count); + } + + /// Waits until holds. + private static async Task EventuallyAsync(Func condition, [System.Runtime.CompilerServices.CallerArgumentExpression(nameof(condition))] string? expression = null) + { + var until = DateTime.UtcNow + Wait; + while (!condition()) + { + Assert.True(DateTime.UtcNow < until, expression); + await Task.Delay(10); + } + } + private static async Task DryRunAsync(HttpClient client, ReorganizeRequest request) { using var response = await client.PostAsJsonAsync(Url, request with { DryRun = true }, ProtocolJson.Default.ReorganizeRequest);