diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index a0ae97b7..5a7f6d8f 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -85,7 +85,7 @@ Dependencies point one way: `Core` ← `Core.Windows`, `Data`, `Protocol` ← `S | `Ui/Controls` | `ChronosWindow` (chrome, language direction), `NavRail`, `JobBar`, `ToastHost` (`Toast.Closed` once a toast leaves the screen), `Field` (placeholder, icon, error), `Chip`, `AutoSlider`, `StatusGlyph`, `EmptyState`, `Icon`, `Bidi` (right-to-left rules), `Motion` (animations off for screenshots) | | `Ui/Interop` | DWM calls of the window chrome; `TrayIcon` (`Shell_NotifyIconW`: the Uploader's tray, the client's Windows notifications) | | `Ui/Visuals` | `DrawingElement` (hover readout, item keyboard and activation) and its charts (`BarChart`, `SeriesChart`, `StackedBar`, `HeatmapStrip`, `TimelineBands`), `Filmstrip`, `PreviewViewport`, `CurveEditor`, `SequenceCard`, `VirtualizingWrapPanel`, `SegmentedControl`, `SplitButton` | -| `Server/Api`, `Server/Events` | Endpoint groups (`XxxEndpoints.Map`; `ScheduleEndpoints`: `/api/v1/schedules`; `LibraryEndpoints`: the Library filters' facets, sequence thumbnail, photo delete, bulk delete/retag/jobs; `TrashEndpoints`: list, restore, empty; `ArchiveEndpoints`: settings, statistics, scan jobs, Reorganize's dry run and `move` job, with `ReorganizePlanCache`: a dry run's plan kept a few minutes while the index's stamp stays, for the same dry run and its confirmation), API errors, the WebSocket `EventHub` | +| `Server/Api`, `Server/Events` | Endpoint groups (`XxxEndpoints.Map`; `ScheduleEndpoints`: `/api/v1/schedules`; `LibraryEndpoints`: the Library filters' facets, sequence thumbnail, photo delete, bulk delete/retag/jobs; `TrashEndpoints`: list, restore, empty; `ArchiveEndpoints`: settings, statistics, scan jobs, Reorganize's dry run and `move` job, with `ReorganizePlanCache`: a dry run's plan, made apart from its request, kept a few minutes while what it reads from the index stays, for the same dry run and its confirmation), API errors, the WebSocket `EventHub` | | `Server/Security` | Accounts, password hashing, tokens, session cookies, login guard, access rules, audit log | | `Server/Storage`, `Server/Jobs` | Archive and processing modules, uploads (`UploadMaintenance`: recovery at start, and the sweep that is the `uploads` clean-up step), render store; `JobEngineHost`, `SchedulerHost` (the `Scheduler` on the engine, started after it; archive.json read every 10 minutes whatever the scan schedule does; every schedule change and run published as a `scheduleChanged` event) and `ServerScheduleHost` (time zone, archive refreshed from archive.json, system schedules), render node service, monitor and availability | | `Server/Hosting`, `Server/Admin/web` | `ServerHost`, `ServerLogging` (console and file levels the logging filters read at every line, the file sink's limits, the engine's level; applied again when Settings › Logs is saved), certificates, discovery beacon, embedded static files (`EmbeddedStaticFiles`: `no-cache` and a content ETag, so an upgrade never mixes old and new modules), Windows service (start and stop failures in the Application event log); the admin panel (ES modules, one per page; `schedules.js` is the Jobs page's Scheduled tab; the Library's views in `pages/library/`: `sequence.js` a sequence and its photo viewer, `trash.js` the archive's trash, `shared.js` cards, selection, grid keyboard and the retry of thumbnails a finished job may have built; `upload.js` the upload dialog the Library opens on demand, hashing in the module worker `hash-worker.js` with the in-house incremental SHA-256 of `sha256.js`, self-tested before its first use, and sending the capture time `capture-time.js` reads: the Exif DateTimeOriginal of a JPEG or TIFF-based file, else the file's time; `reorganize.js` the Reorganize dialog of the Library's selection and of the Archive page's save toast: template switches and fields, a live dry run of `POST /archive/reorganize` one request at a time, Apply that sends the shown plan's fingerprint back) | diff --git a/src/Chronos.Data/Repositories/SequenceRepository.cs b/src/Chronos.Data/Repositories/SequenceRepository.cs index a932e1cc..81b9e4b6 100644 --- a/src/Chronos.Data/Repositories/SequenceRepository.cs +++ b/src/Chronos.Data/Repositories/SequenceRepository.cs @@ -428,21 +428,40 @@ public sealed class SequenceRepository } /// - /// A digest of the index rows of (owner review 2, R7): another value once a sequence is - /// added or removed, or once a row changes in what a Reorganize plans from (folder, title, first capture, camera, lens, - /// frame count) or in what every write of it moves (its versions, verify time and time of change). A change made on - /// disk alone, which the index does not have yet, leaves it as it was. + /// A digest of what a Reorganize of (null: every sequence) plans from in the index of + /// (owner review 2, R7): every sequence of the archive and its folder, which decide the + /// folders that get " (2)" and what a moving folder carries; of the sequences planned, their title, first capture, + /// camera, lens, photo count and frames version; of the others inside their folders, the title. Another value once one + /// of these changes. Any other write (an upload into a sequence not planned, a verify, a status, tag or note change) + /// leaves it as it was, and so does a change made on disk alone, which the index does not have. /// - public async Task GetStampAsync(Guid archiveId, CancellationToken cancellationToken = default) + public async Task GetStampAsync(Guid archiveId, IReadOnlyCollection? sequenceIds, CancellationToken cancellationToken = default) { var rows = await _db.ListAsync( - "SELECT id, folder, title, first_capture, camera_make, camera_model, lens, frame_count, frames_version, settings_version, verified_at, updated_at " + - "FROM sequences WHERE archive_id = $1 ORDER BY id", [archiveId], - r => string.Create(CultureInfo.InvariantCulture, - $"{r.GetGuid(0):N}\u001f{r.GetString(1)}\u001f{r.GetStringOrNull(2)}\u001f{r.GetDateTimeOrNull(3):O}\u001f{r.GetStringOrNull(4)}\u001f" + - $"{r.GetStringOrNull(5)}\u001f{r.GetStringOrNull(6)}\u001f{r.GetInt32(7)}\u001f{r.GetInt32(8)}\u001f{r.GetInt32(9)}\u001f" + - $"{r.GetDateTimeOffsetOrNull(10):O}\u001f{r.GetDateTimeOffset(11):O}"), cancellationToken).ConfigureAwait(false); - return Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(string.Join('\n', rows)))); + "SELECT id, folder, title, first_capture, camera_make, camera_model, lens, frame_count, frames_version FROM sequences WHERE archive_id = $1 ORDER BY id", + [archiveId], r => (Id: r.GetGuid(0), Folder: r.GetString(1), Title: r.GetStringOrNull(2), Read: string.Create(CultureInfo.InvariantCulture, + $"{r.GetStringOrNull(2)}\u001f{r.GetDateTimeOrNull(3):O}\u001f{r.GetStringOrNull(4)}\u001f{r.GetStringOrNull(5)}\u001f{r.GetStringOrNull(6)}\u001f" + + $"{r.GetInt32(7)}\u001f{r.GetInt32(8)}")), cancellationToken).ConfigureAwait(false); + var selected = sequenceIds?.ToHashSet(); + var planned = new HashSet(rows.Where(r => selected?.Contains(r.Id) != false && r.Folder.Length > 0).Select(r => r.Folder), + StringComparer.OrdinalIgnoreCase); + using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + foreach (var row in rows) + { + var read = selected?.Contains(row.Id) != false ? row.Read : Inside(row.Folder) ? row.Title : null; + hash.AppendData(Encoding.UTF8.GetBytes(string.Create(CultureInfo.InvariantCulture, $"{row.Id:N}\u001f{row.Folder}\u001e{read}\n"))); + } + return Convert.ToHexStringLower(hash.GetHashAndReset()); + + // Whether a folder lies inside a planned sequence's folder (RenamePlanner.IsInside), checking each folder around it. + bool Inside(string folder) + { + for (var slash = folder.LastIndexOf('/'); slash > 0; slash = folder.LastIndexOf('/', slash - 1)) + { + if (planned.Contains(folder[..slash])) return true; + } + return false; + } } /// diff --git a/src/Chronos.Protocol/Library.cs b/src/Chronos.Protocol/Library.cs index e9be1216..b7054f69 100644 --- a/src/Chronos.Protocol/Library.cs +++ b/src/Chronos.Protocol/Library.cs @@ -86,9 +86,11 @@ public sealed record LibraryFacetsDto(IReadOnlyList Years, IReadOnlyLi /// POST /api/v1/archive/reorganize (owner review 2, R7; administrators of the archive): new folder, file and title /// templates applied to sequences already in the archive, or every one with /// (exactly one of the two). A null template keeps that part; at least one is given, each valid as the archive settings' -/// templates are. answers the plan () and changes nothing; the change -/// itself sends that plan's back (409 plan_changed when the archive changed since) and runs -/// as a move job (202). +/// templates are. answers the plan () and changes nothing; the server +/// keeps it a few minutes, so the same dry run again (also after one cut off) is answered from it. The change itself sends +/// that plan's back and runs as a move job (202): 409 plan_changed when the index +/// changed since in what the plan reads; a change made on disk alone, which the index does not have yet, is found by the +/// job, which then fails and changes nothing. /// public sealed record ReorganizeRequest(IReadOnlyList? SequenceIds, bool All, string? FolderTemplate, string? FileTemplate, string? TitleTemplate, bool DryRun, string? Fingerprint = null); diff --git a/src/Chronos.Server/Api/ArchiveEndpoints.cs b/src/Chronos.Server/Api/ArchiveEndpoints.cs index cca6f611..42f3d318 100644 --- a/src/Chronos.Server/Api/ArchiveEndpoints.cs +++ b/src/Chronos.Server/Api/ArchiveEndpoints.cs @@ -143,12 +143,15 @@ public static class ArchiveEndpoints /// archive_offline. Audited; the job tells each sequence it changes as sequenceChanged (reorganized) /// once that sequence's step is journaled (a step that failed half-way, when its retry completes it; a plan the server /// completes as it starts, no one: no client is connected yet). Photos a sequence receives between the dry run and the - /// confirmation under the names the plan leaves them keep the fingerprint as it was. A dry run's plan is kept a few - /// minutes while the index stays as it was (): the same dry run again, and its - /// confirmation, take it instead of planning again. + /// confirmation under the names the plan leaves them keep the fingerprint as it was. A dry run's plan is made apart from + /// the request, until the server stops, and kept a few minutes while what it reads from the index stays as it was + /// (): the same dry run again (also after the first was cut off), and its confirmation, + /// take it instead of planning again. So plan_changed answers a change the index has; one made on disk alone since + /// the dry run (a folder made by hand, a desktop writing to a shared archive before a scan) is found by the job, which + /// then fails and changes nothing. /// private static async Task ReorganizeAsync(HttpContext context, Caller caller, ReorganizeRequest request, ChronosDatabase db, ServerArchive archive, - [FromServices] ArchiveService? archives, JobEngineHost host, AuditLog audit, ReorganizePlanCache plans) + [FromServices] ArchiveService? archives, JobEngineHost host, AuditLog audit, ReorganizePlanCache plans, IHostApplicationLifetime lifetime) { var ct = context.RequestAborted; if (request.All == (request.SequenceIds is not null) || request.SequenceIds is { Count: 0 or > LibraryEndpoints.MaxBulkIds }) @@ -161,11 +164,13 @@ public static class ArchiveEndpoints return ApiErrors.Result(context, StatusCodes.Status503ServiceUnavailable, "archive_offline"); var sequences = request.All ? null : request.SequenceIds; // Stamped before planning: a change committed while the plan is made gives the next request another key. - var key = ReorganizePlanCache.Key.Of(archive.Id, await db.Sequences.GetStampAsync(archive.Id, ct), sequences, templates); + var key = ReorganizePlanCache.Key.Of(archive.Id, await db.Sequences.GetStampAsync(archive.Id, sequences, ct), sequences, templates); RenamePlan plan; try { - plan = await plans.GetOrPlanAsync(key, () => LibraryEndpoints.Engine(archives, db).PlanReorganizeAsync(archive.Id, sequences, templates, ct)); + // Made until the server stops, whatever becomes of this request: the same request again takes it. + var stopping = lifetime.ApplicationStopping; + plan = await plans.GetOrPlanAsync(key, () => LibraryEndpoints.Engine(archives, db).PlanReorganizeAsync(archive.Id, sequences, templates, stopping), ct); } catch (NestedRenameException ex) { diff --git a/src/Chronos.Server/Api/ReorganizePlanCache.cs b/src/Chronos.Server/Api/ReorganizePlanCache.cs index 1dbee6ee..93b2df09 100644 --- a/src/Chronos.Server/Api/ReorganizePlanCache.cs +++ b/src/Chronos.Server/Api/ReorganizePlanCache.cs @@ -6,25 +6,31 @@ namespace Chronos.Server.Api; /// /// Reorganize's dry runs kept a short while (owner review 2, R7): the same dry run asked again, and the confirmation that /// follows it, take the plan made then instead of reading every sidecar and photo folder of the selection again (with -/// all, of the whole archive), while the archive's index is as it was then -/// () and for at most . A change made on disk alone, -/// which the index does not have, is not seen by a kept plan: the move job plans again and refuses a plan that no -/// longer matches the archive, so nothing changes that the dry run did not show. +/// all, of the whole archive), while what the plan reads from the index is as it was then +/// () and for at most once made. A plan is made apart +/// from the request that asked for it (until the server stops), once for every request of the same key: a request cut off +/// meanwhile (a proxy's timeout) leaves it to be made, and the same request again waits for it or takes it. A change made +/// on disk alone, which the index does not have, is not seen by a kept plan: the move job plans again and refuses a +/// plan that no longer matches the archive, so nothing changes that the dry run did not show. /// -public sealed class ReorganizePlanCache(TimeProvider time) +public sealed class ReorganizePlanCache(TimeProvider time) : IDisposable { - /// How long a plan is kept, from when its planning started. + /// How long a plan is kept once made; then its timer lets it go. public static readonly TimeSpan Lifetime = TimeSpan.FromMinutes(5); - /// Plans kept at most (a plan of every sequence holds one step per sequence); the oldest goes first. + /// + /// Plans kept or being made at most (each holds a step for every sequence it changes or carries, or whose photos keep + /// their names); the oldest made goes first, then the oldest being made, which is still answered but not kept. + /// public const int Capacity = 8; private readonly Lock _gate = new(); - private readonly Dictionary _plans = []; + private readonly Dictionary _plans = []; + private long _order; /// - /// What a plan is kept by: the archive, the stamp of its index, the sequences (all of them, or the ids in the order - /// given, which decides the folder that gets " (2)") and the templates. + /// What a plan is kept by: the archive, the stamp of what it reads from the index, the sequences (all of them, or the ids + /// in the order given, which decides the folder that gets " (2)") and the templates. /// public readonly record struct Key(Guid ArchiveId, string Stamp, string Sequences, RenameTemplates Templates) { @@ -33,37 +39,108 @@ public sealed class ReorganizePlanCache(TimeProvider time) new(archiveId, stamp, sequenceIds is null ? "*" : string.Join(',', sequenceIds.Select(id => id.ToString("N"))), templates); } + /// Plans kept or being made now. + internal int Count + { + get + { + lock (_gate) return _plans.Count; + } + } + /// - /// The plan kept for , or the one makes now, then kept; a refusal - /// ( throws) is not kept. + /// The plan kept for , the one being made for it, or the one starts making + /// now, apart from the caller, then kept. A refusal ( throws) is passed to every caller waiting for + /// it and not kept. stops the wait only: the plan is still made, and kept. /// - public async Task GetOrPlanAsync(Key key, Func> plan) + public Task GetOrPlanAsync(Key key, Func> plan, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(plan); - DateTimeOffset started; + Entry? entry; + bool start; lock (_gate) { - started = time.GetUtcNow(); - Expire(started); - if (_plans.TryGetValue(key, out var kept)) return kept.Plan; + // Never past its lifetime, whatever its timer. + if (_plans.TryGetValue(key, out entry) && entry.Made is { } made && time.GetUtcNow() - made >= Lifetime) Remove(key); + start = !_plans.TryGetValue(key, out entry); + if (start) + { + if (_plans.Count >= Capacity) Remove(_plans.MinBy(p => (p.Value.Made is null, p.Value.Order)).Key); + entry = new Entry(++_order); + _plans[key] = entry; + } } - var made = await plan().ConfigureAwait(false); - lock (_gate) - { - if (!_plans.ContainsKey(key) && _plans.Count >= Capacity) _plans.Remove(_plans.MinBy(p => p.Value.Started).Key); - _plans[key] = (made, started); - } - return made; + if (start) _ = MakeAsync(key, entry!, plan); + return entry!.Plan.Task.WaitAsync(cancellationToken); } /// Forgets the plan kept for : its job is queued, and what the job changes gives the index another stamp. public void Forget(Key key) { - lock (_gate) _plans.Remove(key); + lock (_gate) Remove(key); } - private void Expire(DateTimeOffset now) + public void Dispose() { - foreach (var key in _plans.Where(p => now - p.Value.Started >= Lifetime).Select(p => p.Key).ToList()) _plans.Remove(key); + lock (_gate) + { + foreach (var key in _plans.Keys.ToList()) Remove(key); + } + } + + private async Task MakeAsync(Key key, Entry entry, Func> plan) + { + RenamePlan made; + try + { + made = await Task.Run(plan).ConfigureAwait(false); + } + catch (Exception ex) + { + // A refusal, a failure, or the server stopping: the next request plans again. + lock (_gate) + { + if (_plans.GetValueOrDefault(key) == entry) Remove(key); + } + if (ex is OperationCanceledException canceled) entry.Plan.SetCanceled(canceled.CancellationToken); + else entry.Plan.SetException(ex); + // Observed here: a request cut off before it ended no longer waits for it. + _ = entry.Plan.Task.Exception; + return; + } + lock (_gate) + { + if (_plans.GetValueOrDefault(key) == entry) + { + entry.Made = time.GetUtcNow(); + entry.Expiry = time.CreateTimer(_ => Expire(key, entry), null, Lifetime, Timeout.InfiniteTimeSpan); + } + } + entry.Plan.SetResult(made); + } + + private void Expire(Key key, Entry entry) + { + lock (_gate) + { + if (_plans.GetValueOrDefault(key) == entry) Remove(key); + } + } + + private void Remove(Key key) + { + if (_plans.Remove(key, out var entry)) entry.Expiry?.Dispose(); + } + + /// A plan being made ( null) or kept, and the order it was asked in. + private sealed class Entry(long order) + { + public long Order { get; } = order; + + public TaskCompletionSource Plan { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public DateTimeOffset? Made { get; set; } + + public ITimer? Expiry { get; set; } } } diff --git a/tests/Chronos.Data.Tests/Repositories/SequenceStampTests.cs b/tests/Chronos.Data.Tests/Repositories/SequenceStampTests.cs index b5610394..ab35011c 100644 --- a/tests/Chronos.Data.Tests/Repositories/SequenceStampTests.cs +++ b/tests/Chronos.Data.Tests/Repositories/SequenceStampTests.cs @@ -6,48 +6,76 @@ using Chronos.Testing; namespace Chronos.Data.Tests.Repositories; /// -/// The stamp of an archive's index rows () on both dialects: the server makes -/// a kept Reorganize plan again once it is another one (owner review 2, R7). +/// The stamp of what a Reorganize plans from in an archive's index () on both +/// dialects: the server makes a kept Reorganize plan again once it is another one, and only then (owner review 2, R7). /// public abstract class SequenceStampTests(TestContext context) : RepositoryTest(context) { [Test] - public async Task TheStampFollowsEveryChangeOfTheArchivesSequencesAndNoOtherArchives() + public async Task TheStampFollowsWhatAReorganizePlansFromAndNothingElse() { var db = await OpenChronosAsync(); var archive = TestData.Archive(); var other = TestData.Archive("Other"); await db.Archives.UpsertAsync(archive, Ct); await db.Archives.UpsertAsync(other, Ct); - var empty = await db.Sequences.GetStampAsync(archive.Id, Ct); - var sequence = TestData.Sequence(archive.Id, new DateTime(2026, 9, 28, 18, 0, 0), "Roof"); - await db.Sequences.UpsertAsync(sequence, Ct); - var stamp = await db.Sequences.GetStampAsync(archive.Id, Ct); + var roof = TestData.Sequence(archive.Id, new DateTime(2026, 9, 28, 18, 0, 0), "Roof"); + var inner = TestData.Sequence(archive.Id, null, "Inner") with { Folder = roof.Folder + "/inner" }; + var ridge = TestData.Sequence(archive.Id, new DateTime(2026, 9, 29, 7, 0, 0), "Ridge"); + IReadOnlyList planned = [roof.Id]; + var empty = await db.Sequences.GetStampAsync(archive.Id, planned, Ct); + foreach (var sequence in new[] { roof, inner, ridge }) await db.Sequences.UpsertAsync(sequence, Ct); + var stamp = await db.Sequences.GetStampAsync(archive.Id, planned, Ct); Assert.NotEqual(empty, stamp); - Assert.Equal(stamp, await db.Sequences.GetStampAsync(archive.Id, Ct)); + Assert.Equal(stamp, await db.Sequences.GetStampAsync(archive.Id, planned, Ct)); + Assert.NotEqual(stamp, await db.Sequences.GetStampAsync(archive.Id, null, Ct)); - // The sequences of another archive are not in it. + // What the plan does not read: another archive, and the writes of every upload, verify, tag or status change elsewhere. await db.Sequences.UpsertAsync(TestData.Sequence(other.Id), Ct); - Assert.Equal(stamp, await db.Sequences.GetStampAsync(archive.Id, Ct)); + await PhotoAsync(db, ridge, 0); + await PhotoAsync(db, inner, 0); + Assert.Equal(SequenceEditOutcome.Updated, (await db.Sequences.EditAsync(ridge.Id, new SequenceEdit { Title = "Ridge at dawn", Notes = "cold" }, null, Ct)).Outcome); + foreach (var id in new[] { roof.Id, ridge.Id }) + { + Assert.True(await db.Sequences.SetTagsAsync(id, ["sky"], Ct)); + Assert.True(await db.Sequences.UpdateStatusAsync(id, SequenceStatus.ProxiesBuilt, SequenceStatus.None, Ct)); + Assert.True(await db.Sequences.RecordVerifiedAsync(id, SequenceStatus.None, SequenceStatus.None, TestData.Now, Ct)); + } + Assert.Equal(stamp, await db.Sequences.GetStampAsync(archive.Id, planned, Ct)); - // Every change of the row gives one never seen before. + // What it reads gives one never seen before: the sequences planned, the titles inside their folders, every folder. var seen = new HashSet(StringComparer.Ordinal) { empty, stamp }; - async Task ChangedAsync(string what) => Assert.True(seen.Add(await db.Sequences.GetStampAsync(archive.Id, Ct)), what); - Assert.Equal(SequenceEditOutcome.Updated, (await db.Sequences.EditAsync(sequence.Id, new SequenceEdit { Title = "Ridge" }, null, Ct)).Outcome); + async Task ChangedAsync(string what) => Assert.True(seen.Add(await db.Sequences.GetStampAsync(archive.Id, planned, Ct)), what); + Assert.Equal(SequenceEditOutcome.Updated, (await db.Sequences.EditAsync(roof.Id, new SequenceEdit { Title = "Roof at noon" }, null, Ct)).Outcome); await ChangedAsync("title"); - Assert.True(await db.Sequences.RelocateAsync(sequence.Id, "2026/moved", null, Ct)); - await ChangedAsync("folder"); - await db.Frames.BulkInsertAsync([TestData.Frame(sequence.Id, 0)], Ct); - Assert.NotNull(await db.Sequences.RefreshStatsAsync(sequence.Id, Ct)); + await PhotoAsync(db, roof, 0); await ChangedAsync("photos"); - // A row indexed again from its sidecar keeps the sidecar's time of change: its folder alone tells. - var row = Assert.NotNull(await db.Sequences.GetAsync(sequence.Id, Ct)); - await db.Sequences.UpsertAsync(row with { Folder = "2026/elsewhere" }, Ct); - await ChangedAsync("folder of a row written back"); - Assert.True(await db.Sequences.RecordVerifiedAsync(sequence.Id, SequenceStatus.None, SequenceStatus.None, TestData.Now, Ct)); - await ChangedAsync("verify"); - Assert.True(await db.Sequences.DeleteAsync(sequence.Id, Ct)); - Assert.Equal(empty, await db.Sequences.GetStampAsync(archive.Id, Ct)); + Assert.Equal(SequenceEditOutcome.Updated, (await db.Sequences.EditAsync(inner.Id, new SequenceEdit { Title = "Inner court" }, null, Ct)).Outcome); + await ChangedAsync("title of a sequence a moving folder would carry"); + Assert.True(await db.Sequences.RelocateAsync(ridge.Id, "2026/ridge", null, Ct)); + await ChangedAsync("folder of a sequence not planned"); + Assert.True(await db.Sequences.RelocateAsync(roof.Id, "2026/moved", null, Ct)); + await ChangedAsync("folder"); + var added = TestData.Sequence(archive.Id); + await db.Sequences.UpsertAsync(added, Ct); + await ChangedAsync("a sequence added"); + Assert.True(await db.Sequences.DeleteAsync(added.Id, Ct)); + Assert.Contains(await db.Sequences.GetStampAsync(archive.Id, planned, Ct), seen); + + // With every sequence planned, a photo anywhere changes the plan read; a verify still does not. + var all = await db.Sequences.GetStampAsync(archive.Id, null, Ct); + Assert.True(await db.Sequences.RecordVerifiedAsync(ridge.Id, SequenceStatus.None, SequenceStatus.None, TestData.Now.AddDays(1), Ct)); + Assert.Equal(all, await db.Sequences.GetStampAsync(archive.Id, null, Ct)); + await PhotoAsync(db, ridge, 1); + Assert.NotEqual(all, await db.Sequences.GetStampAsync(archive.Id, null, Ct)); + } + + /// A photo indexed into a sequence, as an upload does: the frame, then the row's counts and frames version. + private async Task PhotoAsync(ChronosDatabase db, Sequence sequence, int index) + { + var frame = TestData.Frame(sequence.Id, index); + await db.Frames.BulkInsertAsync([frame with { RelativePath = $"{sequence.Folder}/originals/{frame.FileName}" }], Ct); + Assert.NotNull(await db.Sequences.RefreshStatsAsync(sequence.Id, Ct)); } } diff --git a/tests/Chronos.Server.Tests/Api/ReorganizeTests.cs b/tests/Chronos.Server.Tests/Api/ReorganizeTests.cs index 890ef36a..6390cee8 100644 --- a/tests/Chronos.Server.Tests/Api/ReorganizeTests.cs +++ b/tests/Chronos.Server.Tests/Api/ReorganizeTests.cs @@ -237,8 +237,9 @@ public sealed class ReorganizeTests(TestContext context) string fingerprint; using (var dry = await admin.PostAsJsonAsync(Url, request, ProtocolJson.Default.ReorganizeRequest)) { + // No archive engine wrote the sequence's sidecar here: it stays as it is, and says why. var plan = await dry.ReadAsync(ProtocolJson.Default.ReorganizePlanDto); - Assert.Equal("moved/2026-09-28", plan.Steps.Single().ToFolder); + Assert.Equal("unreadable", plan.Steps.Single().Blocked.Single().Reason); fingerprint = plan.Fingerprint; } using (var unavailable = await admin.PostAsJsonAsync(Url, request with { DryRun = false, Fingerprint = fingerprint }, ProtocolJson.Default.ReorganizeRequest)) @@ -286,6 +287,60 @@ public sealed class ReorganizeTests(TestContext context) Assert.Equal(0, (await again.ReadAsync(ProtocolJson.Default.ReorganizePlanDto)).Sequences); } + [Test] + public async Task EverySequenceIsReorganizedByAJobWithTheOnesItCarriesAndTheOnesWhosePhotosStay() + { + await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + var admin = await server.AdminAsync(); + var roof = Guid.Empty; + for (var i = 0; i < 2; i++) roof = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId; + await UntilIdleAsync(server); + await UntilJobAsync(admin, await ConfirmAsync(admin, new ReorganizeRequest([roof], false, null, null, "Roof", DryRun: true)), "done"); + var folder = (await server.Database.Sequences.GetAsync(roof))!.Folder; + // Photos copied in by hand and found by a scan: in the layout inside the roof's folder (a sequence without a title), + // and in place in a folder of their own (titled by it). + var root = server.Options.ArchivePath; + Directory.CreateDirectory(Path.Combine(root, folder, "inner", "originals")); + await File.WriteAllBytesAsync(Path.Combine(root, folder, "inner", "originals", "IMG_0100.JPG"), Photo(3)); + Directory.CreateDirectory(Path.Combine(root, "trips")); + for (var i = 0; i < 2; i++) await File.WriteAllBytesAsync(Path.Combine(root, "trips", $"IMG_{i + 200:0000}.JPG"), Photo(4 + i)); + using (var scan = await admin.PostAsync("/api/v1/archive/scan", null)) await scan.ExpectAsync(HttpStatusCode.Accepted); + await UntilIdleAsync(server); + var locations = await server.Database.Sequences.ListLocationsAsync((await server.Database.Archives.ListAsync()).Single().Id); + var inner = locations.Single(l => l.Folder == folder + "/inner").Id; + var trips = locations.Single(l => l.Folder == "trips").Id; + var tripFiles = Directory.GetFiles(Path.Combine(root, "trips")).Order(StringComparer.Ordinal).ToList(); + + // The roof moves to its title with its photos renamed and carries the sequence inside it, whose title gives no + // folder; the trips keep their folder, and their photos their names, which the plan says why. + var every = new ReorganizeRequest(null, true, "{title}", "{n:000}", null, DryRun: true); + var plan = await DryRunAsync(admin, every); + Assert.SequenceEqual([roof, trips, inner], plan.Steps.Select(s => s.SequenceId)); + Assert.Equal(("Roof", false, 2), (plan.Steps[0].ToFolder, plan.Steps[0].Carried, plan.Steps[0].FileChanges)); + Assert.Equal(("trips", 0), (plan.Steps[1].ToFolder, plan.Steps[1].FileChanges)); + Assert.Equal(("in_place", 2), (plan.Steps[1].Blocked.Single().Reason, plan.Steps[1].Blocked.Single().Photos)); + Assert.Equal(("Roof/inner", true, 0), (plan.Steps[2].ToFolder, plan.Steps[2].Carried, plan.Steps[2].FileChanges)); + Assert.Equal((2, 2), (plan.Sequences, plan.Files)); + + // The job plans again over the sequences with a step, the carried and the kept ones among them, and ends done. + using var events = server.Services.GetRequiredService().Subscribe(); + await UntilJobAsync(admin, await ConfirmAsync(admin, every with { DryRun = false, Fingerprint = plan.Fingerprint }), "done"); + Assert.Equal(("Roof", "Roof"), ((await server.Database.Sequences.GetAsync(roof))!.Folder, (await server.Database.Sequences.GetAsync(roof))!.Title)); + Assert.SequenceEqual(["Roof/originals/001.JPG", "Roof/originals/002.JPG"], (await server.Database.Frames.ListBySequenceAsync(roof)).Select(f => f.RelativePath)); + Assert.Equal("Roof/inner", (await server.Database.Sequences.GetAsync(inner))!.Folder); + Assert.SequenceEqual(["Roof/inner/originals/IMG_0100.JPG"], (await server.Database.Frames.ListBySequenceAsync(inner)).Select(f => f.RelativePath)); + Assert.True(File.Exists(Path.Combine(root, "Roof", "inner", "originals", "IMG_0100.JPG")), "the carried photo moved with its folder"); + Assert.Equal("trips", (await server.Database.Sequences.GetAsync(trips))!.Folder); + Assert.SequenceEqual(tripFiles, Directory.GetFiles(Path.Combine(root, "trips")).Order(StringComparer.Ordinal)); + Assert.False(Directory.Exists(Path.Combine(root, folder)), "nothing is left at the old place"); + Assert.SequenceEqual(new[] { roof, inner }.Order(), (await ReorganizedAsync(events, 2)).Order()); + + // Done: the next dry run plans no change, only the photos that keep their names. + var after = await DryRunAsync(admin, every); + Assert.Equal(0, after.Sequences); + } + [Test] public async Task AnUploadAfterTheDryRunChangesThePlanToConfirm() { @@ -382,13 +437,15 @@ public sealed class ReorganizeTests(TestContext context) } Assert.Equal(plan.Steps[0].ToFolder, (await server.Database.Sequences.GetAsync(first))!.Folder); Assert.Equal(was, (await server.Database.Sequences.GetAsync(second))!.Folder); + // The first sequence was told as soon as its step was done, before the pause. + Assert.SequenceEqual([first], await ReorganizedAsync(events, 1)); - // Resumed, it completes the plan it held and tells the sequence it changed then. + // Resumed, it completes the plan it held and tells the sequence it changed then, and that one only. using (var resume = await admin.PostAsync($"/api/v1/jobs/{job}/resume", null)) await resume.ExpectAsync(HttpStatusCode.OK); await UntilJobAsync(admin, job, "done"); Assert.Equal(plan.Steps[1].ToFolder, (await server.Database.Sequences.GetAsync(second))!.Folder); Assert.Empty(RenameJournal.ListPending(paths)); - Assert.Contains(second, await ReorganizedAsync(events, 1, second)); + Assert.SequenceEqual([second], await ReorganizedAsync(events, 1, second)); } [Test] @@ -404,6 +461,7 @@ public sealed class ReorganizeTests(TestContext context) var admin = await server.AdminAsync(); Guid id = default; for (var i = 0; i < 2; i++) id = (await Uploads.UploadAsync(editor, Photo(i), $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId; + var other = (await Uploads.UploadAsync(editor, Photo(5), "IMG_0100.JPG", sessionKey: "other")).SequenceId; await UntilIdleAsync(server); var request = new ReorganizeRequest([id], false, "moved/{date}", "{n:000}", null, DryRun: true); var target = Path.Combine(server.Options.ArchivePath, "moved", "2026-09-28"); @@ -415,7 +473,16 @@ public sealed class ReorganizeTests(TestContext context) Directory.CreateDirectory(target); Assert.Equal(shown.Fingerprint, (await DryRunAsync(admin, request)).Fingerprint); - // A change of the index plans again at once. + // Neither are what the plan does not read: a photo uploaded into another sequence (and the jobs it queues), a verify. + var versions = (await server.Database.Sequences.GetAsync(other))!.FramesVersion; + await Uploads.UploadAsync(editor, Photo(6), "IMG_0101.JPG", sessionKey: "other"); + using (var verify = await admin.PostAsync("/api/v1/archive/verify", null)) await verify.ExpectAsync(HttpStatusCode.Accepted); + await UntilIdleAsync(server); + Assert.NotEqual(versions, (await server.Database.Sequences.GetAsync(other))!.FramesVersion); + Assert.NotNull((await server.Database.Sequences.GetAsync(id))!.VerifiedAt); + Assert.Equal(shown.Fingerprint, (await DryRunAsync(admin, request)).Fingerprint); + + // A change of what it reads plans again at once. await Uploads.UploadAsync(editor, Photo(2), "IMG_0003.JPG", sessionKey: "roof"); await UntilIdleAsync(server); var beside = await DryRunAsync(admin, request); @@ -448,7 +515,7 @@ public sealed class ReorganizeTests(TestContext context) public async Task KeptPlansAreTheNewestFewWithinTheirLifetime() { var clock = new ManualTimeProvider(); - var plans = new ReorganizePlanCache(clock); + using var plans = new ReorganizePlanCache(clock); var archive = Guid.NewGuid(); var templates = new RenameTemplates(Folder: "moved/{date}"); var made = 0; @@ -482,7 +549,7 @@ public sealed class ReorganizeTests(TestContext context) await plans.GetOrPlanAsync(Key(0), Plan); Assert.Equal(2 + ReorganizePlanCache.Capacity, made); - // A plan is kept for its lifetime, from when its planning started. + // A plan is kept for its lifetime, from when it was made, even before its timer lets it go. clock.Advance(ReorganizePlanCache.Lifetime - TimeSpan.FromSeconds(1)); await plans.GetOrPlanAsync(Key(0), Plan); Assert.Equal(2 + ReorganizePlanCache.Capacity, made); @@ -499,6 +566,56 @@ public sealed class ReorganizeTests(TestContext context) Assert.Equal(5 + ReorganizePlanCache.Capacity, made); } + [Test] + public async Task APlanIsMadeOnceForEveryRequestAndOutlivesOneCutOff() + { + var clock = new SteppedTimeProvider(); + using var plans = new ReorganizePlanCache(clock); + var templates = new RenameTemplates(Folder: "moved/{date}"); + var key = ReorganizePlanCache.Key.Of(Guid.NewGuid(), "stamp", null, templates); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var made = 0; + async Task SlowPlan() + { + Interlocked.Increment(ref made); + await release.Task; + return new RenamePlan(Guid.NewGuid(), key.ArchiveId, []); + } + + // A request cut off while its plan is made (a proxy's timeout) leaves it to be made; asked again, it is waited for. + using (var cutOff = new CancellationTokenSource()) + { + var first = plans.GetOrPlanAsync(key, SlowPlan, cutOff.Token); + await cutOff.CancelAsync(); + await Assert.ThrowsAsync(() => first); + } + var again = plans.GetOrPlanAsync(key, SlowPlan); + var meanwhile = plans.GetOrPlanAsync(key, SlowPlan); + // Being made, it is not the one let go for other plans. + for (var i = 0; i < ReorganizePlanCache.Capacity; i++) + await plans.GetOrPlanAsync(ReorganizePlanCache.Key.Of(key.ArchiveId, "stamp", [new Guid(i, 0, 0, new byte[8])], templates), PlanAtOnce); + release.SetResult(); + var plan = await again; + Assert.True(ReferenceEquals(plan, await meanwhile), "one plan for both"); + Assert.True(ReferenceEquals(plan, await plans.GetOrPlanAsync(key, SlowPlan)), "kept"); + Assert.Equal(1 + ReorganizePlanCache.Capacity, made); + Assert.Equal(ReorganizePlanCache.Capacity, plans.Count); + + // Its timer lets it go at the end of its lifetime, with no request asking. + clock.Advance(ReorganizePlanCache.Lifetime - TimeSpan.FromSeconds(1)); + Assert.Equal(ReorganizePlanCache.Capacity, plans.Count); + clock.Advance(TimeSpan.FromSeconds(1)); + Assert.Equal(0, plans.Count); + await plans.GetOrPlanAsync(key, SlowPlan); + Assert.Equal(2 + ReorganizePlanCache.Capacity, made); + + Task PlanAtOnce() + { + Interlocked.Increment(ref made); + return Task.FromResult(new RenamePlan(Guid.NewGuid(), key.ArchiveId, [])); + } + } + private static async Task DryRunAsync(HttpClient client, ReorganizeRequest request) { using var response = await client.PostAsJsonAsync(Url, request with { DryRun = true }, ProtocolJson.Default.ReorganizeRequest);