From c3756d03c9bf6930de24a61861cb6e429ba1ba85 Mon Sep 17 00:00:00 2001 From: Alberto Balbo Date: Tue, 27 Oct 2026 18:58:47 +0100 Subject: [PATCH] fix: keep render filing retryable and writers out of a folder another index moved Third fix round of owner review 2, phase 4a (Reorganize), cross-cutting items. - Server render filing now runs under the sequence lease, so it can meet a busy sequence or an offline archive. Both completions (client render uploads and render-node results) used to answer 500 and leave the upload stuck in Committing until a restart. They now answer 503 sequence_busy or archive_offline, which clients retry. The upload goes back to receiving with its part file and the node keeps its job (UploadService.ReceiveAgainAsync). - Save both: when this PC's copy cannot be written (another index moved the folder), the server's version is no longer applied, even if this index's scan relocates the row before the apply would run. A copy that has to be dropped is deleted from where the folder is now, as a best effort. - Export rebase on a folder move (scan and Reorganize): the sequence's own export follows the folder unless its file is still at the old path and not at the new one. A file written into a re-created old folder therefore keeps a row that finds it, so the file is never stranded. - A render job refuses to start when its folder is not where the index has it, which also stops it from re-creating the old folder. Renders fetched by sync are now filed under the sequence lease into the current folder, or not at all (new string Errors.Archive.FolderMoved, en and it). - The proxy pass holds the thumbnail lease one batch at a time (a quarter of the lease wait), then lets it go for longer than a waiter's poll and reads the folder and sidecar again. Before, another process's Reorganize, sequence deletion or pack compaction could wait out its lease wait and fail. Updated the stale comments on that lease and ARCHITECTURE.md to match. Tests cover each fix, and each new test fails with its fix reverted. Also covered: proxy store and pass in a moved folder, RenderStore with the folder gone, Take server with the folder moved, and an editor save with the folder moved. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 811509d32a118ed50ad10d47d4b002e534da9d77) --- docs/ARCHITECTURE.md | 15 +- i18n/en/Errors.resx | 1 + i18n/it/Errors.it.resx | 1 + src/Chronos.Core/Archiving/LeaseFile.cs | 5 +- src/Chronos.Data/Archiving/ArchiveImages.cs | 193 ++++++++++-------- src/Chronos.Data/Archiving/ArchiveIndexer.cs | 17 +- src/Chronos.Data/Archiving/FrameTrash.cs | 7 +- src/Chronos.Data/Archiving/SequenceWriter.cs | 2 +- .../Processing/Jobs/RenderJobHandler.cs | 6 +- .../Repositories/RenderRepository.cs | 18 +- src/Chronos.Data/Sync/SyncService.cs | 37 +++- src/Chronos.Data/Sync/SyncTransfers.cs | 46 +++-- src/Chronos.Protocol/Jobs.cs | 6 +- src/Chronos.Protocol/Uploads.cs | 3 +- src/Chronos.Server/Jobs/RenderNodeService.cs | 12 +- src/Chronos.Server/Storage/RenderUploads.cs | 14 +- src/Chronos.Server/Storage/UploadService.cs | 16 ++ .../Editor/ProjectStoreTests.cs | 21 ++ .../Archiving/MassRenameTests.cs | 23 +++ .../Archiving/ProxyTests.cs | 91 +++++++++ .../Processing/RenderJobTests.cs | 28 +++ .../Chronos.Data.Tests/Sync/FakeSyncRemote.cs | 12 +- .../Sync/SyncServiceTests.cs | 71 +++++++ .../Sync/SyncTransferTests.cs | 49 +++++ .../Api/ReorganizeTests.cs | 23 +++ .../Jobs/RenderNodeTests.cs | 83 ++++++++ 26 files changed, 666 insertions(+), 134 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index cd8b37be..ccded0aa 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -58,7 +58,7 @@ Dependencies point one way: `Core` ← `Core.Windows`, `Data`, `Protocol` ← `S | `Jobs` | `JobEngine` (one job at a time; reviews the priority queue and makes a lower running job yield, requeued with its checkpoint; no history purge, the housekeeping schedule's clean-up does it), `JobPriorityQueue` (a level per job kind, the claim order, settings round trip), `IJobHandler`, `IJobHandlerProvider`, `JobContext` (progress, checkpoints, follow-up jobs that never interrupt their parent), `JobThroughput`/`JobEta` (rate and bounded time left) | | `Scheduling` | `Scheduler` (one per host: seeds the host's system schedules, claims due runs by compare-and-set, catch-up, clock changes, create/edit/reset/delete/run now as API v1 DTOs), `ScheduleRecurrence` (the in-house `every:`/`daily:`/`weekly:`/`monthly:` form, wall-clock times across DST), `ScheduleActions` (action names, parameters, availability, the archive jobs a run queues), `IScheduleHost` and `SystemSchedule` (what the server and the desktop supply), `ScheduleOutcome`, `ScheduleMapping` (model ↔ DTO) | | `Housekeeping` | `BackupJobHandler`, `CleanupJobHandler`, `ICleanupStep` with the shared steps (`CleanupSteps`: job history, audit log, trash report that never deletes); hosts add their own steps through `ArchiveServiceOptions.CleanupSteps` | -| `Archiving` | `ArchiveService` (create/adopt/open, archive settings written to `archive.json` then the index), `ArchiveIndexer` (scan, rebuild; a sequence found in another folder takes its exports' paths along), `ArchiveIngest` (uploads and imports), `ImportService`, `ArchiveVerifier`, `ArchiveImages` (thumbnail and proxy passes, several photos at once through `OrderedWork`, which hands results back in frame order), `FrameTiers` (decode tier, `PreviewOnly`/`Undecodable` flags and preview size of frames, written to sidecar and index by the light analysis, deep analysis, renders and image passes), `GprCompanions` (the JPEG of each GoPro GPR, found in the index when an import kept it in another sequence), `ArchiveMaintenance`, `MassRenamer` (mass rename and Reorganize: the dry run reads each sidecar for a file template; each step checked under its leases before anything of it moves (where its sequences are, what is inside and at its new folder, a sidecar of a newer format, the title it was planned from when it changes it or names the folder or files after it), then the folder, every index row (the frame paths and render paths of the exports inside it, its own or another sequence's whose file moved with it, follow it in one transaction; a sidecar that cannot be read or written then still has its row follow, and fails the step), then the files under the plan's journal in `journal/` (a sequence whose photos changed since, such as one still receiving them, or whose files are found changed or in use or have their new names taken keeps its file names, the renames being pinned by digest, and the step and the plan go on); the journal keeps the tally of the whole plan, and a plan its job held (paused, cancelled or failed) is left to that job, only its half-done file renames completed at an archive open, which completes any other plan an interruption left and tells each sequence changed; the file renames of a step: sidecar, a sequence whose sidecar cannot be read staying as it is with every folder around it, index (`FrameRepository.RenameFilesAsync`) and the sequence's journal), `SequenceTrash` (delete into the archive's hidden trash with a dry run, restore; the folder moves back when the index step fails; lists sequence and photo entries; `SequenceTrash.Empty`: the emptying's dry run, fingerprint and erase, each entry checked against its fingerprint again under its lease), `FrameTrash` (photos of a sequence into `trash/photos` with a dry run, a journal that undoes an interrupted deletion whose entry stays unlisted until its sidecar is written, tombstones (a photo never hashed is hashed as it goes; one whose original is missing goes without a file or tombstone; a frame that stays and may be a twin is hashed, its hash kept), positions parked and placed back, proxies and thumbnails moved to follow their frames; `FrameTrash.Restore`: restore re-hashes each file and says why each kept photo stayed (`KeptReason`), under a journal of its own into the sequence that another writer settles only under the entry's lease, into a new sequence through the ingest when theirs is gone, each original staged inside its own entry and the record rewritten after each photo filed), job handlers | +| `Archiving` | `ArchiveService` (create/adopt/open, archive settings written to `archive.json` then the index), `ArchiveIndexer` (scan, rebuild; a sequence found in another folder takes its exports' paths along), `ArchiveIngest` (uploads and imports), `ImportService`, `ArchiveVerifier`, `ArchiveImages` (thumbnail and proxy passes, several photos at once through `OrderedWork`, which hands results back in frame order), `FrameTiers` (decode tier, `PreviewOnly`/`Undecodable` flags and preview size of frames, written to sidecar and index by the light analysis, deep analysis, renders and image passes), `GprCompanions` (the JPEG of each GoPro GPR, found in the index when an import kept it in another sequence), `ArchiveMaintenance`, `MassRenamer` (mass rename and Reorganize: the dry run reads each sidecar for a file template; each step checked under its leases before anything of it moves (where its sequences are, what is inside and at its new folder, a sidecar of a newer format, the title it was planned from when it changes it or names the folder or files after it), then the folder, every index row (the frame paths and render paths of the exports inside it, its own unless its file is still at the old place and another sequence's when its file moved with it, follow it in one transaction; a sidecar that cannot be read or written then still has its row follow, and fails the step), then the files under the plan's journal in `journal/` (a sequence whose photos changed since, such as one still receiving them, or whose files are found changed or in use or have their new names taken keeps its file names, the renames being pinned by digest, and the step and the plan go on); the journal keeps the tally of the whole plan, and a plan its job held (paused, cancelled or failed) is left to that job, only its half-done file renames completed at an archive open, which completes any other plan an interruption left and tells each sequence changed; the file renames of a step: sidecar, a sequence whose sidecar cannot be read staying as it is with every folder around it, index (`FrameRepository.RenameFilesAsync`) and the sequence's journal), `SequenceTrash` (delete into the archive's hidden trash with a dry run, restore; the folder moves back when the index step fails; lists sequence and photo entries; `SequenceTrash.Empty`: the emptying's dry run, fingerprint and erase, each entry checked against its fingerprint again under its lease), `FrameTrash` (photos of a sequence into `trash/photos` with a dry run, a journal that undoes an interrupted deletion whose entry stays unlisted until its sidecar is written, tombstones (a photo never hashed is hashed as it goes; one whose original is missing goes without a file or tombstone; a frame that stays and may be a twin is hashed, its hash kept), positions parked and placed back, proxies and thumbnails moved to follow their frames; `FrameTrash.Restore`: restore re-hashes each file and says why each kept photo stayed (`KeptReason`), under a journal of its own into the sequence that another writer settles only under the entry's lease, into a new sequence through the ingest when theirs is gone, each original staged inside its own entry and the record rewritten after each photo filed), job handlers | | `Processing` | `ProcessingService`, `SequenceAnalyser`, `DeepReports` (a saved deep report while it still describes the frames), `SequenceRender`, `ProjectStore`, `ProjectBundles` (a `.chronos` file for a sequence, and the sequence a file names), `RenderNodeClient`, job handlers | | `Sync` | `SyncService` (sync table of one server: compare, link by content, settle conflicts, enqueue transfers), `SyncRules` (status, settings merge and differences, no I/O), `SyncTombstones` (photos either side removed are not offered back, a replaced photo counts as its replacement), `SyncPositions` (the settings' frame positions translated by photo to the frames of the copy they go to), `SyncLinkStore` (`sync_links`, `sync_state`), `SyncTransfers` and `SyncJobHandler` (resumable, hash-verified uploads and downloads as jobs; downloads commit through the archive ingest), `SyncAgent` (background comparison, server events, backoff, offline proxies), `ProxyCache` and `RemoteSequence` (server sequences opened with proxies), `ISyncRemote` (the server calls; `ChronosSyncRemote` over `ChronosClient`) | @@ -150,11 +150,14 @@ The sidecars are the truth; the index is derived and can always be rebuilt (`Sca snapshot when fresh). Writers coordinate through lease files, so a desktop and a server may share an archive. A writer takes a sequence's folder from its index row read under the sequence's lease (`SequenceWriter.FolderAsync`): a Reorganize moves the folder, its row and the paths of the renders inside it under that lease, so writers of the same index (uploads, proxies and -exports a server files, editor saves, sync applies and "Save both" copies) write where the sequence is. The image -passes hold the thumbnail lease, which a Reorganize takes too. A folder another index moved is found by this index's -next scan, which moves the row, its frames and its exports' paths (`ArchiveIndexer.RebaseRendersAsync`); until then a -writer that finds the folder gone refuses, and a deep analysis that loses it records nothing. A render of another -process whose folder moves while it runs is not stopped: its output can land in a folder created again at the old place. +exports a server files, exports a sync fetches, editor saves, sync applies and "Save both" copies) write where the +sequence is. The image passes hold the thumbnail lease, which a Reorganize takes too: the thumbnail pass for all of it, +the proxy pass a batch at a time (a quarter of the lease wait), reading the folder and sidecar again for each batch. A +folder another index moved is found by this index's next scan, which moves the row, its frames and its exports' paths +(`ArchiveIndexer.RebaseRendersAsync`; an export whose file is still at the old place keeps its path); until then these +writers, a render job as it starts and a proxy pass at each batch refuse when they find the folder gone, and a deep +analysis that loses it records nothing. A render whose folder another index moves while it runs is not stopped: its +output can land in a folder created again at the old place, where its row keeps finding it. Sync (desktop ↔ server, BRIEF §14.5) reuses the same paths: `SyncService` compares the local index with the server's listing (settings and tags carry version counters), uploads go through the chunked protocol into the server's diff --git a/i18n/en/Errors.resx b/i18n/en/Errors.resx index 50a1c81e..47e8adbe 100644 --- a/i18n/en/Errors.resx +++ b/i18n/en/Errors.resx @@ -5,6 +5,7 @@ System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 Sequences not analysed: {0} of {1}; see the job log.long; {0} failed count (2 or more); {1} total count + The folder of this sequence is not where the index has it; try again after the next scan. The archive could not be opened.long The current index could not be backed up or replaced; the restore runs again at the next start.long This backup cannot be restored; choose another one.long diff --git a/i18n/it/Errors.it.resx b/i18n/it/Errors.it.resx index 8230fc73..0574bc79 100644 --- a/i18n/it/Errors.it.resx +++ b/i18n/it/Errors.it.resx @@ -5,6 +5,7 @@ System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 Sequenze non analizzate: {0} su {1}; vedi il registro del job. + La cartella di questa sequenza non è dove la indica l'indice; riprova dopo la prossima scansione. Impossibile aprire l'archivio. Non è stato possibile copiare o sostituire l'indice attuale; il ripristino riparte al prossimo avvio. Questa copia non si può ripristinare; scegline un'altra. diff --git a/src/Chronos.Core/Archiving/LeaseFile.cs b/src/Chronos.Core/Archiving/LeaseFile.cs index d3092607..988ecd2e 100644 --- a/src/Chronos.Core/Archiving/LeaseFile.cs +++ b/src/Chronos.Core/Archiving/LeaseFile.cs @@ -25,6 +25,9 @@ public sealed class LeaseFile : IDisposable { public static readonly TimeSpan DefaultDuration = TimeSpan.FromMinutes(2); + /// The longest a writer waiting for a lease () sleeps between two tries. + public static readonly TimeSpan LongestPoll = TimeSpan.FromMilliseconds(500); + private const string Extension = ".lease"; private const string StaleExtension = ".stale"; private const int ReadAttempts = 5; @@ -84,7 +87,7 @@ public sealed class LeaseFile : IDisposable if (TryAcquire(locksDirectory, name, owner, duration) is { } lease) return lease; if (DateTime.UtcNow >= deadline) throw new LeaseUnavailableException(name, Read(PathFor(locksDirectory, name))); await Task.Delay(delay, cancellationToken).ConfigureAwait(false); - delay = TimeSpan.FromMilliseconds(System.Math.Min(delay.TotalMilliseconds * 2, 500)); + delay = TimeSpan.FromMilliseconds(System.Math.Min(delay.TotalMilliseconds * 2, LongestPoll.TotalMilliseconds)); } } diff --git a/src/Chronos.Data/Archiving/ArchiveImages.cs b/src/Chronos.Data/Archiving/ArchiveImages.cs index 04ddcbb0..e9c7d6f9 100644 --- a/src/Chronos.Data/Archiving/ArchiveImages.cs +++ b/src/Chronos.Data/Archiving/ArchiveImages.cs @@ -5,6 +5,7 @@ using Chronos.Core.Codecs.Jpeg; using Chronos.Core.Imaging; using Chronos.Core.Imaging.Decoders; using Chronos.Core.IO; +using Chronos.Core.Localization; using Chronos.Core.Metadata; using Chronos.Core.Model; @@ -57,6 +58,9 @@ public sealed class ArchiveImages private static readonly TimeSpan CommitInterval = TimeSpan.FromSeconds(10); + /// How long a proxy pass lets the thumbnail lease go between two batches: longer than a waiting writer sleeps between its tries. + private static readonly TimeSpan BatchGap = LeaseFile.LongestPoll * 1.5; + /// /// Age of a proxy file under which its header is not kept: a file system's clock may give a file written again at /// once (with the same length) the same write time, at a granularity of up to two seconds. @@ -214,9 +218,9 @@ public sealed class ArchiveImages int built = 0, reused = 0, undecodable = 0; var complete = true; var tiers = new Dictionary(); - // Thumbnail writers exclude each other only; ingest keeps committing meanwhile. A Reorganize moves the folder under - // this lease too, so the folder is the one the index has once it is held: the pack never opens in a folder the - // sequence left, which opening it would create again. + // Image passes exclude each other only (a proxy pass holds this lease a batch at a time); ingest keeps committing + // meanwhile. A Reorganize moves the folder under this lease too, so the folder is the one the index has once it is + // held: the pack never opens in a folder the sequence left, which opening it would create again. using (await LeaseFile.AcquireAsync(paths.LocksDirectory, $"thumbs-{sequence.Id:N}", _writer.Options.Owner, _writer.Options.LeaseWait, _writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false)) { @@ -292,7 +296,10 @@ public sealed class ArchiveImages /// proxies of the frames an earlier run did again (their headers only). The flag /// of frames this host reads through their proxies follows whether they have one (). Several /// originals are decoded at once (), and a checkpoint still means that every frame before - /// it has its proxy written (). + /// it has its proxy written (). The pass works in batches (), each + /// under the thumbnail lease with the sequence's folder and sidecar read again; a folder that is not where the index + /// has it (another index moved it, until this index's next scan) stops the pass with an + /// , nothing written there. /// public async Task BuildProxiesAsync(Archive archive, Sequence sequence, bool force, int startFrame, Func checkpoint, CancellationToken cancellationToken = default) @@ -301,99 +308,119 @@ public sealed class ArchiveImages ArchiveVolume.EnsureWritable(paths); var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy; if (policy.LongEdge == ProxyPolicy.Disabled) return new ImagePassResult(0, 0, 0); - SequenceManifest manifest; + SequenceManifest? manifest = null; int built = 0, reused = 0, undecodable = 0, waiting = 0; var complete = true; var tiers = new Dictionary(); + var cadence = new Cadence(); // Under the thumbnail lease, as the thumbnail pass: image passes exclude each other only, ingest keeps committing - // meanwhile, and a Reorganize waits for the pass, so the proxies go to the folder the index has once the lease is - // held, never into one the sequence left, which writing them would create again. - using (await LeaseFile.AcquireAsync(paths.LocksDirectory, $"thumbs-{sequence.Id:N}", _writer.Options.Owner, _writer.Options.LeaseWait, - _writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false)) + // meanwhile, and a Reorganize waits for the batch, so the proxies go to the folder the index has once the lease is + // held, never into one the sequence left, which writing them would create again. Between two batches the lease is let + // go for longer than a waiting writer sleeps between its tries, so another process's Reorganize, sequence deletion or + // pack compaction waits for one batch, not for the whole pass. + for (var next = 0; ;) { - var folder = await SequenceWriter.FolderAsync(_db, paths, sequence, cancellationToken).ConfigureAwait(false); - manifest = ManifestStore.ReadSequence(folder.ManifestPath); - Directory.CreateDirectory(folder.Proxies); - var companions = await GprCompanions.FindAsync(_db, archive, manifest.Frames, cancellationToken).ConfigureAwait(false); - var cadence = new Cadence(); - await using (var work = new OrderedWork(ProxyParallelism(manifest.Frames, policy), made => - { - switch (made.Outcome) - { - case ProxyOutcome.Built: - built++; - break; - case ProxyOutcome.Reused: - reused++; - break; - case ProxyOutcome.Waiting: - waiting++; - break; - case ProxyOutcome.Failed: - undecodable++; - complete = false; - break; - default: - undecodable++; - break; - } - if (made.Tier is { } tier) tiers[made.FrameId] = tier; - return cadence.Due() ? checkpoint(made.Index + 1, manifest.Frames.Count) : ValueTask.CompletedTask; - }, cancellationToken)) + using (await LeaseFile.AcquireAsync(paths.LocksDirectory, $"thumbs-{sequence.Id:N}", _writer.Options.Owner, _writer.Options.LeaseWait, + _writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false)) { - for (var i = 0; i < manifest.Frames.Count; i++) + var folder = await SequenceWriter.FolderAsync(_db, paths, sequence, cancellationToken).ConfigureAwait(false); + if (!Directory.Exists(folder.Folder)) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved")); + var read = ManifestStore.ReadSequence(folder.ManifestPath); + // Photos added, removed or renumbered since the last batch: what the earlier batches made may not be theirs. + if (manifest is not null && !SameFrames(read.Frames, manifest.Frames, names: false)) complete = false; + manifest = read; + Directory.CreateDirectory(folder.Proxies); + var companions = await GprCompanions.FindAsync(_db, archive, read.Frames, cancellationToken).ConfigureAwait(false); + var batch = Stopwatch.StartNew(); + await using (var work = new OrderedWork(ProxyParallelism(read.Frames, policy), made => + { + switch (made.Outcome) + { + case ProxyOutcome.Built: + built++; + break; + case ProxyOutcome.Reused: + reused++; + break; + case ProxyOutcome.Waiting: + waiting++; + break; + case ProxyOutcome.Failed: + undecodable++; + complete = false; + break; + default: + undecodable++; + break; + } + if (made.Tier is { } tier) tiers[made.FrameId] = tier; + return cadence.Due() ? checkpoint(made.Index + 1, read.Frames.Count) : ValueTask.CompletedTask; + }, cancellationToken)) { - cancellationToken.ThrowIfCancellationRequested(); - var frame = manifest.Frames[i]; - var path = folder.Proxy(frame.I); - var done = i < startFrame; - if (NeedsProxy(frame.Format)) + // At least one frame per batch, so the pass always moves on. + for (var first = next; next < read.Frames.Count && (next == first || batch.Elapsed < ProxyBatch); next++) { - // Only a client or render node that decodes the original makes this proxy: a preview would make one the - // size of a thumbnail here. One left from a frame renumbered since goes, so it is asked for again, and so - // does one smaller than the policy's (made here from a preview by an earlier version). - var check = Check(path, frame.CapturedAt, frame.W, frame.H, policy).Check; - var usable = check is ProxyCheck.Current or ProxyCheck.Unverifiable; - if (check == ProxyCheck.Stale) Delete(path); - complete &= usable; - if (done) continue; - var outcome = usable ? ProxyOutcome.Reused - : (frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0 ? ProxyOutcome.Unreadable - : ProxyOutcome.Waiting; - await work.AddAsync(new Made(i, frame.Id, outcome, null)).ConfigureAwait(false); - continue; + cancellationToken.ThrowIfCancellationRequested(); + var frame = read.Frames[next]; + var path = folder.Proxy(frame.I); + var done = next < startFrame; + if (NeedsProxy(frame.Format)) + { + // Only a client or render node that decodes the original makes this proxy: a preview would make one the + // size of a thumbnail here. One left from a frame renumbered since goes, so it is asked for again, and so + // does one smaller than the policy's (made here from a preview by an earlier version). + var check = Check(path, frame.CapturedAt, frame.W, frame.H, policy).Check; + var usable = check is ProxyCheck.Current or ProxyCheck.Unverifiable; + if (check == ProxyCheck.Stale) Delete(path); + complete &= usable; + if (done) continue; + var outcome = usable ? ProxyOutcome.Reused + : (frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0 ? ProxyOutcome.Unreadable + : ProxyOutcome.Waiting; + await work.AddAsync(new Made(next, frame.Id, outcome, null)).ConfigureAwait(false); + continue; + } + if (done) + { + // Made by the earlier run, unless the frame did not decode then. + complete &= ReadProxy(path, frame.CapturedAt).Check is ProxyCheck.Current or ProxyCheck.Unverifiable; + continue; + } + if (!force && ReadProxy(path, frame.CapturedAt).Check == ProxyCheck.Current) + { + reused++; + continue; + } + var index = next; + var companion = companions.GetValueOrDefault(frame.Id); + await work.AddAsync(ct => + { + var (image, tier) = Render(folder, frame, companion, recordTier: true, + () => _renderer.Proxy(folder, frame, read.Camera, policy, ct, companion)); + if (image is null) return new Made(index, frame.Id, ProxyOutcome.Failed, tier); + AtomicFile.WriteAllBytes(path, image.Jpeg); + Headers.TryRemove(path, out _); + return new Made(index, frame.Id, ProxyOutcome.Built, tier); + }).ConfigureAwait(false); } - if (done) - { - // Made by the earlier run, unless the frame did not decode then. - complete &= ReadProxy(path, frame.CapturedAt).Check is ProxyCheck.Current or ProxyCheck.Unverifiable; - continue; - } - if (!force && ReadProxy(path, frame.CapturedAt).Check == ProxyCheck.Current) - { - reused++; - continue; - } - var index = i; - var companion = companions.GetValueOrDefault(frame.Id); - await work.AddAsync(ct => - { - var (image, tier) = Render(folder, frame, companion, recordTier: true, - () => _renderer.Proxy(folder, frame, manifest.Camera, policy, ct, companion)); - if (image is null) return new Made(index, frame.Id, ProxyOutcome.Failed, tier); - AtomicFile.WriteAllBytes(path, image.Jpeg); - Headers.TryRemove(path, out _); - return new Made(index, frame.Id, ProxyOutcome.Built, tier); - }).ConfigureAwait(false); + await work.CompleteAsync().ConfigureAwait(false); } - await work.CompleteAsync().ConfigureAwait(false); + if (next >= read.Frames.Count) break; } + await Task.Delay(BatchGap, cancellationToken).ConfigureAwait(false); } await _tiers.RecordAsync(archive, sequence, tiers, proxies: true, cancellationToken).ConfigureAwait(false); await MarkAsync(archive, paths, sequence, manifest, JobKind.Proxies, SequenceStatus.ProxiesBuilt, complete, cancellationToken).ConfigureAwait(false); return new ImagePassResult(built, reused, undecodable, waiting); } + /// + /// How long a proxy pass holds the thumbnail lease for one batch of frames: a quarter of the lease wait, so a writer of + /// another process waiting for the lease meanwhile gets it well before it gives up (the frames the batch started finish + /// first). + /// + private TimeSpan ProxyBatch => _writer.Options.LeaseWait / 4; + /// Photos a thumbnail pass draws at once (): previews of the thumbnail's size serve it. private int ThumbnailParallelism(IReadOnlyList frames, ThumbnailPolicy policy) => PassParallelism(_writer.Options.Parallelism(), _writer.Options.MemoryBudget(), frames, policy.LongEdge, preferPreview: true, _writer.Options.Decoder); @@ -673,13 +700,15 @@ public sealed class ArchiveImages /// removed, renumbered or replaced since have none of what the pass made. A hash learnt since (a verify hashing a /// photo the scan adopted) changes nothing about the photo, and its thumbnail stays its own (). /// - private static bool SameFrames(IReadOnlyList current, IReadOnlyList read) + /// Whether a file renamed since counts as a change (a pass's own batches: it does not, a proxy is + /// kept by frame index and a Reorganize renames files). + private static bool SameFrames(IReadOnlyList current, IReadOnlyList read, bool names = true) { if (current.Count != read.Count) return false; for (var i = 0; i < current.Count; i++) { var (now, then) = (current[i], read[i]); - if (now.Id != then.Id || now.I != then.I || now.File != then.File || now.Size != then.Size || now.CapturedAt != then.CapturedAt + if (now.Id != then.Id || now.I != then.I || (names && now.File != then.File) || now.Size != then.Size || now.CapturedAt != then.CapturedAt || (then.Sha256 is not null && !string.Equals(now.Sha256, then.Sha256, StringComparison.OrdinalIgnoreCase))) return false; } return true; diff --git a/src/Chronos.Data/Archiving/ArchiveIndexer.cs b/src/Chronos.Data/Archiving/ArchiveIndexer.cs index e82b9c38..488e78ad 100644 --- a/src/Chronos.Data/Archiving/ArchiveIndexer.cs +++ b/src/Chronos.Data/Archiving/ArchiveIndexer.cs @@ -187,16 +187,23 @@ public sealed class ArchiveIndexer /// /// The render rows of the archive that follow a sequence's folder from to , - /// in the caller's transaction (): the sequence's own exports - /// inside it, and another sequence's whose file moved with it (at its new path and no longer at the old one, which on a - /// file system that ignores case is the same file when only the case changed). + /// in the caller's transaction (). A file is at its old path + /// still when it is there and the new path is not the same file (on a file system that ignores case it is, when only the + /// case changed). The sequence's own exports inside the folder follow unless their file is at the old path still and not + /// at the new one: written there after the move by a writer that read the old folder, which created it again, so the row + /// keeps naming where the file is. Another sequence's export follows only when its file moved with the folder (at its new + /// path and no longer at the old one). An export missing from both places follows if it is the sequence's own. /// internal static Task RebaseRendersAsync(ChronosRepositories tx, Archive archive, Guid sequenceId, string from, string to, CancellationToken cancellationToken) { var paths = new ArchivePaths(archive.RootPath); - return tx.Renders.RebaseFolderAsync(archive.Id, sequenceId, from, to, (before, after) => File.Exists(paths.FromRelative(after)) - && (string.Equals(before, after, StringComparison.OrdinalIgnoreCase) || !File.Exists(paths.FromRelative(before))), cancellationToken); + return tx.Renders.RebaseFolderAsync(archive.Id, sequenceId, from, to, (before, after, own) => + { + var moved = File.Exists(paths.FromRelative(after)); + var behind = !string.Equals(before, after, StringComparison.OrdinalIgnoreCase) && File.Exists(paths.FromRelative(before)); + return own ? moved || !behind : moved && !behind; + }, cancellationToken); } /// diff --git a/src/Chronos.Data/Archiving/FrameTrash.cs b/src/Chronos.Data/Archiving/FrameTrash.cs index f47e2f57..452d3d4f 100644 --- a/src/Chronos.Data/Archiving/FrameTrash.cs +++ b/src/Chronos.Data/Archiving/FrameTrash.cs @@ -584,8 +584,9 @@ public sealed partial class FrameTrash /// Moves the thumbnails of the frames whose index changed to their new index in thumbs.pack (BRIEF §8.5: the pack /// is keyed by frame index) and drops those of the frames removed, as does for proxies, so the /// thumbnail pass queued afterwards reuses them instead of drawing the rest of the sequence again. Only the pack's index - /// is rewritten. A thumbnail writer holding the pack meanwhile (its lease) leaves it as it is, and so does a failure - /// (logged): each thumbnail carries its original's fingerprint, so one in the wrong place is drawn again. + /// is rewritten. An image pass holding the thumbnail lease meanwhile (a thumbnail pass, or a batch of a proxy pass) + /// leaves the pack as it is, and so does a failure (logged): each thumbnail carries its original's fingerprint, so one + /// in the wrong place is drawn again. /// private void ShiftThumbnails(ArchivePaths paths, SequencePaths folder, Guid sequenceId, IReadOnlyList before, IReadOnlyList after) @@ -598,7 +599,7 @@ public sealed partial class FrameTrash using var lease = LeaseFile.TryAcquire(paths.LocksDirectory, $"thumbs-{sequenceId:N}", _writer.Options.Owner, _writer.Options.LeaseDuration); if (lease is null) { - Log.Info("archive", $"Thumbnails of {folder.RelativeFolder} are being written; the thumbnail pass draws those that moved again"); + Log.Info("archive", $"An image pass holds the thumbnails of {folder.RelativeFolder}; the thumbnail pass draws those that moved again"); return; } using var pack = ThumbPackWriter.Open(folder.ThumbsPackPath); diff --git a/src/Chronos.Data/Archiving/SequenceWriter.cs b/src/Chronos.Data/Archiving/SequenceWriter.cs index 5807a78d..4c6726f0 100644 --- a/src/Chronos.Data/Archiving/SequenceWriter.cs +++ b/src/Chronos.Data/Archiving/SequenceWriter.cs @@ -57,7 +57,7 @@ public sealed record ArchiveServiceOptions /// journaled, or once its folder moved and its index row followed when the step then failed: by its job, or by an archive /// open that completed a plan an interruption left (). /// A server tells its connected clients (BRIEF §14.3); the service it opens its archive with at start sets none, as no - /// client is connected yet. A failure is logged and never fails the job. + /// client is connected yet. The desktop reloads an editor open on the sequence. A failure is logged and never fails the job. /// public Func? SequenceChanged { get; init; } diff --git a/src/Chronos.Data/Processing/Jobs/RenderJobHandler.cs b/src/Chronos.Data/Processing/Jobs/RenderJobHandler.cs index ba951e69..bfa63e57 100644 --- a/src/Chronos.Data/Processing/Jobs/RenderJobHandler.cs +++ b/src/Chronos.Data/Processing/Jobs/RenderJobHandler.cs @@ -21,7 +21,8 @@ namespace Chronos.Data.Processing.Jobs; /// server), named by the template without taking a name another render of this process is writing, and each finished /// output is recorded at once (its row id derives from the job and the entry, so every attempt knows what is recorded). /// Checkpoint: the entries recorded and the output being written, so a crashed attempt's leftovers are removed and only -/// unfinished entries render again. A failed entry fails the job after the others ran; retrying renders it alone. +/// unfinished entries render again. A failed entry fails the job after the others ran; retrying renders it alone. A +/// sequence whose folder is not where the index has it (another index moved it, until this index's next scan) is refused. /// internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions options, FrameTiers tiers) : IJobHandler { @@ -40,6 +41,9 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt var paths = new ArchivePaths(archive.RootPath); ArchiveVolume.EnsureWritable(paths); var folder = paths.Sequence(sequence.Folder); + // A folder another index moved is found by this index's next scan. Until then nothing renders: its photos would all be + // stand-ins, and its outputs would create the old folder again. + if (!Directory.Exists(folder.Folder)) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved")); // A server writes only inside its archive (BRIEF §14.2): an export folder chosen on a client means nothing there. if (options.Host == EncoderHost.Server) export = export with { Folder = null }; var project = (parameters.SettingsJson is { } embedded ? ProjectStore.Parse(embedded) : ProjectStore.Read(sequence, folder)) with { Export = export }; diff --git a/src/Chronos.Data/Repositories/RenderRepository.cs b/src/Chronos.Data/Repositories/RenderRepository.cs index b7b25d14..6ba53e89 100644 --- a/src/Chronos.Data/Repositories/RenderRepository.cs +++ b/src/Chronos.Data/Repositories/RenderRepository.cs @@ -48,17 +48,19 @@ public sealed class RenderRepository /// /// Rewrites the archive-relative output paths of an archive's renders after the folder of sequence /// moved (a Reorganize, or a move another index of the archive or the user made, which a - /// scan finds): an export inside the folder moved with it. The sequence's own exports there follow it, and so does an - /// export of another sequence saved inside it (through the Export folder picker) when says its - /// file moved with it: a sequence that took the old folder since (a later Reorganize another index made) keeps its own. - /// Outputs elsewhere (absolute paths, other folders) stay as they are. + /// scan finds): an export inside the folder moved with it. Each render under the old folder whose file + /// the folder takes the new path: the caller tells the sequence's own exports there (which + /// follow unless their file stayed behind) from an export of another sequence saved inside it through the Export + /// folder picker (which follows only when its file moved with the folder: a sequence that took the old folder since, by + /// a later Reorganize another index made, keeps its own). Outputs elsewhere (absolute paths, other folders) stay as they are. /// - /// Whether the file at the first archive-relative path is at the second one now. + /// Whether the render at the first archive-relative path follows to the second one; the third + /// argument says whether it is one of the sequence's own. /// Only paths still under change, so repeating it after a crash is harmless. - public async Task RebaseFolderAsync(Guid archiveId, Guid sequenceId, string oldFolder, string newFolder, Func moved, + public async Task RebaseFolderAsync(Guid archiveId, Guid sequenceId, string oldFolder, string newFolder, Func follows, CancellationToken cancellationToken = default) { - ArgumentNullException.ThrowIfNull(moved); + ArgumentNullException.ThrowIfNull(follows); // The archive root never moves, and its prefix ("/") would match absolute paths outside the archive. if (oldFolder.Length == 0) return 0; // substr counts characters (code points) on both dialects. @@ -71,7 +73,7 @@ public sealed class RenderRepository foreach (var (id, owner, path) in under) { var to = newFolder.Length == 0 ? path[prefix.Length..] : newFolder + path[oldFolder.Length..]; - if (owner != sequenceId && !moved(path, to)) continue; + if (!follows(path, to, owner == sequenceId)) continue; rebased += await _db.ExecuteAsync("UPDATE renders SET output_path = $2 WHERE id = $1", [id, to], cancellationToken).ConfigureAwait(false); } return rebased; diff --git a/src/Chronos.Data/Sync/SyncService.cs b/src/Chronos.Data/Sync/SyncService.cs index 08d98b04..298dc26d 100644 --- a/src/Chronos.Data/Sync/SyncService.cs +++ b/src/Chronos.Data/Sync/SyncService.cs @@ -177,7 +177,9 @@ public sealed class SyncService /// /// Settles a conflict as the user chose (BRIEF §12.7), against the versions the user was shown: when either side /// changed again since, nothing is written, the sequence is compared again and - /// is thrown. Returns the file this PC's settings were saved to for , else null. + /// is thrown, as it is when cannot keep this PC's version first (the folder is not + /// where the index has it). Returns the file this PC's settings were saved to for , + /// else null. /// public async Task ResolveAsync(ISyncRemote remote, Guid remoteId, SyncResolution resolution, CancellationToken cancellationToken) { @@ -222,9 +224,14 @@ public sealed class SyncService { changed = true; } + else if (resolution == SyncResolution.SaveBoth && (saved = await SaveCopyAsync(local, cancellationToken).ConfigureAwait(false)) is null) + { + // This PC's version could not be kept (another index moved the folder): the server's does not replace it, + // even where this index's scan finds the folder before the server's version would be written. + changed = true; + } else { - if (resolution == SyncResolution.SaveBoth) saved = await SaveCopyAsync(local, cancellationToken).ConfigureAwait(false); var theirs = SyncPayload.Of(dto, manifest!.SettingsJson); if (await ApplyLocallyAsync(local, theirs, Positions(remote, local, manifest), cancellationToken).ConfigureAwait(false) is { } applied) { @@ -234,7 +241,7 @@ public sealed class SyncService { // Edited on this PC since it was read: the copy of a version the user did not choose to keep goes. changed = true; - if (saved is not null) File.Delete(saved); + if (saved is not null) await DeleteCopyAsync(local, saved, cancellationToken).ConfigureAwait(false); saved = null; } } @@ -816,6 +823,30 @@ public sealed class SyncService return path; } + /// + /// Deletes a "Save both" copy of a version the user did not keep from the sequence's folder as the index has it now: a + /// Reorganize may have moved the folder, the copy with it, since the copy was written. Best effort: a copy that cannot + /// be reached (the folder gone, the sequence busy, the archive offline) stays, as one more copy of this PC's version. + /// + private async Task DeleteCopyAsync(Sequence local, string saved, CancellationToken cancellationToken) + { + try + { + var archive = await _db.Archives.GetAsync(local.ArchiveId, cancellationToken).ConfigureAwait(false) + ?? throw new InvalidDataException($"Archive {local.ArchiveId} is not indexed."); + var paths = new ArchivePaths(archive.RootPath); + await using (await _writer.LockAsync(paths, local.Id, cancellationToken).ConfigureAwait(false)) + { + var folder = await SequenceWriter.FolderAsync(_db, paths, local, cancellationToken).ConfigureAwait(false); + if (Directory.Exists(folder.MetaDirectory)) File.Delete(Path.Combine(folder.MetaDirectory, Path.GetFileName(saved))); + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException) + { + Log.Warning(LogCategory, $"The copy {saved} of {local.DisplayTitle}'s settings could not be deleted", ex); + } + } + /// The file name of a "Save both" copy kept at . public static string SavedCopyName(DateTime localTime) => string.Create(CultureInfo.InvariantCulture, $"{SavedCopyPrefix}{localTime.ToString(SavedCopyTimeFormat, CultureInfo.InvariantCulture)}.json"); diff --git a/src/Chronos.Data/Sync/SyncTransfers.cs b/src/Chronos.Data/Sync/SyncTransfers.cs index 9db45a7d..5b60780d 100644 --- a/src/Chronos.Data/Sync/SyncTransfers.cs +++ b/src/Chronos.Data/Sync/SyncTransfers.cs @@ -370,31 +370,43 @@ internal sealed class SyncTransfers(SyncService sync) EnsureSpace(paths.Root, wanted.Sum(r => r.Bytes)); var staging = Path.Combine(paths.StagingDirectory, "exports-" + remoteId.ToString("N")); - var exports = paths.Sequence(local.Folder).Exports; Directory.CreateDirectory(staging); - Directory.CreateDirectory(exports); var progress = new TransferProgress(context, wanted.Count, 0); await progress.ReportAsync().ConfigureAwait(false); foreach (var render in wanted) { var (part, name) = await FetchRenderAsync(remote, render, staging, ct).ConfigureAwait(false); - var target = Unique(exports, name); - File.Move(part, target); - await db.Renders.InsertAsync(new Render + // Filed under the sequence's lease into its folder as the index has it then, as a server files a render: a + // Reorganize may have moved it since the sequence was read, and moves the row's path along when it moves it later. + // One another index moved is found by this index's next scan; until then nothing is filed (the download stays + // staged for the next attempt), so the folder it left is not created again. + var filed = await sync.Archives.UnderLeaseAsync(local, async row => { - Id = render.Id, - SequenceId = local.Id, - PresetName = render.PresetName, - OutputPath = paths.ToRelative(target), - Codec = render.Codec, - Width = render.Width, - Height = render.Height, - Fps = render.Fps, - DurationSeconds = render.DurationSeconds, - Bytes = new FileInfo(target).Length, - Created = render.Created, - ProducedBy = render.ProducedBy, + if (row is null) throw new InvalidOperationException(Loc.L("Errors.Sync.LocalMissing")); + var folder = paths.Sequence(row.Folder); + if (!Directory.Exists(folder.Folder)) return false; + Directory.CreateDirectory(folder.Exports); + var target = Unique(folder.Exports, name); + File.Move(part, target); + // Recorded even if the job is being cancelled: a filed export is never left without its row. + await db.Renders.InsertAsync(new Render + { + Id = render.Id, + SequenceId = row.Id, + PresetName = render.PresetName, + OutputPath = paths.ToRelative(target), + Codec = render.Codec, + Width = render.Width, + Height = render.Height, + Fps = render.Fps, + DurationSeconds = render.DurationSeconds, + Bytes = new FileInfo(target).Length, + Created = render.Created, + ProducedBy = render.ProducedBy, + }, CancellationToken.None).ConfigureAwait(false); + return true; }, ct).ConfigureAwait(false); + if (!filed) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved")); await progress.StepAsync().ConfigureAwait(false); } TryDeleteDirectory(staging); diff --git a/src/Chronos.Protocol/Jobs.cs b/src/Chronos.Protocol/Jobs.cs index a157f990..5a212dbd 100644 --- a/src/Chronos.Protocol/Jobs.cs +++ b/src/Chronos.Protocol/Jobs.cs @@ -56,7 +56,11 @@ public sealed record JobPriorityRequest(string Priority); /// public sealed record RenderJobParameters(string PresetName, string PresetJson, string Codec, int Width, int Height, bool UseProxies, string? SettingsJson); -/// POST /api/v1/nodes/{id}/jobs/{jobId}/result: the finished output a node is about to upload in chunks. +/// +/// POST /api/v1/nodes/{id}/jobs/{jobId}/result: the finished output a node is about to upload in chunks. A +/// result/complete that cannot file it for now answers 503 sequence_busy or archive_offline; the +/// upload and the job stay the node's, to be completed again. +/// public sealed record RenderResultRequest( string FileName, long Size, diff --git a/src/Chronos.Protocol/Uploads.cs b/src/Chronos.Protocol/Uploads.cs index 8b2cf6fd..1df2267c 100644 --- a/src/Chronos.Protocol/Uploads.cs +++ b/src/Chronos.Protocol/Uploads.cs @@ -43,7 +43,8 @@ public sealed record CardImportReport(Guid ImportId, string Card, string State, /// renders are uploaded on request). The render keeps its id on the server, so both sides recognise it; when the /// server already has that id it answers 200 with . Chunks go to /// /uploads/{uploadId}/chunks/{n}; POST /sequences/{id}/renders/{uploadId}/complete verifies the SHA-256 -/// and files the render under the sequence's exports. +/// and files the render under the sequence's exports. A completion that cannot file it for now answers 503 +/// sequence_busy or archive_offline and keeps the upload, to be completed again. /// public sealed record RenderUploadRequest( Guid RenderId, diff --git a/src/Chronos.Server/Jobs/RenderNodeService.cs b/src/Chronos.Server/Jobs/RenderNodeService.cs index 3e0e7e90..7f1bf866 100644 --- a/src/Chronos.Server/Jobs/RenderNodeService.cs +++ b/src/Chronos.Server/Jobs/RenderNodeService.cs @@ -1,5 +1,6 @@ using System.Globalization; using System.Text.Json; +using Chronos.Core.Archiving; using Chronos.Core.Model; using Chronos.Data; using Chronos.Data.Jobs; @@ -175,7 +176,16 @@ public sealed class RenderNodeService( if (await uploads.StageVerifiedAsync(upload, cancellationToken) is not { } staged) return Outcome.Fail(422, "hash_mismatch"); var output = JsonSerializer.Deserialize(upload.RenderJson, ProtocolJson.Default.RenderResultRequest)!; var sequence = await db.Sequences.GetAsync(job.TargetIds[0], cancellationToken); - var render = sequence is null ? null : await renders.CommitAsync(sequence, staged, output, node.ClientName, cancellationToken); + Render? render; + try + { + render = sequence is null ? null : await renders.CommitAsync(sequence, staged, output, node.ClientName, cancellationToken); + } + catch (Exception ex) when (ex is LeaseUnavailableException or ArchiveOfflineException) + { + // The node keeps the job and completes again. + return await uploads.ReceiveAgainAsync(upload, staged, ex); + } if (render is null) { UploadStore.TryDelete(staged); diff --git a/src/Chronos.Server/Storage/RenderUploads.cs b/src/Chronos.Server/Storage/RenderUploads.cs index 02e2214d..33bb017b 100644 --- a/src/Chronos.Server/Storage/RenderUploads.cs +++ b/src/Chronos.Server/Storage/RenderUploads.cs @@ -1,5 +1,7 @@ using System.Globalization; using System.Text.Json; +using Chronos.Core.Archiving; +using Chronos.Core.Model; using Chronos.Data; using Chronos.Data.Repositories; using Chronos.Protocol; @@ -87,8 +89,16 @@ public sealed class RenderUploads( var sequence = await db.Sequences.GetAsync(sequenceId, cancellationToken); var result = new RenderResultRequest(output.FileName, output.Size, output.Sha256, output.PresetName, output.Codec, output.Width, output.Height, output.Fps, output.DurationSeconds); - var render = sequence is null ? null : await renders.CommitAsync(sequence, staged, result, output.ProducedBy ?? caller.ClientName ?? caller.User.Username, - output.RenderId, cancellationToken); + Render? render; + try + { + render = sequence is null ? null : await renders.CommitAsync(sequence, staged, result, output.ProducedBy ?? caller.ClientName ?? caller.User.Username, + output.RenderId, cancellationToken); + } + catch (Exception ex) when (ex is LeaseUnavailableException or ArchiveOfflineException) + { + return await uploads.ReceiveAgainAsync(upload, staged, ex); + } if (render is null) { UploadStore.TryDelete(staged); diff --git a/src/Chronos.Server/Storage/UploadService.cs b/src/Chronos.Server/Storage/UploadService.cs index f3468b48..b46aaf18 100644 --- a/src/Chronos.Server/Storage/UploadService.cs +++ b/src/Chronos.Server/Storage/UploadService.cs @@ -1,6 +1,7 @@ using System.Collections.Concurrent; using System.Globalization; using Microsoft.AspNetCore.Http.Features; +using Chronos.Core.Archiving; using Chronos.Data; using Chronos.Data.Archiving; using Chronos.Data.Repositories; @@ -303,6 +304,21 @@ public sealed class UploadService( return null; } + /// + /// A verified upload whose filing was refused before anything moved, for a reason that passes: its sequence held by + /// another writer past the lease wait (503 sequence_busy), or the archive offline or read-only (503 + /// archive_offline). The staged copy goes and the upload receives again with its part file, so the client + /// completes it again, as it does after any 503. + /// + public async Task> ReceiveAgainAsync(Upload upload, string staged, Exception refusal) + { + ArgumentNullException.ThrowIfNull(upload); + logger.LogWarning("Upload {Upload} ({File}) not filed for now: {Reason}", upload.Id, upload.FileName, refusal.Message); + UploadStore.TryDelete(staged); + await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Receiving, cancellationToken: CancellationToken.None); + return Fail(503, refusal is ArchiveOfflineException ? "archive_offline" : "sequence_busy"); + } + private async Task FailAsync(Upload upload) { await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Failed, cancellationToken: CancellationToken.None); diff --git a/tests/Chronos.App.Tests/Editor/ProjectStoreTests.cs b/tests/Chronos.App.Tests/Editor/ProjectStoreTests.cs index 375994fc..40a7d8fa 100644 --- a/tests/Chronos.App.Tests/Editor/ProjectStoreTests.cs +++ b/tests/Chronos.App.Tests/Editor/ProjectStoreTests.cs @@ -268,6 +268,27 @@ public sealed class ProjectStoreTests(TestContext context) } } + [Test] + public async Task ASaveAfterAnotherIndexMovedTheFolderWritesNothing() + { + var (db, sequenceId, root) = await ArchiveAsync(4); + await using (db) + { + await using var source = new LocalSequenceSource(db, sequenceId, DecoderFactory.CreateClient(null), null, + new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) }); + var content = await source.LoadAsync(context.Cancellation); + + // A server sharing the archive moved the folder; this index finds it there at its next scan. + var left = Path.Combine(root, "Synthetic"); + Directory.Move(left, Path.Combine(root, "Moved")); + var gone = await Assert.ThrowsAsync(() => source.SaveProjectAsync(content.Project with { HeroFrame = 1 }, content.Project, context.Cancellation)); + + Assert.Equal(Loc.L("Editor.Error.Missing"), gone.Message); + Assert.False(Directory.Exists(left), "the folder it left is not created again"); + Assert.Equal(content.Sequence.SettingsVersion, (await db.Sequences.GetAsync(sequenceId))!.SettingsVersion, "nothing recorded"); + } + } + [Test] public async Task JobsGoToTheEngine() { diff --git a/tests/Chronos.Data.Tests/Archiving/MassRenameTests.cs b/tests/Chronos.Data.Tests/Archiving/MassRenameTests.cs index b4494100..a22a7ba6 100644 --- a/tests/Chronos.Data.Tests/Archiving/MassRenameTests.cs +++ b/tests/Chronos.Data.Tests/Archiving/MassRenameTests.cs @@ -305,6 +305,29 @@ public sealed class MassRenameTests(TestContext context) Assert.SequenceEqual(expected, await OutputsAsync(fx, renders)); } + [Test] + public async Task AnExportWrittenWhereItsFolderWasStaysWithItsFile() + { + await using var fx = await CreateAsync(context); + var ids = await PopulateAsync(fx); + List renders = [await RenderAsync(fx, ids[0], "inbox/a/exports/A.mp4")]; + var (server, archive) = await fx.OtherWriterAsync(); + + // The other index moves "inbox/a"; a writer here that read the folder before files an export where it was, creating + // it again. Another export of the sequence is missing from both places. + var plan = await server.PlanRenameAsync(archive.Id, [ids[0]], "moved/{date}", null, fx.Ct); + await server.ApplyRenameAsync(plan, null, fx.Ct); + renders.Add(await RenderAsync(fx, ids[0], "inbox/a/exports/Late.mp4")); + renders.Add(await RenderAsync(fx, ids[0], "inbox/a/exports/Gone.mp4", write: false)); + + // The scan here moves the rows whose file went with the folder or is nowhere, and keeps the one whose file stayed. + await fx.ScanAsync(); + Assert.Equal("moved/2026-09-28", (await fx.Db.Sequences.GetAsync(ids[0], fx.Ct))!.Folder); + string[] expected = ["moved/2026-09-28/exports/A.mp4", "inbox/a/exports/Late.mp4", "moved/2026-09-28/exports/Gone.mp4"]; + Assert.SequenceEqual(expected, await OutputsAsync(fx, renders)); + foreach (var path in expected[..2]) Assert.FileExists(fx.Paths.FromRelative(path)); + } + [Test] public async Task APhotoDeletionWhoseFolderMovedWhileItWaitedForTheLeaseIsRefused() { diff --git a/tests/Chronos.Data.Tests/Archiving/ProxyTests.cs b/tests/Chronos.Data.Tests/Archiving/ProxyTests.cs index 8872b085..0673dd94 100644 --- a/tests/Chronos.Data.Tests/Archiving/ProxyTests.cs +++ b/tests/Chronos.Data.Tests/Archiving/ProxyTests.cs @@ -2,9 +2,11 @@ using Chronos.Core.Archiving; using Chronos.Core.Codecs.Jpeg; using Chronos.Core.Imaging; using Chronos.Core.Imaging.Decoders; +using Chronos.Core.Localization; using Chronos.Core.Metadata; using Chronos.Core.Model; using Chronos.Data.Archiving; +using Chronos.Data.Tests.Processing; using Chronos.Testing; using static Chronos.Data.Tests.Archiving.ArchiveFixture; @@ -88,6 +90,95 @@ public sealed class ProxyTests(TestContext context) Assert.FileMissing(now.Proxy(1)); } + [Test] + public async Task ProxiesOfAFolderAnotherIndexMovedWaitForThisIndexsScan() + { + await using var fx = await CreateAsync(context); + var (sequence, frames) = await HeifSequenceAsync(fx); + var images = fx.Service.Images; + var left = fx.Paths.Sequence(sequence.Folder).Folder; + var (server, archive) = await fx.OtherWriterAsync(); + var plan = await server.PlanRenameAsync(archive.Id, [sequence.Id], "moved/{date}", null, fx.Ct); + await server.ApplyRenameAsync(plan, null, fx.Ct); + var moved = fx.Paths.Sequence(plan.Steps.Single().ToFolder); + + // This index still has the folder where it was: a proxy sent and a proxy pass write nothing, anywhere. + var proxy = Proxy(frames[0].CapturedAt!.Value, 2); + Assert.Equal(ProxyStoreResult.Gone, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], proxy, fx.Ct)); + var refused = await Assert.ThrowsAsync(() => + images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct)); + Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), refused.Message); + Assert.False(Directory.Exists(left), "the folder it left is not created again"); + Assert.FileMissing(moved.Proxy(0)); + + // Once this index's scan found it there, they go where it is. + await fx.ScanAsync(); + Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], proxy, fx.Ct)); + Assert.SequenceEqual(proxy, await File.ReadAllBytesAsync(moved.Proxy(0), fx.Ct)); + } + + [Test] + public async Task AProxyPassLetsAnotherProcessReorganizeBetweenItsBatches() + { + await using var fx = await CreateAsync(context); + Guid id = default; + for (var i = 0; i < 4; i++) + { + var staged = fx.Stage(Start.AddSeconds(10 * i), 60 + i); + id = (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, "batches", staged.Path, $"IMG_{i + 1:0000}.JPG", + staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId; + } + var sequence = (await fx.Db.Sequences.GetAsync(id, fx.Ct))!; + var left = fx.Paths.Sequence(sequence.Folder).Folder; + // The second photo is read as a PNG, through the decoders (JPEGs are decoded without them), and slowly. + var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath); + ManifestStore.WriteSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath, sidecar with + { + Frames = [.. sidecar.Frames.Select(f => f.I == 1 ? f with { Format = ImageFileFormat.Png } : f)], + }); + var gate = new GateDecoder { Hold = Path.GetFileName(sidecar.Frames[1].File) }; + var (server, archive) = await fx.OtherWriterAsync(); + var plan = await server.PlanRenameAsync(archive.Id, [id], "moved/{date}", null, fx.Ct); + var moved = fx.Paths.Sequence(plan.Steps.Single().ToFolder); + + // A proxy pass here, one photo at a time, in batches of a quarter of its lease wait. + var batch = TimeSpan.FromSeconds(TestTiming.Budget(2)); + var here = new ArchiveService(fx.Db, new ArchiveServiceOptions + { + Owner = "test", + LeaseWait = batch * 4, + Decoder = new ImageDecoder([gate]), + Parallelism = () => 1, + }); + var pass = Task.Run(() => here.Images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct)); + await Task.WhenAny(gate.Holding.Task, pass).WaitAsync(TimeSpan.FromSeconds(60), fx.Ct); + if (pass.IsCompleted) Assert.Fail($"The pass ended before the held photo: {await pass}"); + var held = DateTime.UtcNow; + + // Another process sharing the archive reorganizes the sequence: it holds the sequence, then waits for the thumbnail lease. + var move = Task.Run(() => server.ApplyRenameAsync(plan, null, fx.Ct)); + var until = DateTime.UtcNow + TimeSpan.FromSeconds(TestTiming.Budget(30)); + while (LeaseFile.Holder(fx.Paths.LocksDirectory, $"seq-{id:N}")?.Owner != "other") + { + Assert.False(move.IsCompleted, "the Reorganize waits for the pass"); + Assert.True(DateTime.UtcNow < until, "the Reorganize holds the sequence"); + await Task.Delay(20, fx.Ct); + } + var rest = held + batch + TimeSpan.FromSeconds(0.5) - DateTime.UtcNow; + if (rest > TimeSpan.Zero) await Task.Delay(rest, fx.Ct); + gate.Release(); + + // The batch ends with that photo and lets the lease go: the Reorganize moves the folder before the pass ends, and the + // pass, finding the folder gone from where this index has it, stops there. + await move.WaitAsync(TimeSpan.FromSeconds(TestTiming.Budget(60)), fx.Ct); + var stopped = await Assert.ThrowsAsync(() => pass); + Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), stopped.Message); + Assert.False(Directory.Exists(left), "the folder it left is not created again"); + Assert.FileExists(moved.Proxy(0)); + Assert.FileExists(moved.Proxy(1)); + Assert.FileMissing(moved.Proxy(2)); + } + [Test] public async Task AProxyLeftByARenumberedFrameIsAskedForAgain() { diff --git a/tests/Chronos.Data.Tests/Processing/RenderJobTests.cs b/tests/Chronos.Data.Tests/Processing/RenderJobTests.cs index aa3e512b..3e30bbe8 100644 --- a/tests/Chronos.Data.Tests/Processing/RenderJobTests.cs +++ b/tests/Chronos.Data.Tests/Processing/RenderJobTests.cs @@ -126,6 +126,34 @@ public sealed class RenderJobTests(TestContext context) Assert.Empty(await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct)); } + [Test] + public async Task ARenderOfAFolderAnotherIndexMovedWaitsForThisIndexsScan() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(Scene()); + var left = fx.Folder(sequence).Folder; + var (server, archive) = await fx.Archive.OtherWriterAsync(); + var plan = await server.PlanRenameAsync(archive.Id, [sequence.Id], "Moved/{date}", null, fx.Ct); + await server.ApplyRenameAsync(plan, null, fx.Ct); + await fx.StartEngineAsync(); + + // A server sharing the archive moved the folder; this index has not scanned since. Nothing renders, nothing is flagged. + var job = await fx.EnqueueAsync(ProcessingJobs.Render(sequence, Mjpeg, 96, 64)); + var failed = await fx.WaitForAsync(job.Id, JobState.Failed); + Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), failed.Error); + Assert.False(Directory.Exists(left), "the folder it left is not created again"); + Assert.Empty(await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct)); + Assert.True((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).All(f => (f.Flags & (FrameFlags.Missing | FrameFlags.Undecodable)) == 0), "no frame flagged"); + + // Once this index's scan found it there, the render goes into its exports. + await fx.Archive.ScanAsync(); + var again = await fx.EnqueueAsync(ProcessingJobs.Render(sequence, Mjpeg, 96, 64)); + await fx.WaitForAsync(again.Id, JobState.Done); + var moved = fx.Archive.Paths.Sequence(plan.Steps.Single().ToFolder); + Assert.FileExists(Path.Combine(moved.Exports, "clip-MJPEG.avi")); + Assert.False(Directory.Exists(left)); + } + [Test] public async Task AFrameThatNoLongerDecodesIsReplacedByItsNeighbourAndLeftOutNextTime() { diff --git a/tests/Chronos.Data.Tests/Sync/FakeSyncRemote.cs b/tests/Chronos.Data.Tests/Sync/FakeSyncRemote.cs index ce4f9d38..14ffe107 100644 --- a/tests/Chronos.Data.Tests/Sync/FakeSyncRemote.cs +++ b/tests/Chronos.Data.Tests/Sync/FakeSyncRemote.cs @@ -121,12 +121,20 @@ internal sealed class FakeSyncRemote : ISyncRemote } } - public Task> ListFramesAsync(Guid sequenceId, CancellationToken cancellationToken) + /// Runs once, before the next frame listing is answered (frame positions translated against the server's frames). + public Func? OnceBeforeFrames { get; set; } + + public async Task> ListFramesAsync(Guid sequenceId, CancellationToken cancellationToken) { + if (OnceBeforeFrames is { } before) + { + OnceBeforeFrames = null; + await before(); + } lock (_gate) { var sequence = Find(sequenceId); - return Task.FromResult>([.. sequence.Frames.Select(sequence.FrameDto)]); + return [.. sequence.Frames.Select(sequence.FrameDto)]; } } diff --git a/tests/Chronos.Data.Tests/Sync/SyncServiceTests.cs b/tests/Chronos.Data.Tests/Sync/SyncServiceTests.cs index 25319cfb..2ce330d1 100644 --- a/tests/Chronos.Data.Tests/Sync/SyncServiceTests.cs +++ b/tests/Chronos.Data.Tests/Sync/SyncServiceTests.cs @@ -456,6 +456,77 @@ public sealed class SyncServiceTests(TestContext context) Assert.False(Directory.Exists(left), "the folder it left is not created again"); } + [Test] + public async Task SaveBothThatCannotKeepThisPcsVersionTakesNothing() + { + await using var f = await SyncFixture.CreateAsync(context); + // The server holds a photo this PC does not have yet, and its settings name a frame: taking them reads the server's + // frames, after this PC's copy was kept and before the server's version is written. + var photos = f.NewPhotos(3); + var local = await f.LocalAsync("Holy grail", photos[..2]); + var remote = f.Remote.Add("Holy grail", photos); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + local = await f.EditLocalSettingsAsync(local.Id, SyncFixture.Window41); + var theirs = """{"heroFrame":1,"exposure":{"deflickerWindow":{"isAuto":false,"manual":33}}}"""; + f.Remote.EditSettings(remote.Id, theirs); + await f.RefreshAsync(); + Assert.NotNull((await f.EntryAsync("Holy grail")).Link!.ConflictRemote); + var left = f.Archive.Paths.Sequence(local.Folder).Folder; + + // Another index moves the folder while the server's version is read, so this PC's version cannot be kept there; this + // index's scan finds the folder before the server's version would be written. + var (other, archive) = await f.Archive.OtherWriterAsync(); + string? moved = null; + f.Remote.OnceBeforeManifest = async () => + { + var plan = await other.PlanRenameAsync(archive.Id, [local.Id], "moved/{title}", null, f.Ct); + await other.ApplyRenameAsync(plan, null, f.Ct); + moved = plan.Steps.Single().ToFolder; + }; + f.Remote.OnceBeforeFrames = () => f.Archive.ScanAsync(); + await Assert.ThrowsAsync(() => f.Sync.ResolveAsync(f.Remote, remote.Id, SyncResolution.SaveBoth, f.Ct)); + // The server's frames were not read for a version that is not taken: the scan comes now. + f.Remote.OnceBeforeFrames = null; + await f.Archive.ScanAsync(); + + var kept = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!; + Assert.Equal(moved, kept.Folder, "the scan found the folder"); + Assert.True(SyncRules.SameSettings(SyncFixture.Window41, kept.SettingsJson), "this PC's version is not replaced without its copy"); + var meta = f.Archive.Paths.Sequence(kept.Folder).MetaDirectory; + Assert.Empty(Directory.GetFiles(meta, SyncService.SavedCopyPattern)); + Assert.False(Directory.Exists(left), "the folder it left is not created again"); + Assert.NotNull((await f.EntryAsync("Holy grail")).Link!.ConflictRemote, "still a conflict"); + + // Asked again where the sequence is now, both versions are kept. + var copy = await f.Sync.ResolveAsync(f.Remote, remote.Id, SyncResolution.SaveBoth, f.Ct); + Assert.Equal(copy, Assert.Contains(Directory.GetFiles(meta, SyncService.SavedCopyPattern), _ => true)); + Assert.True(SyncRules.SameSettings(SyncFixture.Window41, await File.ReadAllTextAsync(copy!, f.Ct)), "this PC's version is kept"); + } + + [Test] + public async Task TakeServerWhereAnotherIndexMovedTheFolderWritesNothing() + { + await using var f = await SyncFixture.CreateAsync(context); + var (local, remote) = await ConflictAsync(f); + var left = f.Archive.Paths.Sequence(local.Folder).Folder; + + // Another index moves the folder while the server's version is read; this index has not scanned it since. + var (other, archive) = await f.Archive.OtherWriterAsync(); + f.Remote.OnceBeforeManifest = async () => + { + var plan = await other.PlanRenameAsync(archive.Id, [local.Id], "moved/{title}", null, f.Ct); + await other.ApplyRenameAsync(plan, null, f.Ct); + }; + await Assert.ThrowsAsync(() => f.Sync.ResolveAsync(f.Remote, remote.Id, SyncResolution.TakeServer, f.Ct)); + Assert.Null(f.Remote.OnceBeforeManifest, "the folder moved during the resolution"); + + Assert.True(SyncRules.SameSettings(SyncFixture.Window41, (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson), "nothing applied"); + Assert.False(Directory.Exists(left), "the folder it left is not created again"); + Assert.True(SyncRules.SameSettings(SyncFixture.Window41, await File.ReadAllTextAsync(f.Archive.Paths.Sequence("moved/Holy grail").ProjectPath, f.Ct)), + "the moved project is this PC's version"); + } + [Test] public async Task KeepMineRefusesWhenTheServerChangedAfterTheConflictWasShown() { diff --git a/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs b/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs index 7419a590..a748828c 100644 --- a/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs +++ b/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs @@ -600,6 +600,55 @@ public sealed class SyncTransferTests(TestContext context) Assert.Equal(0, (await f.EntryAsync("Star trails")).Link!.ExportsToGet); } + [Test] + public async Task TheServersRendersGoWhereTheSequenceIsNow() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Star trails", photos); + var remote = f.Remote.Add("Star trails", photos); + var first = new FakeRender(Guid.CreateVersion7(), "Star trails 4K.mp4", "4K HEVC", [.. Enumerable.Range(0, 3000).Select(i => (byte)i)]); + remote.Renders.Add(first); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + var engine = await f.StartEngineAsync(); + var left = f.Archive.Paths.Sequence(local.Folder).Folder; + + // A Reorganize here moves the folder while the server's renders are listed: the render goes where the sequence is. + string? moved = null; + f.Remote.OnceBeforeManifest = async () => + { + var plan = await f.Archive.Service.PlanRenameAsync(f.Archive.Archive.Id, [local.Id], "moved/{title}", null, f.Ct); + await f.Archive.Service.ApplyRenameAsync(plan, null, f.Ct); + moved = plan.Steps.Single().ToFolder; + }; + var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null); + await f.Archive.WaitForAsync(job.Id, JobState.Done); + Assert.Equal($"{moved}/exports/Star trails 4K.mp4", (await f.Db.Renders.GetAsync(first.Id, f.Ct))!.OutputPath); + Assert.FileExists(f.Archive.Paths.FromRelative($"{moved}/exports/Star trails 4K.mp4")); + Assert.False(Directory.Exists(left), "the folder it left is not created again"); + + // Another index moves it: nothing is filed until this index's scan finds it there. + var second = new FakeRender(Guid.CreateVersion7(), "Star trails 1080p.mp4", "1080p", [.. Enumerable.Range(0, 2000).Select(i => (byte)(i * 3))]); + remote.Renders.Add(second); + remote.UpdatedAt = DateTimeOffset.UtcNow.AddSeconds(1); + var (other, archive) = await f.Archive.OtherWriterAsync(); + var away = await other.PlanRenameAsync(archive.Id, [local.Id], "elsewhere/{title}", null, f.Ct); + await other.ApplyRenameAsync(away, null, f.Ct); + job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null); + var failed = await f.Archive.WaitForAsync(job.Id, JobState.Failed); + Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), failed.Error); + Assert.False(Directory.Exists(f.Archive.Paths.Sequence(moved!).Folder), "the folder it left is not created again"); + Assert.Null(await f.Db.Renders.GetAsync(second.Id, f.Ct)); + + await f.Archive.ScanAsync(); + job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Star trails", SyncDownloadMode.Exports, null); + await f.Archive.WaitForAsync(job.Id, JobState.Done); + var fetched = (await f.Db.Renders.GetAsync(second.Id, f.Ct))!; + Assert.Equal("elsewhere/Star trails/exports/Star trails 1080p.mp4", fetched.OutputPath); + Assert.SequenceEqual(second.Bytes, await File.ReadAllBytesAsync(f.Archive.Paths.FromRelative(fetched.OutputPath), f.Ct)); + } + [Test] public async Task RendersMadeHereAreSentOnRequestAndAGoneFileIsReported() { diff --git a/tests/Chronos.Server.Tests/Api/ReorganizeTests.cs b/tests/Chronos.Server.Tests/Api/ReorganizeTests.cs index 6390cee8..83e86819 100644 --- a/tests/Chronos.Server.Tests/Api/ReorganizeTests.cs +++ b/tests/Chronos.Server.Tests/Api/ReorganizeTests.cs @@ -146,6 +146,29 @@ public sealed class ReorganizeTests(TestContext context) Assert.False(Directory.Exists(Path.Combine(server.Options.ArchivePath, stale.Folder)), "the folder it left is not created again"); } + [Test] + public async Task AnExportOfAFolderAnotherIndexMovedIsNotFiled() + { + await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + var id = (await Uploads.UploadAsync(editor, Photo(1), "IMG_0001.JPG", sessionKey: "roof")).SequenceId; + await UntilIdleAsync(server); + var sequence = (await server.Database.Sequences.GetAsync(id))!; + + // A desktop sharing the archive moved the folder; this server's index still has it at the old place. + var left = Path.Combine(server.Options.ArchivePath, sequence.Folder); + Directory.Move(left, Path.Combine(server.Options.ArchivePath, "elsewhere")); + var video = Uploads.Photo(4096, 30); + var staged = Path.Combine(context.CreateTempDirectory("staged"), "render.mp4"); + await File.WriteAllBytesAsync(staged, video); + Assert.Null(await server.Services.GetRequiredService().CommitAsync(sequence, staged, + new RenderResultRequest("Roof - 4K.mp4", video.Length, Uploads.Sha(video), "4K HEVC", "hevc", 3840, 2160, 25, 4), "studio-pc", CancellationToken.None)); + + Assert.FileExists(staged); + Assert.False(Directory.Exists(left), "the folder it left is not created again"); + Assert.Empty(await server.Database.Renders.ListBySequenceAsync(id)); + } + [Test] public async Task AReorganizeAsksForTheArchiveAndAValidRequest() { diff --git a/tests/Chronos.Server.Tests/Jobs/RenderNodeTests.cs b/tests/Chronos.Server.Tests/Jobs/RenderNodeTests.cs index b700bbed..654b3c53 100644 --- a/tests/Chronos.Server.Tests/Jobs/RenderNodeTests.cs +++ b/tests/Chronos.Server.Tests/Jobs/RenderNodeTests.cs @@ -2,9 +2,13 @@ using System.Net; using System.Net.Http.Json; using System.Text.Json; using Microsoft.Extensions.DependencyInjection; +using Chronos.Core.Archiving; +using Chronos.Data; +using Chronos.Data.Archiving; using Chronos.Protocol; using Chronos.Protocol.Client; using Chronos.Server.Jobs; +using Chronos.Server.Storage; using Chronos.Testing; namespace Chronos.Server.Tests.Jobs; @@ -120,6 +124,85 @@ public sealed class RenderNodeTests(TestContext context) Assert.Equal(render.Id, (await renders.ReadAsync(ProtocolJson.Default.ListRenderDto)).Single().Id); } + [Test] + public async Task ARenderThatCannotBeFiledForNowIsCompletedAgain() + { + // The archive engine with a short lease wait: a sequence another writer holds refuses the filing at once. + await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = 64 * 1024, s => s.AddSingleton(sp => + new ArchiveService(sp.GetRequiredService(), new ArchiveServiceOptions { Owner = ArchiveModule.Owner, LeaseWait = TimeSpan.FromMilliseconds(200) }))); + var (editor, _, _) = await server.UserAsync("alberto", "editor", renderNode: true); + var frame = await Uploads.UploadAsync(editor, Uploads.Photo(5000, 54), "IMG_0001.CR3"); + var job = await JobApiTests.CreateAsync(editor, "render", frame.SequenceId, HevcJob); + var node = await RegisterAsync(editor, "studio-pc", "hevc", 8192); + using (var claim = await editor.PostAsync($"/api/v1/nodes/{node.Id}/claim", null)) await claim.ExpectAsync(HttpStatusCode.OK); + + // A render node's output, and a render a client sends of the same sequence, both fully received. + var output = Uploads.Photo(3000, 55); + var result = new RenderResultRequest("Roof - 4K HEVC.mp4", output.Length, Uploads.Sha(output), "4K HEVC", "hevc", 3840, 2160, 25, 120); + Guid fromNode; + using (var started = await editor.PostAsJsonAsync($"/api/v1/nodes/{node.Id}/jobs/{job.Id}/result", result, ProtocolJson.Default.RenderResultRequest)) + { + await started.ExpectAsync(HttpStatusCode.Created); + fromNode = Assert.NotNull((await started.ReadAsync(ProtocolJson.Default.StartUploadResponse)).UploadId); + } + using (var put = await Uploads.PutChunkAsync(editor, fromNode, 0, output)) await put.ExpectAsync(HttpStatusCode.OK); + var sent = Uploads.Photo(2000, 56); + var offered = new RenderUploadRequest(Guid.CreateVersion7(), "Roof.mp4", sent.Length, Uploads.Sha(sent), "1080p", "h264", 1920, 1080, 30, 10, null); + Guid fromClient; + using (var started = await editor.PostAsJsonAsync($"/api/v1/sequences/{frame.SequenceId}/renders", offered, ProtocolJson.Default.RenderUploadRequest)) + { + await started.ExpectAsync(HttpStatusCode.Created); + fromClient = Assert.NotNull((await started.ReadAsync(ProtocolJson.Default.StartUploadResponse)).UploadId); + } + using (var put = await Uploads.PutChunkAsync(editor, fromClient, 0, sent)) await put.ExpectAsync(HttpStatusCode.OK); + + var completions = new[] { $"/api/v1/nodes/{node.Id}/jobs/{job.Id}/result/complete", $"/api/v1/sequences/{frame.SequenceId}/renders/{fromClient}/complete" }; + var paths = new ArchivePaths(server.Options.ArchivePath); + async Task RefusedAsync(string code) + { + foreach (var complete in completions) + { + using var refused = await editor.PostAsync(complete, null); + await refused.ExpectErrorAsync(HttpStatusCode.ServiceUnavailable, code); + } + // Nothing filed, nothing staged: both uploads receive again with their chunks, and the node keeps its job. + foreach (var upload in new[] { fromNode, fromClient }) + { + Assert.Equal(Chronos.Data.Repositories.UploadState.Receiving, (await server.Database.Uploads.GetAsync(upload))!.State); + Assert.True(server.Services.GetRequiredService().Exists(upload)); + } + Assert.False(Directory.Exists(server.Options.StagingPath) && Directory.EnumerateFiles(server.Options.StagingPath).Any()); + Assert.Equal(0, (await server.Database.Sequences.GetAsync(frame.SequenceId))!.RenderedCount); + var waiting = await JobApiTests.GetAsync(editor, job.Id); + Assert.Equal(("running", node.Id.ToString()), (waiting.State, waiting.NodeId)); + } + + // Another writer holds the sequence past the lease wait (a Reorganize step, a photo deletion). + using (Assert.NotNull(LeaseFile.TryAcquire(paths.LocksDirectory, $"seq-{frame.SequenceId:N}", "tests"))) + await RefusedAsync("sequence_busy"); + // The archive drive is gone. + File.Move(paths.ManifestPath, paths.ManifestPath + ".away"); + try + { + await RefusedAsync("archive_offline"); + } + finally + { + File.Move(paths.ManifestPath + ".away", paths.ManifestPath); + } + + foreach (var complete in completions) + { + using var filed = await editor.PostAsync(complete, null); + await filed.ExpectAsync(HttpStatusCode.OK); + } + Assert.Equal("done", (await JobApiTests.GetAsync(editor, job.Id)).State); + var sequence = (await server.Database.Sequences.GetAsync(frame.SequenceId))!; + Assert.Equal(2, sequence.RenderedCount); + await Http.BytesEqualAsync(output, File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, sequence.Folder, "exports", "Roof - 4K HEVC.mp4"))); + await Http.BytesEqualAsync(sent, File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, sequence.Folder, "exports", "Roof.mp4"))); + } + [Test] public async Task NodesReportFailuresAndNeedTheRight() {