From 6e99a4b2695924f5dab7c74ee0893d083da102d2 Mon Sep 17 00:00:00 2001 From: Alberto Balbo Date: Tue, 27 Oct 2026 17:42:21 +0100 Subject: [PATCH] fix(archive): exports follow a folder any index moved; proxies, exports and copies no longer recreate it Second review of owner review 2, phase 4a (Reorganize), cross-cutting findings. - A second index of the archive (a desktop sharing a server's archive, or a folder moved by hand) moved the sequence row at its next scan but left renders.output_path at the old folder: the export answered 404 there. The scan now rebases the render rows in the same transaction when it finds a sequence in another folder. - The rebase covers every render of the archive under the old folder: the sequence's own exports always, and another sequence's export saved inside it (through the Export folder picker) when its file moved with the folder. That check keeps the exports of a sequence that took the old folder since (a later Reorganize on the other index) where they are. A change of case only counts as a move. - The desktop now tells an open editor after each Reorganize step (ArchiveServiceOptions.SequenceChanged -> AppServices.SequenceReorganized), and the editor reads its sequence again where it is now, so frames no longer decode blank until the next save. LocalSequenceSource.LoadAsync also requires the same folder on both row reads, so a move that commits during the frame listing is never taken as current. - Writers that wrote without the lease, or trusted a row another index had moved, could create the old folder again. A proxy sent by a client is now filed under the sequence's lease into the folder the index has then, and a gone sequence or folder answers 404 (sequence_busy 503 after the lease wait). The server files a render a client or render node sends under the lease (ArchiveService.UnderLeaseAsync). "Save both" keeps its copy under the lease. The proxy pass holds the thumbnail lease, as the thumbnail pass does. The deep analysis opens its checkpoint under the sequence lock only while the folder is still there, and a pass whose folder another process moved records nothing (the job logs Advisor.Log.FramesChangedElsewhere). Editor saves and sync applies refuse when the folder is gone. - ARCHITECTURE.md no longer claims more than this: a render of another process whose folder moves while it runs is still not stopped. - ArchiveVerifier.FlushAsync: corrected the comment. A move keeps each file's stamp, so only a check that found the file missing no longer applies. Tests cover the scan path with a later swap into the vacated folder, another sequence's export inside a moving folder, a photo deletion refused when the folder moved while it waited for the lease, proxies sent and made after a move, Save both during a move, a deep analysis whose folder another index moved, a render filed after a move on the server, and an open editor reloaded by a Move job. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit c857b98f795b22956c060f1c5acf4d94b6d1c994) --- docs/ARCHITECTURE.md | 11 +- src/Chronos.App/Editor/EditorViewModel.cs | 11 ++ src/Chronos.App/Editor/LocalSequenceSource.cs | 9 +- src/Chronos.App/Infrastructure/AppServices.cs | 12 ++ src/Chronos.Data/Archiving/ArchiveImages.cs | 181 ++++++++++-------- src/Chronos.Data/Archiving/ArchiveIndexer.cs | 18 ++ src/Chronos.Data/Archiving/ArchiveService.cs | 19 ++ src/Chronos.Data/Archiving/ArchiveVerifier.cs | 3 +- src/Chronos.Data/Archiving/MassRenamer.cs | 5 +- .../Processing/SequenceAnalyser.cs | 56 +++++- .../Repositories/RenderRepository.cs | 33 +++- src/Chronos.Data/Sync/SyncService.cs | 30 ++- src/Chronos.Server/Api/FileEndpoints.cs | 11 +- src/Chronos.Server/Storage/RenderStore.cs | 64 ++++--- .../Editor/EditorSmokeTests.cs | 53 +++++ .../Archiving/MassRenameTests.cs | 80 +++++++- .../Archiving/ProxyTests.cs | 48 +++-- .../Processing/AnalysisJobTests.cs | 27 +++ .../Sync/SyncServiceTests.cs | 24 +++ .../Api/ReorganizeTests.cs | 26 +++ 20 files changed, 561 insertions(+), 160 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 8cea1695..a0ae97b7 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -58,7 +58,7 @@ Dependencies point one way: `Core` ← `Core.Windows`, `Data`, `Protocol` ← `S | `Jobs` | `JobEngine` (one job at a time; reviews the priority queue and makes a lower running job yield, requeued with its checkpoint; no history purge, the housekeeping schedule's clean-up does it), `JobPriorityQueue` (a level per job kind, the claim order, settings round trip), `IJobHandler`, `IJobHandlerProvider`, `JobContext` (progress, checkpoints, follow-up jobs that never interrupt their parent), `JobThroughput`/`JobEta` (rate and bounded time left) | | `Scheduling` | `Scheduler` (one per host: seeds the host's system schedules, claims due runs by compare-and-set, catch-up, clock changes, create/edit/reset/delete/run now as API v1 DTOs), `ScheduleRecurrence` (the in-house `every:`/`daily:`/`weekly:`/`monthly:` form, wall-clock times across DST), `ScheduleActions` (action names, parameters, availability, the archive jobs a run queues), `IScheduleHost` and `SystemSchedule` (what the server and the desktop supply), `ScheduleOutcome`, `ScheduleMapping` (model ↔ DTO) | | `Housekeeping` | `BackupJobHandler`, `CleanupJobHandler`, `ICleanupStep` with the shared steps (`CleanupSteps`: job history, audit log, trash report that never deletes); hosts add their own steps through `ArchiveServiceOptions.CleanupSteps` | -| `Archiving` | `ArchiveService` (create/adopt/open, archive settings written to `archive.json` then the index), `ArchiveIndexer` (scan, rebuild), `ArchiveIngest` (uploads and imports), `ImportService`, `ArchiveVerifier`, `ArchiveImages` (thumbnail and proxy passes, several photos at once through `OrderedWork`, which hands results back in frame order), `FrameTiers` (decode tier, `PreviewOnly`/`Undecodable` flags and preview size of frames, written to sidecar and index by the light analysis, deep analysis, renders and image passes), `GprCompanions` (the JPEG of each GoPro GPR, found in the index when an import kept it in another sequence), `ArchiveMaintenance`, `MassRenamer` (mass rename and Reorganize: the dry run reads each sidecar for a file template; each step checked under its leases before anything of it moves (where its sequences are, what is inside and at its new folder, a sidecar of a newer format, the title it was planned from when it changes it or names the folder or files after it), then the folder, every index row (the frame and render paths follow it in one transaction; a sidecar that cannot be read or written then still has its row follow, and fails the step), then the files under the plan's journal in `journal/` (a sequence whose photos changed since, such as one still receiving them, or whose files are in use or have their new names taken keeps its file names, the renames being pinned by digest, and the step and the plan go on); the journal keeps the tally of the whole plan, and a plan its job held (paused, cancelled or failed) is left to that job, only its half-done file renames completed at an archive open, which completes any other plan an interruption left and tells each sequence changed; the file renames of a step: sidecar, a sequence whose sidecar cannot be read staying as it is with every folder around it, index (`FrameRepository.RenameFilesAsync`) and the sequence's journal), `SequenceTrash` (delete into the archive's hidden trash with a dry run, restore; the folder moves back when the index step fails; lists sequence and photo entries; `SequenceTrash.Empty`: the emptying's dry run, fingerprint and erase, each entry checked against its fingerprint again under its lease), `FrameTrash` (photos of a sequence into `trash/photos` with a dry run, a journal that undoes an interrupted deletion whose entry stays unlisted until its sidecar is written, tombstones (a photo never hashed is hashed as it goes; one whose original is missing goes without a file or tombstone; a frame that stays and may be a twin is hashed, its hash kept), positions parked and placed back, proxies and thumbnails moved to follow their frames; `FrameTrash.Restore`: restore re-hashes each file and says why each kept photo stayed (`KeptReason`), under a journal of its own into the sequence that another writer settles only under the entry's lease, into a new sequence through the ingest when theirs is gone, each original staged inside its own entry and the record rewritten after each photo filed), job handlers | +| `Archiving` | `ArchiveService` (create/adopt/open, archive settings written to `archive.json` then the index), `ArchiveIndexer` (scan, rebuild; a sequence found in another folder takes its exports' paths along), `ArchiveIngest` (uploads and imports), `ImportService`, `ArchiveVerifier`, `ArchiveImages` (thumbnail and proxy passes, several photos at once through `OrderedWork`, which hands results back in frame order), `FrameTiers` (decode tier, `PreviewOnly`/`Undecodable` flags and preview size of frames, written to sidecar and index by the light analysis, deep analysis, renders and image passes), `GprCompanions` (the JPEG of each GoPro GPR, found in the index when an import kept it in another sequence), `ArchiveMaintenance`, `MassRenamer` (mass rename and Reorganize: the dry run reads each sidecar for a file template; each step checked under its leases before anything of it moves (where its sequences are, what is inside and at its new folder, a sidecar of a newer format, the title it was planned from when it changes it or names the folder or files after it), then the folder, every index row (the frame paths and render paths of the exports inside it, its own or another sequence's whose file moved with it, follow it in one transaction; a sidecar that cannot be read or written then still has its row follow, and fails the step), then the files under the plan's journal in `journal/` (a sequence whose photos changed since, such as one still receiving them, or whose files are found changed or in use or have their new names taken keeps its file names, the renames being pinned by digest, and the step and the plan go on); the journal keeps the tally of the whole plan, and a plan its job held (paused, cancelled or failed) is left to that job, only its half-done file renames completed at an archive open, which completes any other plan an interruption left and tells each sequence changed; the file renames of a step: sidecar, a sequence whose sidecar cannot be read staying as it is with every folder around it, index (`FrameRepository.RenameFilesAsync`) and the sequence's journal), `SequenceTrash` (delete into the archive's hidden trash with a dry run, restore; the folder moves back when the index step fails; lists sequence and photo entries; `SequenceTrash.Empty`: the emptying's dry run, fingerprint and erase, each entry checked against its fingerprint again under its lease), `FrameTrash` (photos of a sequence into `trash/photos` with a dry run, a journal that undoes an interrupted deletion whose entry stays unlisted until its sidecar is written, tombstones (a photo never hashed is hashed as it goes; one whose original is missing goes without a file or tombstone; a frame that stays and may be a twin is hashed, its hash kept), positions parked and placed back, proxies and thumbnails moved to follow their frames; `FrameTrash.Restore`: restore re-hashes each file and says why each kept photo stayed (`KeptReason`), under a journal of its own into the sequence that another writer settles only under the entry's lease, into a new sequence through the ingest when theirs is gone, each original staged inside its own entry and the record rewritten after each photo filed), job handlers | | `Processing` | `ProcessingService`, `SequenceAnalyser`, `DeepReports` (a saved deep report while it still describes the frames), `SequenceRender`, `ProjectStore`, `ProjectBundles` (a `.chronos` file for a sequence, and the sequence a file names), `RenderNodeClient`, job handlers | | `Sync` | `SyncService` (sync table of one server: compare, link by content, settle conflicts, enqueue transfers), `SyncRules` (status, settings merge and differences, no I/O), `SyncTombstones` (photos either side removed are not offered back, a replaced photo counts as its replacement), `SyncPositions` (the settings' frame positions translated by photo to the frames of the copy they go to), `SyncLinkStore` (`sync_links`, `sync_state`), `SyncTransfers` and `SyncJobHandler` (resumable, hash-verified uploads and downloads as jobs; downloads commit through the archive ingest), `SyncAgent` (background comparison, server events, backoff, offline proxies), `ProxyCache` and `RemoteSequence` (server sequences opened with proxies), `ISyncRemote` (the server calls; `ChronosSyncRemote` over `ChronosClient`) | @@ -148,8 +148,13 @@ after an import or a scan that indexed something; screens never queue it again. The sidecars are the truth; the index is derived and can always be rebuilt (`ScanMode.Rebuild`, using the index snapshot when fresh). Writers coordinate through lease files, so a desktop and a server may share an archive. A writer takes a sequence's -folder from its index row read under the sequence's lease (`SequenceWriter.FolderAsync`): a Reorganize moves the folder -and its row under that lease, and a folder the sequence left is never created again. +folder from its index row read under the sequence's lease (`SequenceWriter.FolderAsync`): a Reorganize moves the folder, +its row and the paths of the renders inside it under that lease, so writers of the same index (uploads, proxies and +exports a server files, editor saves, sync applies and "Save both" copies) write where the sequence is. The image +passes hold the thumbnail lease, which a Reorganize takes too. A folder another index moved is found by this index's +next scan, which moves the row, its frames and its exports' paths (`ArchiveIndexer.RebaseRendersAsync`); until then a +writer that finds the folder gone refuses, and a deep analysis that loses it records nothing. A render of another +process whose folder moves while it runs is not stopped: its output can land in a folder created again at the old place. Sync (desktop ↔ server, BRIEF §14.5) reuses the same paths: `SyncService` compares the local index with the server's listing (settings and tags carry version counters), uploads go through the chunked protocol into the server's diff --git a/src/Chronos.App/Editor/EditorViewModel.cs b/src/Chronos.App/Editor/EditorViewModel.cs index 1f54ee17..2dd816a7 100644 --- a/src/Chronos.App/Editor/EditorViewModel.cs +++ b/src/Chronos.App/Editor/EditorViewModel.cs @@ -150,6 +150,7 @@ public sealed class EditorViewModel : LocalizedViewModel, IAsyncDisposable session.Source.JobChanged += OnJobChanged; session.Source.ReadOnlyChanged += OnReadOnlyChanged; shell.Services.Sync.Changed += OnSyncChanged; + shell.Services.SequenceReorganized += OnSequenceReorganized; UpdateMarkers(); RefreshSteps(); RequestCorrections(); @@ -708,6 +709,7 @@ public sealed class EditorViewModel : LocalizedViewModel, IAsyncDisposable Session.Source.JobChanged -= OnJobChanged; Session.Source.ReadOnlyChanged -= OnReadOnlyChanged; Shell.Services.Sync.Changed -= OnSyncChanged; + Shell.Services.SequenceReorganized -= OnSequenceReorganized; foreach (var tab in Tabs) tab.Dispose(); base.Dispose(); await Session.DisposeAsync(); @@ -1182,6 +1184,15 @@ public sealed class EditorViewModel : LocalizedViewModel, IAsyncDisposable if (ServerJob is { } job) await Session.Source.CancelJobAsync(job.Id, CancellationToken.None); } + /// + /// A Reorganize of this PC moved the folder of a sequence or renamed it or its files (any thread): the open sequence of + /// this PC is read again where it is now, so its frames decode and its project goes there. + /// + private void OnSequenceReorganized(Guid sequenceId) => _dispatcher.BeginInvoke(() => + { + if (!_disposed && !IsRemote && sequenceId == Session.Content.Sequence.Id) _ = ReloadAsync(null); + }); + /// The session, the sync links or the rights changed (any thread): whether the Export "Sync" option applies. private void OnSyncChanged() => _dispatcher.BeginInvoke(() => _ = OnSyncChangedAsync()); diff --git a/src/Chronos.App/Editor/LocalSequenceSource.cs b/src/Chronos.App/Editor/LocalSequenceSource.cs index 0aa4fb58..77abdcb7 100644 --- a/src/Chronos.App/Editor/LocalSequenceSource.cs +++ b/src/Chronos.App/Editor/LocalSequenceSource.cs @@ -179,6 +179,8 @@ public sealed class LocalSequenceSource : ISequenceSource // it too: moved since the read, the project goes to the new folder once the editor read the sequence there, // and the folder it left is never created again. if (row.FramesVersion != framesVersion || row.Folder != relativeFolder) throw new FramesChangedException(Loc.L("Editor.Error.FramesArriving")); + // Moved by another index of the archive (a server's Reorganize), which this index finds at its next scan. + if (!Directory.Exists(folder.Folder)) throw Missing(); var stored = ReadProject(row, folder); var merged = ProjectMerge.SameValues(stored, basis) ? project : ProjectMerge.Merge(basis, project, stored); var json = merged.ToJson(); @@ -227,9 +229,10 @@ public sealed class LocalSequenceSource : ISequenceSource /// /// The sequence row with its frames as one state: the frames are listed between two reads of the row that find the /// same frames version (an insertion renumbers the frames and moves the row's positions in one transaction, and - /// bumps the version), so the project's positions name these frames. + /// bumps the version), so the project's positions name these frames, and the same folder (a Reorganize moves the + /// folder and rewrites the frames' paths in one transaction, the version kept), so the files are where they were read. /// - /// The sequence is gone, or an insertion landed during every attempt. + /// The sequence is gone, or an insertion or a move landed during every attempt. private async Task<(Sequence Row, List Frames)> ReadFramesAsync(CancellationToken cancellationToken) { for (var attempt = 0; attempt < ReadAttempts; attempt++) @@ -243,7 +246,7 @@ public sealed class LocalSequenceSource : ISequenceSource if (page.Count < PageSize) break; } var after = await _db.Sequences.GetAsync(_sequenceId, cancellationToken) ?? throw Missing(); - if (after.FramesVersion == before.FramesVersion) return (after, frames); + if (after.FramesVersion == before.FramesVersion && after.Folder == before.Folder) return (after, frames); } throw new EditorException(Loc.L("Editor.Error.FramesArriving")); } diff --git a/src/Chronos.App/Infrastructure/AppServices.cs b/src/Chronos.App/Infrastructure/AppServices.cs index fa2a549d..9e5ce17e 100644 --- a/src/Chronos.App/Infrastructure/AppServices.cs +++ b/src/Chronos.App/Infrastructure/AppServices.cs @@ -111,6 +111,12 @@ public sealed class AppServices : IAsyncDisposable /// Sync with the signed-in server: engine, job connection, background comparison. public SyncHost Sync { get; } + /// + /// A job of this PC moved a sequence's folder or changed its title or file names (a Reorganize step), with the + /// sequence's id (any thread): an open editor reads it again where it is now. + /// + public event Action? SequenceReorganized; + public IEncoderInventory Encoders { get; set; } /// Decode tier per format on this PC and the missing codec extensions (Settings › Formats). @@ -452,6 +458,12 @@ public sealed class AppServices : IAsyncDisposable // The housekeeping schedule trims the cache to Settings › Storage › Cache limit (owner review 2, R10). CleanupSteps = [new CacheTrimStep(() => CacheStore.Folder(Environment, Settings), () => Settings.CacheLimitGb * (1L << 30))], BackupDirectory = Environment.BackupDirectory, + // Told after each step of a Reorganize, while the rest of the job runs. + SequenceChanged = (id, _, _) => + { + SequenceReorganized?.Invoke(id); + return Task.CompletedTask; + }, }; Archives = new ArchiveService(database, archive); Processing = new ProcessingService(database, ProcessingOptions(archive)); diff --git a/src/Chronos.Data/Archiving/ArchiveImages.cs b/src/Chronos.Data/Archiving/ArchiveImages.cs index 7f1a069e..04ddcbb0 100644 --- a/src/Chronos.Data/Archiving/ArchiveImages.cs +++ b/src/Chronos.Data/Archiving/ArchiveImages.cs @@ -15,7 +15,7 @@ namespace Chronos.Data.Archiving; /// client or render node that can (BRIEF §2.3): unlike ones, nothing is wrong with them. public sealed record ImagePassResult(int Built, int Reused, int Undecodable, int Waiting = 0); -/// What became of a proxy sent by a client or render node (). +/// What became of a proxy sent by a client or render node (). public enum ProxyStoreResult { Stored, @@ -31,6 +31,9 @@ public enum ProxyStoreResult /// Smaller than the archive's proxies of that frame: made from an embedded preview, not the original. TooSmall, + + /// The frame's sequence left the index meanwhile (deleted), or its folder (moved by another index): nothing was written. + Gone, } /// @@ -38,7 +41,7 @@ public enum ProxyStoreResult /// regenerable and self-validating: a thumbnail carries its source fingerprint, a proxy its capture time, so after /// frames were renumbered the stale ones are rebuilt and the rest reused. A frame whose original this host cannot /// decode () gets its proxy from a client or render node that can -/// (, BRIEF §2.3) and its thumbnail from that proxy. A frame nothing decodes is flagged +/// (, BRIEF §2.3) and its thumbnail from that proxy. A frame nothing decodes is flagged /// undecodable (BRIEF §8.2 tier 3); one flagged so that decodes after all gets its tier probed again. /// public sealed class ArchiveImages @@ -122,7 +125,7 @@ public sealed class ArchiveImages Check(proxy, capturedAt, width, height, policy).Check is ProxyCheck.Current or ProxyCheck.Unverifiable; /// - /// A proxy of the original at for a server that cannot decode it (): + /// A proxy of the original at for a server that cannot decode it (): /// at (0 = full size) and , with 's capture /// data (the server's record, which it checks) and in its Exif. Null when /// cannot decode the original either. @@ -159,10 +162,13 @@ public sealed class ArchiveImages /// Files a proxy made elsewhere (a client or render node whose decoders read the original) as the frame's proxy, /// replacing the one there: a JPEG of at most that decodes, whose Exif capture time is the /// frame's (proxies carry it, so the proxy counts as current and cannot belong to another photo) and whose size is - /// the policy's (), written atomically. + /// the policy's (), written atomically under the sequence's lease into its folder as the index has + /// it then: a Reorganize moves the folder under that lease, and the folder it left is never created again (one another + /// index moved, which this index finds at its next scan, takes nothing: ). /// /// The archive is offline or read-only. - public ProxyStoreResult StoreProxy(Archive archive, Sequence sequence, Frame frame, byte[] jpeg) + /// Another writer held the sequence for longer than the lease wait. + public async Task StoreProxyAsync(Archive archive, Sequence sequence, Frame frame, byte[] jpeg, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(jpeg); var paths = new ArchivePaths(archive.RootPath); @@ -177,11 +183,16 @@ public sealed class ArchiveImages using var stream = new MemoryStream(jpeg, writable: false); if (!SameCapture(MetadataReader.Read(stream, "proxy.jpg").CapturedAt, captured)) return ProxyStoreResult.WrongFrame; } - var folder = paths.Sequence(sequence.Folder); - Directory.CreateDirectory(folder.Proxies); - var path = folder.Proxy(frame.Index); - AtomicFile.WriteAllBytes(path, jpeg); - Headers.TryRemove(path, out _); + await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false)) + { + if (await _db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) is not { } row) return ProxyStoreResult.Gone; + var folder = paths.Sequence(row.Folder); + if (!Directory.Exists(folder.Folder)) return ProxyStoreResult.Gone; + Directory.CreateDirectory(folder.Proxies); + var path = folder.Proxy(frame.Index); + AtomicFile.WriteAllBytes(path, jpeg); + Headers.TryRemove(path, out _); + } return ProxyStoreResult.Stored; } @@ -290,85 +301,93 @@ public sealed class ArchiveImages ArchiveVolume.EnsureWritable(paths); var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy; if (policy.LongEdge == ProxyPolicy.Disabled) return new ImagePassResult(0, 0, 0); - var folder = paths.Sequence(sequence.Folder); - var manifest = ManifestStore.ReadSequence(folder.ManifestPath); - Directory.CreateDirectory(folder.Proxies); + SequenceManifest manifest; int built = 0, reused = 0, undecodable = 0, waiting = 0; var complete = true; var tiers = new Dictionary(); - var companions = await GprCompanions.FindAsync(_db, archive, manifest.Frames, cancellationToken).ConfigureAwait(false); - var cadence = new Cadence(); - await using (var work = new OrderedWork(ProxyParallelism(manifest.Frames, policy), made => - { - switch (made.Outcome) - { - case ProxyOutcome.Built: - built++; - break; - case ProxyOutcome.Reused: - reused++; - break; - case ProxyOutcome.Waiting: - waiting++; - break; - case ProxyOutcome.Failed: - undecodable++; - complete = false; - break; - default: - undecodable++; - break; - } - if (made.Tier is { } tier) tiers[made.FrameId] = tier; - return cadence.Due() ? checkpoint(made.Index + 1, manifest.Frames.Count) : ValueTask.CompletedTask; - }, cancellationToken)) + // Under the thumbnail lease, as the thumbnail pass: image passes exclude each other only, ingest keeps committing + // meanwhile, and a Reorganize waits for the pass, so the proxies go to the folder the index has once the lease is + // held, never into one the sequence left, which writing them would create again. + using (await LeaseFile.AcquireAsync(paths.LocksDirectory, $"thumbs-{sequence.Id:N}", _writer.Options.Owner, _writer.Options.LeaseWait, + _writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false)) { - for (var i = 0; i < manifest.Frames.Count; i++) + var folder = await SequenceWriter.FolderAsync(_db, paths, sequence, cancellationToken).ConfigureAwait(false); + manifest = ManifestStore.ReadSequence(folder.ManifestPath); + Directory.CreateDirectory(folder.Proxies); + var companions = await GprCompanions.FindAsync(_db, archive, manifest.Frames, cancellationToken).ConfigureAwait(false); + var cadence = new Cadence(); + await using (var work = new OrderedWork(ProxyParallelism(manifest.Frames, policy), made => + { + switch (made.Outcome) + { + case ProxyOutcome.Built: + built++; + break; + case ProxyOutcome.Reused: + reused++; + break; + case ProxyOutcome.Waiting: + waiting++; + break; + case ProxyOutcome.Failed: + undecodable++; + complete = false; + break; + default: + undecodable++; + break; + } + if (made.Tier is { } tier) tiers[made.FrameId] = tier; + return cadence.Due() ? checkpoint(made.Index + 1, manifest.Frames.Count) : ValueTask.CompletedTask; + }, cancellationToken)) { - cancellationToken.ThrowIfCancellationRequested(); - var frame = manifest.Frames[i]; - var path = folder.Proxy(frame.I); - var done = i < startFrame; - if (NeedsProxy(frame.Format)) + for (var i = 0; i < manifest.Frames.Count; i++) { - // Only a client or render node that decodes the original makes this proxy: a preview would make one the - // size of a thumbnail here. One left from a frame renumbered since goes, so it is asked for again, and so - // does one smaller than the policy's (made here from a preview by an earlier version). - var check = Check(path, frame.CapturedAt, frame.W, frame.H, policy).Check; - var usable = check is ProxyCheck.Current or ProxyCheck.Unverifiable; - if (check == ProxyCheck.Stale) Delete(path); - complete &= usable; - if (done) continue; - var outcome = usable ? ProxyOutcome.Reused - : (frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0 ? ProxyOutcome.Unreadable - : ProxyOutcome.Waiting; - await work.AddAsync(new Made(i, frame.Id, outcome, null)).ConfigureAwait(false); - continue; + cancellationToken.ThrowIfCancellationRequested(); + var frame = manifest.Frames[i]; + var path = folder.Proxy(frame.I); + var done = i < startFrame; + if (NeedsProxy(frame.Format)) + { + // Only a client or render node that decodes the original makes this proxy: a preview would make one the + // size of a thumbnail here. One left from a frame renumbered since goes, so it is asked for again, and so + // does one smaller than the policy's (made here from a preview by an earlier version). + var check = Check(path, frame.CapturedAt, frame.W, frame.H, policy).Check; + var usable = check is ProxyCheck.Current or ProxyCheck.Unverifiable; + if (check == ProxyCheck.Stale) Delete(path); + complete &= usable; + if (done) continue; + var outcome = usable ? ProxyOutcome.Reused + : (frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0 ? ProxyOutcome.Unreadable + : ProxyOutcome.Waiting; + await work.AddAsync(new Made(i, frame.Id, outcome, null)).ConfigureAwait(false); + continue; + } + if (done) + { + // Made by the earlier run, unless the frame did not decode then. + complete &= ReadProxy(path, frame.CapturedAt).Check is ProxyCheck.Current or ProxyCheck.Unverifiable; + continue; + } + if (!force && ReadProxy(path, frame.CapturedAt).Check == ProxyCheck.Current) + { + reused++; + continue; + } + var index = i; + var companion = companions.GetValueOrDefault(frame.Id); + await work.AddAsync(ct => + { + var (image, tier) = Render(folder, frame, companion, recordTier: true, + () => _renderer.Proxy(folder, frame, manifest.Camera, policy, ct, companion)); + if (image is null) return new Made(index, frame.Id, ProxyOutcome.Failed, tier); + AtomicFile.WriteAllBytes(path, image.Jpeg); + Headers.TryRemove(path, out _); + return new Made(index, frame.Id, ProxyOutcome.Built, tier); + }).ConfigureAwait(false); } - if (done) - { - // Made by the earlier run, unless the frame did not decode then. - complete &= ReadProxy(path, frame.CapturedAt).Check is ProxyCheck.Current or ProxyCheck.Unverifiable; - continue; - } - if (!force && ReadProxy(path, frame.CapturedAt).Check == ProxyCheck.Current) - { - reused++; - continue; - } - var index = i; - var companion = companions.GetValueOrDefault(frame.Id); - await work.AddAsync(ct => - { - var (image, tier) = Render(folder, frame, companion, recordTier: true, - () => _renderer.Proxy(folder, frame, manifest.Camera, policy, ct, companion)); - if (image is null) return new Made(index, frame.Id, ProxyOutcome.Failed, tier); - AtomicFile.WriteAllBytes(path, image.Jpeg); - Headers.TryRemove(path, out _); - return new Made(index, frame.Id, ProxyOutcome.Built, tier); - }).ConfigureAwait(false); + await work.CompleteAsync().ConfigureAwait(false); } - await work.CompleteAsync().ConfigureAwait(false); } await _tiers.RecordAsync(archive, sequence, tiers, proxies: true, cancellationToken).ConfigureAwait(false); await MarkAsync(archive, paths, sequence, manifest, JobKind.Proxies, SequenceStatus.ProxiesBuilt, complete, cancellationToken).ConfigureAwait(false); diff --git a/src/Chronos.Data/Archiving/ArchiveIndexer.cs b/src/Chronos.Data/Archiving/ArchiveIndexer.cs index d9abad40..e82b9c38 100644 --- a/src/Chronos.Data/Archiving/ArchiveIndexer.cs +++ b/src/Chronos.Data/Archiving/ArchiveIndexer.cs @@ -156,6 +156,10 @@ public sealed class ArchiveIndexer if (row.HeroFrameId is { } hero && !manifest.Frames.Any(f => f.Id == hero)) row = row with { HeroFrameId = null }; await tx.Sequences.UpsertAsync(row with { Status = row.Status & ~clear }, cancellationToken).ConfigureAwait(false); await tx.Frames.SyncSequenceAsync(manifest.Id, SequenceMapping.ToFrames(manifest, sequence), cancellationToken).ConfigureAwait(false); + // Found in another folder (a Reorganize of another index of the archive, or a folder moved by hand): the exports + // inside it moved with it. + if (existing is not null && existing.Folder != sequence.RelativeFolder) + await RebaseRendersAsync(tx, archive, manifest.Id, existing.Folder, sequence.RelativeFolder, cancellationToken).ConfigureAwait(false); await tx.FrameRemovals.SyncSequenceAsync(manifest.Id, SequenceMapping.ToRemovals(manifest), cancellationToken).ConfigureAwait(false); foreach (var key in manifest.SessionKeys) await tx.Sequences.MapSessionKeyAsync(archive.Id, key, manifest.Id, cancellationToken).ConfigureAwait(false); @@ -181,6 +185,20 @@ public sealed class ArchiveIndexer return manifest; } + /// + /// The render rows of the archive that follow a sequence's folder from to , + /// in the caller's transaction (): the sequence's own exports + /// inside it, and another sequence's whose file moved with it (at its new path and no longer at the old one, which on a + /// file system that ignores case is the same file when only the case changed). + /// + internal static Task RebaseRendersAsync(ChronosRepositories tx, Archive archive, Guid sequenceId, string from, string to, + CancellationToken cancellationToken) + { + var paths = new ArchivePaths(archive.RootPath); + return tx.Renders.RebaseFolderAsync(archive.Id, sequenceId, from, to, (before, after) => File.Exists(paths.FromRelative(after)) + && (string.Equals(before, after, StringComparison.OrdinalIgnoreCase) || !File.Exists(paths.FromRelative(before))), cancellationToken); + } + /// /// Remembers the current stamps of a folder whose sidecar and index were just updated together, so the next /// incremental scan does not re-read what is already indexed. diff --git a/src/Chronos.Data/Archiving/ArchiveService.cs b/src/Chronos.Data/Archiving/ArchiveService.cs index 4f01ae4b..c6d00cd0 100644 --- a/src/Chronos.Data/Archiving/ArchiveService.cs +++ b/src/Chronos.Data/Archiving/ArchiveService.cs @@ -305,6 +305,25 @@ public sealed class ArchiveService : IJobHandlerProvider } } + /// + /// Runs under the lease of , with its index row read under it (null + /// when the sequence left the index). A Reorganize moves a folder, its row and the paths of the renders inside it under + /// that lease, so a writer that files into the folder and records what it filed (a render a client or render node sends) + /// does both where the sequence is then, never in a folder it left, and what it records follows a later move. + /// + /// The sequence's archive is not in the index. + /// The archive is offline or read-only. + /// Another writer held the sequence for longer than the lease wait. + public async Task UnderLeaseAsync(Sequence sequence, Func> write, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(sequence); + ArgumentNullException.ThrowIfNull(write); + var paths = new ArchivePaths((await LoadArchiveAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false)).RootPath); + ArchiveVolume.EnsureWritable(paths); + await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false)) + return await write(await _db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + } + public Task> FindDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) => Maintenance.FindDuplicatesAsync(archiveId, cancellationToken); diff --git a/src/Chronos.Data/Archiving/ArchiveVerifier.cs b/src/Chronos.Data/Archiving/ArchiveVerifier.cs index 65c4a456..e62d3ae6 100644 --- a/src/Chronos.Data/Archiving/ArchiveVerifier.cs +++ b/src/Chronos.Data/Archiving/ArchiveVerifier.cs @@ -114,7 +114,8 @@ public sealed class ArchiveVerifier if (pending.Count == 0) return; await using (await _writer.LockAsync(paths, target.Id, cancellationToken).ConfigureAwait(false)) { - // Where the sequence is now (a Reorganize may have moved it); a check of a file that moved since no longer applies. + // Where the sequence is now (a Reorganize may have moved it). A folder move keeps each file's size and time, so a + // check made at the old place still applies; one that found a file missing while it moved does not (StillApplies). var sequence = await SequenceWriter.FolderAsync(_db, paths, target, cancellationToken).ConfigureAwait(false); var manifest = SequenceWriter.ReadLocked(sequence, target.Id); var current = manifest.Frames.ToDictionary(f => f.Id); diff --git a/src/Chronos.Data/Archiving/MassRenamer.cs b/src/Chronos.Data/Archiving/MassRenamer.cs index 23a9abf7..57528239 100644 --- a/src/Chronos.Data/Archiving/MassRenamer.cs +++ b/src/Chronos.Data/Archiving/MassRenamer.cs @@ -653,7 +653,8 @@ public sealed class MassRenamer /// /// Sidecar title, then index (folder, title, frames' paths and the paths of renders whose export moved with the - /// folder, in one transaction); each part is idempotent so a resumed step can repeat it. + /// folder, the sequence's own or another's saved inside it, in one transaction); each part is idempotent so a resumed + /// step can repeat it. /// The index row changes in its folder and title only (): a settings edit /// committed meanwhile stays, and frames another process listed in the sidecar are left to the next scan, for which /// the folder keeps no stamps unless the index held it as it was @@ -700,7 +701,7 @@ public sealed class MassRenamer if (step.MovesFolder) { await tx.Frames.RebaseFolderAsync(step.SequenceId, step.FromFolder, step.ToFolder, cancellationToken).ConfigureAwait(false); - await tx.Renders.RebaseFolderAsync(step.SequenceId, step.FromFolder, step.ToFolder, cancellationToken).ConfigureAwait(false); + await ArchiveIndexer.RebaseRendersAsync(tx, archive, step.SequenceId, step.FromFolder, step.ToFolder, cancellationToken).ConfigureAwait(false); } await tx.Archives.DeleteFolderStateAsync(archive.Id, step.FromFolder, cancellationToken).ConfigureAwait(false); await tx.CommitAsync(cancellationToken).ConfigureAwait(false); diff --git a/src/Chronos.Data/Processing/SequenceAnalyser.cs b/src/Chronos.Data/Processing/SequenceAnalyser.cs index 60149d91..45451e87 100644 --- a/src/Chronos.Data/Processing/SequenceAnalyser.cs +++ b/src/Chronos.Data/Processing/SequenceAnalyser.cs @@ -35,7 +35,8 @@ internal enum FramesChange /// /// Changed in the sidecar by another process (a desktop sharing a server's archive, or that server), which analyses - /// them; this index holds them only after its next scan, which on a server queues their analysis here too. + /// them, or moved with their folder by its Reorganize while a deep pass read them; this index holds them only after its + /// next scan, which on a server queues their analysis here too. /// NotIndexed, } @@ -78,7 +79,18 @@ internal sealed class SequenceAnalyser(ChronosDatabase db, ProcessingOptions opt { // Proxies may have arrived during the pass: the flags below follow the frames as they are at its end, and the // report is checked against them under the lock. - (report, var decoded) = await AnalyseDeepAsync(archive, frames, metadata, folder, progress, cancellationToken).ConfigureAwait(false); + Dictionary decoded; + try + { + (report, decoded) = await AnalyseDeepAsync(archive, paths, sequence, frames, metadata, folder, progress, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is FolderMovedException || (ex is IOException && Directory.Exists(paths.Root) && !Directory.Exists(folder.Folder))) + { + // Moved while the pass read it (a Reorganize of another process sharing the archive, whose checkpoint then + // cannot be written there): what it read after the move was not there, so nothing is recorded, and this index + // finds the folder at its next scan. + return new AnalysisOutcome(sequence, frames.Count, 0, SettingsChanged: false, SettingsUnreadable: false, [], FramesChange.NotIndexed); + } frames = WithTiers(frames, await _tiers.RecordAsync(archive, sequence, decoded, proxies: true, cancellationToken).ConfigureAwait(false)); metadata = [.. metadata.Select((m, i) => m with { Flags = frames.Frames[i].Flags })]; } @@ -281,8 +293,10 @@ internal sealed class SequenceAnalyser(ChronosDatabase db, ProcessingOptions opt private static DeepAnalysisCheckpointFile Checkpoint(SequencePaths folder) => new(Path.Combine(folder.MetaDirectory, DeepAnalysisCheckpointFile.FileName)); - private async Task<(DeepAnalysisReport Report, Dictionary Decoded)> AnalyseDeepAsync(Archive archive, SequenceFrames frames, - IReadOnlyList metadata, SequencePaths folder, IProgress? progress, CancellationToken cancellationToken) + /// The folder moved before the pass opened its checkpoint (). + /// The folder moved while the pass ran: its checkpoint can no longer be written there. + private async Task<(DeepAnalysisReport Report, Dictionary Decoded)> AnalyseDeepAsync(Archive archive, ArchivePaths paths, Sequence sequence, + SequenceFrames frames, IReadOnlyList metadata, SequencePaths folder, IProgress? progress, CancellationToken cancellationToken) { var file = Checkpoint(folder); // The host's analysis parameters as they are now (Settings › Analysis on the desktop). Those that change per-frame @@ -294,7 +308,8 @@ internal sealed class SequenceAnalyser(ChronosDatabase db, ProcessingOptions opt Dictionary decoded; await using (var source = new SequenceFrameSource(frames, metadata, options.Archive.Decoder) { Originals = true, Companions = companions }) { - report = await DeepAnalyzer.AnalyzeAsync(source, analysis, new FramesCheckpoint(file, ContentKey(frames)), progress, cancellationToken) + report = await DeepAnalyzer.AnalyzeAsync(source, analysis, + new FramesCheckpoint(file, ContentKey(frames), ct => InPlaceAsync(paths, sequence, folder, ct)), progress, cancellationToken) .ConfigureAwait(false); decoded = Decoded(frames, report, source, companions, cancellationToken); } @@ -343,11 +358,36 @@ internal sealed class SequenceAnalyser(ChronosDatabase db, ProcessingOptions opt return Convert.ToHexStringLower(hash.GetHashAndReset())[..16]; } - private sealed class FramesCheckpoint(DeepAnalysisCheckpointFile file, string contentKey) : IDeepAnalysisCheckpoint + /// + /// The sequence lock, taken while , where the pass reads the frames, is still there: a + /// Reorganize, of this process or another sharing the archive, moves the folder under that lock's lease. + /// + /// The folder moved since the pass read the sequence. + private async Task InPlaceAsync(ArchivePaths paths, Sequence sequence, SequencePaths folder, CancellationToken cancellationToken) { - public ValueTask> LoadAsync(string fingerprint, CancellationToken cancellationToken) => - file.LoadAsync($"{fingerprint};content={contentKey}", cancellationToken); + var held = await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false); + if (Directory.Exists(folder.Folder)) return held; + await held.DisposeAsync().ConfigureAwait(false); + throw new FolderMovedException(); + } + + /// + /// The pass's checkpoint, for the frames it reads (). It is opened, and created when there is + /// none, under the sequence lock while the sequence is still where the pass reads it (): + /// creating it in a folder a Reorganize moved would create that folder again. Appends never create a folder. + /// + private sealed class FramesCheckpoint(DeepAnalysisCheckpointFile file, string contentKey, Func> inPlace) + : IDeepAnalysisCheckpoint + { + public async ValueTask> LoadAsync(string fingerprint, CancellationToken cancellationToken) + { + await using (await inPlace(cancellationToken).ConfigureAwait(false)) + return await file.LoadAsync($"{fingerprint};content={contentKey}", cancellationToken).ConfigureAwait(false); + } public ValueTask AppendAsync(IReadOnlyList frames, CancellationToken cancellationToken) => file.AppendAsync(frames, cancellationToken); } + + /// A deep pass found the folder it reads moved (a Reorganize) before it opened its checkpoint: nothing was written. + private sealed class FolderMovedException : Exception; } diff --git a/src/Chronos.Data/Repositories/RenderRepository.cs b/src/Chronos.Data/Repositories/RenderRepository.cs index 0defa177..b7b25d14 100644 --- a/src/Chronos.Data/Repositories/RenderRepository.cs +++ b/src/Chronos.Data/Repositories/RenderRepository.cs @@ -46,20 +46,35 @@ public sealed class RenderRepository }, cancellationToken); /// - /// Rewrites the output paths of a sequence's renders after its folder moved (Reorganize): an export inside the folder - /// moved with it. Outputs elsewhere (absolute paths, other folders of the archive) stay as they are. + /// Rewrites the archive-relative output paths of an archive's renders after the folder of sequence + /// moved (a Reorganize, or a move another index of the archive or the user made, which a + /// scan finds): an export inside the folder moved with it. The sequence's own exports there follow it, and so does an + /// export of another sequence saved inside it (through the Export folder picker) when says its + /// file moved with it: a sequence that took the old folder since (a later Reorganize another index made) keeps its own. + /// Outputs elsewhere (absolute paths, other folders) stay as they are. /// + /// Whether the file at the first archive-relative path is at the second one now. /// Only paths still under change, so repeating it after a crash is harmless. - public Task RebaseFolderAsync(Guid sequenceId, string oldFolder, string newFolder, CancellationToken cancellationToken = default) + public async Task RebaseFolderAsync(Guid archiveId, Guid sequenceId, string oldFolder, string newFolder, Func moved, + CancellationToken cancellationToken = default) { + ArgumentNullException.ThrowIfNull(moved); // The archive root never moves, and its prefix ("/") would match absolute paths outside the archive. - if (oldFolder.Length == 0) return Task.FromResult(0); - // substr counts characters (code points) on both dialects; the tail starts with the '/' after the folder. + if (oldFolder.Length == 0) return 0; + // substr counts characters (code points) on both dialects. var prefix = oldFolder + "/"; - var length = prefix.EnumerateRunes().Count(); - return _db.ExecuteAsync( - "UPDATE renders SET output_path = $2 || substr(output_path, $3) WHERE sequence_id = $1 AND substr(output_path, 1, $4) = $5", - [sequenceId, newFolder, length, length, prefix], cancellationToken); + var under = await _db.ListAsync( + "SELECT r.id, r.sequence_id, r.output_path FROM renders r JOIN sequences s ON s.id = r.sequence_id WHERE s.archive_id = $1 AND substr(r.output_path, 1, $2) = $3", + [archiveId, prefix.EnumerateRunes().Count(), prefix], r => (Id: r.GetGuid(0), SequenceId: r.GetGuid(1), Path: r.GetString(2)), cancellationToken) + .ConfigureAwait(false); + var rebased = 0; + foreach (var (id, owner, path) in under) + { + var to = newFolder.Length == 0 ? path[prefix.Length..] : newFolder + path[oldFolder.Length..]; + if (owner != sequenceId && !moved(path, to)) continue; + rebased += await _db.ExecuteAsync("UPDATE renders SET output_path = $2 WHERE id = $1", [id, to], cancellationToken).ConfigureAwait(false); + } + return rebased; } /// diff --git a/src/Chronos.Data/Sync/SyncService.cs b/src/Chronos.Data/Sync/SyncService.cs index 325d61e4..08d98b04 100644 --- a/src/Chronos.Data/Sync/SyncService.cs +++ b/src/Chronos.Data/Sync/SyncService.cs @@ -721,8 +721,9 @@ public sealed class SyncService /// sequence's lease, which the editor's saves and the archive's frame commits take too, and only while the local copy /// is still at the settings version was read at. The settings' frame positions are moved to /// this copy's frames as they are under the lease (). Null when an edit was saved on this - /// PC since, or the frames of either copy could not be read as one state: nothing is written then. A payload the local - /// copy already holds writes nothing, so it needs neither a writable archive nor the lease. + /// PC since, the frames of either copy could not be read as one state, or the copy is not in the folder this index has + /// (another index moved it): nothing is written then. A payload the local copy already holds writes nothing, so it + /// needs neither a writable archive nor the lease. /// private async Task ApplyLocallyAsync(Sequence local, SyncPayload payload, SyncPositions positions, CancellationToken cancellationToken) { @@ -739,8 +740,10 @@ public sealed class SyncService // The frames cannot change under the lease: the positions are moved to them as they are now. if (await positions.ToLocalAsync(payload.SettingsJson, local.SettingsVersion, cancellationToken).ConfigureAwait(false) is not { } current) return null; var row = current.Local; - // The folder too: a Reorganize moves it and its row under this lease. + // The folder too: a Reorganize moves it and its row under this lease. One another index moved is found by the next + // scan; nothing is written until then, so the folder it left is not created again. var folder = paths.Sequence(row.Folder); + if (!Directory.Exists(folder.Folder)) return null; payload = payload with { SettingsJson = current.SettingsJson }; var metadata = !payload.SameMetadata(SyncPayload.Of(row)); var settings = !SyncRules.SameSettings(payload.SettingsJson, row.SettingsJson); @@ -790,16 +793,25 @@ public sealed class SyncService /// /// "Save both": this PC's project is kept next to project.json as project.this-pc-<local time>.json - /// before the server's replaces it; returns that file. + /// before the server's replaces it; returns that file. Written under the sequence's lease into its folder as the index + /// has it then: a Reorganize may have moved it while the server was asked, and the folder it left is never created again. + /// Null, with nothing written, when the sequence is not in that folder (another index moved it; the next scan finds it). /// - private async Task SaveCopyAsync(Sequence local, CancellationToken cancellationToken) + private async Task SaveCopyAsync(Sequence local, CancellationToken cancellationToken) { var archive = await _db.Archives.GetAsync(local.ArchiveId, cancellationToken).ConfigureAwait(false) ?? throw new InvalidDataException($"Archive {local.ArchiveId} is not indexed."); - var folder = new ArchivePaths(archive.RootPath).Sequence(local.Folder); - Directory.CreateDirectory(folder.MetaDirectory); - var path = Path.Combine(folder.MetaDirectory, SavedCopyName(DateTime.Now)); - AtomicFile.WriteAllText(path, SyncRules.NormalizeSettings(local.SettingsJson)); + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + string path; + await using (await _writer.LockAsync(paths, local.Id, cancellationToken).ConfigureAwait(false)) + { + var folder = await SequenceWriter.FolderAsync(_db, paths, local, cancellationToken).ConfigureAwait(false); + if (!Directory.Exists(folder.Folder)) return null; + Directory.CreateDirectory(folder.MetaDirectory); + path = Path.Combine(folder.MetaDirectory, SavedCopyName(DateTime.Now)); + AtomicFile.WriteAllText(path, SyncRules.NormalizeSettings(local.SettingsJson)); + } Log.Info(LogCategory, $"Settings of {local.DisplayTitle} kept as {path}"); return path; } diff --git a/src/Chronos.Server/Api/FileEndpoints.cs b/src/Chronos.Server/Api/FileEndpoints.cs index 84dbeff8..f6eed718 100644 --- a/src/Chronos.Server/Api/FileEndpoints.cs +++ b/src/Chronos.Server/Api/FileEndpoints.cs @@ -63,7 +63,8 @@ public static class FileEndpoints /// (); the sequence's thumbnails, proxy state and light analysis follow as background jobs. /// 204; 413 too_large; 422 proxy_not_wanted (the server decodes that original itself, or it is missing or /// damaged), invalid_proxy, proxy_mismatch or proxy_too_small, which concern that proxy only; 409 - /// proxies_disabled, which concerns them all; 503 archive_offline. + /// proxies_disabled, which concerns them all; 503 archive_offline, or sequence_busy while another writer + /// holds the sequence (a Reorganize moving its folder) for longer than the lease wait. /// private static async Task StoreProxyAsync(HttpContext context, Guid id, ChronosDatabase db, ServerArchive archive, [FromServices] ArchiveService? archives, SequenceJobs follow) @@ -83,14 +84,20 @@ public static class FileEndpoints ProxyStoreResult result; try { - result = images.StoreProxy(archive.Current, sequence, frame, body.ToArray()); + result = await images.StoreProxyAsync(archive.Current, sequence, frame, body.ToArray(), ct); } catch (ArchiveOfflineException) { return ApiErrors.Result(context, StatusCodes.Status503ServiceUnavailable, "archive_offline"); } + catch (LeaseUnavailableException) + { + return ApiErrors.Result(context, StatusCodes.Status503ServiceUnavailable, "sequence_busy"); + } switch (result) { + case ProxyStoreResult.Gone: + return ApiErrors.NotFound(context, "frame"); case ProxyStoreResult.Disabled: return ApiErrors.Conflict(context, "proxies_disabled"); case ProxyStoreResult.Invalid: diff --git a/src/Chronos.Server/Storage/RenderStore.cs b/src/Chronos.Server/Storage/RenderStore.cs index f77a5713..349b26af 100644 --- a/src/Chronos.Server/Storage/RenderStore.cs +++ b/src/Chronos.Server/Storage/RenderStore.cs @@ -1,47 +1,57 @@ using Chronos.Core.IO; using Chronos.Core.Model; using Chronos.Data; +using Chronos.Data.Archiving; using Chronos.Protocol; namespace Chronos.Server.Storage; /// Files a verified render output under the sequence's exports/ folder (BRIEF §7) and records it. -public sealed class RenderStore(ChronosDatabase db, ServerArchive archive, TimeProvider time) +/// The archive engine whose sequence lease the filing takes; one of its own when none is registered. +public sealed class RenderStore(ChronosDatabase db, ServerArchive archive, TimeProvider time, ArchiveService? archives = null) { + private readonly ArchiveService _archives = archives ?? new ArchiveService(db, new ArchiveServiceOptions { Owner = ArchiveModule.Owner }); + /// /// Moves (on the archive volume) to <sequence folder>/exports/, never - /// replacing an existing export, and inserts the render row. Null when the sequence has no folder in the archive yet. + /// replacing an existing export, and inserts the render row, both under the sequence's lease with its folder as the + /// index has it then: a Reorganize may have moved it since was read, and moves the row's + /// path along when it moves it later. Null when the sequence left the index, has no folder in the archive yet or is not + /// in the folder the index has (another index moved it); nothing is moved then. /// public Task CommitAsync(Sequence sequence, string stagedFile, RenderResultRequest output, string producedBy, CancellationToken cancellationToken) => CommitAsync(sequence, stagedFile, output, producedBy, Guid.CreateVersion7(), cancellationToken); /// As above, keeping : a render a client sends keeps the id it has on that client. - public async Task CommitAsync(Sequence sequence, string stagedFile, RenderResultRequest output, string producedBy, Guid id, - CancellationToken cancellationToken) - { - if (string.IsNullOrEmpty(sequence.Folder) || ArchivePathGuard.Resolve(archive.Root, sequence.Folder.TrimEnd('/') + "/exports") is not { } exports) - return null; - Directory.CreateDirectory(exports); - var target = Unique(exports, output.FileName); - File.Move(stagedFile, target); - var render = new Render + public Task CommitAsync(Sequence sequence, string stagedFile, RenderResultRequest output, string producedBy, Guid id, + CancellationToken cancellationToken) => + _archives.UnderLeaseAsync(sequence, async row => { - Id = id, - SequenceId = sequence.Id, - PresetName = output.PresetName, - OutputPath = LongPath.ToPortableRelative(archive.Root, target), - Codec = output.Codec, - Width = output.Width, - Height = output.Height, - Fps = output.Fps, - DurationSeconds = output.DurationSeconds, - Bytes = output.Size, - Created = time.GetUtcNow(), - ProducedBy = producedBy, - }; - await db.Renders.InsertAsync(render, cancellationToken); - return render; - } + // A folder another index moved is found by this index's next scan; until then nothing is filed, so it is not created again. + if (row is null || string.IsNullOrEmpty(row.Folder) || ArchivePathGuard.Resolve(archive.Root, row.Folder.TrimEnd('/') + "/exports") is not { } exports + || !Directory.Exists(Path.GetDirectoryName(exports))) + return null; + Directory.CreateDirectory(exports); + var target = Unique(exports, output.FileName); + File.Move(stagedFile, target); + var render = new Render + { + Id = id, + SequenceId = row.Id, + PresetName = output.PresetName, + OutputPath = LongPath.ToPortableRelative(archive.Root, target), + Codec = output.Codec, + Width = output.Width, + Height = output.Height, + Fps = output.Fps, + DurationSeconds = output.DurationSeconds, + Bytes = output.Size, + Created = time.GetUtcNow(), + ProducedBy = producedBy, + }; + await db.Renders.InsertAsync(render, cancellationToken); + return (Render?)render; + }, cancellationToken); private static string Unique(string folder, string fileName) { diff --git a/tests/Chronos.App.Tests/Editor/EditorSmokeTests.cs b/tests/Chronos.App.Tests/Editor/EditorSmokeTests.cs index 8ee354e5..76b0d6b9 100644 --- a/tests/Chronos.App.Tests/Editor/EditorSmokeTests.cs +++ b/tests/Chronos.App.Tests/Editor/EditorSmokeTests.cs @@ -330,6 +330,59 @@ public sealed class EditorSmokeTests } }); + /// + /// A Reorganize of this PC moves the folder of the sequence open in the editor: the editor reads it again where it is + /// now as soon as its step is done, so its frames decode and its project goes there. + /// + [Test] + public Task AnOpenSequenceIsReadAgainWhereAReorganizeMovedIt() => ScreenHost.RunAsync(async () => + { + await using var app = await ScreenSmokeTests.SampleApp.StartAsync(); + var id = await app.Sample.FillEditorAsync(); + var db = app.Services.Database!; + var archives = app.Services.Archives!; + var source = new LocalSequenceSource(db, id, Chronos.Core.Imaging.Decoders.DecoderFactory.CreateClient(null), null); + var content = await source.LoadAsync(CancellationToken.None); + var editor = new EditorViewModel(app.Shell, new EditorSession(source, content, []), [], EncoderHost.Client); + await using (editor) + { + var archive = (await db.Archives.GetAsync(content.Sequence.ArchiveId))!; + var templates = new Chronos.Core.Archiving.RenameTemplates("Moved/{date}"); + var plan = await archives.PlanReorganizeAsync(archive.Id, [id], templates, CancellationToken.None); + var moved = plan.Steps.Single().ToFolder; + await using var engine = new Chronos.Data.Jobs.JobEngine(db, archives.CreateJobHandlers(), new Chronos.Data.Jobs.JobEngineOptions + { + WorkerId = "test", + Kinds = [JobKind.Move], + PollInterval = TimeSpan.FromMilliseconds(50), + }); + await engine.StartAsync(CancellationToken.None); + var job = await Chronos.Data.Archiving.ArchiveJobs.EnqueueAsync(engine, Chronos.Data.Archiving.ArchiveJobs.Reorganize(archive, plan, templates)); + // The editor reads its photos as it opens; on Windows the folder cannot move while one is open, and the job that + // failed holds the plan for its retry. + for (var attempt = 1; ; attempt++) + { + Job? ended = null; + for (var i = 0; i < 1200 && ended is null; i++) + { + await Task.Delay(25); + if (await db.Jobs.GetAsync(job.Id) is { State: JobState.Done or JobState.Failed or JobState.Cancelled } now) ended = now; + } + Assert.NotNull(ended, "the move job ends"); + if (ended!.State == JobState.Done) break; + Assert.True(ended.State == JobState.Failed && attempt < 20, ended.Error); + await Task.Delay(250); + await engine.ResumeAsync(job.Id, CancellationToken.None); + } + + for (var i = 0; i < 400 && editor.Session.Content.Sequence.Folder != moved; i++) await Task.Delay(25); + Assert.Equal(moved, editor.Session.Content.Sequence.Folder); + Assert.True(await source.FramesCurrentAsync(CancellationToken.None), "the editor holds the frames where they are now"); + using var frame = await source.DecodeAsync(0, 32, Chronos.Core.Imaging.DecodePurpose.Thumbnail, CancellationToken.None); + Assert.NotNull(frame, "its frames decode"); + } + }); + /// /// At the 1200 px minimum beside the 200 px rail (owner review 1, R1), the preview's tools stay inside the preview in /// every language, crop mask on (the crop-result tool enabled): they take a second line rather than lose Play and Full screen. diff --git a/tests/Chronos.Data.Tests/Archiving/MassRenameTests.cs b/tests/Chronos.Data.Tests/Archiving/MassRenameTests.cs index 47cc5e03..b4494100 100644 --- a/tests/Chronos.Data.Tests/Archiving/MassRenameTests.cs +++ b/tests/Chronos.Data.Tests/Archiving/MassRenameTests.cs @@ -241,23 +241,30 @@ public sealed class MassRenameTests(TestContext context) { await using var fx = await CreateAsync(context); var ids = await NestedAsync(fx); - // Exports of "trips" and of "trips/day2" in their folders, one of "trips" outside the archive, and a row whose - // path only starts with the same letters. + // Exports of "trips" and of "trips/day2" in their folders, one of "trips/day2/dawn" saved in the exports of "trips" + // (the Export tab's folder picker), one of "trips" outside the archive, a row whose path only starts with the same + // letters, and one of "trips/day2" in "trips" whose file is gone. var outside = Path.Combine(context.CreateTempDirectory("exports"), "Trips elsewhere.mp4"); List renders = [ await RenderAsync(fx, ids[0], "trips/exports/Trips.mp4"), await RenderAsync(fx, ids[1], "trips/day2/exports/Day 2.mp4"), + await RenderAsync(fx, ids[2], "trips/exports/Dawn.mp4"), await RenderAsync(fx, ids[0], outside), await RenderAsync(fx, ids[0], "trips 2/exports/Other.mp4", write: false), + await RenderAsync(fx, ids[1], "trips/exports/Gone.mp4", write: false), ]; var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [ids[0]], "moved/{date}", null, fx.Ct); Assert.SequenceEqual([false, true, true], plan.Steps.Select(s => s.Carried)); await fx.Service.ApplyRenameAsync(plan, null, fx.Ct); - string[] expected = ["moved/2026-09-28/exports/Trips.mp4", "moved/2026-09-28/day2/exports/Day 2.mp4", outside, "trips 2/exports/Other.mp4"]; + string[] expected = + [ + "moved/2026-09-28/exports/Trips.mp4", "moved/2026-09-28/day2/exports/Day 2.mp4", "moved/2026-09-28/exports/Dawn.mp4", outside, + "trips 2/exports/Other.mp4", "trips/exports/Gone.mp4", + ]; Assert.SequenceEqual(expected, await OutputsAsync(fx, renders)); - foreach (var path in expected[..3]) Assert.FileExists(Path.IsPathRooted(path) ? path : fx.Paths.FromRelative(path)); + foreach (var path in expected[..4]) Assert.FileExists(Path.IsPathRooted(path) ? path : fx.Paths.FromRelative(path)); // A crash after the steps were recorded but before the journal said so: completed again, and again, nothing moves twice. RenameJournal.Write(fx.Paths, new RenameJournal(plan, [])); @@ -266,6 +273,71 @@ public sealed class MassRenameTests(TestContext context) Assert.SequenceEqual(expected, await OutputsAsync(fx, renders)); } + [Test] + public async Task AnotherIndexFindsTheExportsWhereAReorganizeMovedTheirFolder() + { + await using var fx = await CreateAsync(context); + var ids = await PopulateAsync(fx); + // This index's exports: of "inbox/a" in its folder, of "inbox/b" saved in the exports of "inbox/a", of "inbox/c" in its own. + List renders = + [ + await RenderAsync(fx, ids[0], "inbox/a/exports/A.mp4"), + await RenderAsync(fx, ids[1], "inbox/a/exports/B.mp4"), + await RenderAsync(fx, ids[2], "inbox/c/exports/C.mp4"), + ]; + var (server, archive) = await fx.OtherWriterAsync(); + + // The other index (a server sharing this desktop's archive) moves "inbox/a", then "inbox/c" into the folder it left. + var first = await server.PlanRenameAsync(archive.Id, [ids[0]], "moved/{date}", null, fx.Ct); + await server.ApplyRenameAsync(first, null, fx.Ct); + var second = await server.PlanRenameAsync(archive.Id, [ids[2]], "inbox/a", null, fx.Ct); + Assert.Equal("inbox/a", second.Steps.Single().ToFolder); + await server.ApplyRenameAsync(second, null, fx.Ct); + + // A scan here finds both where they are now (the new "inbox/a" first): each export follows the folder its file moved with. + await fx.ScanAsync(); + Assert.Equal("moved/2026-09-28", (await fx.Db.Sequences.GetAsync(ids[0], fx.Ct))!.Folder); + Assert.Equal("inbox/a", (await fx.Db.Sequences.GetAsync(ids[2], fx.Ct))!.Folder); + string[] expected = ["moved/2026-09-28/exports/A.mp4", "moved/2026-09-28/exports/B.mp4", "inbox/a/exports/C.mp4"]; + Assert.SequenceEqual(expected, await OutputsAsync(fx, renders)); + foreach (var path in expected) Assert.FileExists(fx.Paths.FromRelative(path)); + Assert.Equal(0, (await fx.ScanAsync()).Indexed); + Assert.SequenceEqual(expected, await OutputsAsync(fx, renders)); + } + + [Test] + public async Task APhotoDeletionWhoseFolderMovedWhileItWaitedForTheLeaseIsRefused() + { + await using var fx = await CreateAsync(context); + var ids = await PopulateAsync(fx); + var frames = await fx.Db.Frames.ListBySequenceAsync(ids[0], cancellationToken: fx.Ct); + var deletion = await fx.Service.FrameTrash.PlanDeleteAsync(ids[0], [frames[1].Id], cancellationToken: fx.Ct); + Assert.True(deletion.CanDelete); + var pause = TimeSpan.FromSeconds(TestTiming.Budget(0.5)); + + // Another process holds the sequence and moves its folder (a Reorganize), while the deletion waits for the lease. + Task delete; + using (await LeaseFile.AcquireAsync(fx.Paths.LocksDirectory, $"seq-{ids[0]:N}", "other", TimeSpan.FromSeconds(5), cancellationToken: fx.Ct)) + { + delete = fx.Service.FrameTrash.DeleteAsync(deletion, cancellationToken: fx.Ct); + await Task.Delay(pause, fx.Ct); + Assert.False(delete.IsCompleted, "the deletion waits for the other process"); + Directory.CreateDirectory(fx.Paths.FromRelative("moved")); + Directory.Move(fx.Paths.FromRelative("inbox/a"), fx.Paths.FromRelative("moved/a")); + await using var tx = await fx.Db.BeginAsync(fx.Ct); + await tx.Sequences.RelocateAsync(ids[0], "moved/a", null, fx.Ct); + await tx.Frames.RebaseFolderAsync(ids[0], "inbox/a", "moved/a", fx.Ct); + await tx.CommitAsync(fx.Ct); + } + + await Assert.ThrowsAsync(() => delete); + Assert.Count(3, await fx.Db.Frames.ListBySequenceAsync(ids[0], cancellationToken: fx.Ct)); + Assert.Count(3, ManifestStore.ReadSequence(fx.Paths.Sequence("moved/a").ManifestPath).Frames); + foreach (var f in frames) Assert.FileExists(fx.Paths.FromRelative(f.RelativePath.Replace("inbox/a/", "moved/a/", StringComparison.Ordinal))); + Assert.Empty(await fx.Service.Trash.ListAsync(fx.Archive, fx.Ct)); + Assert.False(Directory.Exists(fx.Paths.FromRelative("inbox/a")), "the folder it left is not created again"); + } + [Test] public async Task AnInterruptedMoveTakesTheExportsAlongWhenItResumes() { diff --git a/tests/Chronos.Data.Tests/Archiving/ProxyTests.cs b/tests/Chronos.Data.Tests/Archiving/ProxyTests.cs index f0789eb8..8872b085 100644 --- a/tests/Chronos.Data.Tests/Archiving/ProxyTests.cs +++ b/tests/Chronos.Data.Tests/Archiving/ProxyTests.cs @@ -37,11 +37,11 @@ public sealed class ProxyTests(TestContext context) var folder = fx.Paths.Sequence(sequence.Folder); var images = fx.Service.Images; - Assert.Equal(ProxyStoreResult.Invalid, images.StoreProxy(fx.Archive, sequence, frames[0], "not a photo"u8.ToArray())); - Assert.Equal(ProxyStoreResult.WrongFrame, images.StoreProxy(fx.Archive, sequence, frames[0], Proxy(frames[1].CapturedAt!.Value, 1))); + Assert.Equal(ProxyStoreResult.Invalid, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], "not a photo"u8.ToArray(), fx.Ct)); + Assert.Equal(ProxyStoreResult.WrongFrame, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], Proxy(frames[1].CapturedAt!.Value, 1), fx.Ct)); Assert.FileMissing(folder.Proxy(0)); var proxy = Proxy(frames[0].CapturedAt!.Value, 2); - Assert.Equal(ProxyStoreResult.Stored, images.StoreProxy(fx.Archive, sequence, frames[0], proxy)); + Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], proxy, fx.Ct)); Assert.SequenceEqual(proxy, await File.ReadAllBytesAsync(folder.Proxy(0), fx.Ct)); // The proxy pass keeps what was sent and makes none from the originals here, even when forced. @@ -62,6 +62,32 @@ public sealed class ProxyTests(TestContext context) Assert.Equal((0, 1, 1), (again.Built, again.Reused, again.Undecodable)); } + [Test] + public async Task ProxiesSentOrMadeAfterTheFolderMovedGoWhereTheSequenceIsNow() + { + await using var fx = await CreateAsync(context); + var (stale, frames) = await HeifSequenceAsync(fx); + var images = fx.Service.Images; + var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [stale.Id], "moved/{date}", null, fx.Ct); + await fx.Service.ApplyRenameAsync(plan, null, fx.Ct); + var now = fx.Paths.Sequence(plan.Steps.Single().ToFolder); + var left = fx.Paths.Sequence(stale.Folder).Folder; + Assert.False(Directory.Exists(left)); + + // A proxy upload and a proxy pass that read the sequence before the move write once they hold its lease. + var proxy = Proxy(frames[0].CapturedAt!.Value, 2); + Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, stale, frames[0], proxy, fx.Ct)); + Assert.SequenceEqual(proxy, await File.ReadAllBytesAsync(now.Proxy(0), fx.Ct)); + var pass = await images.BuildProxiesAsync(fx.Archive, stale, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct); + Assert.Equal((0, 1, 0, 1), (pass.Built, pass.Reused, pass.Undecodable, pass.Waiting)); + Assert.False(Directory.Exists(left), "the folder it left is not created again"); + + // Deleted since: nothing is written. + Assert.True(await fx.Db.Sequences.DeleteAsync(stale.Id, fx.Ct)); + Assert.Equal(ProxyStoreResult.Gone, await images.StoreProxyAsync(fx.Archive, stale, frames[1], Proxy(frames[1].CapturedAt!.Value, 3), fx.Ct)); + Assert.FileMissing(now.Proxy(1)); + } + [Test] public async Task AProxyLeftByARenumberedFrameIsAskedForAgain() { @@ -99,8 +125,8 @@ public sealed class ProxyTests(TestContext context) // A client or render node sending one that small is refused; one of the policy's size (1920 px) is kept. var frame = frames[0] with { Width = 4032, Height = 3024 }; - Assert.Equal(ProxyStoreResult.TooSmall, images.StoreProxy(fx.Archive, sequence, frame, Proxy(frame.CapturedAt!.Value, 6, 160, 120))); - Assert.Equal(ProxyStoreResult.Stored, images.StoreProxy(fx.Archive, sequence, frame, Proxy(frame.CapturedAt!.Value, 7, 1920, 1440))); + Assert.Equal(ProxyStoreResult.TooSmall, await images.StoreProxyAsync(fx.Archive, sequence, frame, Proxy(frame.CapturedAt!.Value, 6, 160, 120), fx.Ct)); + Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frame, Proxy(frame.CapturedAt!.Value, 7, 1920, 1440), fx.Ct)); var again = await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct); Assert.Equal((0, 1, 0, 1), (again.Built, again.Reused, again.Undecodable, again.Waiting)); Assert.FileExists(folder.Proxy(0)); @@ -114,7 +140,7 @@ public sealed class ProxyTests(TestContext context) var manifest = ManifestStore.ReadArchive(fx.Paths.ManifestPath); ManifestStore.WriteArchive(fx.Paths.ManifestPath, manifest with { ProxyPolicy = new ProxyPolicy { LongEdge = ProxyPolicy.Disabled } }); - Assert.Equal(ProxyStoreResult.Disabled, fx.Service.Images.StoreProxy(fx.Archive, sequence, frames[0], Proxy(frames[0].CapturedAt!.Value, 4))); + Assert.Equal(ProxyStoreResult.Disabled, await fx.Service.Images.StoreProxyAsync(fx.Archive, sequence, frames[0], Proxy(frames[0].CapturedAt!.Value, 4), fx.Ct)); Assert.FileMissing(fx.Paths.Sequence(sequence.Folder).Proxy(0)); } @@ -170,7 +196,7 @@ public sealed class ProxyTests(TestContext context) // A client makes one from the index's time: it is kept, and the next pass reuses it rather than making its own. var sent = Proxy(frame.CapturedAt!.Value, 1); - Assert.Equal(ProxyStoreResult.Stored, images.StoreProxy(fx.Archive, sequence, frame, sent)); + Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frame, sent, fx.Ct)); var again = await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct); Assert.Equal((0, 1), (again.Built, again.Reused)); Assert.SequenceEqual(sent, await File.ReadAllBytesAsync(folder.Proxy(0), fx.Ct)); @@ -264,7 +290,7 @@ public sealed class ProxyTests(TestContext context) await using var fx = await CreateAsync(context); var (sequence, frames) = await HeifSequenceAsync(fx); var images = fx.Service.Images; - Assert.Equal(ProxyStoreResult.Stored, images.StoreProxy(fx.Archive, sequence, frames[1], Proxy(frames[1].CapturedAt!.Value, 1))); + Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frames[1], Proxy(frames[1].CapturedAt!.Value, 1), fx.Ct)); // A pass resumed at frame 1 (paused, retried or restarted) finds that frame done, but frame 0 still waits: the // status must not tell a server's clients that nothing is wanted. @@ -272,7 +298,7 @@ public sealed class ProxyTests(TestContext context) Assert.Equal((0, 1, 0, 0), (resumed.Built, resumed.Reused, resumed.Undecodable, resumed.Waiting)); Assert.False(await ProxiesBuiltAsync(fx, sequence), "frame 0 has no proxy"); - Assert.Equal(ProxyStoreResult.Stored, images.StoreProxy(fx.Archive, sequence, frames[0], Proxy(frames[0].CapturedAt!.Value, 2))); + Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], Proxy(frames[0].CapturedAt!.Value, 2), fx.Ct)); await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 1, (_, _) => ValueTask.CompletedTask, fx.Ct); Assert.True(await ProxiesBuiltAsync(fx, sequence), "every frame has its proxy"); } @@ -290,7 +316,7 @@ public sealed class ProxyTests(TestContext context) { Frames = [.. sidecar.Frames.Select(f => f with { Tier = DecodeTier.EmbeddedPreview, Flags = f.Flags | FrameFlags.PreviewOnly, PreviewW = 160, PreviewH = 120 })], }); - Assert.Equal(ProxyStoreResult.Stored, images.StoreProxy(fx.Archive, sequence, frames[0], Proxy(frames[0].CapturedAt!.Value, 1))); + Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], Proxy(frames[0].CapturedAt!.Value, 1), fx.Ct)); // Frame 0 now renders from its proxy, not from the thumbnail: it caps nothing. Its tier stays the original's. await images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct); @@ -315,7 +341,7 @@ public sealed class ProxyTests(TestContext context) var (sequence, frames) = await HeifSequenceAsync(fx); var folder = fx.Paths.Sequence(sequence.Folder); var images = fx.Service.Images; - Assert.Equal(ProxyStoreResult.Stored, images.StoreProxy(fx.Archive, sequence, frames[0], Proxy(frames[0].CapturedAt!.Value, 1, 640, 480))); + Assert.Equal(ProxyStoreResult.Stored, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], Proxy(frames[0].CapturedAt!.Value, 1, 640, 480), fx.Ct)); Task ThumbnailsAsync(bool force = false) => images.BuildThumbnailsAsync(fx.Archive, sequence, force, 0, (_, _) => ValueTask.CompletedTask, fx.Ct); (int, int) Entry() diff --git a/tests/Chronos.Data.Tests/Processing/AnalysisJobTests.cs b/tests/Chronos.Data.Tests/Processing/AnalysisJobTests.cs index c630c6f6..c259345b 100644 --- a/tests/Chronos.Data.Tests/Processing/AnalysisJobTests.cs +++ b/tests/Chronos.Data.Tests/Processing/AnalysisJobTests.cs @@ -455,6 +455,33 @@ public sealed class AnalysisJobTests(TestContext context) Assert.Contains("13", ProjectSettings.FromJson(await ProjectJsonAsync(fx, folder)).Timing.FrameRate.ReasonArgs!); } + [Test] + public async Task ADeepAnalysisWhoseFolderAnotherChronosMovedRecordsNothingAndCreatesNoFolder() + { + // One decode at a time, so the held one is the only photo being read while the folder moves. + await using var fx = await ProcessingFixture.CreateAsync(context, o => o with { DeepAnalysis = o.DeepAnalysis with { MaxDegreeOfParallelism = 1 } }); + var sequence = await fx.AddAsync(Flickering(12)); + var left = fx.Folder(sequence).Folder; + var (server, archive) = await fx.Archive.OtherWriterAsync(); + await fx.StartEngineAsync(); + + // A server sharing this desktop's archive reorganizes the sequence while the deep pass reads its photos. + fx.Decoder.HoldFrame(5); + var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id])); + await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct); + var plan = await server.PlanRenameAsync(archive.Id, [sequence.Id], "Moved/{date}", null, fx.Ct); + await server.ApplyRenameAsync(plan, null, fx.Ct); + fx.Decoder.Hold = null; + fx.Decoder.Release(); + var done = await fx.WaitForAsync(deep.Id, JobState.Done); + + Assert.Contains(Loc.L("Advisor.Log.FramesChangedElsewhere", sequence.DisplayTitle), done.LogTail); + Assert.False(Directory.Exists(left), "the folder it left is not created again"); + var moved = fx.Archive.Paths.Sequence(plan.Steps.Single().ToFolder); + Assert.FileMissing(moved.AnalysisPath); + Assert.False((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.Status.HasFlag(SequenceStatus.AnalyzedDeep), "nothing is recorded"); + } + [Test] public async Task ALightAnalysisOvertakenByAScanMarksNothingAnalysedAndRunsAgain() { diff --git a/tests/Chronos.Data.Tests/Sync/SyncServiceTests.cs b/tests/Chronos.Data.Tests/Sync/SyncServiceTests.cs index 6fa3d23b..25319cfb 100644 --- a/tests/Chronos.Data.Tests/Sync/SyncServiceTests.cs +++ b/tests/Chronos.Data.Tests/Sync/SyncServiceTests.cs @@ -432,6 +432,30 @@ public sealed class SyncServiceTests(TestContext context) Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Holy grail")).Status); } + [Test] + public async Task SaveBothWhileAReorganizeMovesTheFolderKeepsTheCopyWhereTheSequenceIsNow() + { + await using var f = await SyncFixture.CreateAsync(context); + var (local, remote) = await ConflictAsync(f); + var left = f.Archive.Paths.Sequence(local.Folder).Folder; + + // A Reorganize moves the folder while the server's version is read. + f.Remote.OnceBeforeManifest = async () => + { + var plan = await f.Archive.Service.PlanRenameAsync(f.Archive.Archive.Id, [local.Id], "moved/{title}", null, f.Ct); + await f.Archive.Service.ApplyRenameAsync(plan, null, f.Ct); + }; + var copy = await f.Sync.ResolveAsync(f.Remote, remote.Id, SyncResolution.SaveBoth, f.Ct); + Assert.Null(f.Remote.OnceBeforeManifest, "the folder moved during the resolution"); + + var taken = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!; + Assert.Equal("moved/Holy grail", taken.Folder); + Assert.Equal(copy, Assert.Contains(Directory.GetFiles(f.Archive.Paths.Sequence(taken.Folder).MetaDirectory, "project.this-pc-*.json"), _ => true)); + Assert.True(SyncRules.SameSettings(SyncFixture.Window41, await File.ReadAllTextAsync(copy!, f.Ct)), "this PC's version is kept"); + Assert.True(SyncRules.SameSettings(SyncFixture.Window33, taken.SettingsJson)); + Assert.False(Directory.Exists(left), "the folder it left is not created again"); + } + [Test] public async Task KeepMineRefusesWhenTheServerChangedAfterTheConflictWasShown() { diff --git a/tests/Chronos.Server.Tests/Api/ReorganizeTests.cs b/tests/Chronos.Server.Tests/Api/ReorganizeTests.cs index 8bed604b..890ef36a 100644 --- a/tests/Chronos.Server.Tests/Api/ReorganizeTests.cs +++ b/tests/Chronos.Server.Tests/Api/ReorganizeTests.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.DependencyInjection; using Chronos.Core.Archiving; using Chronos.Core.Codecs.Jpeg; using Chronos.Core.Model; +using Chronos.Data.Archiving; using Chronos.Protocol; using Chronos.Server.Api; using Chronos.Server.Events; @@ -120,6 +121,31 @@ public sealed class ReorganizeTests(TestContext context) Assert.False(Directory.Exists(Path.Combine(server.Options.ArchivePath, folder)), "nothing is left at the old place"); } + [Test] + public async Task AnExportFiledAfterItsFolderMovedGoesWhereTheSequenceIsNow() + { + await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + var id = (await Uploads.UploadAsync(editor, Photo(1), "IMG_0001.JPG", sessionKey: "roof")).SequenceId; + await UntilIdleAsync(server); + var stale = (await server.Database.Sequences.GetAsync(id))!; + + // A render node's output read the sequence before a Reorganize moved its folder, and is filed after it. + var archives = server.Services.GetRequiredService(); + var plan = await archives.PlanRenameAsync(stale.ArchiveId, [id], "reorganized/{date}", null, CancellationToken.None); + await archives.ApplyRenameAsync(plan, null, CancellationToken.None); + var folder = plan.Steps.Single().ToFolder; + var video = Uploads.Photo(4096, 29); + var staged = Path.Combine(context.CreateTempDirectory("staged"), "render.mp4"); + await File.WriteAllBytesAsync(staged, video); + var render = Assert.NotNull(await server.Services.GetRequiredService().CommitAsync(stale, staged, + new RenderResultRequest("Roof - 4K.mp4", video.Length, Uploads.Sha(video), "4K HEVC", "hevc", 3840, 2160, 25, 4), "studio-pc", CancellationToken.None)); + + Assert.Equal($"{folder}/exports/Roof - 4K.mp4", render.OutputPath); + await Http.BytesEqualAsync(video, File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, render.OutputPath))); + Assert.False(Directory.Exists(Path.Combine(server.Options.ArchivePath, stale.Folder)), "the folder it left is not created again"); + } + [Test] public async Task AReorganizeAsksForTheArchiveAndAValidRequest() {