diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index a46f3f7c..54cb9267 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -42,8 +42,8 @@ Dependencies point one way: `Core` ← `Core.Windows`, `Data`, `Protocol` ← `S | `Video` (+ `Mp4`, `Avi`, `Bitstream`, `Yuv`, `Managed`) | `IVideoEncoderProvider`, MP4 muxer and inspector, AVI-MJPEG, NV12/P010, managed encoders (MJPEG-AVI, GIF, APNG, image sequences) | | `Archiving` | Archive layout and paths, sidecars (`sequence.json`, `project.json`), scanner, index snapshot, leases, thumbnail packs, verify, adopt, rename plans (`RenamePlanner`: folder, title and file templates, `RenameTemplates`, the plan's fingerprint, which pins every file rename by its digest and not the frames version; `FileRenames`: a sequence's new file names; `FileRenameJournal`: two-phase renames through temporary names, `.chronos/rename-files.json`, a recovery safe to run again from wherever it stopped, a new name taken meanwhile sending that photo and its companions back together to their old names or a free `_2`, those that took their new names already with them; `FrameCompanions`: the XMP files that follow a photo, one rule for the trash and Reorganize, matched whatever their case and each folder listed once by `FrameCompanions.Index`) | | `Import` | Card scan, session detection, import plan (photos named through `FrameNaming` with the camera and lens of their session's first photo), hash-verified copy, card eraser (re-hash before delete) | -| `Conversion` | Mass conversion after import (owner review 2, R7): `ConvertOptions` (target JPEG 4:4:4 / TIFF-16 / linear DNG, JPEG quality, the converted name: same stem, the target's extension in the old one's case), `FrameConverter` (native-tier decode only, in-house encoders, Exif from the sidecar and the original via `ExifWriter`, hashed as made, written atomically, verified: read back, metadata again, decoded by the server's decoders), `ConvertPlanner` (per sequence the frames converted and skipped with `ConvertSkip`, size estimates per pixel, `ConvertPlan` with its space and fingerprint) | -| `Naming` | Folder/file templates and tokens (folder levels scans skip escaped with `_`; `ExpandText` for titles; `ExpandFile` shortens a long file name before its `.{ext}`); `FrameNaming` (the naming context and unique file name of a frame, shared by import, upload and Reorganize: the sequence's camera and lens, the first frame for folders; `UniqueNames`: `_2`, `_3`… in linear time); `TemplateRules` (what every place a template is typed or sent refuses, the archive engine's new or changed templates included: a folder scans skip, a lost extension, names that repeat) | +| `Conversion` | Mass conversion after import (owner review 2, R7): `ConvertOptions` (target JPEG 4:4:4 / TIFF-16 / linear DNG, JPEG quality, the converted name: the archive's file template with the frame's `FrameNaming` context in its sequence, as filing names it, `{ext}` the target's), `FrameConverter` (native-tier decode only, in-house encoders, Exif from the sidecar and the original via `ExifWriter`, hashed as made, written atomically, verified: read back, metadata again, decoded by the server's decoders), `ConvertPlanner` (per sequence the frames converted and skipped with `ConvertSkip`, size estimates per pixel, `ConvertPlan` with its space and fingerprint) | +| `Naming` | Folder/file templates and tokens (folder levels scans skip escaped with `_`; `ExpandText` for titles; `ExpandFile` shortens a long file name before its `.{ext}`); `FrameNaming` (the naming context and unique file name of a frame, shared by import, upload, Reorganize and conversion: the sequence's camera and lens, the first frame for folders; `UniqueNames`: `_2`, `_3`… in linear time); `TemplateRules` (what every place a template is typed or sent refuses, the archive engine's new or changed templates included: a folder scans skip, a lost extension, names that repeat) | | `Model` | Domain entities and enums (§5) shared by Data, Server and clients | | `Packaging` | TPK1 installer payload writer/reader, staged install transaction | | `Diagnostics` | `Log` (never throws: a line its sink cannot write is dropped), `FileLogSink` with `LogFileOptions` (level, daily or hourly files, numbered parts past a size, pruning by age and by the total size, never the file being written), synthetic sequences, `SecretScrubber`, `DiagnosticsArchive` (scrubbed diagnostics zip) | @@ -59,7 +59,7 @@ Dependencies point one way: `Core` ← `Core.Windows`, `Data`, `Protocol` ← `S | `Jobs` | `JobEngine` (one job at a time; reviews the priority queue and makes a lower running job yield, requeued with its checkpoint; no history purge, the housekeeping schedule's clean-up does it), `JobPriorityQueue` (a level per job kind, the claim order, settings round trip), `IJobHandler`, `IJobHandlerProvider`, `JobContext` (progress, checkpoints, follow-up jobs that never interrupt their parent), `JobThroughput`/`JobEta` (rate and bounded time left) | | `Scheduling` | `Scheduler` (one per host: seeds the host's system schedules, claims due runs by compare-and-set, catch-up, clock changes, create/edit/reset/delete/run now as API v1 DTOs), `ScheduleRecurrence` (the in-house `every:`/`daily:`/`weekly:`/`monthly:` form, wall-clock times across DST), `ScheduleActions` (action names, parameters, availability, the archive jobs a run queues), `IScheduleHost` and `SystemSchedule` (what the server and the desktop supply), `ScheduleOutcome`, `ScheduleMapping` (model ↔ DTO) | | `Housekeeping` | `BackupJobHandler`, `CleanupJobHandler`, `ICleanupStep` with the shared steps (`CleanupSteps`: job history, audit log, trash report that never deletes); hosts add their own steps through `ArchiveServiceOptions.CleanupSteps` | -| `Archiving` | `ArchiveService` (create/adopt/open, archive settings written to `archive.json` then the index, `ArchiveSettings.CheckTemplates` and `TemplateRuleException` holding a template given anew to the template rules; `UnderLeaseAsync`: a write under a sequence's lease with its row read under it), `ArchiveIndexer` (scan, rebuild; a sequence found in another folder takes its exports' paths along), `ArchiveIngest` (uploads and imports, named through `FrameNaming`), `ImportService`, `ArchiveVerifier`, `ArchiveImages` (thumbnail and proxy passes, several photos at once through `OrderedWork`, which hands results back in frame order), `FrameTiers` (decode tier, `PreviewOnly`/`Undecodable` flags and preview size of frames, written to sidecar and index by the light analysis, deep analysis, renders and image passes), `GprCompanions` (the JPEG of each GoPro GPR, found in the index when an import kept it in another sequence), `ArchiveMaintenance`, `MassRenamer` (mass rename and Reorganize: the dry run reads each sidecar for a file template; each step checked under its leases before anything of it moves (where its sequences are, what is inside and at its new folder, a sidecar of a newer format, the title it was planned from when it changes it or names the folder or files after it), then the folder, with the sequences of the plan an earlier step left as they were inside it (taken along, their title and file names kept), every index row (the frame paths and render paths of the exports inside it, a file or an image-sequence folder, its own unless the export is still at the old place and another sequence's when it moved with it, follow it in one transaction; a sidecar that cannot be read or written then still has its row follow, and fails the step), then the files under the plan's journal in `journal/` (a sequence whose photos changed since, such as one still receiving them, or whose files are found changed or in use or have their new names taken keeps its file names, the renames being pinned by digest, and the step and the plan go on); the journal keeps the tally of the whole plan, and a plan its job held (paused, cancelled or failed) is left to that job, only its half-done file renames completed at an archive open, which completes any other plan an interruption left and tells each sequence changed; the file renames of a step: sidecar, a sequence whose sidecar cannot be read staying as it is with every folder around it, index (`FrameRepository.RenameFilesAsync`) and the sequence's journal), `SequenceFolders` (`Holds`: the folder the index has for a sequence is there and its sidecar names no other sequence; every writer into a sequence's folder asks it), `SequenceTrash` (delete into the archive's hidden trash with a dry run, restore; the folder moves back when the index step fails; lists sequence and photo entries; `SequenceTrash.Empty`: the emptying's dry run, fingerprint and erase, each entry checked against its fingerprint again under its lease), `FrameTrash` (photos of a sequence into `trash/photos` with a dry run, a journal that undoes an interrupted deletion whose entry stays unlisted until its sidecar is written, tombstones (a photo never hashed is hashed as it goes; one whose original is missing goes without a file or tombstone; a frame that stays and may be a twin is hashed, its hash kept), positions parked and placed back, proxies and thumbnails moved to follow their frames; `FrameTrash.Restore`: restore re-hashes each file and says why each kept photo stayed (`KeptReason`), under a journal of its own into the sequence that another writer settles only under the entry's lease, into a new sequence through the ingest when theirs is gone, each original staged inside its own entry and the record rewritten after each photo filed; a replaced entry swaps back while its converted frames are unchanged; `FrameTrash.Replace`: the files of frames swapped in place, same ids and indices, the old files into one `Replaced` entry with their replacements' hashes and a copy of `analysis.json`, tombstones aliasing old content to new, under `.chronos/replace.pending.json`, which the next writer undoes or completes), `MassConverter` (conversion's dry run with busy, offline and unreadable sequences and the space against `ArchiveServiceOptions.FreeSpace`; the job's stage A converts into `staging/convert-/` with `OrderedWork`, each staged file appended to `journal/convert-.staged`; stage B per sequence, all or nothing: replace through `FrameTrash.ReplaceAsync`, or a new sequence through the ingest with the source's tags, rating, notes and project), job handlers | +| `Archiving` | `ArchiveService` (create/adopt/open, archive settings written to `archive.json` then the index, `ArchiveSettings.CheckTemplates` and `TemplateRuleException` holding a template given anew to the template rules; `UnderLeaseAsync`: a write under a sequence's lease with its row read under it), `ArchiveIndexer` (scan, rebuild; a sequence found in another folder takes its exports' paths along), `ArchiveIngest` (uploads and imports, named through `FrameNaming`), `ImportService`, `ArchiveVerifier`, `ArchiveImages` (thumbnail and proxy passes, several photos at once through `OrderedWork`, which hands results back in frame order), `FrameTiers` (decode tier, `PreviewOnly`/`Undecodable` flags and preview size of frames, written to sidecar and index by the light analysis, deep analysis, renders and image passes), `GprCompanions` (the JPEG of each GoPro GPR, found in the index when an import kept it in another sequence), `ArchiveMaintenance`, `MassRenamer` (mass rename and Reorganize: the dry run reads each sidecar for a file template; each step checked under its leases before anything of it moves (where its sequences are, what is inside and at its new folder, a sidecar of a newer format, the title it was planned from when it changes it or names the folder or files after it), then the folder, with the sequences of the plan an earlier step left as they were inside it (taken along, their title and file names kept), every index row (the frame paths and render paths of the exports inside it, a file or an image-sequence folder, its own unless the export is still at the old place and another sequence's when it moved with it, follow it in one transaction; a sidecar that cannot be read or written then still has its row follow, and fails the step), then the files under the plan's journal in `journal/` (a sequence whose photos changed since, such as one still receiving them, or whose files are found changed or in use or have their new names taken keeps its file names, the renames being pinned by digest, and the step and the plan go on); the journal keeps the tally of the whole plan, and a plan its job held (paused, cancelled or failed) is left to that job, only its half-done file renames completed at an archive open, which completes any other plan an interruption left and tells each sequence changed; the file renames of a step: sidecar, a sequence whose sidecar cannot be read staying as it is with every folder around it, index (`FrameRepository.RenameFilesAsync`) and the sequence's journal), `SequenceFolders` (`Holds`: the folder the index has for a sequence is there and its sidecar names no other sequence; every writer into a sequence's folder asks it), `SequenceTrash` (delete into the archive's hidden trash with a dry run, restore; the folder moves back when the index step fails; lists sequence and photo entries; `SequenceTrash.Empty`: the emptying's dry run, fingerprint and erase, each entry checked against its fingerprint again under its lease), `FrameTrash` (photos of a sequence into `trash/photos` with a dry run, a journal that undoes an interrupted deletion whose entry stays unlisted until its sidecar is written, tombstones (a photo never hashed is hashed as it goes; one whose original is missing goes without a file or tombstone; a frame that stays and may be a twin is hashed, its hash kept), positions parked and placed back, proxies and thumbnails moved to follow their frames; `FrameTrash.Restore`: restore re-hashes each file and says why each kept photo stayed (`KeptReason`), under a journal of its own into the sequence that another writer settles only under the entry's lease, into a new sequence through the ingest when theirs is gone, each original staged inside its own entry and the record rewritten after each photo filed; a replaced entry swaps back while its converted frames are unchanged; `FrameTrash.Replace`: the files of frames swapped in place, same ids and indices, the old files into one `Replaced` entry with their replacements' hashes and a copy of `analysis.json`, tombstones aliasing old content to new, under `.chronos/replace.pending.json`, which the next writer undoes or completes), `MassConverter` (conversion's dry run with busy, offline and unreadable sequences and the space against `ArchiveServiceOptions.FreeSpace`; the job's stage A converts into `staging/convert-/` with `OrderedWork`, each staged file appended to `journal/convert-.staged`; stage B per sequence, all or nothing: replace through `FrameTrash.ReplaceAsync` (the sequence's folder read again under its lease, as every writer does), or a new sequence through the ingest with the source's tags, rating, notes and project; each sequence replaced or made told as it completes), job handlers | | `Processing` | `ProcessingService`, `SequenceAnalyser`, `DeepReports` (a saved deep report while it still describes the frames), `SequenceRender`, `ProjectStore`, `ProjectBundles` (a `.chronos` file for a sequence, and the sequence a file names), `RenderNodeClient`, job handlers | | `Sync` | `SyncService` (sync table of one server: compare, link by content, settle conflicts, enqueue transfers), `SyncRules` (status, settings merge and differences, no I/O), `SyncTombstones` (photos either side removed are not offered back, a replaced photo counts as its replacement), `SyncPositions` (the settings' frame positions translated by photo to the frames of the copy they go to), `SyncLinkStore` (`sync_links`, `sync_state`), `SyncTransfers` and `SyncJobHandler` (resumable, hash-verified uploads and downloads as jobs; downloads commit through the archive ingest), `SyncAgent` (background comparison, server events, backoff, offline proxies), `ProxyCache` and `RemoteSequence` (server sequences opened with proxies), `ISyncRemote` (the server calls; `ChronosSyncRemote` over `ChronosClient`) | diff --git a/i18n/en/Admin.resx b/i18n/en/Admin.resx index 995367c7..4f2e1fa8 100644 --- a/i18n/en/Admin.resx +++ b/i18n/en/Admin.resx @@ -124,6 +124,7 @@ Open job Show the job and its log on the Jobs page {0} · {1} · {2}invariant; a sequence's title, the photos it converts, and their size before and after + The archive changed during the confirmation; check the new preview and apply again.long Planning… Preview What each sequence converts and leaves as it is, with the size before and after (an estimate); nothing has changed yet diff --git a/i18n/it/Admin.it.resx b/i18n/it/Admin.it.resx index 2f3b154f..51b06637 100644 --- a/i18n/it/Admin.it.resx +++ b/i18n/it/Admin.it.resx @@ -121,6 +121,7 @@ Niente da convertire: ogni foto resta com'è, per i motivi indicati. Apri job Mostra il job e il suo log nella pagina Job + L'archivio è cambiato durante la conferma; controlla la nuova anteprima e applica di nuovo. Pianificazione… Anteprima Cosa converte e cosa lascia com'è ogni sequenza, con la dimensione prima e dopo (una stima); niente è ancora cambiato diff --git a/src/Chronos.App/Archive/ConvertDialogViewModel.cs b/src/Chronos.App/Archive/ConvertDialogViewModel.cs index 96c86ca3..9bcb007b 100644 --- a/src/Chronos.App/Archive/ConvertDialogViewModel.cs +++ b/src/Chronos.App/Archive/ConvertDialogViewModel.cs @@ -55,7 +55,7 @@ public sealed class ConvertDialogViewModel : ArchiveDialogViewModel _confirm = confirm; // Only a JPEG has a quality: for the other formats the field neither counts nor blocks. Quality = new EditableField(ConvertOptions.DefaultQuality.ToString(Loc.Culture), t => _target == ConvertTarget.Jpeg ? ConvertPlans.ValidateQuality(t) : null, - committed => _ = PlanAsync(), () => Loc.L("Tooltips.Archive.ConvertQuality")); + _ => OnCommitted(), () => Loc.L("Tooltips.Archive.ConvertQuality")); Quality.PropertyChanged += (_, e) => { if (e.PropertyName == nameof(EditableField.Text)) OnTyped(); @@ -251,6 +251,23 @@ public sealed class ConvertDialogViewModel : ArchiveDialogViewModel _delay.Start(); } + /// + /// Enter, or the focus leaving the quality, plans at once, unless the dry run on screen or the one under way is already of + /// these options: a click on Apply takes the focus from the field first, and an identical dry run started then would turn + /// Apply off under the click. + /// + private void OnCommitted() + { + if (IsPlanned()) _delay.Stop(); + else _ = PlanAsync(); + } + + /// The options on screen have their plan, or a dry run of them is under way; false after a failed one. + private bool IsPlanned() + { + lock (_gate) return _run.For is { } options && options == Current() && (_run.Plan is not null || _planning is not null); + } + private async Task DryRunAsync() { var options = Current(); diff --git a/src/Chronos.App/Infrastructure/AppServices.cs b/src/Chronos.App/Infrastructure/AppServices.cs index 9e5ce17e..baf4fc5c 100644 --- a/src/Chronos.App/Infrastructure/AppServices.cs +++ b/src/Chronos.App/Infrastructure/AppServices.cs @@ -112,8 +112,8 @@ public sealed class AppServices : IAsyncDisposable public SyncHost Sync { get; } /// - /// A job of this PC moved a sequence's folder or changed its title or file names (a Reorganize step), with the - /// sequence's id (any thread): an open editor reads it again where it is now. + /// A job of this PC moved a sequence's folder or changed its title or file names (a Reorganize step), or replaced its + /// frames' files (a conversion), with the sequence's id (any thread): an open editor reads it again where and as it is now. /// public event Action? SequenceReorganized; @@ -458,7 +458,7 @@ public sealed class AppServices : IAsyncDisposable // The housekeeping schedule trims the cache to Settings › Storage › Cache limit (owner review 2, R10). CleanupSteps = [new CacheTrimStep(() => CacheStore.Folder(Environment, Settings), () => Settings.CacheLimitGb * (1L << 30))], BackupDirectory = Environment.BackupDirectory, - // Told after each step of a Reorganize, while the rest of the job runs. + // Told after each step of a Reorganize, and each sequence a conversion replaced or made, while the rest of the job runs. SequenceChanged = (id, _, _) => { SequenceReorganized?.Invoke(id); diff --git a/src/Chronos.App/Sections/SettingsViewModel.cs b/src/Chronos.App/Sections/SettingsViewModel.cs index e2e924cb..ecc62aec 100644 --- a/src/Chronos.App/Sections/SettingsViewModel.cs +++ b/src/Chronos.App/Sections/SettingsViewModel.cs @@ -1176,7 +1176,7 @@ public sealed class SettingsViewModel : SectionViewModel [ JobKind.Render, JobKind.Import, JobKind.Download, JobKind.UploadCommit, JobKind.Scan, JobKind.Rebuild, JobKind.Verify, JobKind.LightAnalyse, JobKind.DeepAnalyse, JobKind.Thumbs, JobKind.Proxies, JobKind.Maintenance, JobKind.Duplicates, JobKind.Orphans, JobKind.Adopt, JobKind.Move, - JobKind.Backup, JobKind.Cleanup, JobKind.EmptyTrash, + JobKind.Backup, JobKind.Cleanup, JobKind.EmptyTrash, JobKind.Convert, ]; /// Stores for , keeping only the levels that differ from the defaults. diff --git a/src/Chronos.Core/Conversion/ConvertOptions.cs b/src/Chronos.Core/Conversion/ConvertOptions.cs index 42ebaa55..25e0c063 100644 --- a/src/Chronos.Core/Conversion/ConvertOptions.cs +++ b/src/Chronos.Core/Conversion/ConvertOptions.cs @@ -1,5 +1,7 @@ using System.Globalization; +using Chronos.Core.Archiving; using Chronos.Core.Imaging; +using Chronos.Core.Naming; namespace Chronos.Core.Conversion; @@ -70,15 +72,30 @@ public sealed record ConvertOptions(ConvertTarget Target, int Quality = ConvertO public string Key => string.Create(CultureInfo.InvariantCulture, $"{Target}|{EffectiveQuality}"); /// - /// The name a converted file of (a sequence-relative path) gets: the same folder and stem, this - /// target's extension, upper case when the old extension was (IMG_0001.CR2 → IMG_0001.DNG). + /// with this target's extension in place of its own (originals/IMG_0001.CR2 → + /// originals/IMG_0001.dng): the file a converted copy takes its {ext} from when it is named + /// (, or the ingest filing it into a new sequence). /// - public string NameFor(string file) + public string WithExtension(string file) { ArgumentNullException.ThrowIfNull(file); - var old = Path.GetExtension(file); - var stem = file[..^old.Length]; - var upper = old.Length > 1 && old.Skip(1).Where(char.IsLetter).All(char.IsUpper) && old.Skip(1).Any(char.IsLetter); - return stem + (upper ? Extension.ToUpperInvariant() : Extension); + return file[..^Path.GetExtension(file).Length] + Extension; + } + + /// + /// The name the converted copy of is filed under in (relative to its + /// folder), as filing names a photo there (: an import, an upload and Reorganize name it alike): + /// the archive's file with the frame's context in the sequence (the sequence's camera and lens, + /// {n} = its index + 1, the sequence's title), {name} the name the photo had on the card and {ext} this + /// target's extension, a long name shortened before its extension; in the folder of the frame's file. Not made unique + /// ( does). + /// + public string FileName(PathTemplate template, SidecarFrame frame, SequenceManifest sequence) + { + ArgumentNullException.ThrowIfNull(template); + ArgumentNullException.ThrowIfNull(frame); + ArgumentNullException.ThrowIfNull(sequence); + var directory = frame.File[..(frame.File.LastIndexOf('/') + 1)]; + return directory + template.ExpandFile(FrameNaming.Context(frame with { File = WithExtension(frame.File) }, sequence, sequence.Title)); } } diff --git a/src/Chronos.Data/Archiving/ArchiveIndexer.cs b/src/Chronos.Data/Archiving/ArchiveIndexer.cs index f3c3934a..f2232d8f 100644 --- a/src/Chronos.Data/Archiving/ArchiveIndexer.cs +++ b/src/Chronos.Data/Archiving/ArchiveIndexer.cs @@ -292,11 +292,12 @@ public sealed class ArchiveIndexer /// private static SequenceManifest? Reconcile(SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) { - // A restore from the trash that could not be settled yet (its entry busy) may have photos in the folder its sidecar - // does not list: they are its own, not photos to adopt, until a later writer settles it. - if (File.Exists(sequence.RestoreJournalPath)) + // A restore from the trash or a frame replacement that could not be settled yet (its entry busy, a file that could not + // go back) may have photos in the folder its sidecar does not list: they are its own, not photos to adopt, until a + // later writer settles it. + if (File.Exists(sequence.RestoreJournalPath) || File.Exists(sequence.ReplaceJournalPath)) { - Log.Info("archive", $"{sequence.RelativeFolder} is not reconciled while a restore from the trash into it waits"); + Log.Info("archive", $"{sequence.RelativeFolder} is not reconciled while a restore from the trash or a replacement of its frames waits"); return null; } return SequenceAdopter.Reconcile(sequence, manifest, cancellationToken) is { } reconciled @@ -369,8 +370,10 @@ public sealed class ArchiveIndexer ManifestStore.WriteSequence(sequence.ManifestPath, manifest); bytes = null; } - // A journal of an ingest, a photo deletion or a restore a crash interrupted is settled under the lease first (ReadLocked). - if (reconcile || File.Exists(sequence.IngestJournalPath) || File.Exists(sequence.RemovalJournalPath) || File.Exists(sequence.RestoreJournalPath)) + // A journal of an ingest, a photo deletion, a frame replacement or a restore a crash interrupted is settled under the + // lease first (ReadLocked). + if (reconcile || File.Exists(sequence.IngestJournalPath) || File.Exists(sequence.RemovalJournalPath) || File.Exists(sequence.RestoreJournalPath) + || File.Exists(sequence.ReplaceJournalPath)) { // Photos added, gone or back clear what covered the frames, and the scan's follow-ups make it again. manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, m => Reconcile(sequence, m, cancellationToken), cancellationToken) diff --git a/src/Chronos.Data/Archiving/ArchiveIngest.cs b/src/Chronos.Data/Archiving/ArchiveIngest.cs index 95ac7368..a7073225 100644 --- a/src/Chronos.Data/Archiving/ArchiveIngest.cs +++ b/src/Chronos.Data/Archiving/ArchiveIngest.cs @@ -14,7 +14,18 @@ namespace Chronos.Data.Archiving; /// Name chosen by an import plan; null = the archive's file template. /// See . internal sealed record StagedFrame(string StagedPath, string OriginalFileName, long Size, string Sha256, FrameMetadata Metadata, - DateTime? ClientCapturedAt, string? PlannedName, bool RefuseRemoved = false); + DateTime? ClientCapturedAt, string? PlannedName, bool RefuseRemoved = false) +{ + /// + /// The file name the template names the photo from (its {ext}) when it is not , + /// which stays its {name} and the frame's original: a conversion's copy, the original's name with the target's + /// extension (owner review 2, R7). Null: . + /// + public string? NamedAs { get; init; } + + /// The frame's file before it is named ( as the templates see it). + public string NamingFile => NamedAs ?? OriginalFileName; +} /// A sequence found or created by key (upload session, card session, or a new title). /// Folder of an import plan; null = the archive's folder template. @@ -224,7 +235,7 @@ public sealed class ArchiveIngest : IArchiveIngest private async Task ClaimNewFolderAsync(Archive archive, ArchivePaths paths, string? preferredFolder, StagedFrame first, SequenceManifest manifest, CancellationToken cancellationToken) { - var frame = SequenceFrames.FromMetadata(Guid.Empty, first.OriginalFileName, first.OriginalFileName, first.Size, first.Sha256, first.Metadata, + var frame = SequenceFrames.FromMetadata(Guid.Empty, first.NamingFile, first.OriginalFileName, first.Size, first.Sha256, first.Metadata, first.ClientCapturedAt); var wanted = preferredFolder ?? PathTemplate.Parse(archive.FolderTemplate).ExpandFolder(FrameNaming.Context(frame, manifest with { Frames = [frame] }, manifest.Title)); @@ -337,7 +348,7 @@ public sealed class ArchiveIngest : IArchiveIngest repairs.Add((p, damaged)); continue; } - var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), p.Frame.OriginalFileName, p.Frame.OriginalFileName, p.Frame.Size, + var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), p.Frame.NamingFile, p.Frame.OriginalFileName, p.Frame.Size, p.Frame.Sha256, p.Frame.Metadata, p.Frame.ClientCapturedAt); filed[p.Frame.Sha256] = frame.Id; accepted.Add((p, frame)); diff --git a/src/Chronos.Data/Archiving/FrameTrash.Replace.cs b/src/Chronos.Data/Archiving/FrameTrash.Replace.cs index 6f2a96c2..b5c326d5 100644 --- a/src/Chronos.Data/Archiving/FrameTrash.Replace.cs +++ b/src/Chronos.Data/Archiving/FrameTrash.Replace.cs @@ -15,13 +15,14 @@ namespace Chronos.Data.Archiving; /// The file that comes in, relative to the archive root: a staged conversion, or an original in a /// trash entry. /// Files that come in with it (the XMP companions of an original back from the trash). -/// The name it is filed under, from the frame as the sidecar has it under the lease, relative to the -/// sequence folder; made unique with _2, _3… when it, or one of its companions' names, is taken. +/// The name it is filed under, from the frame and the sequence as the sidecar has them under the lease, +/// relative to the sequence folder; made unique with _2, _3… when it, or one of its companions' names, is +/// taken. /// The frame's record after the swap, from the frame as the sidecar has it under the lease (its /// names the incoming file): the replacement keeps the frame's id and index, files it /// under its name, and keeps the flags a user or the capture set ( come from the /// record). -internal sealed record FrameSwap(Guid FrameId, string Incoming, IReadOnlyList Companions, Func Name, +internal sealed record FrameSwap(Guid FrameId, string Incoming, IReadOnlyList Companions, Func Name, Func Record); /// A companion that comes in with its photo: where it is (relative to the archive root), and what its name adds to the photo's stem. @@ -124,14 +125,20 @@ public sealed partial class FrameTrash if (swaps.Count == 0) throw new ArgumentException("No frame is swapped.", nameof(swaps)); var paths = new ArchivePaths(archive.RootPath); ArchiveVolume.EnsureWritable(paths); - if (await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) is not { } row || row.ArchiveId != archive.Id) return null; - var folder = paths.Sequence(row.Folder); - var had = row.Status; + if (await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) is not { } listed || listed.ArchiveId != archive.Id) return null; + Sequence row; + SequencePaths folder; + SequenceStatus had; TrashEntry entry; SequenceManifest updated; bool indexed; await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false)) { + // Where the sequence is now: a Reorganize moves its folder and row under this lease, maybe while this waited for it. + if (await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) is not { } moved || moved.ArchiveId != archive.Id) return null; + row = moved; + folder = paths.Sequence(row.Folder); + had = row.Status; var manifest = SequenceWriter.ReadLocked(folder, sequenceId, consumed?.Lease); EnsureSettled(folder); EnsureReplaced(folder); @@ -275,7 +282,7 @@ public sealed partial class FrameTrash var names = new Dictionary(); foreach (var swap in swaps) { - var file = unique.Claim(swap.Name(frames[swap.FrameId]), c => swap.Companions.All(t => !taken.Contains(StemOf(c) + t.Tail))); + var file = unique.Claim(swap.Name(frames[swap.FrameId], manifest), c => swap.Companions.All(t => !taken.Contains(StemOf(c) + t.Tail))); names[swap.FrameId] = file; moves.Add(new SwapMove(file, swap.Incoming)); foreach (var companion in swap.Companions) @@ -499,7 +506,7 @@ public sealed partial class FrameTrash if (record.Photos.Count == 0 || record.Photos.Any(p => p.Files.Count == 0 || p.Frame.Sha256 is null || p.Replacement is null)) return null; var swaps = record.Photos.Select(p => new FrameSwap(p.Frame.Id, paths.ToRelative(Path.Combine(entryFolder, p.Files[0].Path)), [.. p.Files.Skip(1).Select(c => new IncomingCompanion(paths.ToRelative(Path.Combine(entryFolder, c.Path)), FrameCompanions.Tail(c.From, p.Files[0].From)))], - _ => p.Files[0].From, _ => p.Frame)).ToList(); + (_, _) => p.Files[0].From, _ => p.Frame)).ToList(); var copy = Path.Combine(entryFolder, AnalysisCopy); var analysis = record.Leftovers?.Any(f => f.Path == AnalysisCopy) == true && File.Exists(copy) ? paths.ToRelative(copy) : null; var consumed = new ConsumedEntry(paths.ToRelative(recordPath), paths.ToRelative(entryFolder), lease, [.. record.Photos.Select(p => p.Frame.Id)], analysis); diff --git a/src/Chronos.Data/Archiving/FrameTrash.Restore.cs b/src/Chronos.Data/Archiving/FrameTrash.Restore.cs index 538599b2..e89d4e34 100644 --- a/src/Chronos.Data/Archiving/FrameTrash.Restore.cs +++ b/src/Chronos.Data/Archiving/FrameTrash.Restore.cs @@ -62,10 +62,11 @@ public sealed partial class FrameTrash var into = sequence is not null && sequence.ArchiveId == archive.Id ? sequence : null; // What an earlier restore a crash interrupted left half-way is settled first, so the record read next is true. Unstage(entryFolder); - if (into is not null && File.Exists(paths.Sequence(into.Folder).RestoreJournalPath)) + // A replacement too (a conversion or its undo), whose completion may wait for this entry's lease. + if (into is not null && (File.Exists(paths.Sequence(into.Folder).RestoreJournalPath) || File.Exists(paths.Sequence(into.Folder).ReplaceJournalPath))) { await using (await _writer.LockAsync(paths, into.Id, cancellationToken).ConfigureAwait(false)) - SequenceWriter.ReadLocked(paths.Sequence(into.Folder), into.Id, lease); + SequenceWriter.ReadLocked(await SequenceWriter.FolderAsync(_db, paths, into, cancellationToken).ConfigureAwait(false), into.Id, lease); } if (!System.IO.Directory.Exists(entryFolder) || ReadRecord(recordPath) is not { } record) throw new DirectoryNotFoundException($"{name} is not in the trash of {archive.Name}."); @@ -179,8 +180,9 @@ public sealed partial class FrameTrash { var folder = await SequenceWriter.FolderAsync(_db, paths, sequence, cancellationToken).ConfigureAwait(false); var manifest = SequenceWriter.ReadLocked(folder, sequence.Id, lease); - // A restore of another entry, busy now, is not settled yet: its journal is not written over. + // A restore of another entry, or a replacement, busy now, is not settled yet: its journal is not written over. EnsureSettled(folder); + EnsureReplaced(folder); var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); var photosChanged = !inStep && await _indexer.PhotosChangedAsync(archive, folder, cancellationToken).ConfigureAwait(false); var ids = manifest.Frames.Select(f => f.Id).ToHashSet(); diff --git a/src/Chronos.Data/Archiving/FrameTrash.cs b/src/Chronos.Data/Archiving/FrameTrash.cs index 6b3ef7dd..b6ea48aa 100644 --- a/src/Chronos.Data/Archiving/FrameTrash.cs +++ b/src/Chronos.Data/Archiving/FrameTrash.cs @@ -237,6 +237,8 @@ public sealed partial class FrameTrash if ((await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false))?.Folder != revised.Sequence.Folder) throw new StaleDeletePlanException(id); var manifest = SequenceWriter.ReadLocked(folder, id); + // A replacement not settled yet tells by the frames version whether its sidecar was written: none moves under it. + EnsureReplaced(folder); var plannedPaths = plan.Frames.ToDictionary(f => f.Id, f => f.RelativePath); var planned = plannedPaths.Keys.ToHashSet(); var removed = manifest.Frames.Where(f => planned.Contains(f.Id)).ToList(); diff --git a/src/Chronos.Data/Archiving/Jobs/ConvertJobHandler.cs b/src/Chronos.Data/Archiving/Jobs/ConvertJobHandler.cs index 14112130..b3ae0416 100644 --- a/src/Chronos.Data/Archiving/Jobs/ConvertJobHandler.cs +++ b/src/Chronos.Data/Archiving/Jobs/ConvertJobHandler.cs @@ -12,9 +12,9 @@ namespace Chronos.Data.Archiving.Jobs; /// volume lacks the space, and converts one sequence at a time, each all or nothing; its journal makes every stage /// resumable. Progress counts photos; the checkpoint only marks a resume, the journal says where. Each sequence whose frames /// were replaced is told as converted, each new sequence as created (), -/// and the passes they need (light analysis, a deep one when the source had it, thumbnails and proxies) are queued as -/// follow-ups. Sequences left as they were (changed after the plan, busy, a photo that failed) are logged and fail the job -/// at the end, after the others are converted; a retry tries them again. +/// as soon as it is done; the passes they need (light analysis, a deep one when the source had it, thumbnails and +/// proxies) are queued as follow-ups. Sequences left as they were (changed after the plan, busy, a photo that failed) are +/// logged and fail the job at the end, after the others are converted; a retry tries them again. /// internal sealed class ConvertJobHandler(ChronosDatabase db, MassConverter converter, Func? sequenceChanged) : IJobHandler { @@ -39,11 +39,10 @@ internal sealed class ConvertJobHandler(ChronosDatabase db, MassConverter conver var result = await converter.RunAsync(archive, parameters, context.Job.Id, by, new ConvertCallbacks( (done, total) => context.ReportAsync(done, total), (done, total) => context.CheckpointAsync(Checkpoint, done, total), - context.Log), context.Cancellation).ConfigureAwait(false); + context.Log, TellAsync), context.Cancellation).ConfigureAwait(false); await QueueAsync(context, result).ConfigureAwait(false); context.Log(Loc.L("Archive.Log.ConvertDone", result.Sequences.Count(s => s.Outcome == ConvertOutcome.Converted), result.Sequences.Count(s => s.Outcome == ConvertOutcome.Created), result.Frames)); - await TellAsync(result).ConfigureAwait(false); var kept = result.Kept.Count(); if (kept > 0) throw new InvalidOperationException(Loc.Plural("Archive.Error.ConvertKept", kept)); } @@ -65,27 +64,27 @@ internal sealed class ConvertJobHandler(ChronosDatabase db, MassConverter conver } } - /// Tells each sequence changed or made; a failure is logged, never passed on to the job. - private async Task TellAsync(ConvertResult result) + /// + /// Tells a sequence whose frames were replaced, or the one made, as soon as its outcome is journaled, while the rest of the + /// job runs (an editor open on it reads it again); a failure is logged, never passed on to the job. + /// + private async ValueTask TellAsync(ConvertSequenceResult sequence) { if (sequenceChanged is null) return; - foreach (var sequence in result.Sequences) + var (id, change) = sequence.Outcome switch { - var (id, change) = sequence.Outcome switch - { - ConvertOutcome.Converted => (sequence.SequenceId, Converted), - ConvertOutcome.Created when sequence.NewSequenceId is { } made => (made, Created), - _ => (Guid.Empty, string.Empty), - }; - if (id == Guid.Empty) continue; - try - { - await sequenceChanged(id, change, CancellationToken.None).ConfigureAwait(false); - } - catch (Exception ex) - { - Log.Warning("archive", $"The change of sequence {id} was not told", ex); - } + ConvertOutcome.Converted => (sequence.SequenceId, Converted), + ConvertOutcome.Created when sequence.NewSequenceId is { } made => (made, Created), + _ => (Guid.Empty, string.Empty), + }; + if (id == Guid.Empty) return; + try + { + await sequenceChanged(id, change, CancellationToken.None).ConfigureAwait(false); + } + catch (Exception ex) + { + Log.Warning("archive", $"The change of sequence {id} was not told", ex); } } } diff --git a/src/Chronos.Data/Archiving/MassConverter.cs b/src/Chronos.Data/Archiving/MassConverter.cs index ffaa8810..d8ac4acf 100644 --- a/src/Chronos.Data/Archiving/MassConverter.cs +++ b/src/Chronos.Data/Archiving/MassConverter.cs @@ -10,6 +10,7 @@ using Chronos.Core.IO; using Chronos.Core.Localization; using Chronos.Core.Metadata; using Chronos.Core.Model; +using Chronos.Core.Naming; using Chronos.Core.Project; using Chronos.Data.Archiving.Jobs; using Chronos.Data.Processing; @@ -55,20 +56,27 @@ public sealed record ConvertResult(IReadOnlyList Sequence public IEnumerable Kept => Sequences.Where(s => !s.Final); } -/// How a running conversion reports: progress in photos, a checkpoint after each sequence, and job log lines. -internal sealed record ConvertCallbacks(Func Progress, Func Checkpoint, Action Log); +/// +/// How a running conversion reports: progress in photos, a checkpoint after each sequence, job log lines, and each sequence +/// it came to an end with in this run, once its outcome is journaled (to tell the sequences it changed or made while the +/// rest of the job runs, as a Reorganize tells each step). +/// +internal sealed record ConvertCallbacks(Func Progress, Func Checkpoint, Action Log, + Func? Done = null); /// /// Mass conversion of existing sequences (owner review 2, R7; BRIEF §12.5): /// is the dry run ( per sequence, what blocks it, the space needed against the free space of /// the archive's volume, ), and a job applies the -/// confirmed plan one sequence at a time. Stage A, without a lease: each photo is hashed against its record and converted -/// (, several at once as the host's parallelism and memory allow) into +/// confirmed plan one sequence at a time. Stage A, once the sequence's sidecar is settled under its lease where the index +/// has it then (), without the lease: each photo is hashed against its record and +/// converted (, several at once as the host's parallelism and memory allow) into /// .chronos/staging/convert-<job>/<sequence>/, each staged file appended to the job's journal /// (), so a resumed job keeps the staged files whose hash still verifies. Stage B, all or nothing, /// under the sequence's lease and while its frames are still at the version planned: /// swaps the frames' files (: the originals into one trash entry of kind -/// , same ids and indices, tombstones naming the replacements); +/// , same ids and indices, tombstones naming the replacements; the converted files named as +/// filing names a photo of the sequence, ); /// files the converted copies into a new sequence through the archive ingest (session key convert:<job>:<source>, /// so a resume finds it again), titled "title (FORMAT)", with the source's tags, rating, notes and project, the source left /// as it is. A sequence that changed after the plan, is busy, or has a photo that fails is left as it was and reported. @@ -231,6 +239,7 @@ public sealed class MassConverter journal.Write(paths); if (!result.Final) DeleteStaging(StagingOf(paths, jobId, step.Id)); Log(callbacks, result); + if (noted?.Final != true && callbacks.Done is { } done) await done(result).ConfigureAwait(false); await callbacks.Checkpoint(progress.Done, progress.Total).ConfigureAwait(false); } DeleteStaging(Path.Combine(paths.StagingDirectory, StagingFolder(jobId))); @@ -244,8 +253,25 @@ public sealed class MassConverter ConvertSequencePlan planned, ConvertOptions options, ConvertMode mode, Guid jobId, string? createdBy, ConvertJournal journal, StagedLog staged, Progress progress, ConvertCallbacks callbacks, CancellationToken cancellationToken) { - var folder = paths.Sequence(row.Folder); - if (Read(folder, row.Id) is not { } manifest) return (new ConvertSequenceResult(row.Id, row.DisplayTitle, ConvertOutcome.Blocked, 0), []); + Sequence now; + SequencePaths folder; + SequenceManifest manifest; + try + { + if (await SettledAsync(archive, paths, row.Id, cancellationToken).ConfigureAwait(false) is not { } settled) + return (new ConvertSequenceResult(row.Id, row.DisplayTitle, ConvertOutcome.Changed, 0), []); + (now, folder, manifest) = settled; + } + catch (ArchiveOfflineException) + { + throw; + } + catch (Exception ex) when (ex is LeaseUnavailableException or IOException or UnauthorizedAccessException or InvalidDataException or InvalidOperationException) + { + Core.Diagnostics.Log.Warning("archive", $"The sidecar of {row.DisplayTitle} cannot be read; it is not converted", ex); + return (new ConvertSequenceResult(row.Id, row.DisplayTitle, ConvertOutcome.Blocked, 0, Error: ex.Message), []); + } + row = now; if (manifest.Versions.Frames != planned.FramesVersion) return (new ConvertSequenceResult(row.Id, row.DisplayTitle, ConvertOutcome.Changed, 0), []); var staging = StagingOf(paths, jobId, row.Id); var key = SessionKey(jobId, row.Id); @@ -378,15 +404,20 @@ public sealed class MassConverter } } - /// Stage B of : the staged files take their frames' place (). + /// + /// Stage B of : the staged files take their frames' place (), + /// each named as filing names a photo of the sequence (: the archive's file template, + /// the frame and the sequence as the sidecar has them under the lease), made unique among every file there. + /// private async Task<(ConvertSequenceResult, IReadOnlyList)> ReplaceAsync(Archive archive, ArchivePaths paths, Sequence row, ConvertSequencePlan planned, SequenceManifest manifest, Dictionary ready, string staging, ConvertOptions options, string? createdBy, CancellationToken cancellationToken) { + var template = PathTemplate.Parse(archive.FileTemplate); var swaps = manifest.Frames.Where(f => ready.ContainsKey(f.Id)).Select(f => { var conversion = ready[f.Id]; - return new FrameSwap(f.Id, paths.ToRelative(Path.Combine(staging, conversion.File)), [], current => options.NameFor(current.File), + return new FrameSwap(f.Id, paths.ToRelative(Path.Combine(staging, conversion.File)), [], (current, sequence) => options.FileName(template, current, sequence), current => Converted(current, conversion, options)); }).ToList(); // The frames hold the content that was converted: a frames version that did not move says so, and each hash agrees. @@ -422,8 +453,9 @@ public sealed class MassConverter /// /// Stage B of : the staged files filed into the sequence of - /// (made on first use, titled "title (FORMAT)"), named as their originals with the target's extension, with the source's - /// capture facts; then the source's tags, rating, notes and project (its positions translated to the new frames). + /// (made on first use, titled "title (FORMAT)") and named as the ingest names every photo it files there + /// (: {name} the original's name on the card, {ext} the target's), with + /// the source's capture facts; then the source's tags, rating, notes and project (its positions translated to the new frames). /// private async Task<(ConvertSequenceResult, IReadOnlyList)> CreateAsync(Archive archive, ArchivePaths paths, Sequence row, SequenceManifest manifest, Dictionary ready, string staging, ConvertOptions options, string key, string? createdBy, @@ -434,8 +466,10 @@ public sealed class MassConverter { var conversion = ready[f.Id]; var path = Path.Combine(staging, conversion.File); - return new StagedFrame(path, f.Original ?? Path.GetFileName(f.File), conversion.Size, conversion.Sha256, Metadata(path, f), null, - Path.GetFileName(options.NameFor(f.File))); + return new StagedFrame(path, f.Original ?? Path.GetFileName(f.File), conversion.Size, conversion.Sha256, Metadata(path, f), null, null) + { + NamedAs = options.WithExtension(Path.GetFileName(f.File)), + }; }).ToList(); var title = $"{manifest.Title ?? row.DisplayTitle} ({options.Label})"; // Once the first file moves, the commit runs to its end whatever the caller cancels meanwhile. @@ -526,6 +560,24 @@ public sealed class MassConverter : null; } + /// + /// The sequence before its stage A, under its lease: where the index has it now (a Reorganize, another host's or one this + /// job made way for, may have moved its folder since the job planned) and its sidecar as every writer reads it + /// (: a rename of its files, a deletion, a replacement, a restore or an ingest a + /// crash left half-way is settled first), so stage A converts the files the sidecar names. Stage B reads it again under + /// that lease. Null when the sequence left the archive. + /// + private async Task<(Sequence Row, SequencePaths Folder, SequenceManifest Manifest)?> SettledAsync(Archive archive, ArchivePaths paths, Guid id, + CancellationToken cancellationToken) + { + await using (await _writer.LockAsync(paths, id, cancellationToken).ConfigureAwait(false)) + { + if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } row || row.ArchiveId != archive.Id) return null; + var folder = paths.Sequence(row.Folder); + return (row, folder, SequenceWriter.ReadLocked(folder, id)); + } + } + /// The sidecar of a sequence, read without its lease; null when it cannot be read or names another sequence. private static SequenceManifest? Read(SequencePaths folder, Guid id) { diff --git a/src/Chronos.Data/Archiving/MassRenamer.cs b/src/Chronos.Data/Archiving/MassRenamer.cs index 923977f3..97401b30 100644 --- a/src/Chronos.Data/Archiving/MassRenamer.cs +++ b/src/Chronos.Data/Archiving/MassRenamer.cs @@ -582,17 +582,19 @@ public sealed class MassRenamer /// /// Settles, under the step's leases and before anything of it moves, what a crash or a busy trash entry left in the /// sequence, as every writer does (): an interrupted rename of its files, photo - /// deletion, restore from the trash or ingest. A restore whose trash entry is busy is not settled yet, and the step - /// waits for it () as a deletion of the sequence does: its journal names the - /// files it moved by the names they had, so neither the folder nor those names change under it. Returns the sidecar as - /// settled. + /// deletion, frame replacement (a conversion or its undo), restore from the trash or ingest. A restore or a replacement + /// whose trash entry is busy is not settled yet, and the step waits for it (, + /// ) as a deletion of the sequence does: its journal names the files it moved by + /// the names they had, so neither the folder nor those names change under it. Returns the sidecar as settled. /// - /// A restore from the trash into the sequence is not settled; nothing of the step changed. + /// A restore from the trash into the sequence, or a replacement of its frames, + /// is not settled; nothing of the step changed. private static SequenceManifest Settle(ArchivePaths paths, RenameStep step) { var sequence = Where(paths, step); var manifest = SequenceWriter.ReadLocked(sequence, step.SequenceId); FrameTrash.EnsureSettled(sequence); + FrameTrash.EnsureReplaced(sequence); return manifest; } diff --git a/src/Chronos.Data/Archiving/SequenceTrash.cs b/src/Chronos.Data/Archiving/SequenceTrash.cs index 8873fa61..7b2f7823 100644 --- a/src/Chronos.Data/Archiving/SequenceTrash.cs +++ b/src/Chronos.Data/Archiving/SequenceTrash.cs @@ -373,15 +373,17 @@ public sealed partial class SequenceTrash var folder = paths.Sequence(sequence.Folder); if (!Directory.Exists(folder.Folder) || (File.Exists(folder.ManifestPath) && SidecarId(folder.ManifestPath) != sequence.Id)) throw new StaleDeletePlanException(sequence.Id); - // A photo deletion, a restore from the trash or an ingest a crash interrupted is settled first, as any writer does - // (SequenceWriter.ReadLocked): neither its journal nor the files it moved travel into this entry, where emptying it - // would erase them while their own entry still lists them. A restore whose entry is busy keeps the folder here. + // A photo deletion, a frame replacement, a restore from the trash or an ingest a crash interrupted is settled first, + // as any writer does (SequenceWriter.ReadLocked): neither its journal nor the files it moved travel into this entry, + // where emptying it would erase them while their own entry still lists them. A restore or a replacement whose entry + // is busy keeps the folder here. var (files, bytes) = (step.Files, step.Bytes); if (File.Exists(folder.ManifestPath) && (File.Exists(folder.RemovalJournalPath) || File.Exists(folder.RestoreJournalPath) - || File.Exists(folder.IngestJournalPath))) + || File.Exists(folder.ReplaceJournalPath) || File.Exists(folder.IngestJournalPath))) { SequenceWriter.ReadLocked(folder, sequence.Id); FrameTrash.EnsureSettled(folder); + FrameTrash.EnsureReplaced(folder); // Files came back into the folder, or left it: the record counts what moves. (files, bytes) = Measure(folder.Folder, cancellationToken); } diff --git a/src/Chronos.Server/Admin/web/convert.js b/src/Chronos.Server/Admin/web/convert.js index b917d50d..d062f167 100644 --- a/src/Chronos.Server/Admin/web/convert.js +++ b/src/Chronos.Server/Admin/web/convert.js @@ -309,7 +309,9 @@ export function openConvert({ api, target, names = new Map() } = {}) { /** * Replace asks for a word to be typed (the originals leave the sequence for the trash); New sequence for a plain - * confirmation. The plan sent back is the one the table shows (the fields cannot change meanwhile). + * confirmation. The plan sent back is the one the table showed. The fields cannot change meanwhile, but a dry run + * scheduled before may answer during it: the same plan again (same request and fingerprint) is applied all the same; a + * different one is shown instead, and Apply waits for the user. */ async function apply() { if (applyButton.disabled) return; @@ -339,7 +341,12 @@ export function openConvert({ api, target, names = new Map() } = {}) { cancel: t("Admin.Common.Keep"), cancelTip: t("Admin.Common.Keep.Tip"), }); - if (!confirmed || !dialog.open || plan !== shownPlan) return; + if (!confirmed || !dialog.open) return; + if (plan !== shownPlan && (plan?.key !== shownPlan.key || plan.fingerprint !== shownPlan.fingerprint)) { + // A failed dry run already says why; a new plan is on screen to be checked. + if (plan) showError(t("Admin.Convert.PlanChanged")); + return; + } busy = true; showError(""); drawButtons(); diff --git a/tests/Chronos.App.Tests/Archive/ConvertDialogTests.cs b/tests/Chronos.App.Tests/Archive/ConvertDialogTests.cs index 279f0268..5df20959 100644 --- a/tests/Chronos.App.Tests/Archive/ConvertDialogTests.cs +++ b/tests/Chronos.App.Tests/Archive/ConvertDialogTests.cs @@ -151,6 +151,14 @@ public sealed class ConvertDialogTests(TestContext context) await dialog.PlanAsync(); Assert.SequenceEqual([Loc.L("Archive.Convert.Skipped.SameFormat") + Loc.Separator + Loc.Plural("Shell.Status.Photos", 2)], dialog.Rows[0].Kept); Assert.Equal(Loc.L("Archive.Blocked.NothingToConvert"), dialog.ApplyToolTip); + // A click on Apply takes the focus from the quality first, which commits it: the dry run on screen is of that + // quality already, so no other one starts under the click. + dialog.Quality.Text = "90"; + await dialog.PlanAsync(); + Assert.True(dialog.Quality.IsDirty, "typed, not committed"); + Assert.True(dialog.Quality.Commit()); + Assert.Null(dialog.BusyText); + Assert.Equal(new ConvertOptions(ConvertTarget.Jpeg, 90), dialog.Plan!.Options); dialog.Quality.Text = "0"; Assert.Null(dialog.Current()); Assert.Equal(dialog.Quality.Error, dialog.ApplyToolTip); diff --git a/tests/Chronos.App.Tests/Archive/TrashDialogTests.cs b/tests/Chronos.App.Tests/Archive/TrashDialogTests.cs index 4d4cbdb2..2fccedb4 100644 --- a/tests/Chronos.App.Tests/Archive/TrashDialogTests.cs +++ b/tests/Chronos.App.Tests/Archive/TrashDialogTests.cs @@ -103,7 +103,8 @@ public sealed class TrashDialogTests(TestContext context) // Kept: the confirmation was dismissed, nothing is queued. dialog.EmptyCommand.Execute(null); - await UntilAsync(() => asked is not null && !dialog.IsBusy); + // Until the command has ended, not only the dialog's work: a command still running ignores the next Execute. + await UntilAsync(() => asked is not null && !dialog.IsBusy && dialog.EmptyCommand.CanExecute(null)); Assert.Equal("1", asked!.TypedConfirmation); Assert.False(queued.Task.IsCompleted); Assert.False(closed); diff --git a/tests/Chronos.Core.Tests/Conversion/FrameConverterTests.cs b/tests/Chronos.Core.Tests/Conversion/FrameConverterTests.cs index 91234d8d..ae4cb0fc 100644 --- a/tests/Chronos.Core.Tests/Conversion/FrameConverterTests.cs +++ b/tests/Chronos.Core.Tests/Conversion/FrameConverterTests.cs @@ -7,6 +7,7 @@ using Chronos.Core.Imaging; using Chronos.Core.Imaging.Decoders; using Chronos.Core.Metadata; using Chronos.Core.Model; +using Chronos.Core.Naming; using Chronos.Testing; namespace Chronos.Core.Tests.Conversion; @@ -253,12 +254,35 @@ public sealed class FrameConverterTests(TestContext context) } [Test] - public void ConvertedNamesKeepTheStemAndTheCaseOfTheExtension() + public void ConvertedNamesAreTheNamesFilingGives() { - Assert.Equal("originals/IMG_0001.DNG", new ConvertOptions(ConvertTarget.LinearDng).NameFor("originals/IMG_0001.CR2")); - Assert.Equal("originals/img_0001.tif", new ConvertOptions(ConvertTarget.Tiff16).NameFor("originals/img_0001.cr3")); - Assert.Equal("originals/2026-09-28_0001.jpg", new ConvertOptions(ConvertTarget.Jpeg).NameFor("originals/2026-09-28_0001.Nef")); - Assert.Equal("frame.jpg", new ConvertOptions(ConvertTarget.Jpeg).NameFor("frame")); + var dng = new ConvertOptions(ConvertTarget.LinearDng); + Assert.Equal("originals/IMG_0001.dng", dng.WithExtension("originals/IMG_0001.CR2")); + Assert.Equal("frame.jpg", new ConvertOptions(ConvertTarget.Jpeg).WithExtension("frame")); + + // The second frame of a sequence filed by a Canon, named with the sequence's camera, its number, the title and the + // name it had on the card, as an upload or a Reorganize names it; {ext} is the target's. + var first = Frame("2026-09-28_0001.CR3", ImageFileFormat.Cr3, 96, 64) with { I = 0, Original = "IMG_0001.CR3" }; + var second = Frame("2026-09-28_0002.CR3", ImageFileFormat.Cr3, 96, 64) with { I = 1, Original = "IMG_0002.CR3", CapturedAt = Captured.AddSeconds(10) }; + var sequence = new SequenceManifest + { + Id = Guid.NewGuid(), + Title = "Harbour", + Camera = new SidecarCamera { Make = "Canon", Model = "Canon EOS R5" }, + Frames = [first, second], + }; + Assert.Equal("originals/Harbour_Canon EOS R5_0002_IMG_0002.dng", + dng.FileName(PathTemplate.Parse("{title}_{camera}_{n}_{name}.{ext}"), second, sequence)); + Assert.Equal("originals/2026-09-28_0002.tif", + new ConvertOptions(ConvertTarget.Tiff16).FileName(PathTemplate.Parse("{date}_{n}"), second, sequence)); + Assert.Equal(PathTemplate.Parse("{date}_{n}").ExpandFile(FrameNaming.Context(second with { File = "originals/x.dng" }, sequence, "Harbour")), + Path.GetFileName(dng.FileName(PathTemplate.Parse("{date}_{n}"), second, sequence))); + // A frame adopted in place stays in its folder; a long name keeps the target's extension whole. + var adopted = second with { File = "IMG_0002.CR3", Original = new string('x', 300) + ".CR3" }; + var named = dng.FileName(PathTemplate.Parse("{name}.{ext}"), adopted, sequence); + Assert.False(named.Contains('/')); + Assert.True(named.EndsWith(".dng", StringComparison.Ordinal) && named.Length <= PathTemplate.MaxSegmentLength, named); + Assert.Equal(ConvertTarget.LinearDng, ConvertOptions.ArchivalTarget); Assert.False(new ConvertOptions(ConvertTarget.Jpeg, 0).IsValid); Assert.False(new ConvertOptions(ConvertTarget.Jpeg, 101).IsValid); diff --git a/tests/Chronos.Data.Tests/Archiving/ConvertTests.cs b/tests/Chronos.Data.Tests/Archiving/ConvertTests.cs index 59cfe849..e2a3c1b5 100644 --- a/tests/Chronos.Data.Tests/Archiving/ConvertTests.cs +++ b/tests/Chronos.Data.Tests/Archiving/ConvertTests.cs @@ -152,10 +152,10 @@ public sealed class ConvertTests(TestContext context) Assert.Equal(before.Versions.Frames + 1, row.FramesVersion); Assert.SequenceEqual(originals.Select(f => (f.Id, f.Index)), frames.Select(f => (f.Id, f.Index))); var after = Sidecar(fx, row); - var options = new ConvertOptions(ConvertTarget.LinearDng); foreach (var (was, now) in before.Frames.Zip(after.Frames)) { - Assert.Equal((options.NameFor(was.File), ImageFileFormat.Dng, DecodeTier.Native, 1), (now.File, now.Format, now.Tier, now.Orientation)); + // Named as filing named the original, with the target's extension: the same template, frame and sequence. + Assert.Equal((Path.ChangeExtension(was.File, ".dng"), ImageFileFormat.Dng, DecodeTier.Native, 1), (now.File, now.Format, now.Tier, now.Orientation)); Assert.Equal((was.CapturedAt, was.Original, was.Flags), (now.CapturedAt, now.Original, now.Flags)); var source = Assert.NotNull(now.Source); Assert.Equal((was.Sha256, was.Format, was.File, was.Size), (source.Sha256, source.Format, source.File, source.Size)); @@ -282,7 +282,9 @@ public sealed class ConvertTests(TestContext context) var frames = await FramesAsync(fx, made); Assert.SequenceEqual(originals.Select(f => f.CapturedAt), frames.Select(f => f.CapturedAt)); Assert.True(frames.All(f => f.Format == ImageFileFormat.Tiff)); - Assert.SequenceEqual(originals.Select(f => Path.ChangeExtension(f.FileName, ".tif")), frames.Select(f => Path.ChangeExtension(f.FileName, ".tif"))); + // Filed as the ingest files every photo: the archive's template in the new sequence, {name} the card's, {ext} the target's. + Assert.SequenceEqual(originals.Select(f => Path.ChangeExtension(f.FileName, ".tif")), frames.Select(f => f.FileName)); + Assert.SequenceEqual(originals.Select(f => f.OriginalFileName), frames.Select(f => f.OriginalFileName)); var settings = ProjectSettings.FromJson(copy.SettingsJson!); Assert.SequenceEqual([1], settings.ExcludedFrames); Assert.Equal((2, frames[2].Id), (settings.HeroFrame, copy.HeroFrameId)); @@ -577,4 +579,96 @@ public sealed class ConvertTests(TestContext context) _ = JsonSerializer.Serialize(new ConvertJobParameters(fx.Archive.Id, ConvertTarget.Jpeg, 80, ConvertMode.NewSequence, [new ConvertSequence(first, 2)], "f"), ArchiveJobJson.Default.ConvertJobParameters); } + + [Test] + public async Task ConvertedFilesAreNamedAsFilingNamesThemAndNothingIsOverwritten() + { + await using var fx = await FixtureAsync(); + await fx.Service.UpdateSettingsAsync(fx.Archive.Id, s => s with { Templates = s.Templates with { File = "{name}_{n:00}" } }, fx.Ct); + var roof = await UploadedAsync(fx, 2); + var lake = await UploadedAsync(fx, 2, "upload-2", seed: 200); + Assert.SequenceEqual(["IMG_0001_01.JPG", "IMG_0002_02.JPG"], (await FramesAsync(fx, roof)).Select(f => f.FileName)); + await fx.Service.SetSequenceInfoAsync(roof, "Roof", [], 0, null, fx.Ct); + await fx.Service.SetSequenceInfoAsync(lake, "Lake", [], 0, null, fx.Ct); + // Filing names a photo with the template of the moment: a conversion names its files as filing would now. + await fx.Service.UpdateSettingsAsync(fx.Archive.Id, s => s with { Templates = s.Templates with { File = "{title}_{n:00}" } }, fx.Ct); + // A file that is no frame holds the first name: it is never written over, the converted file takes the next. + var stray = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(roof))!.Folder).File("originals/Roof_01.dng"); + File.WriteAllBytes(stray, [1, 2, 3]); + var engine = await fx.StartEngineAsync(); + + await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, roof)); + var replaced = await FramesAsync(fx, roof); + Assert.SequenceEqual(["Roof_01_2.dng", "Roof_02.dng"], replaced.Select(f => f.FileName)); + Assert.SequenceEqual(["IMG_0001.JPG", "IMG_0002.JPG"], replaced.Select(f => f.OriginalFileName)); + Assert.SequenceEqual(new byte[] { 1, 2, 3 }, File.ReadAllBytes(stray)); + + var job = await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, lake)); + var made = Assert.NotNull(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, lake))); + var copies = await FramesAsync(fx, made); + Assert.SequenceEqual(["Lake (TIFF)_01.tif", "Lake (TIFF)_02.tif"], copies.Select(f => f.FileName)); + Assert.SequenceEqual(["IMG_0001.JPG", "IMG_0002.JPG"], copies.Select(f => f.OriginalFileName)); + } + + [Test] + public async Task AFolderMovedWhileTheJobWaitsForItsLeaseIsWhereTheConvertedFilesGo() + { + await using var fx = await FixtureAsync(); + var id = await UploadedAsync(fx, 2); + var engine = await fx.StartEngineAsync(); + var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id); + var row = (await fx.Db.Sequences.GetAsync(id))!; + var from = fx.Paths.Sequence(row.Folder).Folder; + Job job; + var planned = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + void OnProgress(JobProgressInfo info) + { + if (info.Kind == JobKind.Convert && info.Total == 2) planned.TrySetResult(); + } + engine.Progress += OnProgress; + using (await LeaseFile.AcquireAsync(fx.Paths.LocksDirectory, $"seq-{id:N}", "editor", TimeSpan.FromSeconds(5), cancellationToken: fx.Ct)) + { + job = await engine.EnqueueAsync(ArchiveJobs.Convert(fx.Archive, plan), cancellationToken: fx.Ct); + // Planned again where the index had the sequence then: the job now waits for its lease to convert it. + await planned.Task.WaitAsync(TimeSpan.FromSeconds(TestTiming.Budget(30)), fx.Ct); + // What a Reorganize step does under that lease meanwhile: the folder moves, and its index row follows. + var to = fx.Paths.Sequence("moved/Roof").Folder; + Directory.CreateDirectory(Path.GetDirectoryName(to)!); + Directory.Move(from, to); + Assert.True(await fx.Db.Sequences.RelocateAsync(id, "moved/Roof", null, fx.Ct)); + } + engine.Progress -= OnProgress; + + await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60)); + var frames = await FramesAsync(fx, id); + Assert.True(frames.All(f => f.Format == ImageFileFormat.Tiff && f.RelativePath.StartsWith("moved/Roof/", StringComparison.Ordinal))); + Assert.False(Directory.Exists(from), "the folder the sequence left is never made again"); + Assert.Equal("moved/Roof", (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Folder); + } + + [Test] + public async Task AReorganizeTheArchiveWaitsForKeepsTheConversionOffItsSequences() + { + await using var fx = await FixtureAsync(); + var id = await UploadedAsync(fx, 2); + var originals = await FramesAsync(fx, id); + var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id); + // A Reorganize paused by its user holds the archive: its plan waits for it. + await fx.Db.Jobs.EnqueueAsync(new Job + { + Id = Guid.CreateVersion7(), + Kind = JobKind.Move, + TargetIds = [fx.Archive.Id], + State = JobState.Paused, + Created = DateTimeOffset.UtcNow, + }); + Assert.SequenceEqual([ConvertBlock.Busy], (await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id)).Sequences.Single().Blocked); + + var engine = await fx.StartEngineAsync(); + var job = await RunAsync(fx, engine, plan, JobState.Failed); + Assert.Equal(Loc.Plural("Archive.Error.ConvertKept", 1), job.Error); + Assert.Contains(Loc.L("Archive.Log.ConvertBlocked", (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle), job.LogTail); + Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256, f.RelativePath)), (await FramesAsync(fx, id)).Select(f => (f.Id, f.Sha256, f.RelativePath))); + Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); + } } diff --git a/tests/Chronos.Server.Tests/Hosting/AdminConvertTests.cs b/tests/Chronos.Server.Tests/Hosting/AdminConvertTests.cs index 7fb863ef..60a535e4 100644 --- a/tests/Chronos.Server.Tests/Hosting/AdminConvertTests.cs +++ b/tests/Chronos.Server.Tests/Hosting/AdminConvertTests.cs @@ -62,6 +62,10 @@ public sealed partial class AdminConvertTests(TestContext context) Assert.True(Regex.IsMatch(dialog, @"confirmDialog\(replace \? \{[^}]*typed: t\(""Admin\.Convert\.ReplaceWord""\),\s+typedKind: ""word"",\s+\} : \{[^}]*\}\)", RegexOptions.Singleline), "Replace asks for the word, New sequence does not"); Assert.Contains("#/jobs?job=", dialog); + // A dry run answered during the confirmation: the same plan again is applied, a different one shown. + Assert.Contains("if (plan !== shownPlan && (plan?.key !== shownPlan.key || plan.fingerprint !== shownPlan.fingerprint)) {", dialog); + Assert.Contains(@"showError(t(""Admin.Convert.PlanChanged""))", dialog); + Assert.True(!dialog.Contains("plan !== shownPlan) return;", StringComparison.Ordinal), "a confirmed apply is dropped when the same plan is answered again"); // The panel runs under a strict CSP: no inline handlers or styles. Assert.True(!Regex.IsMatch(dialog, @"\son[a-z]+="""), "inline handler"); Assert.True(!dialog.Contains(" style=", StringComparison.Ordinal), "inline style");