From 833d7cf8e98b158111abb99158a6bb097bda1112 Mon Sep 17 00:00:00 2001 From: Alberto Balbo Date: Tue, 3 Nov 2026 10:19:56 +0100 Subject: [PATCH] fix(workspace): every cut of a sync is recoverable, renders name only versions the home holds A push now moves the base before its local file reads as sent; a replay journals itself in sync.json, writes the replayed versions, then the base (the commit point), and only then sets the old ones aside and adopts the local commits the home already held; a conflict met past such commits moves the base onto them first. A save-both journals its copy and the ids it sends under, so a retry completes the same copy instead of making another. The next push, pull or resolve finishes or undoes whatever a cut left, so no version is orphaned and no false conflict appears. The editor reads the working state and the media index under the project lease; workspace.json is written whole and moved into place (an empty one is made again); project files may hold 64 MiB and one that would be larger is refused before it is written, so large projects check out. The workspace copy checks the drive's free space before and during the copy, keeps the folder in its checkpoint and stores a reset when a pass starts again, and runs at High level in the priority queue. Renders record a version only when the index holds it with the settings rendered; an unsent version renders as the project. Co-Authored-By: Claude Opus 5.5 --- i18n/en/Errors.resx | 2 + i18n/it/Errors.it.resx | 2 + src/Chronos.App/Archive/ArchiveFailures.cs | 2 +- .../Editor/WorkspaceSequenceSource.cs | 10 +- src/Chronos.App/Shell/ErrorText.cs | 2 +- src/Chronos.Core/Project/ProjectBundle.cs | 30 +- src/Chronos.Data/Archiving/ArchiveService.cs | 3 + src/Chronos.Data/Jobs/JobPriorityQueue.cs | 6 +- .../Processing/Jobs/RenderJobHandler.cs | 26 +- .../Workspaces/WorkspaceCopyJobHandler.cs | 164 +++++++-- src/Chronos.Data/Workspaces/WorkspaceStore.cs | 141 ++++++-- src/Chronos.Data/Workspaces/WorkspaceSync.cs | 216 +++++++++-- .../Workspace/WorkspaceEditorTests.cs | 6 + .../Project/ProjectBundleTests.cs | 18 + .../Jobs/JobPriorityQueueTests.cs | 5 + .../Processing/RenderJobTests.cs | 12 + .../Workspaces/WorkspaceFixture.cs | 63 ++++ .../Workspaces/WorkspaceRecoveryTests.cs | 334 ++++++++++++++++++ 18 files changed, 950 insertions(+), 92 deletions(-) create mode 100644 tests/Chronos.Data.Tests/Workspaces/WorkspaceRecoveryTests.cs diff --git a/i18n/en/Errors.resx b/i18n/en/Errors.resx index 916a0c5c..2c90826f 100644 --- a/i18n/en/Errors.resx +++ b/i18n/en/Errors.resx @@ -16,6 +16,7 @@ This path is a file, not a folder. The drive of this folder is not available. Enter a full path that starts with a drive or a network share. + The project's settings are larger than {0} MB; Chronos cannot store them.{0} megabytes {0} could not be downloaded intact.{0} file name Exports failed: {0} of {1}; see the job log.{0} failed count; {1} total count Render nodes send back one file per job; post each preset of a batch as its own job.long @@ -51,6 +52,7 @@ Settle the conflict first: Sync shows which settings differ. {0} did not match its source after it was copied.{0} file path A file of this project in the workspace cannot be read; the log names it. + The workspace drive needs {0} GB more for this copy.{0} gigabytes missing The last edit could not be stored yet; it is tried again, try once more in a moment. The library's copy of this project does not match what was sent; nothing was removed. The workspace folder is inside a library; nothing is copied into it. diff --git a/i18n/it/Errors.it.resx b/i18n/it/Errors.it.resx index d2a4b811..e6671d4d 100644 --- a/i18n/it/Errors.it.resx +++ b/i18n/it/Errors.it.resx @@ -16,6 +16,7 @@ Questo percorso è un file, non una cartella. L'unità di questa cartella non è disponibile. Inserisci un percorso completo che inizi con un'unità o una condivisione di rete. + Le impostazioni del progetto superano {0} MB; Chronos non può salvarle. {0} non è stato scaricato integro. Esportazioni non riuscite: {0} su {1}; vedi il registro del job. I nodi di rendering restituiscono un file per job; invia ogni preset di un gruppo come job a sé. @@ -51,6 +52,7 @@ Risolvi prima il conflitto: Sincronizza mostra quali impostazioni sono diverse. {0} non corrispondeva all'originale dopo la copia. Un file di questo progetto nello spazio di lavoro non è leggibile; il log lo indica. + Al disco dello spazio di lavoro servono altri {0} GB per questa copia. L'ultima modifica non è ancora stata memorizzata; viene ritentata, riprova tra un momento. La copia di questo progetto nella libreria non corrisponde a quanto inviato; non è stato rimosso nulla. La cartella dello spazio di lavoro è dentro una libreria; non vi viene copiato nulla. diff --git a/src/Chronos.App/Archive/ArchiveFailures.cs b/src/Chronos.App/Archive/ArchiveFailures.cs index 2957eda4..fc4e484e 100644 --- a/src/Chronos.App/Archive/ArchiveFailures.cs +++ b/src/Chronos.App/Archive/ArchiveFailures.cs @@ -20,7 +20,7 @@ public static class ArchiveFailures // any more (FrameTrash.RestoreAsync). Other IOExceptions and InvalidDataExceptions (a sidecar of another sequence) // carry technical text. LeaseUnavailableException or NewerFormatException or StaleRenamePlanException or NestedRenameException or StaleDeletePlanException - or SequenceInLibraryException or TrashMismatchException => ex.Message, + or SequenceInLibraryException or TrashMismatchException or Core.Project.ProjectTooLargeException => ex.Message, UnauthorizedAccessException => Loc.L("Archive.Failed.Access"), DatabaseException => Loc.L("Archive.Failed.Index"), ArgumentException => Loc.L("Archive.Failed.Gone"), diff --git a/src/Chronos.App/Editor/WorkspaceSequenceSource.cs b/src/Chronos.App/Editor/WorkspaceSequenceSource.cs index 9ae8554f..87426f32 100644 --- a/src/Chronos.App/Editor/WorkspaceSequenceSource.cs +++ b/src/Chronos.App/Editor/WorkspaceSequenceSource.cs @@ -228,7 +228,7 @@ public sealed class WorkspaceSequenceSource : ISequenceSource, IProjectRenders /// The render at the home (owner review 2, R8: edit with proxies, render where the originals are): one job of this PC's /// engine per preset, with the project's portable settings (positions parked by photo, placed on the home's frames by /// the render) and the project recorded on the render; the version too when the settings are those of the last - /// version saved (an unsaved working state renders as it is, without a version). + /// version the home holds (an unsaved working state, or one saved here and not sent yet, renders as it is, without a version). /// public IReadOnlyList RenderRequests(Sequence sequence, ProjectSettings project, ExportSettings export, int sourceWidth, int sourceHeight) { @@ -293,12 +293,16 @@ public sealed class WorkspaceSequenceSource : ISequenceSource, IProjectRenders } } - /// The last version saved: the newest one not sent yet, else the base; with its content hash. + /// + /// The version a render may name, with its content hash: the base, which the home holds, while no version waits to be + /// sent; none otherwise. A version not sent yet may never reach the home under its id (a replay or a save-both sends it + /// under another), so a render of it names the project alone. + /// private (Guid? Id, string? Hash) Tip() { try { - if (_sync.Unpushed(ProjectId) is { Count: > 0 } unsent) return (unsent[^1].Id, unsent[^1].ContentHash); + if (_sync.Unpushed(ProjectId).Count > 0) return (null, null); return Project.ReadBundle().Workspace?.Base is { } basis ? (basis.VersionId, basis.ContentHash) : (null, null); } catch (Exception ex) when (ex is IOException or InvalidDataException or InvalidOperationException) diff --git a/src/Chronos.App/Shell/ErrorText.cs b/src/Chronos.App/Shell/ErrorText.cs index 4349d7dd..02e3dcad 100644 --- a/src/Chronos.App/Shell/ErrorText.cs +++ b/src/Chronos.App/Shell/ErrorText.cs @@ -43,7 +43,7 @@ public static class ErrorText private static string? Describe(Exception ex) => ex switch { LeaseUnavailableException or ArchiveOfflineException or NewerFormatException or ImportSourceUnavailableException - or CopyVerificationException or StaleRenamePlanException or SyncConflictChangedException => ex.Message, + or CopyVerificationException or StaleRenamePlanException or SyncConflictChangedException or Core.Project.ProjectTooLargeException => ex.Message, // A template an archive refused (Settings › Archive, or this PC's templates for a new archive): the rule it breaks. TemplateRuleException rule => Loc.L("Errors.Template." + rule.Rule), // The workspace (owner review 2, R8): its own sentences, a home out of reach, a project the home refused. diff --git a/src/Chronos.Core/Project/ProjectBundle.cs b/src/Chronos.Core/Project/ProjectBundle.cs index 6782811f..6f520ec0 100644 --- a/src/Chronos.Core/Project/ProjectBundle.cs +++ b/src/Chronos.Core/Project/ProjectBundle.cs @@ -2,10 +2,19 @@ using System.Globalization; using System.Text; using System.Text.Json; using Chronos.Core.IO; +using Chronos.Core.Localization; using Chronos.Core.Naming; namespace Chronos.Core.Project; +/// A project file that would be larger than : refused before it is written, with a sentence for the user. +public sealed class ProjectTooLargeException(long bytes) + : IOException(Loc.L("Errors.Project.TooLarge", ProjectBundle.MaxBytes >> 20)) +{ + /// The size the file would have had. + public long Bytes { get; } = bytes; +} + /// The archive a project file's sequence lives in: its id, and its name and folder when the file was saved. public sealed record BundleArchive(Guid Id, string Name, string? Path); @@ -70,8 +79,12 @@ public sealed record ProjectBundle(BundleArchive Archive, BundleSequence Sequenc /// The settings come from a newer Chronos (): show them, never edit them. public bool IsReadOnly => Project.IsNewerFormat; - /// Project settings take a few kilobytes; a larger file is not a project file. - public const int MaxBytes = 4 << 20; + /// + /// The largest project file read or written, one limit for both: a workspace's working state parks every position by + /// photo until its media are copied (some 260 bytes each), so a 100,000-frame sequence with every frame excluded takes + /// some 26 MB; a larger file is not a project file. + /// + public const int MaxBytes = 64 << 20; private static readonly JsonWriterOptions Indented = new() { Indented = true }; @@ -165,8 +178,17 @@ public sealed record ProjectBundle(BundleArchive Archive, BundleSequence Sequenc return Parse(reader.ReadToEnd()); } - /// Writes the file atomically (temp, flush, rename): an interrupted save never leaves half a file. - public void Save(string path) => AtomicFile.WriteAllText(path, ToJson()); + /// + /// Writes the file atomically (temp, flush, rename): an interrupted save never leaves half a file. A file larger than + /// is refused before anything is written, as no read could take it back. + /// + /// The file would be larger than . + public void Save(string path) + { + var bytes = new UTF8Encoding(false).GetBytes(ToJson()); + if (bytes.Length > MaxBytes) throw new ProjectTooLargeException(bytes.Length); + AtomicFile.WriteAllBytes(path, bytes); + } /// /// The settings to give the sequence when the file is opened, the sequence having diff --git a/src/Chronos.Data/Archiving/ArchiveService.cs b/src/Chronos.Data/Archiving/ArchiveService.cs index f128acea..49188aff 100644 --- a/src/Chronos.Data/Archiving/ArchiveService.cs +++ b/src/Chronos.Data/Archiving/ArchiveService.cs @@ -76,6 +76,9 @@ public sealed class ArchiveService : IJobHandlerProvider /// The decoders of this engine's image passes (thumbnails, proxies). public Core.Imaging.ImageDecoder Decoder => _writer.Options.Decoder; + /// How the host measures free space under a folder (); null: the folder's drive. + public Func? FreeSpace => _writer.Options.FreeSpace; + /// /// The steps this host's jobs can run: the shared ones (), /// then the host's (). A schedule host lists their names. diff --git a/src/Chronos.Data/Jobs/JobPriorityQueue.cs b/src/Chronos.Data/Jobs/JobPriorityQueue.cs index 2b0c9156..fca96fd0 100644 --- a/src/Chronos.Data/Jobs/JobPriorityQueue.cs +++ b/src/Chronos.Data/Jobs/JobPriorityQueue.cs @@ -22,14 +22,16 @@ public enum JobKindPriority public sealed record JobPriorityQueue { /// - /// The levels a kind has unless the user changed it: renders, imports and downloads first; the long passes that resume - /// anywhere, and the housekeeping of the schedules (backups, clean-ups), last. + /// The levels a kind has unless the user changed it: renders, imports and downloads first, and the copy of a project's + /// media into the workspace, which Edit waits for (owner review 2, R8); the long passes that resume anywhere, and the + /// housekeeping of the schedules (backups, clean-ups), last. /// public static IReadOnlyDictionary DefaultLevels { get; } = new Dictionary { [JobKind.Render] = JobKindPriority.High, [JobKind.Import] = JobKindPriority.High, [JobKind.Download] = JobKindPriority.High, + [JobKind.WorkspaceCopy] = JobKindPriority.High, [JobKind.Verify] = JobKindPriority.Low, [JobKind.Proxies] = JobKindPriority.Low, [JobKind.Backup] = JobKindPriority.Low, diff --git a/src/Chronos.Data/Processing/Jobs/RenderJobHandler.cs b/src/Chronos.Data/Processing/Jobs/RenderJobHandler.cs index 0148c616..ecf525e1 100644 --- a/src/Chronos.Data/Processing/Jobs/RenderJobHandler.cs +++ b/src/Chronos.Data/Processing/Jobs/RenderJobHandler.cs @@ -53,6 +53,7 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt var aliases = embedded is not null ? await Projects.HomePhotos.AliasesAsync(db, sequence.Id, cancellation).ConfigureAwait(false) : []; var project = (embedded is not null ? ProjectStore.Parse(Placed(embedded, frames, aliases)) : ProjectStore.Read(sequence, folder)) with { Export = export }; var settingsJson = project.ToJson(); + var (projectId, projectVersionId) = await ProvenanceAsync(parameters, sequence, cancellation).ConfigureAwait(false); using var names = new OutputNames(); var all = Targets(export, parameters.PresetName); @@ -90,8 +91,8 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt { Id = RenderId(context.Job.Id, index), OutputPath = Stored(paths, result.OutputPath), - ProjectId = parameters.ProjectId, - ProjectVersionId = parameters.ProjectVersionId, + ProjectId = projectId, + ProjectVersionId = projectVersionId, }; await db.Renders.InsertAsync(entity, CancellationToken.None).ConfigureAwait(false); done.Add(index); @@ -137,6 +138,27 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt : throw new InvalidOperationException(Loc.L("Errors.Render.ProjectVersion")); } + /// + /// The project and the version a render records (owner review 2, R8: a render never names a version whose settings it + /// did not use, nor one the index does not hold): the project when it is one of the sequence's; the version when the index + /// holds it, for that project of the sequence, with the content hash of the settings the job embeds. A job naming a + /// version the home never received (one saved on a PC and sent later under another id) records the project alone. + /// + private async Task<(Guid? Project, Guid? Version)> ProvenanceAsync(Protocol.RenderJobParameters parameters, Sequence sequence, CancellationToken cancellationToken) + { + Guid? project = parameters.ProjectId is { } id && await db.Projects.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } owner + && owner.SequenceId == sequence.Id + ? id + : null; + if (parameters.ProjectVersionId is not { } versionId + || await db.Projects.GetVersionAsync(versionId, cancellationToken).ConfigureAwait(false) is not { } version + || (project is { } named && version.ProjectId != named) + || await db.Projects.GetAsync(version.ProjectId, cancellationToken).ConfigureAwait(false) is not { } of || of.SequenceId != sequence.Id + || (parameters.SettingsJson is { } embedded && !ProjectContent.Verifies(embedded, version.ContentHash))) + return (project, null); + return (version.ProjectId, versionId); + } + /// /// Settings whose positions wait for their photos (a project version's portable form, settings synced from another copy) /// with them placed on the frames rendered, a photo parked on an encoding a conversion replaced since found through diff --git a/src/Chronos.Data/Workspaces/WorkspaceCopyJobHandler.cs b/src/Chronos.Data/Workspaces/WorkspaceCopyJobHandler.cs index fe7f75d6..945be6b3 100644 --- a/src/Chronos.Data/Workspaces/WorkspaceCopyJobHandler.cs +++ b/src/Chronos.Data/Workspaces/WorkspaceCopyJobHandler.cs @@ -11,6 +11,7 @@ using Chronos.Data.Archiving; using Chronos.Data.Archiving.Jobs; using Chronos.Data.Jobs; using Chronos.Data.Projects; +using Chronos.Data.Sync; namespace Chronos.Data.Workspaces; @@ -47,7 +48,10 @@ public static class WorkspaceJobs /// Frames changed at the home during the copy, or the folder moved (a Reorganize), make it start again (three times at /// most; a folder still moving fails it, to be retried). The working state's positions follow the new frames: by frame /// where a frame stays (one whose file a conversion replaced keeps its positions), by photo where frames came or went. -/// Checkpoint: copy:<frame>:<frames version>; a run resumed on other frames checks every file again. +/// Before each pass and at each checkpoint the workspace's drive must hold what is still to copy plus a reserve, else the +/// job stops saying how much is missing (the workspace shares this PC's drive with the index and the logs). +/// Checkpoint: copy:<frame>:<frames version>:<folder>; a run resumed on other frames or in another folder +/// checks every file again, and a pass started again stores that before it reads anything. /// Originals come in phase 5c: asked for, the job fails saying so. /// internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService archives) : IJobHandler @@ -57,6 +61,17 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService private static readonly TimeSpan LockWait = TimeSpan.FromMinutes(2); private static readonly TimeSpan CheckpointEvery = TimeSpan.FromSeconds(5); + /// Bytes left free on the workspace's drive besides the copy, as a download keeps (). + private const long SpaceReserve = 512L << 20; + + /// A proxy at px, when the archive has none to measure: some 1.5 MB. + private const long ProxyEstimate = 3L << 19; + + private const int ProxyEstimateEdge = 1920; + + /// The long edge taken for a full-size proxy of a frame whose size is not known. + private const int UnknownFullSize = 6000; + public JobKind Kind => JobKind.WorkspaceCopy; public async Task RunAsync(JobContext context) @@ -80,20 +95,29 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService var paths = new ArchivePaths(archive.RootPath); // Read only: a read-only archive is copied from like any other. if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(archive.RootPath, ArchiveAvailability.Offline); - var (copied, checkpointVersion) = Parse(context.Checkpoint); + var (copied, checkpointVersion, checkpointFolder) = Parse(context.Checkpoint); for (var pass = 1; ; pass++) { var (row, frames) = await FramesAsync(sequenceId, ct).ConfigureAwait(false); var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy; var folder = paths.Sequence(row.Folder); - // Another index moved the sequence away, maybe another one in: its files are not this sequence's photos. - if (!SequenceFolders.Holds(folder, sequenceId)) throw new SequenceFolderMovedException(); - // Files kept from an earlier run are this frames version's only (a conversion keeps a frame's id, not its photo). - if (checkpointVersion is { } version && version != row.FramesVersion) copied = 0; + // Another index moved the sequence away, maybe another one in: its files are not this sequence's photos. A retry + // checks every file again (the files of this run may be of that folder). + if (!SequenceFolders.Holds(folder, sequenceId)) + { + await context.CheckpointAsync(Checkpoint(Stage.Copy, 0, row.FramesVersion, row.Folder), 0, frames.Count).ConfigureAwait(false); + throw new SequenceFolderMovedException(); + } + // Files kept from an earlier run are this frames version's and this folder's only (a conversion keeps a frame's id, + // not its photo; a sequence moved meanwhile may have left its photos' names to another sequence's). + if ((checkpointVersion is { } version && version != row.FramesVersion) || (checkpointFolder is { } read && read != row.Folder)) copied = 0; var total = (long)frames.Count; var tally = new Tally(); var entries = new WorkspaceFrame[frames.Count]; Directory.CreateDirectory(project.ProxiesDirectory); + // The workspace is on this PC's own drive, with the index, the settings and the logs: the copy never fills it. + var remaining = Remaining(project, folder, frames, policy); + EnsureSpace(parameters.WorkspaceRoot, remaining[0]); var clock = System.Diagnostics.Stopwatch.StartNew(); for (var i = 0; i < frames.Count; i++) { @@ -101,7 +125,9 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService entries[i] = Copy(project, folder, paths, frames[i], policy, done: i < copied, tally, ct); if (clock.Elapsed >= CheckpointEvery) { - await context.CheckpointAsync(Checkpoint(Stage.Copy, i + 1, row.FramesVersion), i + 1, total).ConfigureAwait(false); + await context.CheckpointAsync(Checkpoint(Stage.Copy, i + 1, row.FramesVersion, row.Folder), i + 1, total).ConfigureAwait(false); + // Another program may take the space meanwhile. + EnsureSpace(parameters.WorkspaceRoot, remaining[i + 1]); clock.Restart(); } else @@ -114,15 +140,21 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService var moved = after.Folder != row.Folder || !SequenceFolders.Holds(paths.Sequence(after.Folder), sequenceId) || Renamed(frames, now, entries); if (after.FramesVersion != row.FramesVersion || moved) { - // The files read may be of other frames, or missing where a Reorganize took them: read them again. + // The files read may be of other frames, or missing where a Reorganize took them: read them again, and so + // does a run resumed after a crash in the next pass (the checkpoint stored says so). if (pass < Passes) { context.Log(Loc.L("Archive.Log.WorkspaceFramesChanged", row.DisplayTitle)); - (copied, checkpointVersion) = (0, null); + await context.CheckpointAsync(Checkpoint(Stage.Copy, 0, after.FramesVersion, after.Folder), 0, total).ConfigureAwait(false); + (copied, checkpointVersion, checkpointFolder) = (0, null, null); continue; } // Frames still arriving are copied as they were read (Update media takes the rest); a folder still moving is not. - if (moved) throw new SequenceFolderMovedException(); + if (moved) + { + await context.CheckpointAsync(Checkpoint(Stage.Copy, 0, after.FramesVersion, after.Folder), 0, total).ConfigureAwait(false); + throw new SequenceFolderMovedException(); + } } var aliases = await HomePhotos.AliasesAsync(db, sequenceId, ct).ConfigureAwait(false); var media = new WorkspaceMedia(WorkspaceMedia.FormatName, WorkspaceMedia.CurrentFormatVersion, BundleMedia.Proxies, sequenceId, row.FramesVersion, entries, @@ -134,7 +166,7 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService RemoveStale(project, media); } context.Log(Loc.L("Archive.Log.WorkspaceCopied", row.DisplayTitle, tally.Copied, tally.Made, tally.Kept, tally.Missing)); - await context.CheckpointAsync(Checkpoint(Stage.Done, frames.Count, row.FramesVersion), total, total).ConfigureAwait(false); + await context.CheckpointAsync(Checkpoint(Stage.Done, frames.Count, row.FramesVersion, row.Folder), total, total).ConfigureAwait(false); return; } } @@ -205,6 +237,9 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService } if (bytes is null) { + // A file an earlier copy left for the frame (of other frames, or another folder's) is never kept as its photo by a + // resumed run: a regenerable copy, it goes. + if (File.Exists(target)) TryDelete(target); tally.Missing++; return WorkspaceFrame.Of(frame, null, null, 0); } @@ -292,15 +327,81 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService var listed = media.Frames.Where(f => f.File is not null).Select(f => Path.GetFileName(f.File!)).ToHashSet(StringComparer.OrdinalIgnoreCase); foreach (var file in Directory.EnumerateFiles(project.ProxiesDirectory, "*.jpg")) { - if (listed.Contains(Path.GetFileName(file))) continue; - try - { - File.Delete(file); - } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) - { - Log.Warning("workspace", $"{file} could not be removed", ex); - } + if (!listed.Contains(Path.GetFileName(file))) TryDelete(file); + } + } + + private static void TryDelete(string file) + { + try + { + File.Delete(file); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warning("workspace", $"{file} could not be removed", ex); + } + } + + /// + /// The bytes the copy still writes from each frame on (index i: frames i to the last; one more entry, 0): + /// nothing for a frame the workspace holds a file of already (kept, or replaced by one of about its size), the archive's + /// proxy's size where it has one, else an estimate at the proxy size the copy makes (). + /// + private static long[] Remaining(WorkspaceProject project, SequencePaths folder, List frames, ProxyPolicy policy) + { + var held = Directory.Exists(project.ProxiesDirectory) + ? Directory.EnumerateFiles(project.ProxiesDirectory).Select(Path.GetFileName).ToHashSet(StringComparer.OrdinalIgnoreCase) + : []; + var proxies = Directory.Exists(folder.Proxies) + ? new DirectoryInfo(folder.Proxies).EnumerateFiles().ToDictionary(f => f.Name, f => f.Length, StringComparer.OrdinalIgnoreCase) + : []; + var remaining = new long[frames.Count + 1]; + for (var i = frames.Count - 1; i >= 0; i--) + { + var frame = frames[i]; + var bytes = held.Contains(ProxyCache.FileName(frame.Id)) ? 0 + : proxies.TryGetValue(ArchiveLayout.ProxyFileName(frame.Index), out var length) ? length + : (frame.Flags & FrameFlags.Missing) != 0 ? 0 + : Estimate(frame, policy); + remaining[i] = remaining[i + 1] + bytes; + } + return remaining; + } + + /// A proxy's size when the archive has none to measure: some 1.5 MB at 1920 px, scaled by the area at the copy's long edge. + private static long Estimate(Frame frame, ProxyPolicy policy) + { + var edge = policy.LongEdge == ProxyPolicy.Disabled ? new ProxyPolicy().LongEdge : policy.LongEdge; + var longest = Math.Max(frame.Width, frame.Height); + var side = edge == ProxyPolicy.FullSize ? (longest > 0 ? longest : UnknownFullSize) : longest > 0 ? Math.Min(edge, longest) : edge; + return (long)(ProxyEstimate * Math.Pow(side / (double)ProxyEstimateEdge, 2)); + } + + /// + /// Refuses to go on while the workspace's drive lacks besides , with the + /// gigabytes missing; a drive that cannot be measured (a network share) is not refused, the copy itself then says so. + /// + /// The drive lacks the space (Errors.Workspace.DiskFull). + private void EnsureSpace(string root, long bytes) + { + if (bytes <= 0 || FreeBytes(root) is not { } free || free >= bytes + SpaceReserve) return; + throw new IOException(Loc.L("Errors.Workspace.DiskFull", ((bytes + SpaceReserve - free) / 1e9).ToString("0.0", Loc.Culture))); + } + + /// Free bytes on the volume of , as the host measures them (), else its drive's. + private long? FreeBytes(string folder) + { + if (archives.FreeSpace is { } measure) return measure(folder); + try + { + var full = Path.GetFullPath(folder); + // Windows wants a drive root; Unix accepts any directory and reports the volume it is mounted on. + return new DriveInfo(OperatingSystem.IsWindows() ? Path.GetPathRoot(full)! : full).AvailableFreeSpace; + } + catch (Exception ex) when (ex is ArgumentException or IOException or UnauthorizedAccessException) + { + return null; } } @@ -361,22 +462,23 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService Done, } - private static string Checkpoint(Stage stage, int frame, int framesVersion) => - string.Create(CultureInfo.InvariantCulture, $"{stage.ToString().ToLowerInvariant()}:{frame}:{framesVersion}"); + private static string Checkpoint(Stage stage, int frame, int framesVersion, string folder) => + string.Create(CultureInfo.InvariantCulture, $"{stage.ToString().ToLowerInvariant()}:{frame}:{framesVersion}:{folder}"); /// - /// The frames copied by an earlier run (copy:<frame>[:<frames version>]) and the frames version they - /// were of (null when the checkpoint does not say); nothing for a finished run, or another stage (an earlier build's - /// archive proxy pass), whose files the copy checks again. + /// The frames copied by an earlier run (copy:<frame>[:<frames version>[:<folder>]], the folder + /// last as it may hold a colon), the frames version and the sequence's folder they were of (null when the checkpoint does + /// not say); nothing for a finished run, or another stage (an earlier build's archive proxy pass), whose files the copy + /// checks again. /// - private static (int Copied, int? FramesVersion) Parse(string? checkpoint) + private static (int Copied, int? FramesVersion, string? Folder) Parse(string? checkpoint) { - var parts = checkpoint?.Split(':'); - if (parts is not ([_, _] or [_, _, _]) || !Enum.TryParse(parts[0], ignoreCase: true, out var stage) || stage != Stage.Copy + var parts = checkpoint?.Split(':', 4); + if (parts is not ([_, _] or [_, _, _] or [_, _, _, _]) || !Enum.TryParse(parts[0], ignoreCase: true, out var stage) || stage != Stage.Copy || !int.TryParse(parts[1], NumberStyles.None, CultureInfo.InvariantCulture, out var next)) - return (0, null); - int? version = parts.Length == 3 && int.TryParse(parts[2], NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture, out var v) ? v : null; - return (next, version); + return (0, null, null); + int? version = parts.Length >= 3 && int.TryParse(parts[2], NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture, out var v) ? v : null; + return (next, version, parts.Length == 4 ? parts[3] : null); } private sealed class Tally diff --git a/src/Chronos.Data/Workspaces/WorkspaceStore.cs b/src/Chronos.Data/Workspaces/WorkspaceStore.cs index b3448e14..4229825f 100644 --- a/src/Chronos.Data/Workspaces/WorkspaceStore.cs +++ b/src/Chronos.Data/Workspaces/WorkspaceStore.cs @@ -13,14 +13,41 @@ namespace Chronos.Data.Workspaces; /// The manifest of a workspace folder (workspace.json). public sealed record WorkspaceManifest(string Format, int FormatVersion, Guid Id, DateTimeOffset Created); -/// A workspace's sync state of one project besides its versions (sync.json): a restore to record, a conflict to settle. +/// +/// A workspace's sync state of one project besides its versions (sync.json): a restore to record, a conflict to +/// settle, and the journal of a replay or a save-both under way, which the next sync finishes or undoes after a crash. +/// /// The version a restore brought back; the next commit is a "restore" of it. /// The conflict a pull or push stopped at, until the user settles it. -public sealed record WorkspaceSyncState(Guid? RestoredFrom, WorkspaceConflict? Conflict) +/// A replay (or a base moved onto local commits the home holds) written in steps; null when none is under way. +/// A save-both that made its copy at the home and has not finished; null when none is under way. +public sealed record WorkspaceSyncState(Guid? RestoredFrom, WorkspaceConflict? Conflict, WorkspaceReplay? Replay = null, WorkspaceSaveBoth? SaveBoth = null) { public static WorkspaceSyncState None { get; } = new(null, null); + + /// Nothing to keep: the file can go. + [JsonIgnore] + public bool IsEmpty => RestoredFrom is null && Conflict is null && Replay is null && SaveBoth is null; } +/// +/// The journal of a replay (): the base it moves the project onto, the local versions it +/// replaces, the replayed ones it writes and the local versions the home already holds (their answer lost), whose files +/// take the home's copy. The bundle's base is the commit point: once it is the replay is finished +/// (the replaced set aside, the held ones adopted), before it the replayed ones are set aside and nothing else moved. +/// +public sealed record WorkspaceReplay(Guid OntoBase, IReadOnlyList Replaced, IReadOnlyList Replayed, IReadOnlyList Adopted); + +/// +/// A save-both under way (): the copy made at the home and its first version (the base the +/// local versions are sent on top of), the base this project had when it started, and the id each local version is sent +/// under, so a retry sends to the same copy what it lacks and the home answers "already stored" for the rest. +/// +public sealed record WorkspaceSaveBoth(Guid CopyId, Guid CopyBaseId, int CopyBaseNumber, Guid? FromBase, IReadOnlyList Moves); + +/// A local version () and the id it is sent under to a save-both's copy. +public sealed record WorkspaceMove(Guid LocalId, Guid CopyVersionId); + /// /// A pull or push that met real conflicts (owner review 2, R8): the home's head it met, the version the local commits were /// made from, the local commits (oldest first) and the members both sides changed (mine, theirs). Settled by keeping mine, @@ -58,7 +85,11 @@ public sealed class WorkspaceStore /// The folder name of the workspace inside the program's data folder. public const string DefaultFolderName = "workspace"; - /// Opens the workspace at , making the folder and its manifest when there is none. + /// + /// Opens the workspace at , making the folder and its manifest when there is none. The manifest + /// is written whole to a temporary file and moved into place, so a crash never leaves half of one; a manifest left + /// empty, or unreadable in a folder that holds no project yet (an earlier build's interrupted first write), is made again. + /// /// The folder's workspace.json is not a workspace manifest, or a newer one. /// The folder cannot be made or read. public WorkspaceStore(string root) @@ -67,20 +98,8 @@ public sealed class WorkspaceStore Root = LongPath.Normalize(Path.TrimEndingDirectorySeparator(Path.GetFullPath(root))); Directory.CreateDirectory(Root); var path = Path.Combine(Root, ManifestFile); - if (!File.Exists(path)) - { - var manifest = new WorkspaceManifest(FormatName, FormatVersion, Guid.CreateVersion7(), DateTimeOffset.UtcNow); - // Another program opening the same folder at the same moment: the first manifest wins. - try - { - using var stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.None); - JsonSerializer.Serialize(stream, manifest, WorkspaceJson.Default.WorkspaceManifest); - stream.Flush(flushToDisk: true); - } - catch (IOException) when (File.Exists(path)) - { - } - } + if (!File.Exists(path)) Create(path, replace: false); + else if (Unfinished(path)) Create(path, replace: true); Manifest = Read(path); } @@ -130,6 +149,70 @@ public sealed class WorkspaceStore return bundles; } + /// + /// Writes a new manifest. Without another program opening the same folder at the same moment + /// may have made one first: that one wins (the move never replaces a file). + /// + private static void Create(string path, bool replace) + { + var manifest = new WorkspaceManifest(FormatName, FormatVersion, Guid.CreateVersion7(), DateTimeOffset.UtcNow); + var text = JsonSerializer.Serialize(manifest, WorkspaceJson.Default.WorkspaceManifest); + if (replace) + { + Log.Warning("workspace", $"{path} was left unfinished; a new manifest is written"); + AtomicFile.WriteAllText(path, text); + return; + } + var temp = AtomicFile.TempPathFor(path); + try + { + using (var stream = new FileStream(temp, FileMode.CreateNew, FileAccess.Write, FileShare.None)) + { + stream.Write(System.Text.Encoding.UTF8.GetBytes(text)); + stream.Flush(flushToDisk: true); + } + try + { + File.Move(temp, path, overwrite: false); + } + catch (IOException) when (File.Exists(path)) + { + // Another program's manifest came first. + } + } + finally + { + if (File.Exists(temp)) File.Delete(temp); + } + } + + /// + /// A manifest an interrupted first write left (an earlier build wrote it in place): empty, or not JSON in a folder that + /// holds no project yet. A readable manifest, or any in a folder with projects, is never written over. + /// + private bool Unfinished(string path) + { + string text; + try + { + text = AtomicFile.ReadAllText(path); + } + catch (IOException) + { + return false; + } + if (string.IsNullOrWhiteSpace(text.Trim('\0'))) return true; + try + { + using var _ = JsonDocument.Parse(text); + return false; + } + catch (JsonException) + { + return List().Count == 0; + } + } + private static WorkspaceManifest Read(string path) { WorkspaceManifest? manifest; @@ -274,7 +357,10 @@ public sealed class WorkspaceProject internal void WriteVersion(ProjectVersion version, bool replace = false) => ProjectSidecars.WriteVersion(VersionPath(version), version, replace); - /// Moves a version file aside into versions/.replaced (a local commit replayed under another id). + /// + /// Moves a version file aside into versions/.replaced (a local commit replayed under another id), trying again + /// for a moment while another program (an indexer, an antivirus) holds it open. + /// internal void SetAside(ProjectVersion version) { var path = VersionPath(version); @@ -283,9 +369,23 @@ public sealed class WorkspaceProject Directory.CreateDirectory(aside); var target = Path.Combine(aside, Path.GetFileName(path)); if (File.Exists(target)) target += "." + DateTime.UtcNow.Ticks.ToString(System.Globalization.CultureInfo.InvariantCulture); - File.Move(path, target); + for (var attempt = 1; ; attempt++) + { + try + { + File.Move(path, target); + return; + } + catch (IOException) when (attempt < SetAsideAttempts && File.Exists(path) && !File.Exists(target)) + { + Thread.Sleep(10 * attempt); + } + } } + /// Moves tried, some 1 s in all, before a version file held open fails to move aside. + private const int SetAsideAttempts = 15; + public WorkspaceSyncState ReadSync() { if (!File.Exists(SyncPath)) return WorkspaceSyncState.None; @@ -301,7 +401,8 @@ public sealed class WorkspaceProject internal void WriteSync(WorkspaceSyncState state) { - if (state == WorkspaceSyncState.None || state is { RestoredFrom: null, Conflict: null }) + ArgumentNullException.ThrowIfNull(state); + if (state.IsEmpty) { if (File.Exists(SyncPath)) File.Delete(SyncPath); return; diff --git a/src/Chronos.Data/Workspaces/WorkspaceSync.cs b/src/Chronos.Data/Workspaces/WorkspaceSync.cs index 77ab7612..d9fd6a63 100644 --- a/src/Chronos.Data/Workspaces/WorkspaceSync.cs +++ b/src/Chronos.Data/Workspaces/WorkspaceSync.cs @@ -1,4 +1,5 @@ using System.Text.Json; +using Chronos.Core.Diagnostics; using Chronos.Core.Model; using Chronos.Core.Project; using Chronos.Data.Projects; @@ -84,6 +85,11 @@ public sealed class WorkspaceSyncException(PushOutcome outcome, string message) /// travel in the portable form (), so frames inserted at the home never move a position. Every /// operation holds the project's lease; nothing is ever deleted: a restore or save-both first saves uncommitted changes /// as a version. +/// Every write sequence survives a cut at any step: the working state's base is the commit point. A push moves the base +/// before its local file reads as sent; a replay journals itself in sync.json (), writes +/// the replayed versions, then the base, and only then sets the old ones aside; a save-both journals its copy and the ids +/// it sends under (). The next push, pull or resolve finishes or undoes what a cut left +/// (). /// public sealed class WorkspaceSync(WorkspaceStore store, string clientName) { @@ -100,6 +106,11 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName) /// How long an operation waits for the project's lease (another program working on it). public TimeSpan LockWait { get; init; } = TimeSpan.FromSeconds(30); + /// For tests: called with a step's name between the writes of a sync, where it may throw as a crash would. + internal Action? Cut { get; init; } + + private void Step(string name) => Cut?.Invoke(name); + /// /// The project of brought into the workspace: its folder, the home's /// head as the base and the working state (positions waiting for the media, which a @@ -126,30 +137,41 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName) ?? throw new ProjectException(ProjectError.NotFound, $"The home has no project {projectId}."); var head = await home.HeadAsync(projectId, cancellationToken).ConfigureAwait(false); Verify(head); - project.WriteVersion(head, replace: true); - Directory.CreateDirectory(project.ExportsDirectory); var bundle = new ProjectBundle(sequence.Archive, sequence.Sequence, ProjectSettings.FromJson(head.SettingsJson, allowNewer: true), DateTimeOffset.UtcNow) { Workspace = new BundleWorkspace(projectId, listed.Name, listed.IsMain, sequence.Home, Base(head), new BundleMedia(media, 0)), }; + // The working state first: one larger than a project file holds is refused before anything is written + // (ProjectTooLargeException), never left as a file no later open could read. Its base is fetched again if a cut + // comes before the version's own file. project.WriteBundle(bundle); + project.WriteVersion(head, replace: true); + Directory.CreateDirectory(project.ExportsDirectory); return project; } } - /// The working state for the editor: its settings placed on the media's frames, and the media's sequence. + /// + /// The working state for the editor: its settings placed on the media's frames, and the media's sequence. Both files are + /// read under the project's lease, as one state: a copy committing new media rewrites them in turn. + /// /// The project is not in the workspace. - public Task OpenAsync(Guid projectId, CancellationToken cancellationToken = default) => - Task.Run(() => + /// Another writer kept the project for longer than . + public async Task OpenAsync(Guid projectId, CancellationToken cancellationToken = default) + { + var project = Existing(projectId); + ProjectBundle bundle; + WorkspaceMedia? media; + using (await project.LockAsync(LockWait, cancellationToken).ConfigureAwait(false)) + (bundle, media) = await Task.Run(() => (project.ReadBundle(), project.ReadMedia()), cancellationToken).ConfigureAwait(false); + return await Task.Run(() => { - var project = Existing(projectId); - var bundle = project.ReadBundle(); - var media = project.ReadMedia(); var working = bundle.Project; if (media is not null && !working.IsNewerFormat) working = ProjectSettings.FromJson(ProjectContent.Place(working.ToJson(), media.Photos), allowNewer: true); return new WorkspaceSession(project, bundle, working, media, media is null ? null : SequenceOf(project, bundle, media)); - }, cancellationToken); + }, cancellationToken).ConfigureAwait(false); + } /// Stores the working state (the editor's settings, positions on the media's frames); no version is made. /// The project is read-only (a newer Chronos's settings) or not in the workspace. @@ -192,6 +214,7 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName) var project = Existing(projectId); using (await project.LockAsync(LockWait, cancellationToken).ConfigureAwait(false)) { + await SettleAsync(project, home, cancellationToken).ConfigureAwait(false); var pushed = new List(); var pulled = new List(); for (var attempt = 0; attempt < PushAttempts; attempt++) @@ -206,10 +229,13 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName) { case PushOutcome.Stored or PushOutcome.AlreadyStored when result.Version is { } stored: Verify(stored); - if (stored.Number != local.Number) project.SetAside(local); - project.WriteVersion(stored, replace: true); + // The base moves first: cut after it, the local file still reads as unsent and the next sync + // adopts it; cut before it, the home answers "already stored" next time. Marked sent first, it + // would leave the chain from the old base and take the versions after it along. bundle = WithBase(bundle, stored); project.WriteBundle(bundle); + Step("push.base"); + Adopt(project, local, stored); pushed.Add(stored); continue; case PushOutcome.Diverged: @@ -240,6 +266,7 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName) var project = Existing(projectId); using (await project.LockAsync(LockWait, cancellationToken).ConfigureAwait(false)) { + await SettleAsync(project, home, cancellationToken).ConfigureAwait(false); IReadOnlyList pulled = []; if (project.ReadSync().Conflict is null) { @@ -267,6 +294,7 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName) var project = Existing(projectId); using (await project.LockAsync(LockWait, cancellationToken).ConfigureAwait(false)) { + await SettleAsync(project, home, cancellationToken).ConfigureAwait(false); var bundle = project.ReadBundle(); var sync = project.ReadSync(); if (sync.Conflict is null) return new WorkspaceSyncResult(await StatusLockedAsync(project, home, cancellationToken).ConfigureAwait(false), [], []); @@ -328,15 +356,19 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName) /// /// Pulls the home's versions after the base and replays the local ones and the working state on its head. With /// null a conflict is recorded and nothing else moves (the versions pulled are kept as - /// files); with a choice the conflicts are settled by it. + /// files; the base only moves onto local commits the home already holds); with a choice the conflicts are settled by it. + /// Written in steps a cut can stop at any point (): the journal, the replayed versions, the + /// base with the merged working state (the commit point), then the old versions set aside and the held ones adopted. /// private async Task<(ProjectBundle Bundle, WorkspaceConflict? Conflict, IReadOnlyList Pulled)> ReplayAsync(WorkspaceProject project, ProjectBundle bundle, IProjectHome home, IReadOnlySet? takeTheirs, CancellationToken cancellationToken) { var basis = await BaseVersionAsync(project, bundle, home, cancellationToken).ConfigureAwait(false); - // Read before the home's versions are written over them: a local commit the home stored although its answer was lost - // comes back among them, and the commits after it were made on it. + // A local commit the home stored although its answer was lost comes back among the home's versions, and the commits + // after it were made on it. Its file keeps reading as unsent until the base has moved onto it or past it: written + // over now, it would leave the chain from the base, and the commits after it with it. var locals = Unpushed(project, bundle); + var unsent = project.Versions().Where(v => v.StoredAt is null).Select(v => v.Id).ToHashSet(); var theirs = new List(); for (var after = basis?.Number ?? 0; ;) { @@ -344,13 +376,14 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName) foreach (var version in page) { Verify(version); - project.WriteVersion(version, replace: true); + if (!unsent.Contains(version.Id)) project.WriteVersion(version, replace: true); } theirs.AddRange(page); if (page.Count < PullPage) break; after = page[^1].Number; } if (theirs.Count == 0) return (bundle, null, theirs); + var held = theirs.Where(v => unsent.Contains(v.Id)).ToList(); var head = theirs[^1]; var atHome = theirs.Select(v => v.Id).ToHashSet(); var ours = locals.TakeWhile(v => atHome.Contains(v.Id)).ToList(); @@ -377,15 +410,28 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName) if (differences.Count > 0 && takeTheirs is null) { var conflict = new WorkspaceConflict(head.Id, head.Number, basis?.Id ?? Guid.Empty, [.. locals.Select(v => v.Id)], differences, DateTimeOffset.UtcNow); - project.WriteSync(project.ReadSync() with { Conflict = conflict }); + if (ours.Count > 0) + { + // The conflict is met past local commits the home holds: the base moves onto the last of them (their files + // adopted once it has), so the commits after them stay in the chain and the conflict is settled against it. + var onto = new WorkspaceReplay(basis!.Id, [], [], [.. held.Select(v => v.Id)]); + project.WriteSync(project.ReadSync() with { Replay = onto }); + Step("replay.journal"); + bundle = WithBase(bundle, basis); + project.WriteBundle(bundle); + Step("replay.base"); + await FinishReplayAsync(project, onto, held, home, cancellationToken).ConfigureAwait(false); + } + project.WriteSync(project.ReadSync() with { Conflict = conflict, Replay = null }); return (bundle, conflict, theirs); } - var parent = head; + var rebased = new List(locals.Count); + ProjectVersion parent = head; for (var i = 0; i < locals.Count; i++) { var local = locals[i]; var portable = ProjectContent.Canonical(replay.Commits[i].ToJson()); - var rebased = local with + parent = local with { Id = Guid.CreateVersion7(), ParentId = parent.Id, @@ -396,48 +442,162 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName) SettingsFormat = ProjectContent.FormatOf(portable), StoredAt = null, }; - project.SetAside(local); - project.WriteVersion(rebased); - parent = rebased; + rebased.Add(parent); } var placed = ProjectSettings.FromJson(ProjectContent.Place(ProjectContent.Canonical(merge.Merged.ToJson()), photos), allowNewer: true); var moved = WithBase(bundle, head) with { Project = placed, Saved = DateTimeOffset.UtcNow }; + var journal = new WorkspaceReplay(head.Id, [.. locals.Select(v => v.Id)], [.. rebased.Select(v => v.Id)], [.. held.Select(v => v.Id)]); + project.WriteSync(project.ReadSync() with { Replay = journal }); + Step("replay.journal"); + foreach (var version in rebased) + { + project.WriteVersion(version); + Step("replay.version"); + } + // The commit point: before it the old chain hangs from the old base, after it the replayed one from the head. project.WriteBundle(moved); + Step("replay.base"); + await FinishReplayAsync(project, journal, held, home, cancellationToken).ConfigureAwait(false); + project.WriteSync(project.ReadSync() with { Replay = null }); return (moved, null, theirs); } - /// Both sides kept: this workspace's versions to a new project at the home, this project onto the home's head. + /// + /// Finishes a replay whose base moved (): each local version the home holds takes + /// its copy (from , a file of the workspace, else the home), the replaced ones are set aside. + /// + private static async Task FinishReplayAsync(WorkspaceProject project, WorkspaceReplay replay, IReadOnlyList known, IProjectHome home, + CancellationToken cancellationToken) + { + foreach (var id in replay.Adopted) + { + var locals = project.Versions().Where(v => v.Id == id && v.StoredAt is null).ToList(); + if (locals.Count == 0) continue; + if ((known.FirstOrDefault(v => v.Id == id) ?? await StoredCopyAsync(project, id, home, cancellationToken).ConfigureAwait(false)) is not { } stored) continue; + foreach (var local in locals) Adopt(project, local, stored); + } + foreach (var id in replay.Replaced) SetAsideUnsent(project, id); + } + + /// + /// Finishes or undoes what a cut left (see the class remarks), before anything reads the chain: a replay's journal is + /// finished once the base is its target, else its replayed versions are set aside (the old chain still hangs from the + /// base); a save-both whose project moved onto the home's head sets its sent versions aside and clears its conflict; a + /// local file of the version the base names (a push cut after it moved the base) takes the home's copy. + /// + private static async Task SettleAsync(WorkspaceProject project, IProjectHome home, CancellationToken cancellationToken) + { + var sync = project.ReadSync(); + var basis = project.ReadBundle().Workspace!.Base; + if (sync.Replay is { } replay) + { + if (basis?.VersionId == replay.OntoBase) await FinishReplayAsync(project, replay, [], home, cancellationToken).ConfigureAwait(false); + else foreach (var id in replay.Replayed) SetAsideUnsent(project, id); + sync = sync with { Replay = null }; + project.WriteSync(sync); + Log.Info("workspace", $"A sync of project {project.ProjectId} cut short was {(basis?.VersionId == replay.OntoBase ? "finished" : "undone")}"); + } + if (sync.SaveBoth is { } both && basis?.VersionId != both.FromBase) + { + foreach (var move in both.Moves) SetAsideUnsent(project, move.LocalId); + var settled = sync.Conflict is { } conflict && conflict.BaseVersionId == (both.FromBase ?? Guid.Empty); + sync = sync with { SaveBoth = null, Conflict = settled ? null : sync.Conflict }; + project.WriteSync(sync); + Log.Info("workspace", $"A save-both of project {project.ProjectId} cut short was finished"); + } + if (basis is not null && project.Versions().Where(v => v.Id == basis.VersionId && v.StoredAt is null).ToList() is { Count: > 0 } lingering + && await StoredCopyAsync(project, basis.VersionId, home, cancellationToken).ConfigureAwait(false) is { } stored) + { + foreach (var local in lingering) Adopt(project, local, stored); + } + } + + /// The home's copy of version : a file of the workspace marked sent, else asked from the home (kept as a file); null when the home has none. + private static async Task StoredCopyAsync(WorkspaceProject project, Guid id, IProjectHome home, CancellationToken cancellationToken) + { + if (project.Versions().FirstOrDefault(v => v.Id == id && v.StoredAt is not null) is { } file) return file; + if (await home.GetVersionAsync(project.ProjectId, id, cancellationToken).ConfigureAwait(false) is not { StoredAt: not null } fetched) return null; + Verify(fetched); + return fetched; + } + + /// + /// The local file of a version the home holds takes the home's copy (sent, maybe under another number). Called only once + /// the base names the version or one after it: before, the copy would take it out of the chain of unsent versions. + /// + private static void Adopt(WorkspaceProject project, ProjectVersion local, ProjectVersion stored) + { + project.WriteVersion(stored, replace: true); + if (stored.Number != local.Number) project.SetAside(local); + } + + /// Sets aside the unsent local files of version . + private static void SetAsideUnsent(WorkspaceProject project, Guid id) + { + foreach (var version in project.Versions().Where(v => v.Id == id && v.StoredAt is null)) project.SetAside(version); + } + + /// + /// Both sides kept: this workspace's versions to a new project at the home, this project onto the home's head. The copy + /// and the ids its versions are sent under are journaled before the first is sent (): a + /// retry after a cut sends to the same copy what it lacks (the home answers "already stored" for the rest), whatever name + /// it is asked for, and the base moving onto the home's head is the commit point. + /// private async Task SaveBothAsync(WorkspaceProject project, ProjectBundle bundle, IProjectHome home, string? name, CancellationToken cancellationToken) { var workspace = bundle.Workspace!; if (!bundle.IsReadOnly) CommitLocked(project, bundle, null, null); bundle = project.ReadBundle(); var locals = Unpushed(project, bundle); - var basis = await BaseVersionAsync(project, bundle, home, cancellationToken).ConfigureAwait(false); + var sync = project.ReadSync(); + var pending = sync.SaveBoth; SequenceProject? copy = null; + if (pending is not null) + { + copy = (await home.ListAsync(cancellationToken).ConfigureAwait(false)).FirstOrDefault(p => p.Id == pending.CopyId); + // A copy deleted at the home meanwhile: a new one is made. + if (copy is null) pending = null; + } var pushed = new List(); if (locals.Count > 0) { - copy = await CreateNamedAsync(home, name, workspace.Name, basis?.Id, cancellationToken).ConfigureAwait(false); - var parent = await home.HeadAsync(copy.Id, cancellationToken).ConfigureAwait(false); + if (pending is null) + { + var basis = await BaseVersionAsync(project, bundle, home, cancellationToken).ConfigureAwait(false); + copy = await CreateNamedAsync(home, name, workspace.Name, basis?.Id, cancellationToken).ConfigureAwait(false); + var first = await home.HeadAsync(copy.Id, cancellationToken).ConfigureAwait(false); + pending = new WorkspaceSaveBoth(copy.Id, first.Id, first.Number, workspace.Base?.VersionId, []); + } + var known = pending.Moves.Select(m => m.LocalId).ToHashSet(); + if (locals.Any(l => !known.Contains(l.Id)) || !ReferenceEquals(pending, sync.SaveBoth)) + { + pending = pending with { Moves = [.. pending.Moves, .. locals.Where(l => !known.Contains(l.Id)).Select(l => new WorkspaceMove(l.Id, Guid.CreateVersion7()))] }; + project.WriteSync(sync with { SaveBoth = pending }); + } + Step("saveboth.journal"); + var ids = pending.Moves.ToDictionary(m => m.LocalId, m => m.CopyVersionId); + var (parentId, parentNumber) = (pending.CopyBaseId, pending.CopyBaseNumber); foreach (var local in locals) { - var moved = local with { Id = Guid.CreateVersion7(), ProjectId = copy.Id, ParentId = parent.Id, Number = parent.Number + 1, StoredAt = null }; + var moved = local with { Id = ids[local.Id], ProjectId = copy!.Id, ParentId = parentId, Number = parentNumber + 1, StoredAt = null }; var result = await home.PushAsync(copy.Id, moved, cancellationToken).ConfigureAwait(false); if (result is not { Outcome: PushOutcome.Stored or PushOutcome.AlreadyStored, Version: { } stored }) throw new WorkspaceSyncException(result.Outcome, $"The home refused version {local.Number} saved as project '{copy.Name}': {result.Outcome}."); pushed.Add(stored); - parent = stored; + (parentId, parentNumber) = (stored.Id, stored.Number); + Step("saveboth.pushed"); } } // This project takes the home's head: the local versions now live in the copy (and their files in versions/.replaced). var head = await home.HeadAsync(project.ProjectId, cancellationToken).ConfigureAwait(false); Verify(head); project.WriteVersion(head, replace: true); - foreach (var local in locals) project.SetAside(local); var media = project.ReadMedia(); var working = ProjectSettings.FromJson(ProjectContent.Place(head.SettingsJson, media?.Photos ?? NoFrames), allowNewer: true); project.WriteBundle(WithBase(bundle, head) with { Project = working, Saved = DateTimeOffset.UtcNow }); + Step("saveboth.base"); + foreach (var local in locals) project.SetAside(local); + foreach (var move in pending?.Moves ?? []) SetAsideUnsent(project, move.LocalId); project.WriteSync(WorkspaceSyncState.None); return new WorkspaceSyncResult(await StatusLockedAsync(project, home, cancellationToken).ConfigureAwait(false), pushed, [head], copy); } diff --git a/tests/Chronos.App.Tests/Workspace/WorkspaceEditorTests.cs b/tests/Chronos.App.Tests/Workspace/WorkspaceEditorTests.cs index 642fc007..099da7c1 100644 --- a/tests/Chronos.App.Tests/Workspace/WorkspaceEditorTests.cs +++ b/tests/Chronos.App.Tests/Workspace/WorkspaceEditorTests.cs @@ -139,7 +139,13 @@ public sealed class WorkspaceEditorTests(TestContext context) Assert.Null(portable.HeroFrame, "positions travel by photo"); Assert.Contains(portable.PendingFrames!, f => f.Hero); + // A version saved here and not sent yet may reach the home under another id (a replay): the render names none. var saved = (await host.SaveVersionAsync(projectId, null)).Version!; + var (unsent, _) = ProcessingJobs.ReadRender(session.RenderRequests(session.Project.Export)[0].ParametersJson); + Assert.Null(unsent.ProjectVersionId, "a version not sent names no version"); + Assert.Equal(projectId, unsent.ProjectId); + // Sent, the home holds it under its id: the render names it. + await host.SyncAsync(projectId); var (versioned, _) = ProcessingJobs.ReadRender(session.RenderRequests(session.Project.Export)[0].ParametersJson); Assert.Equal(saved.Id, versioned.ProjectVersionId); Assert.Equal(parameters.SettingsJson, versioned.SettingsJson); diff --git a/tests/Chronos.Core.Tests/Project/ProjectBundleTests.cs b/tests/Chronos.Core.Tests/Project/ProjectBundleTests.cs index d9f8f049..711b9123 100644 --- a/tests/Chronos.Core.Tests/Project/ProjectBundleTests.cs +++ b/tests/Chronos.Core.Tests/Project/ProjectBundleTests.cs @@ -1,3 +1,4 @@ +using Chronos.Core.Localization; using Chronos.Core.Project; using Chronos.Testing; @@ -107,6 +108,23 @@ public sealed class ProjectBundleTests(TestContext context) Assert.Throws(() => ProjectBundle.Load(path)); } + [Test] + public void AFileNoReadCouldTakeBackIsRefusedBeforeItIsWritten() + { + var folder = context.CreateTempDirectory("bundle"); + var path = Path.Combine(folder, "huge.chronos"); + var sample = Sample(); + var huge = sample with { Sequence = sample.Sequence with { Title = new string('x', ProjectBundle.MaxBytes) } }; + var refused = Assert.Throws(() => huge.Save(path)); + Assert.Equal(Loc.L("Errors.Project.TooLarge", ProjectBundle.MaxBytes >> 20), refused.Message); + Assert.False(File.Exists(path)); + Assert.Empty(Directory.GetFiles(folder)); + + // Within the limit, the file is written and read back. + sample.Save(path); + Assert.Equal(sample.Sequence.Title, ProjectBundle.Load(path).Sequence.Title); + } + [Test] public void FramePositionsApplyOnlyToTheFramesTheyWereSavedFor() { diff --git a/tests/Chronos.Data.Tests/Jobs/JobPriorityQueueTests.cs b/tests/Chronos.Data.Tests/Jobs/JobPriorityQueueTests.cs index ffa74cd8..9f87aa39 100644 --- a/tests/Chronos.Data.Tests/Jobs/JobPriorityQueueTests.cs +++ b/tests/Chronos.Data.Tests/Jobs/JobPriorityQueueTests.cs @@ -17,6 +17,11 @@ public sealed class JobPriorityQueueTests Assert.Equal(JobKindPriority.High, queue.LevelOf(JobKind.Render)); Assert.Equal(JobKindPriority.High, queue.LevelOf(JobKind.Import)); Assert.Equal(JobKindPriority.High, queue.LevelOf(JobKind.Download)); + // Edit waits for the copy of a project's media into the workspace: it goes before scans and analyses. + Assert.Equal(JobKindPriority.High, queue.LevelOf(JobKind.WorkspaceCopy)); + var on = queue with { Enabled = true }; + Assert.True(on.Interrupts(JobKind.WorkspaceCopy, JobKind.Scan)); + Assert.False(on.Interrupts(JobKind.WorkspaceCopy, JobKind.Render)); Assert.Equal(JobKindPriority.Low, queue.LevelOf(JobKind.Verify)); Assert.Equal(JobKindPriority.Low, queue.LevelOf(JobKind.Proxies)); foreach (var kind in new[] { JobKind.Scan, JobKind.Thumbs, JobKind.LightAnalyse, JobKind.DeepAnalyse, JobKind.Maintenance }) diff --git a/tests/Chronos.Data.Tests/Processing/RenderJobTests.cs b/tests/Chronos.Data.Tests/Processing/RenderJobTests.cs index f30c947f..c764a9f1 100644 --- a/tests/Chronos.Data.Tests/Processing/RenderJobTests.cs +++ b/tests/Chronos.Data.Tests/Processing/RenderJobTests.cs @@ -108,6 +108,18 @@ public sealed class RenderJobTests(TestContext context) var second = await fx.EnqueueAsync(requests[0] with { ParametersJson = System.Text.Json.JsonSerializer.Serialize(parameters, named) }); await fx.Archive.WaitForAsync(second.Id, JobState.Done, (int)TestTiming.Budget(60)); Assert.True((await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct)).All(r => r.ProjectVersionId == version.Id)); + + // A version the index does not hold (saved on a PC, sent later under another id), or one whose settings the job does + // not embed: the render records the project alone, never a version it did not render. + var embedded = System.Text.Json.JsonSerializer.Deserialize(requests[0].ParametersJson!, named)!; + var other = ProjectContent.ToPortable(settings with { ExcludedFrames = [3] }, photos); + foreach (var wrong in new[] { embedded with { ProjectVersionId = Guid.CreateVersion7() }, embedded with { SettingsJson = other } }) + { + var refused = await fx.EnqueueAsync(requests[0] with { ParametersJson = System.Text.Json.JsonSerializer.Serialize(wrong, named) }); + await fx.Archive.WaitForAsync(refused.Id, JobState.Done, (int)TestTiming.Budget(60)); + var recorded = (await fx.Db.Renders.GetAsync(Chronos.Data.Processing.Jobs.RenderJobHandler.RenderId(refused.Id, 0), fx.Ct))!; + Assert.Equal<(Guid?, Guid?)>((vertical.Id, null), (recorded.ProjectId, recorded.ProjectVersionId)); + } } [Test] diff --git a/tests/Chronos.Data.Tests/Workspaces/WorkspaceFixture.cs b/tests/Chronos.Data.Tests/Workspaces/WorkspaceFixture.cs index f6422baf..487d1332 100644 --- a/tests/Chronos.Data.Tests/Workspaces/WorkspaceFixture.cs +++ b/tests/Chronos.Data.Tests/Workspaces/WorkspaceFixture.cs @@ -44,6 +44,23 @@ internal sealed class WorkspaceFixture : IAsyncDisposable return (store, new WorkspaceSync(store, "Chronos on " + name) { LockWait = TimeSpan.FromSeconds(TestTiming.Budget(10)) }); } + /// + /// Another sync of that stops, as a crash would, at the -th time it + /// reaches step (), throwing . + /// + public static WorkspaceSync Cutting(WorkspaceStore store, string step, int occurrence = 1) + { + var seen = 0; + return new WorkspaceSync(store, "Chronos cut") + { + LockWait = TimeSpan.FromSeconds(TestTiming.Budget(10)), + Cut = name => + { + if (name == step && ++seen == occurrence) throw new CutException(step); + }, + }; + } + /// Main of the sequence checked out into 's workspace, its media copied. public async Task CheckoutAsync(WorkspaceSync sync, Guid? projectId = null) { @@ -81,6 +98,52 @@ internal sealed class WorkspaceFixture : IAsyncDisposable } } +/// A crash between two writes of a sync (). +internal sealed class CutException(string step) : Exception($"Cut at {step}."); + +/// +/// A home whose pushes can be cut: one refused as out of reach before the home sees it, or one stored whose answer is lost +/// on the way back (the program killed, the network dropped). +/// +internal sealed class CuttingHome(IProjectHome home) : IProjectHome +{ + public int Pushes { get; private set; } + + /// The push (counted from 1) refused as out of reach. + public int? Refuse { get; set; } + + /// The push (counted from 1) the home stores and whose answer is lost. + public int? LoseAnswer { get; set; } + + public async Task PushAsync(Guid projectId, ProjectVersion version, CancellationToken cancellationToken) + { + var push = ++Pushes; + if (push == Refuse) throw new HomeUnavailableException("The home cannot be reached."); + var result = await home.PushAsync(projectId, version, cancellationToken); + if (push == LoseAnswer) throw new HomeUnavailableException("The answer was lost."); + return result; + } + + public Task SequenceAsync(CancellationToken cancellationToken) => home.SequenceAsync(cancellationToken); + public Task> ListAsync(CancellationToken cancellationToken) => home.ListAsync(cancellationToken); + + public Task CreateAsync(string name, Guid? fromVersionId, string? author, CancellationToken cancellationToken) => + home.CreateAsync(name, fromVersionId, author, cancellationToken); + + public Task HeadAsync(Guid projectId, CancellationToken cancellationToken) => home.HeadAsync(projectId, cancellationToken); + + public Task GetVersionAsync(Guid projectId, Guid versionId, CancellationToken cancellationToken) => + home.GetVersionAsync(projectId, versionId, cancellationToken); + + public Task> VersionsAfterAsync(Guid projectId, int afterNumber, int limit, CancellationToken cancellationToken) => + home.VersionsAfterAsync(projectId, afterNumber, limit, cancellationToken); + + public Task> HistoryAsync(Guid projectId, int? beforeNumber, int limit, CancellationToken cancellationToken) => + home.HistoryAsync(projectId, beforeNumber, limit, cancellationToken); + + public Task FramesAsync(CancellationToken cancellationToken) => home.FramesAsync(cancellationToken); +} + /// A home that cannot be reached, standing for a server out of reach or an archive offline. internal sealed class UnreachableHome(IProjectHome home) : IProjectHome { diff --git a/tests/Chronos.Data.Tests/Workspaces/WorkspaceRecoveryTests.cs b/tests/Chronos.Data.Tests/Workspaces/WorkspaceRecoveryTests.cs new file mode 100644 index 00000000..774f215f --- /dev/null +++ b/tests/Chronos.Data.Tests/Workspaces/WorkspaceRecoveryTests.cs @@ -0,0 +1,334 @@ +using System.Security.Cryptography; +using Chronos.Core.Archiving; +using Chronos.Core.Localization; +using Chronos.Core.Model; +using Chronos.Core.Project; +using Chronos.Data.Archiving; +using Chronos.Data.Projects; +using Chronos.Data.Tests.Archiving; +using Chronos.Data.Tests.Infrastructure; +using Chronos.Data.Tests.Projects; +using Chronos.Data.Workspaces; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Workspaces; + +/// +/// The workspace survives a cut at every step (owner review 2, R8): a push, a replay and a save-both stopped between two +/// writes lose no version and make no false conflict, and the next sync finishes them; the files it writes are whole. +/// +[Category("workspace")] +public abstract class WorkspaceRecoveryTests(TestContext context) : DatabaseTest(context) +{ + private async Task FixtureAsync(int frames = 5, Func? configure = null) + { + var archive = Own(await ArchiveFixture.CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true), configure: configure)); + return Own(await WorkspaceFixture.CreateAsync(Context, archive, frames)); + } + + private static async Task EditAsync(WorkspaceSync sync, Guid projectId, Func edit, CancellationToken ct) + { + var session = await sync.OpenAsync(projectId, ct); + await sync.SaveAsync(projectId, edit(session.Working), ct); + } + + private static ProjectSettings Offset(ProjectSettings s, double ev) => s with { Exposure = s.Exposure with { ExposureOffsetEv = ev } }; + + private static ProjectSettings Settings(ProjectVersion version) => ProjectSettings.FromJson(version.SettingsJson); + + private static async Task> NotesAsync(WorkspaceFixture fx, Guid projectId) => + [.. (await fx.Home.HistoryAsync(projectId, null, 50, fx.Ct)).OrderBy(v => v.Number).Select(v => v.Note)]; + + [Test] + public async Task APushCutAfterTheBaseMovedLosesNoVersionAndMakesNoFalseConflict() + { + var fx = await FixtureAsync(); + var (store, sync) = fx.Workspace(); + var project = await fx.CheckoutAsync(sync); + var id = project.ProjectId; + await EditAsync(sync, id, s => Offset(s, 0.5), fx.Ct); + var first = (await sync.CommitAsync(id, "first", cancellationToken: fx.Ct))!; + await EditAsync(sync, id, s => Offset(s, 0.8), fx.Ct); + var second = (await sync.CommitAsync(id, "second", cancellationToken: fx.Ct))!; + + // Stored at the home, the base moved onto it, then the program died: 'second' still waits after it. + await Assert.ThrowsAsync(() => WorkspaceFixture.Cutting(store, "push.base").PushAsync(id, fx.Home, fx.Ct)); + var status = await sync.StatusAsync(id, fx.Home, fx.Ct); + Assert.Equal((WorkspaceState.Ahead, 1, 0, first.Id), (status.State, status.Ahead, status.Behind, status.Base!.VersionId)); + Assert.Equal(second.Id, sync.Unpushed(id).Single().Id); + + // The next sync, as the app runs it (pull, then push): no conflict, both versions at the home, the local file adopted. + Assert.Null((await sync.PullAsync(id, fx.Home, fx.Ct)).Status.Conflict); + Assert.Equal(WorkspaceState.InSync, (await sync.PushAsync(id, fx.Home, fx.Ct)).Status.State); + Assert.SequenceEqual(["first", "second"], (await NotesAsync(fx, id)).Skip(1)); + Assert.Equal(0.8, Settings(await fx.Home.HeadAsync(id, fx.Ct)).Exposure.ExposureOffsetEv); + Assert.True(project.Versions().Where(v => v.Id == first.Id).All(v => v.StoredAt is not null), "the local file reads as sent"); + Assert.Empty(project.Versions().Where(v => v.StoredAt is null)); + } + + [Test] + public async Task APushWhoseAnswerWasLostIsFinishedByTheNextSync() + { + var fx = await FixtureAsync(); + var (_, sync) = fx.Workspace(); + var project = await fx.CheckoutAsync(sync); + var id = project.ProjectId; + await EditAsync(sync, id, s => Offset(s, 0.5), fx.Ct); + await sync.CommitAsync(id, "first", cancellationToken: fx.Ct); + await EditAsync(sync, id, s => Offset(s, 0.8), fx.Ct); + await sync.CommitAsync(id, "second", cancellationToken: fx.Ct); + + await Assert.ThrowsAsync(() => sync.PushAsync(id, new CuttingHome(fx.Home) { LoseAnswer = 1 }, fx.Ct)); + Assert.Equal(2, sync.Unpushed(id).Count); + Assert.Null((await sync.PullAsync(id, fx.Home, fx.Ct)).Status.Conflict); + Assert.Equal(WorkspaceState.InSync, (await sync.PushAsync(id, fx.Home, fx.Ct)).Status.State); + Assert.SequenceEqual(["first", "second"], (await NotesAsync(fx, id)).Skip(1)); + Assert.Equal(0.8, Settings(await fx.Home.HeadAsync(id, fx.Ct)).Exposure.ExposureOffsetEv); + } + + [Test] + public async Task AReplayCutAtAnyStepLosesNoVersionEvenWhenTheHomeMovesAgain() + { + var fx = await FixtureAsync(); + var cuts = new[] { ("replay.journal", 1), ("replay.version", 1), ("replay.version", 2), ("replay.base", 1) }; + foreach (var (step, occurrence) in cuts) + { + var label = $"{step} #{occurrence}"; + var id = (await fx.Archive.Service.Projects.CreateAsync(fx.Sequence.Id, $"Cut {step} {occurrence}", cancellationToken: fx.Ct)).Id; + var (store, one) = fx.Workspace("one"); + var (_, two) = fx.Workspace("two"); + var project = await fx.CheckoutAsync(one, id); + await fx.CheckoutAsync(two, id); + await EditAsync(two, id, s => s with { Exposure = s.Exposure with { DeflickerStrength = new Auto().WithManual(0.6) } }, fx.Ct); + await two.CommitAsync(id, cancellationToken: fx.Ct); + await two.PushAsync(id, fx.Home, fx.Ct); + await EditAsync(one, id, s => Offset(s, 0.3), fx.Ct); + await one.CommitAsync(id, "offset", cancellationToken: fx.Ct); + await EditAsync(one, id, s => s with { ExcludedFrames = [3], Stack = new StackSettings { Mode = StackSetting.Mean } }, fx.Ct); + await one.CommitAsync(id, "stack", cancellationToken: fx.Ct); + + // The push meets the home moved: the replay is cut, then the home moves again before this PC syncs. + await Assert.ThrowsAsync(() => WorkspaceFixture.Cutting(store, step, occurrence).PushAsync(id, fx.Home, fx.Ct)); + Assert.Equal(2, one.Unpushed(id).Count, label); + await two.PullAsync(id, fx.Home, fx.Ct); + await EditAsync(two, id, s => s with { Exposure = s.Exposure with { DeflickerStrength = new Auto().WithManual(0.7) } }, fx.Ct); + await two.CommitAsync(id, "again", cancellationToken: fx.Ct); + await two.PushAsync(id, fx.Home, fx.Ct); + + Assert.Null((await one.PullAsync(id, fx.Home, fx.Ct)).Status.Conflict, label); + Assert.Equal(WorkspaceState.InSync, (await one.PushAsync(id, fx.Home, fx.Ct)).Status.State, label); + var notes = await NotesAsync(fx, id); + Assert.Equal((1, 1), (notes.Count(n => n == "offset"), notes.Count(n => n == "stack")), label); + var head = ProjectSettings.FromJson(ProjectContent.Place((await fx.Home.HeadAsync(id, fx.Ct)).SettingsJson, ProjectServiceTests.Photos(fx.Frames))); + Assert.Equal((0.7, 0.3, StackSetting.Mean), (head.Exposure.DeflickerStrength.Resolve(1), head.Exposure.ExposureOffsetEv, head.Stack.Mode), label); + Assert.SequenceEqual([3], head.ExcludedFrames, label); + Assert.Null(project.ReadSync().Replay, label); + Assert.Empty(project.Versions().Where(v => v.StoredAt is null)); + } + } + + [Test] + public async Task AConflictMetPastAVersionTheHomeHeldKeepsTheVersionsAfterIt() + { + var fx = await FixtureAsync(); + var (_, one) = fx.Workspace("one"); + var (_, two) = fx.Workspace("two"); + var id = (await fx.CheckoutAsync(one)).ProjectId; + await fx.CheckoutAsync(two, id); + await EditAsync(one, id, s => Offset(s, 0.5), fx.Ct); + var first = (await one.CommitAsync(id, "first", cancellationToken: fx.Ct))!; + await EditAsync(one, id, s => s with { Stack = new StackSettings { Mode = StackSetting.Mean } }, fx.Ct); + var second = (await one.CommitAsync(id, "second", cancellationToken: fx.Ct))!; + // 'first' is stored, its answer lost; another PC builds on it. + await Assert.ThrowsAsync(() => one.PushAsync(id, new CuttingHome(fx.Home) { LoseAnswer = 1 }, fx.Ct)); + await two.PullAsync(id, fx.Home, fx.Ct); + await EditAsync(two, id, s => Offset(s, 1.0), fx.Ct); + await two.CommitAsync(id, "other", cancellationToken: fx.Ct); + await two.PushAsync(id, fx.Home, fx.Ct); + + // This PC changed the offset too, without saving a version: a real conflict, met past 'first'. + await EditAsync(one, id, s => Offset(s, 2.0), fx.Ct); + var stopped = await one.PullAsync(id, fx.Home, fx.Ct); + Assert.Equal(WorkspaceState.Conflict, stopped.Status.State); + Assert.Equal((first.Id, 1), (stopped.Status.Base!.VersionId, stopped.Status.Ahead)); + Assert.Equal(second.Id, one.Unpushed(id).Single().Id); + Assert.Equal("exposure.exposureOffsetEv", stopped.Status.Conflict!.Differences.Single().Path); + + // The server's offset; 'second''s own change, which nobody else made, reaches the home. + await one.ResolveAsync(id, fx.Home, ConflictChoice.TakeServer, cancellationToken: fx.Ct); + Assert.Equal(WorkspaceState.InSync, (await one.PushAsync(id, fx.Home, fx.Ct)).Status.State); + var head = Settings(await fx.Home.HeadAsync(id, fx.Ct)); + Assert.Equal((1.0, StackSetting.Mean), (head.Exposure.ExposureOffsetEv, head.Stack.Mode)); + Assert.SequenceEqual(["first", "other", "second"], (await NotesAsync(fx, id)).Skip(1)); + } + + [Test] + public async Task ASaveBothCutShortIsFinishedIntoTheSameCopy() + { + var fx = await FixtureAsync(); + var (_, one) = fx.Workspace("one"); + var (store, three) = fx.Workspace("three"); + var id = (await fx.CheckoutAsync(one)).ProjectId; + var project = await fx.CheckoutAsync(three, id); + await EditAsync(one, id, s => Offset(s, 0.5), fx.Ct); + await one.CommitAsync(id, cancellationToken: fx.Ct); + await one.PushAsync(id, fx.Home, fx.Ct); + await EditAsync(three, id, s => Offset(s, 2), fx.Ct); + await three.CommitAsync(id, "bright", cancellationToken: fx.Ct); + await EditAsync(three, id, s => s with { Stack = new StackSettings { Mode = StackSetting.StarTrails } }, fx.Ct); + Assert.Equal(WorkspaceState.Conflict, (await three.PushAsync(id, fx.Home, fx.Ct)).Status.State); + + // The copy is made and the first version sent, then the home goes out of reach. + var cut = new CuttingHome(fx.Home) { Refuse = 2 }; + await Assert.ThrowsAsync(() => three.ResolveAsync(id, cut, ConflictChoice.SaveBoth, saveBothName: "Main (PC)", cancellationToken: fx.Ct)); + var partial = (await fx.Home.ListAsync(fx.Ct)).Single(p => p.Name == "Main (PC)"); + Assert.Equal(WorkspaceState.Conflict, (await three.StatusAsync(id, fx.Home, fx.Ct)).State); + + // The retry, asked another name as the app asks once the first is taken, sends the rest to the same copy. + var both = await three.ResolveAsync(id, fx.Home, ConflictChoice.SaveBoth, saveBothName: "Main (other)", cancellationToken: fx.Ct); + Assert.Equal(partial.Id, both.SavedAs!.Id); + Assert.Equal(2, (await fx.Home.ListAsync(fx.Ct)).Count); + var kept = Settings(await fx.Home.HeadAsync(partial.Id, fx.Ct)); + Assert.Equal((2, StackSetting.StarTrails), (kept.Exposure.ExposureOffsetEv, kept.Stack.Mode)); + Assert.Equal(3, (await fx.Home.HistoryAsync(partial.Id, null, 10, fx.Ct)).Count); + Assert.Equal(WorkspaceState.InSync, both.Status.State); + Assert.Equal(WorkspaceSyncState.None, project.ReadSync()); + + // Cut once this project moved onto the home's head: the next sync finishes it, the conflict settled. + await EditAsync(one, id, s => Offset(s, 0.7), fx.Ct); + await one.CommitAsync(id, cancellationToken: fx.Ct); + await one.PushAsync(id, fx.Home, fx.Ct); + await EditAsync(three, id, s => Offset(s, 3), fx.Ct); + await three.CommitAsync(id, "brighter", cancellationToken: fx.Ct); + Assert.Equal(WorkspaceState.Conflict, (await three.PushAsync(id, fx.Home, fx.Ct)).Status.State); + await Assert.ThrowsAsync(() => WorkspaceFixture.Cutting(store, "saveboth.base").ResolveAsync(id, fx.Home, ConflictChoice.SaveBoth, cancellationToken: fx.Ct)); + var settled = await three.PushAsync(id, fx.Home, fx.Ct); + Assert.Equal((WorkspaceState.InSync, 0), (settled.Status.State, settled.Status.Ahead)); + Assert.Equal(0.7, (await three.OpenAsync(id, fx.Ct)).Working.Exposure.ExposureOffsetEv); + var second = (await fx.Home.ListAsync(fx.Ct)).Single(p => p.Name == "Main (2)"); + Assert.Equal(3, Settings(await fx.Home.HeadAsync(second.Id, fx.Ct)).Exposure.ExposureOffsetEv); + Assert.Equal(WorkspaceSyncState.None, project.ReadSync()); + Assert.Empty(project.Versions().Where(v => v.StoredAt is null)); + } + + [Test] + public async Task AProjectWithTwentyThousandParkedExclusionsIsCheckedOutEditedAndSent() + { + var fx = await FixtureAsync(3); + var projects = fx.Archive.Service.Projects; + var large = await projects.CreateAsync(fx.Sequence.Id, "Large", cancellationToken: fx.Ct); + var head = await projects.HeadAsync(large.Id, fx.Ct); + // Exclusions of photos this sequence does not hold (taken from another copy): they wait, parked by photo. + var pending = Enumerable.Range(0, 20_000).Select(i => new PendingFrame + { + Sha256 = Convert.ToHexStringLower(SHA256.HashData(BitConverter.GetBytes(i))), + Size = 1_000_000 + i, + CapturedAt = new DateTime(2024, 1, 1).AddMinutes(i), + Excluded = true, + }).ToList(); + var portable = ProjectContent.Canonical((Settings(head) with { PendingFrames = pending }).ToJson()); + var draft = head with + { + Id = Guid.CreateVersion7(), + ParentId = head.Id, + Number = head.Number + 1, + Kind = ProjectVersionKind.Commit, + SettingsJson = portable, + ContentHash = ProjectContent.Hash(portable), + StoredAt = null, + }; + Assert.Equal(PushOutcome.Stored, (await projects.PushAsync(large.Id, draft, fx.Ct)).Outcome); + + var (_, sync) = fx.Workspace(); + var project = await fx.CheckoutAsync(sync, large.Id); + Assert.True(new FileInfo(project.BundlePath).Length > 4 << 20, "larger than a project file could be before"); + var session = await sync.OpenAsync(large.Id, fx.Ct); + Assert.Equal(20_000, session.Working.PendingFrames!.Count); + await sync.SaveAsync(large.Id, Offset(session.Working, 0.2), fx.Ct); + Assert.NotNull(await sync.CommitAsync(large.Id, cancellationToken: fx.Ct)); + Assert.Equal(WorkspaceState.InSync, (await sync.PushAsync(large.Id, fx.Home, fx.Ct)).Status.State); + Assert.Equal(20_000, Settings(await fx.Home.HeadAsync(large.Id, fx.Ct)).PendingFrames!.Count); + } + + [Test] + public async Task TheCopyStopsBeforeItFillsTheWorkspaceDrive() + { + var fx = await FixtureAsync(3, options => options with { FreeSpace = _ => 1 << 20 }); + var (store, sync) = fx.Workspace(); + var main = await fx.Archive.Service.Projects.MainAsync(fx.Sequence.Id, fx.Ct); + var project = await sync.CheckoutAsync(fx.Home, main.Id, cancellationToken: fx.Ct); + var job = await fx.CopyAsync(store, main.Id, JobState.Failed); + var parts = Loc.L("Errors.Workspace.DiskFull", "\u0001").Split('\u0001'); + Assert.True(job.Error!.StartsWith(parts[0], StringComparison.Ordinal) && job.Error.EndsWith(parts[1], StringComparison.Ordinal), job.Error); + Assert.Null(project.ReadMedia()); + Assert.Empty(Directory.GetFiles(project.ProxiesDirectory)); + } + + [Test] + public async Task ACopyResumedAfterTheSequenceMovedChecksEveryFileAgain() + { + var fx = await FixtureAsync(); + var (store, sync) = fx.Workspace(); + var main = await fx.Archive.Service.Projects.MainAsync(fx.Sequence.Id, fx.Ct); + var project = await sync.CheckoutAsync(fx.Home, main.Id, cancellationToken: fx.Ct); + await fx.Archive.Service.Images.BuildProxiesAsync(fx.Archive.Archive, fx.Sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct); + await fx.CopyAsync(store, main.Id); + var copied = project.ReadMedia()!; + var damaged = project.MediaFile(copied.Frames[0].File!); + + // A run of the same frames in the same folder keeps what it had copied. + File.WriteAllText(damaged, "not a photo"); + var job = await fx.CopyAsync(store, main.Id, checkpoint: $"copy:{copied.Frames.Count}:{fx.Sequence.FramesVersion}:{fx.Sequence.Folder}"); + Assert.Contains(Loc.L("Archive.Log.WorkspaceCopied", fx.Sequence.DisplayTitle, 0, 0, 5, 0), job.LogTail!); + // One read in another folder (the sequence moved since, another may hold its old one) checks every file again. + job = await fx.CopyAsync(store, main.Id, checkpoint: $"copy:{copied.Frames.Count}:{fx.Sequence.FramesVersion}:elsewhere/folder"); + Assert.Contains(Loc.L("Archive.Log.WorkspaceCopied", fx.Sequence.DisplayTitle, 1, 0, 4, 0), job.LogTail!); + Assert.Equal(copied.Frames[0].FileSha256, Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(damaged)))); + } + + [Test] + public async Task TheEditorReadsTheWorkingStateAndTheMediaUnderTheLease() + { + var fx = await FixtureAsync(2); + var (store, sync) = fx.Workspace(); + var project = await fx.CheckoutAsync(sync); + var impatient = new WorkspaceSync(store, "Chronos test") { LockWait = TimeSpan.FromMilliseconds(200) }; + using (await project.LockAsync(TimeSpan.FromSeconds(TestTiming.Budget(5)), fx.Ct)) + await Assert.ThrowsAsync(() => impatient.OpenAsync(project.ProjectId, fx.Ct)); + Assert.NotNull((await impatient.OpenAsync(project.ProjectId, fx.Ct)).Sequence); + } + + [Test] + public void AManifestLeftUnfinishedIsMadeAgainOnlyWhileTheFolderHoldsNoProject() + { + var root = Path.Combine(Context.CreateTempDirectory("ws"), "workspace"); + Directory.CreateDirectory(root); + var manifest = Path.Combine(root, WorkspaceStore.ManifestFile); + File.WriteAllBytes(manifest, []); + var store = new WorkspaceStore(root); + Assert.Equal(WorkspaceStore.FormatName, store.Manifest.Format); + Assert.Empty(Directory.GetFiles(root, "*.tmp")); + Assert.Equal(store.Id, new WorkspaceStore(root).Id); + + File.WriteAllText(manifest, """{ "format": "chronos-wor"""); + Assert.NotEqual(store.Id, new WorkspaceStore(root).Id); + + // With a project in the folder, a damaged manifest is never written over: it is reported. + var project = Path.Combine(root, Guid.CreateVersion7().ToString("N")); + Directory.CreateDirectory(project); + File.WriteAllText(Path.Combine(project, WorkspaceProject.BundleFile), "{}"); + File.WriteAllText(manifest, """{ "format": "chronos-wor"""); + Assert.Throws(() => new WorkspaceStore(root)); + } +} + +public sealed class SqliteWorkspaceRecoveryTests(TestContext context) : WorkspaceRecoveryTests(context) +{ + protected override Backend Backend => Backend.SqliteFile; +} + +[RequiresEnvironment(PgTestServer.Variable)] +public sealed class PostgresWorkspaceRecoveryTests(TestContext context) : WorkspaceRecoveryTests(context) +{ + protected override Backend Backend => Backend.Postgres; +}