diff --git a/i18n/en/Errors.resx b/i18n/en/Errors.resx index 916a0c5c..2c90826f 100644 --- a/i18n/en/Errors.resx +++ b/i18n/en/Errors.resx @@ -16,6 +16,7 @@ This path is a file, not a folder. The drive of this folder is not available. Enter a full path that starts with a drive or a network share. + The project's settings are larger than {0} MB; Chronos cannot store them.{0} megabytes {0} could not be downloaded intact.{0} file name Exports failed: {0} of {1}; see the job log.{0} failed count; {1} total count Render nodes send back one file per job; post each preset of a batch as its own job.long @@ -51,6 +52,7 @@ Settle the conflict first: Sync shows which settings differ. {0} did not match its source after it was copied.{0} file path A file of this project in the workspace cannot be read; the log names it. + The workspace drive needs {0} GB more for this copy.{0} gigabytes missing The last edit could not be stored yet; it is tried again, try once more in a moment. The library's copy of this project does not match what was sent; nothing was removed. The workspace folder is inside a library; nothing is copied into it. diff --git a/i18n/it/Errors.it.resx b/i18n/it/Errors.it.resx index d2a4b811..e6671d4d 100644 --- a/i18n/it/Errors.it.resx +++ b/i18n/it/Errors.it.resx @@ -16,6 +16,7 @@ Questo percorso è un file, non una cartella. L'unità di questa cartella non è disponibile. Inserisci un percorso completo che inizi con un'unità o una condivisione di rete. + Le impostazioni del progetto superano {0} MB; Chronos non può salvarle. {0} non è stato scaricato integro. Esportazioni non riuscite: {0} su {1}; vedi il registro del job. I nodi di rendering restituiscono un file per job; invia ogni preset di un gruppo come job a sé. @@ -51,6 +52,7 @@ Risolvi prima il conflitto: Sincronizza mostra quali impostazioni sono diverse. {0} non corrispondeva all'originale dopo la copia. Un file di questo progetto nello spazio di lavoro non è leggibile; il log lo indica. + Al disco dello spazio di lavoro servono altri {0} GB per questa copia. L'ultima modifica non è ancora stata memorizzata; viene ritentata, riprova tra un momento. La copia di questo progetto nella libreria non corrisponde a quanto inviato; non è stato rimosso nulla. La cartella dello spazio di lavoro è dentro una libreria; non vi viene copiato nulla. diff --git a/src/Chronos.App/Archive/ArchiveFailures.cs b/src/Chronos.App/Archive/ArchiveFailures.cs index 2957eda4..fc4e484e 100644 --- a/src/Chronos.App/Archive/ArchiveFailures.cs +++ b/src/Chronos.App/Archive/ArchiveFailures.cs @@ -20,7 +20,7 @@ public static class ArchiveFailures // any more (FrameTrash.RestoreAsync). Other IOExceptions and InvalidDataExceptions (a sidecar of another sequence) // carry technical text. LeaseUnavailableException or NewerFormatException or StaleRenamePlanException or NestedRenameException or StaleDeletePlanException - or SequenceInLibraryException or TrashMismatchException => ex.Message, + or SequenceInLibraryException or TrashMismatchException or Core.Project.ProjectTooLargeException => ex.Message, UnauthorizedAccessException => Loc.L("Archive.Failed.Access"), DatabaseException => Loc.L("Archive.Failed.Index"), ArgumentException => Loc.L("Archive.Failed.Gone"), diff --git a/src/Chronos.App/Editor/WorkspaceSequenceSource.cs b/src/Chronos.App/Editor/WorkspaceSequenceSource.cs index 9ae8554f..87426f32 100644 --- a/src/Chronos.App/Editor/WorkspaceSequenceSource.cs +++ b/src/Chronos.App/Editor/WorkspaceSequenceSource.cs @@ -228,7 +228,7 @@ public sealed class WorkspaceSequenceSource : ISequenceSource, IProjectRenders /// The render at the home (owner review 2, R8: edit with proxies, render where the originals are): one job of this PC's /// engine per preset, with the project's portable settings (positions parked by photo, placed on the home's frames by /// the render) and the project recorded on the render; the version too when the settings are those of the last - /// version saved (an unsaved working state renders as it is, without a version). + /// version the home holds (an unsaved working state, or one saved here and not sent yet, renders as it is, without a version). /// public IReadOnlyList RenderRequests(Sequence sequence, ProjectSettings project, ExportSettings export, int sourceWidth, int sourceHeight) { @@ -293,12 +293,16 @@ public sealed class WorkspaceSequenceSource : ISequenceSource, IProjectRenders } } - /// The last version saved: the newest one not sent yet, else the base; with its content hash. + /// + /// The version a render may name, with its content hash: the base, which the home holds, while no version waits to be + /// sent; none otherwise. A version not sent yet may never reach the home under its id (a replay or a save-both sends it + /// under another), so a render of it names the project alone. + /// private (Guid? Id, string? Hash) Tip() { try { - if (_sync.Unpushed(ProjectId) is { Count: > 0 } unsent) return (unsent[^1].Id, unsent[^1].ContentHash); + if (_sync.Unpushed(ProjectId).Count > 0) return (null, null); return Project.ReadBundle().Workspace?.Base is { } basis ? (basis.VersionId, basis.ContentHash) : (null, null); } catch (Exception ex) when (ex is IOException or InvalidDataException or InvalidOperationException) diff --git a/src/Chronos.App/Shell/ErrorText.cs b/src/Chronos.App/Shell/ErrorText.cs index 4349d7dd..02e3dcad 100644 --- a/src/Chronos.App/Shell/ErrorText.cs +++ b/src/Chronos.App/Shell/ErrorText.cs @@ -43,7 +43,7 @@ public static class ErrorText private static string? Describe(Exception ex) => ex switch { LeaseUnavailableException or ArchiveOfflineException or NewerFormatException or ImportSourceUnavailableException - or CopyVerificationException or StaleRenamePlanException or SyncConflictChangedException => ex.Message, + or CopyVerificationException or StaleRenamePlanException or SyncConflictChangedException or Core.Project.ProjectTooLargeException => ex.Message, // A template an archive refused (Settings › Archive, or this PC's templates for a new archive): the rule it breaks. TemplateRuleException rule => Loc.L("Errors.Template." + rule.Rule), // The workspace (owner review 2, R8): its own sentences, a home out of reach, a project the home refused. diff --git a/src/Chronos.Core/Project/ProjectBundle.cs b/src/Chronos.Core/Project/ProjectBundle.cs index 6782811f..6f520ec0 100644 --- a/src/Chronos.Core/Project/ProjectBundle.cs +++ b/src/Chronos.Core/Project/ProjectBundle.cs @@ -2,10 +2,19 @@ using System.Globalization; using System.Text; using System.Text.Json; using Chronos.Core.IO; +using Chronos.Core.Localization; using Chronos.Core.Naming; namespace Chronos.Core.Project; +/// A project file that would be larger than : refused before it is written, with a sentence for the user. +public sealed class ProjectTooLargeException(long bytes) + : IOException(Loc.L("Errors.Project.TooLarge", ProjectBundle.MaxBytes >> 20)) +{ + /// The size the file would have had. + public long Bytes { get; } = bytes; +} + /// The archive a project file's sequence lives in: its id, and its name and folder when the file was saved. public sealed record BundleArchive(Guid Id, string Name, string? Path); @@ -70,8 +79,12 @@ public sealed record ProjectBundle(BundleArchive Archive, BundleSequence Sequenc /// The settings come from a newer Chronos (): show them, never edit them. public bool IsReadOnly => Project.IsNewerFormat; - /// Project settings take a few kilobytes; a larger file is not a project file. - public const int MaxBytes = 4 << 20; + /// + /// The largest project file read or written, one limit for both: a workspace's working state parks every position by + /// photo until its media are copied (some 260 bytes each), so a 100,000-frame sequence with every frame excluded takes + /// some 26 MB; a larger file is not a project file. + /// + public const int MaxBytes = 64 << 20; private static readonly JsonWriterOptions Indented = new() { Indented = true }; @@ -165,8 +178,17 @@ public sealed record ProjectBundle(BundleArchive Archive, BundleSequence Sequenc return Parse(reader.ReadToEnd()); } - /// Writes the file atomically (temp, flush, rename): an interrupted save never leaves half a file. - public void Save(string path) => AtomicFile.WriteAllText(path, ToJson()); + /// + /// Writes the file atomically (temp, flush, rename): an interrupted save never leaves half a file. A file larger than + /// is refused before anything is written, as no read could take it back. + /// + /// The file would be larger than . + public void Save(string path) + { + var bytes = new UTF8Encoding(false).GetBytes(ToJson()); + if (bytes.Length > MaxBytes) throw new ProjectTooLargeException(bytes.Length); + AtomicFile.WriteAllBytes(path, bytes); + } /// /// The settings to give the sequence when the file is opened, the sequence having diff --git a/src/Chronos.Data/Archiving/ArchiveService.cs b/src/Chronos.Data/Archiving/ArchiveService.cs index f128acea..49188aff 100644 --- a/src/Chronos.Data/Archiving/ArchiveService.cs +++ b/src/Chronos.Data/Archiving/ArchiveService.cs @@ -76,6 +76,9 @@ public sealed class ArchiveService : IJobHandlerProvider /// The decoders of this engine's image passes (thumbnails, proxies). public Core.Imaging.ImageDecoder Decoder => _writer.Options.Decoder; + /// How the host measures free space under a folder (); null: the folder's drive. + public Func? FreeSpace => _writer.Options.FreeSpace; + /// /// The steps this host's jobs can run: the shared ones (), /// then the host's (). A schedule host lists their names. diff --git a/src/Chronos.Data/Jobs/JobPriorityQueue.cs b/src/Chronos.Data/Jobs/JobPriorityQueue.cs index 2b0c9156..fca96fd0 100644 --- a/src/Chronos.Data/Jobs/JobPriorityQueue.cs +++ b/src/Chronos.Data/Jobs/JobPriorityQueue.cs @@ -22,14 +22,16 @@ public enum JobKindPriority public sealed record JobPriorityQueue { /// - /// The levels a kind has unless the user changed it: renders, imports and downloads first; the long passes that resume - /// anywhere, and the housekeeping of the schedules (backups, clean-ups), last. + /// The levels a kind has unless the user changed it: renders, imports and downloads first, and the copy of a project's + /// media into the workspace, which Edit waits for (owner review 2, R8); the long passes that resume anywhere, and the + /// housekeeping of the schedules (backups, clean-ups), last. /// public static IReadOnlyDictionary DefaultLevels { get; } = new Dictionary { [JobKind.Render] = JobKindPriority.High, [JobKind.Import] = JobKindPriority.High, [JobKind.Download] = JobKindPriority.High, + [JobKind.WorkspaceCopy] = JobKindPriority.High, [JobKind.Verify] = JobKindPriority.Low, [JobKind.Proxies] = JobKindPriority.Low, [JobKind.Backup] = JobKindPriority.Low, diff --git a/src/Chronos.Data/Processing/Jobs/RenderJobHandler.cs b/src/Chronos.Data/Processing/Jobs/RenderJobHandler.cs index 0148c616..ecf525e1 100644 --- a/src/Chronos.Data/Processing/Jobs/RenderJobHandler.cs +++ b/src/Chronos.Data/Processing/Jobs/RenderJobHandler.cs @@ -53,6 +53,7 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt var aliases = embedded is not null ? await Projects.HomePhotos.AliasesAsync(db, sequence.Id, cancellation).ConfigureAwait(false) : []; var project = (embedded is not null ? ProjectStore.Parse(Placed(embedded, frames, aliases)) : ProjectStore.Read(sequence, folder)) with { Export = export }; var settingsJson = project.ToJson(); + var (projectId, projectVersionId) = await ProvenanceAsync(parameters, sequence, cancellation).ConfigureAwait(false); using var names = new OutputNames(); var all = Targets(export, parameters.PresetName); @@ -90,8 +91,8 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt { Id = RenderId(context.Job.Id, index), OutputPath = Stored(paths, result.OutputPath), - ProjectId = parameters.ProjectId, - ProjectVersionId = parameters.ProjectVersionId, + ProjectId = projectId, + ProjectVersionId = projectVersionId, }; await db.Renders.InsertAsync(entity, CancellationToken.None).ConfigureAwait(false); done.Add(index); @@ -137,6 +138,27 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt : throw new InvalidOperationException(Loc.L("Errors.Render.ProjectVersion")); } + /// + /// The project and the version a render records (owner review 2, R8: a render never names a version whose settings it + /// did not use, nor one the index does not hold): the project when it is one of the sequence's; the version when the index + /// holds it, for that project of the sequence, with the content hash of the settings the job embeds. A job naming a + /// version the home never received (one saved on a PC and sent later under another id) records the project alone. + /// + private async Task<(Guid? Project, Guid? Version)> ProvenanceAsync(Protocol.RenderJobParameters parameters, Sequence sequence, CancellationToken cancellationToken) + { + Guid? project = parameters.ProjectId is { } id && await db.Projects.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } owner + && owner.SequenceId == sequence.Id + ? id + : null; + if (parameters.ProjectVersionId is not { } versionId + || await db.Projects.GetVersionAsync(versionId, cancellationToken).ConfigureAwait(false) is not { } version + || (project is { } named && version.ProjectId != named) + || await db.Projects.GetAsync(version.ProjectId, cancellationToken).ConfigureAwait(false) is not { } of || of.SequenceId != sequence.Id + || (parameters.SettingsJson is { } embedded && !ProjectContent.Verifies(embedded, version.ContentHash))) + return (project, null); + return (version.ProjectId, versionId); + } + /// /// Settings whose positions wait for their photos (a project version's portable form, settings synced from another copy) /// with them placed on the frames rendered, a photo parked on an encoding a conversion replaced since found through diff --git a/src/Chronos.Data/Workspaces/WorkspaceCopyJobHandler.cs b/src/Chronos.Data/Workspaces/WorkspaceCopyJobHandler.cs index fe7f75d6..945be6b3 100644 --- a/src/Chronos.Data/Workspaces/WorkspaceCopyJobHandler.cs +++ b/src/Chronos.Data/Workspaces/WorkspaceCopyJobHandler.cs @@ -11,6 +11,7 @@ using Chronos.Data.Archiving; using Chronos.Data.Archiving.Jobs; using Chronos.Data.Jobs; using Chronos.Data.Projects; +using Chronos.Data.Sync; namespace Chronos.Data.Workspaces; @@ -47,7 +48,10 @@ public static class WorkspaceJobs /// Frames changed at the home during the copy, or the folder moved (a Reorganize), make it start again (three times at /// most; a folder still moving fails it, to be retried). The working state's positions follow the new frames: by frame /// where a frame stays (one whose file a conversion replaced keeps its positions), by photo where frames came or went. -/// Checkpoint: copy:<frame>:<frames version>; a run resumed on other frames checks every file again. +/// Before each pass and at each checkpoint the workspace's drive must hold what is still to copy plus a reserve, else the +/// job stops saying how much is missing (the workspace shares this PC's drive with the index and the logs). +/// Checkpoint: copy:<frame>:<frames version>:<folder>; a run resumed on other frames or in another folder +/// checks every file again, and a pass started again stores that before it reads anything. /// Originals come in phase 5c: asked for, the job fails saying so. /// internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService archives) : IJobHandler @@ -57,6 +61,17 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService private static readonly TimeSpan LockWait = TimeSpan.FromMinutes(2); private static readonly TimeSpan CheckpointEvery = TimeSpan.FromSeconds(5); + /// Bytes left free on the workspace's drive besides the copy, as a download keeps (). + private const long SpaceReserve = 512L << 20; + + /// A proxy at px, when the archive has none to measure: some 1.5 MB. + private const long ProxyEstimate = 3L << 19; + + private const int ProxyEstimateEdge = 1920; + + /// The long edge taken for a full-size proxy of a frame whose size is not known. + private const int UnknownFullSize = 6000; + public JobKind Kind => JobKind.WorkspaceCopy; public async Task RunAsync(JobContext context) @@ -80,20 +95,29 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService var paths = new ArchivePaths(archive.RootPath); // Read only: a read-only archive is copied from like any other. if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(archive.RootPath, ArchiveAvailability.Offline); - var (copied, checkpointVersion) = Parse(context.Checkpoint); + var (copied, checkpointVersion, checkpointFolder) = Parse(context.Checkpoint); for (var pass = 1; ; pass++) { var (row, frames) = await FramesAsync(sequenceId, ct).ConfigureAwait(false); var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy; var folder = paths.Sequence(row.Folder); - // Another index moved the sequence away, maybe another one in: its files are not this sequence's photos. - if (!SequenceFolders.Holds(folder, sequenceId)) throw new SequenceFolderMovedException(); - // Files kept from an earlier run are this frames version's only (a conversion keeps a frame's id, not its photo). - if (checkpointVersion is { } version && version != row.FramesVersion) copied = 0; + // Another index moved the sequence away, maybe another one in: its files are not this sequence's photos. A retry + // checks every file again (the files of this run may be of that folder). + if (!SequenceFolders.Holds(folder, sequenceId)) + { + await context.CheckpointAsync(Checkpoint(Stage.Copy, 0, row.FramesVersion, row.Folder), 0, frames.Count).ConfigureAwait(false); + throw new SequenceFolderMovedException(); + } + // Files kept from an earlier run are this frames version's and this folder's only (a conversion keeps a frame's id, + // not its photo; a sequence moved meanwhile may have left its photos' names to another sequence's). + if ((checkpointVersion is { } version && version != row.FramesVersion) || (checkpointFolder is { } read && read != row.Folder)) copied = 0; var total = (long)frames.Count; var tally = new Tally(); var entries = new WorkspaceFrame[frames.Count]; Directory.CreateDirectory(project.ProxiesDirectory); + // The workspace is on this PC's own drive, with the index, the settings and the logs: the copy never fills it. + var remaining = Remaining(project, folder, frames, policy); + EnsureSpace(parameters.WorkspaceRoot, remaining[0]); var clock = System.Diagnostics.Stopwatch.StartNew(); for (var i = 0; i < frames.Count; i++) { @@ -101,7 +125,9 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService entries[i] = Copy(project, folder, paths, frames[i], policy, done: i < copied, tally, ct); if (clock.Elapsed >= CheckpointEvery) { - await context.CheckpointAsync(Checkpoint(Stage.Copy, i + 1, row.FramesVersion), i + 1, total).ConfigureAwait(false); + await context.CheckpointAsync(Checkpoint(Stage.Copy, i + 1, row.FramesVersion, row.Folder), i + 1, total).ConfigureAwait(false); + // Another program may take the space meanwhile. + EnsureSpace(parameters.WorkspaceRoot, remaining[i + 1]); clock.Restart(); } else @@ -114,15 +140,21 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService var moved = after.Folder != row.Folder || !SequenceFolders.Holds(paths.Sequence(after.Folder), sequenceId) || Renamed(frames, now, entries); if (after.FramesVersion != row.FramesVersion || moved) { - // The files read may be of other frames, or missing where a Reorganize took them: read them again. + // The files read may be of other frames, or missing where a Reorganize took them: read them again, and so + // does a run resumed after a crash in the next pass (the checkpoint stored says so). if (pass < Passes) { context.Log(Loc.L("Archive.Log.WorkspaceFramesChanged", row.DisplayTitle)); - (copied, checkpointVersion) = (0, null); + await context.CheckpointAsync(Checkpoint(Stage.Copy, 0, after.FramesVersion, after.Folder), 0, total).ConfigureAwait(false); + (copied, checkpointVersion, checkpointFolder) = (0, null, null); continue; } // Frames still arriving are copied as they were read (Update media takes the rest); a folder still moving is not. - if (moved) throw new SequenceFolderMovedException(); + if (moved) + { + await context.CheckpointAsync(Checkpoint(Stage.Copy, 0, after.FramesVersion, after.Folder), 0, total).ConfigureAwait(false); + throw new SequenceFolderMovedException(); + } } var aliases = await HomePhotos.AliasesAsync(db, sequenceId, ct).ConfigureAwait(false); var media = new WorkspaceMedia(WorkspaceMedia.FormatName, WorkspaceMedia.CurrentFormatVersion, BundleMedia.Proxies, sequenceId, row.FramesVersion, entries, @@ -134,7 +166,7 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService RemoveStale(project, media); } context.Log(Loc.L("Archive.Log.WorkspaceCopied", row.DisplayTitle, tally.Copied, tally.Made, tally.Kept, tally.Missing)); - await context.CheckpointAsync(Checkpoint(Stage.Done, frames.Count, row.FramesVersion), total, total).ConfigureAwait(false); + await context.CheckpointAsync(Checkpoint(Stage.Done, frames.Count, row.FramesVersion, row.Folder), total, total).ConfigureAwait(false); return; } } @@ -205,6 +237,9 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService } if (bytes is null) { + // A file an earlier copy left for the frame (of other frames, or another folder's) is never kept as its photo by a + // resumed run: a regenerable copy, it goes. + if (File.Exists(target)) TryDelete(target); tally.Missing++; return WorkspaceFrame.Of(frame, null, null, 0); } @@ -292,15 +327,81 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService var listed = media.Frames.Where(f => f.File is not null).Select(f => Path.GetFileName(f.File!)).ToHashSet(StringComparer.OrdinalIgnoreCase); foreach (var file in Directory.EnumerateFiles(project.ProxiesDirectory, "*.jpg")) { - if (listed.Contains(Path.GetFileName(file))) continue; - try - { - File.Delete(file); - } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) - { - Log.Warning("workspace", $"{file} could not be removed", ex); - } + if (!listed.Contains(Path.GetFileName(file))) TryDelete(file); + } + } + + private static void TryDelete(string file) + { + try + { + File.Delete(file); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warning("workspace", $"{file} could not be removed", ex); + } + } + + /// + /// The bytes the copy still writes from each frame on (index i: frames i to the last; one more entry, 0): + /// nothing for a frame the workspace holds a file of already (kept, or replaced by one of about its size), the archive's + /// proxy's size where it has one, else an estimate at the proxy size the copy makes (). + /// + private static long[] Remaining(WorkspaceProject project, SequencePaths folder, List frames, ProxyPolicy policy) + { + var held = Directory.Exists(project.ProxiesDirectory) + ? Directory.EnumerateFiles(project.ProxiesDirectory).Select(Path.GetFileName).ToHashSet(StringComparer.OrdinalIgnoreCase) + : []; + var proxies = Directory.Exists(folder.Proxies) + ? new DirectoryInfo(folder.Proxies).EnumerateFiles().ToDictionary(f => f.Name, f => f.Length, StringComparer.OrdinalIgnoreCase) + : []; + var remaining = new long[frames.Count + 1]; + for (var i = frames.Count - 1; i >= 0; i--) + { + var frame = frames[i]; + var bytes = held.Contains(ProxyCache.FileName(frame.Id)) ? 0 + : proxies.TryGetValue(ArchiveLayout.ProxyFileName(frame.Index), out var length) ? length + : (frame.Flags & FrameFlags.Missing) != 0 ? 0 + : Estimate(frame, policy); + remaining[i] = remaining[i + 1] + bytes; + } + return remaining; + } + + /// A proxy's size when the archive has none to measure: some 1.5 MB at 1920 px, scaled by the area at the copy's long edge. + private static long Estimate(Frame frame, ProxyPolicy policy) + { + var edge = policy.LongEdge == ProxyPolicy.Disabled ? new ProxyPolicy().LongEdge : policy.LongEdge; + var longest = Math.Max(frame.Width, frame.Height); + var side = edge == ProxyPolicy.FullSize ? (longest > 0 ? longest : UnknownFullSize) : longest > 0 ? Math.Min(edge, longest) : edge; + return (long)(ProxyEstimate * Math.Pow(side / (double)ProxyEstimateEdge, 2)); + } + + /// + /// Refuses to go on while the workspace's drive lacks besides , with the + /// gigabytes missing; a drive that cannot be measured (a network share) is not refused, the copy itself then says so. + /// + /// The drive lacks the space (Errors.Workspace.DiskFull). + private void EnsureSpace(string root, long bytes) + { + if (bytes <= 0 || FreeBytes(root) is not { } free || free >= bytes + SpaceReserve) return; + throw new IOException(Loc.L("Errors.Workspace.DiskFull", ((bytes + SpaceReserve - free) / 1e9).ToString("0.0", Loc.Culture))); + } + + /// Free bytes on the volume of , as the host measures them (), else its drive's. + private long? FreeBytes(string folder) + { + if (archives.FreeSpace is { } measure) return measure(folder); + try + { + var full = Path.GetFullPath(folder); + // Windows wants a drive root; Unix accepts any directory and reports the volume it is mounted on. + return new DriveInfo(OperatingSystem.IsWindows() ? Path.GetPathRoot(full)! : full).AvailableFreeSpace; + } + catch (Exception ex) when (ex is ArgumentException or IOException or UnauthorizedAccessException) + { + return null; } } @@ -361,22 +462,23 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService Done, } - private static string Checkpoint(Stage stage, int frame, int framesVersion) => - string.Create(CultureInfo.InvariantCulture, $"{stage.ToString().ToLowerInvariant()}:{frame}:{framesVersion}"); + private static string Checkpoint(Stage stage, int frame, int framesVersion, string folder) => + string.Create(CultureInfo.InvariantCulture, $"{stage.ToString().ToLowerInvariant()}:{frame}:{framesVersion}:{folder}"); /// - /// The frames copied by an earlier run (copy:<frame>[:<frames version>]) and the frames version they - /// were of (null when the checkpoint does not say); nothing for a finished run, or another stage (an earlier build's - /// archive proxy pass), whose files the copy checks again. + /// The frames copied by an earlier run (copy:<frame>[:<frames version>[:<folder>]], the folder + /// last as it may hold a colon), the frames version and the sequence's folder they were of (null when the checkpoint does + /// not say); nothing for a finished run, or another stage (an earlier build's archive proxy pass), whose files the copy + /// checks again. /// - private static (int Copied, int? FramesVersion) Parse(string? checkpoint) + private static (int Copied, int? FramesVersion, string? Folder) Parse(string? checkpoint) { - var parts = checkpoint?.Split(':'); - if (parts is not ([_, _] or [_, _, _]) || !Enum.TryParse(parts[0], ignoreCase: true, out var stage) || stage != Stage.Copy + var parts = checkpoint?.Split(':', 4); + if (parts is not ([_, _] or [_, _, _] or [_, _, _, _]) || !Enum.TryParse(parts[0], ignoreCase: true, out var stage) || stage != Stage.Copy || !int.TryParse(parts[1], NumberStyles.None, CultureInfo.InvariantCulture, out var next)) - return (0, null); - int? version = parts.Length == 3 && int.TryParse(parts[2], NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture, out var v) ? v : null; - return (next, version); + return (0, null, null); + int? version = parts.Length >= 3 && int.TryParse(parts[2], NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture, out var v) ? v : null; + return (next, version, parts.Length == 4 ? parts[3] : null); } private sealed class Tally diff --git a/src/Chronos.Data/Workspaces/WorkspaceStore.cs b/src/Chronos.Data/Workspaces/WorkspaceStore.cs index b3448e14..4229825f 100644 --- a/src/Chronos.Data/Workspaces/WorkspaceStore.cs +++ b/src/Chronos.Data/Workspaces/WorkspaceStore.cs @@ -13,14 +13,41 @@ namespace Chronos.Data.Workspaces; /// The manifest of a workspace folder (workspace.json). public sealed record WorkspaceManifest(string Format, int FormatVersion, Guid Id, DateTimeOffset Created); -/// A workspace's sync state of one project besides its versions (sync.json): a restore to record, a conflict to settle. +/// +/// A workspace's sync state of one project besides its versions (sync.json): a restore to record, a conflict to +/// settle, and the journal of a replay or a save-both under way, which the next sync finishes or undoes after a crash. +/// /// The version a restore brought back; the next commit is a "restore" of it. /// The conflict a pull or push stopped at, until the user settles it. -public sealed record WorkspaceSyncState(Guid? RestoredFrom, WorkspaceConflict? Conflict) +/// A replay (or a base moved onto local commits the home holds) written in steps; null when none is under way. +/// A save-both that made its copy at the home and has not finished; null when none is under way. +public sealed record WorkspaceSyncState(Guid? RestoredFrom, WorkspaceConflict? Conflict, WorkspaceReplay? Replay = null, WorkspaceSaveBoth? SaveBoth = null) { public static WorkspaceSyncState None { get; } = new(null, null); + + /// Nothing to keep: the file can go. + [JsonIgnore] + public bool IsEmpty => RestoredFrom is null && Conflict is null && Replay is null && SaveBoth is null; } +/// +/// The journal of a replay (): the base it moves the project onto, the local versions it +/// replaces, the replayed ones it writes and the local versions the home already holds (their answer lost), whose files +/// take the home's copy. The bundle's base is the commit point: once it is the replay is finished +/// (the replaced set aside, the held ones adopted), before it the replayed ones are set aside and nothing else moved. +/// +public sealed record WorkspaceReplay(Guid OntoBase, IReadOnlyList Replaced, IReadOnlyList Replayed, IReadOnlyList Adopted); + +/// +/// A save-both under way (): the copy made at the home and its first version (the base the +/// local versions are sent on top of), the base this project had when it started, and the id each local version is sent +/// under, so a retry sends to the same copy what it lacks and the home answers "already stored" for the rest. +/// +public sealed record WorkspaceSaveBoth(Guid CopyId, Guid CopyBaseId, int CopyBaseNumber, Guid? FromBase, IReadOnlyList Moves); + +/// A local version () and the id it is sent under to a save-both's copy. +public sealed record WorkspaceMove(Guid LocalId, Guid CopyVersionId); + /// /// A pull or push that met real conflicts (owner review 2, R8): the home's head it met, the version the local commits were /// made from, the local commits (oldest first) and the members both sides changed (mine, theirs). Settled by keeping mine, @@ -58,7 +85,11 @@ public sealed class WorkspaceStore /// The folder name of the workspace inside the program's data folder. public const string DefaultFolderName = "workspace"; - /// Opens the workspace at , making the folder and its manifest when there is none. + /// + /// Opens the workspace at , making the folder and its manifest when there is none. The manifest + /// is written whole to a temporary file and moved into place, so a crash never leaves half of one; a manifest left + /// empty, or unreadable in a folder that holds no project yet (an earlier build's interrupted first write), is made again. + /// /// The folder's workspace.json is not a workspace manifest, or a newer one. /// The folder cannot be made or read. public WorkspaceStore(string root) @@ -67,20 +98,8 @@ public sealed class WorkspaceStore Root = LongPath.Normalize(Path.TrimEndingDirectorySeparator(Path.GetFullPath(root))); Directory.CreateDirectory(Root); var path = Path.Combine(Root, ManifestFile); - if (!File.Exists(path)) - { - var manifest = new WorkspaceManifest(FormatName, FormatVersion, Guid.CreateVersion7(), DateTimeOffset.UtcNow); - // Another program opening the same folder at the same moment: the first manifest wins. - try - { - using var stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.None); - JsonSerializer.Serialize(stream, manifest, WorkspaceJson.Default.WorkspaceManifest); - stream.Flush(flushToDisk: true); - } - catch (IOException) when (File.Exists(path)) - { - } - } + if (!File.Exists(path)) Create(path, replace: false); + else if (Unfinished(path)) Create(path, replace: true); Manifest = Read(path); } @@ -130,6 +149,70 @@ public sealed class WorkspaceStore return bundles; } + /// + /// Writes a new manifest. Without another program opening the same folder at the same moment + /// may have made one first: that one wins (the move never replaces a file). + /// + private static void Create(string path, bool replace) + { + var manifest = new WorkspaceManifest(FormatName, FormatVersion, Guid.CreateVersion7(), DateTimeOffset.UtcNow); + var text = JsonSerializer.Serialize(manifest, WorkspaceJson.Default.WorkspaceManifest); + if (replace) + { + Log.Warning("workspace", $"{path} was left unfinished; a new manifest is written"); + AtomicFile.WriteAllText(path, text); + return; + } + var temp = AtomicFile.TempPathFor(path); + try + { + using (var stream = new FileStream(temp, FileMode.CreateNew, FileAccess.Write, FileShare.None)) + { + stream.Write(System.Text.Encoding.UTF8.GetBytes(text)); + stream.Flush(flushToDisk: true); + } + try + { + File.Move(temp, path, overwrite: false); + } + catch (IOException) when (File.Exists(path)) + { + // Another program's manifest came first. + } + } + finally + { + if (File.Exists(temp)) File.Delete(temp); + } + } + + /// + /// A manifest an interrupted first write left (an earlier build wrote it in place): empty, or not JSON in a folder that + /// holds no project yet. A readable manifest, or any in a folder with projects, is never written over. + /// + private bool Unfinished(string path) + { + string text; + try + { + text = AtomicFile.ReadAllText(path); + } + catch (IOException) + { + return false; + } + if (string.IsNullOrWhiteSpace(text.Trim('\0'))) return true; + try + { + using var _ = JsonDocument.Parse(text); + return false; + } + catch (JsonException) + { + return List().Count == 0; + } + } + private static WorkspaceManifest Read(string path) { WorkspaceManifest? manifest; @@ -274,7 +357,10 @@ public sealed class WorkspaceProject internal void WriteVersion(ProjectVersion version, bool replace = false) => ProjectSidecars.WriteVersion(VersionPath(version), version, replace); - /// Moves a version file aside into versions/.replaced (a local commit replayed under another id). + /// + /// Moves a version file aside into versions/.replaced (a local commit replayed under another id), trying again + /// for a moment while another program (an indexer, an antivirus) holds it open. + /// internal void SetAside(ProjectVersion version) { var path = VersionPath(version); @@ -283,9 +369,23 @@ public sealed class WorkspaceProject Directory.CreateDirectory(aside); var target = Path.Combine(aside, Path.GetFileName(path)); if (File.Exists(target)) target += "." + DateTime.UtcNow.Ticks.ToString(System.Globalization.CultureInfo.InvariantCulture); - File.Move(path, target); + for (var attempt = 1; ; attempt++) + { + try + { + File.Move(path, target); + return; + } + catch (IOException) when (attempt < SetAsideAttempts && File.Exists(path) && !File.Exists(target)) + { + Thread.Sleep(10 * attempt); + } + } } + /// Moves tried, some 1 s in all, before a version file held open fails to move aside. + private const int SetAsideAttempts = 15; + public WorkspaceSyncState ReadSync() { if (!File.Exists(SyncPath)) return WorkspaceSyncState.None; @@ -301,7 +401,8 @@ public sealed class WorkspaceProject internal void WriteSync(WorkspaceSyncState state) { - if (state == WorkspaceSyncState.None || state is { RestoredFrom: null, Conflict: null }) + ArgumentNullException.ThrowIfNull(state); + if (state.IsEmpty) { if (File.Exists(SyncPath)) File.Delete(SyncPath); return; diff --git a/src/Chronos.Data/Workspaces/WorkspaceSync.cs b/src/Chronos.Data/Workspaces/WorkspaceSync.cs index 77ab7612..d9fd6a63 100644 --- a/src/Chronos.Data/Workspaces/WorkspaceSync.cs +++ b/src/Chronos.Data/Workspaces/WorkspaceSync.cs @@ -1,4 +1,5 @@ using System.Text.Json; +using Chronos.Core.Diagnostics; using Chronos.Core.Model; using Chronos.Core.Project; using Chronos.Data.Projects; @@ -84,6 +85,11 @@ public sealed class WorkspaceSyncException(PushOutcome outcome, string message) /// travel in the portable form (), so frames inserted at the home never move a position. Every /// operation holds the project's lease; nothing is ever deleted: a restore or save-both first saves uncommitted changes /// as a version. +/// Every write sequence survives a cut at any step: the working state's base is the commit point. A push moves the base +/// before its local file reads as sent; a replay journals itself in sync.json (), writes +/// the replayed versions, then the base, and only then sets the old ones aside; a save-both journals its copy and the ids +/// it sends under (). The next push, pull or resolve finishes or undoes what a cut left +/// (). /// public sealed class WorkspaceSync(WorkspaceStore store, string clientName) { @@ -100,6 +106,11 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName) /// How long an operation waits for the project's lease (another program working on it). public TimeSpan LockWait { get; init; } = TimeSpan.FromSeconds(30); + /// For tests: called with a step's name between the writes of a sync, where it may throw as a crash would. + internal Action? Cut { get; init; } + + private void Step(string name) => Cut?.Invoke(name); + /// /// The project of brought into the workspace: its folder, the home's /// head as the base and the working state (positions waiting for the media, which a @@ -126,30 +137,41 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName) ?? throw new ProjectException(ProjectError.NotFound, $"The home has no project {projectId}."); var head = await home.HeadAsync(projectId, cancellationToken).ConfigureAwait(false); Verify(head); - project.WriteVersion(head, replace: true); - Directory.CreateDirectory(project.ExportsDirectory); var bundle = new ProjectBundle(sequence.Archive, sequence.Sequence, ProjectSettings.FromJson(head.SettingsJson, allowNewer: true), DateTimeOffset.UtcNow) { Workspace = new BundleWorkspace(projectId, listed.Name, listed.IsMain, sequence.Home, Base(head), new BundleMedia(media, 0)), }; + // The working state first: one larger than a project file holds is refused before anything is written + // (ProjectTooLargeException), never left as a file no later open could read. Its base is fetched again if a cut + // comes before the version's own file. project.WriteBundle(bundle); + project.WriteVersion(head, replace: true); + Directory.CreateDirectory(project.ExportsDirectory); return project; } } - /// The working state for the editor: its settings placed on the media's frames, and the media's sequence. + /// + /// The working state for the editor: its settings placed on the media's frames, and the media's sequence. Both files are + /// read under the project's lease, as one state: a copy committing new media rewrites them in turn. + /// /// The project is not in the workspace. - public Task OpenAsync(Guid projectId, CancellationToken cancellationToken = default) => - Task.Run(() => + /// Another writer kept the project for longer than . + public async Task OpenAsync(Guid projectId, CancellationToken cancellationToken = default) + { + var project = Existing(projectId); + ProjectBundle bundle; + WorkspaceMedia? media; + using (await project.LockAsync(LockWait, cancellationToken).ConfigureAwait(false)) + (bundle, media) = await Task.Run(() => (project.ReadBundle(), project.ReadMedia()), cancellationToken).ConfigureAwait(false); + return await Task.Run(() => { - var project = Existing(projectId); - var bundle = project.ReadBundle(); - var media = project.ReadMedia(); var working = bundle.Project; if (media is not null && !working.IsNewerFormat) working = ProjectSettings.FromJson(ProjectContent.Place(working.ToJson(), media.Photos), allowNewer: true); return new WorkspaceSession(project, bundle, working, media, media is null ? null : SequenceOf(project, bundle, media)); - }, cancellationToken); + }, cancellationToken).ConfigureAwait(false); + } /// Stores the working state (the editor's settings, positions on the media's frames); no version is made. /// The project is read-only (a newer Chronos's settings) or not in the workspace. @@ -192,6 +214,7 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName) var project = Existing(projectId); using (await project.LockAsync(LockWait, cancellationToken).ConfigureAwait(false)) { + await SettleAsync(project, home, cancellationToken).ConfigureAwait(false); var pushed = new List(); var pulled = new List(); for (var attempt = 0; attempt < PushAttempts; attempt++) @@ -206,10 +229,13 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName) { case PushOutcome.Stored or PushOutcome.AlreadyStored when result.Version is { } stored: Verify(stored); - if (stored.Number != local.Number) project.SetAside(local); - project.WriteVersion(stored, replace: true); + // The base moves first: cut after it, the local file still reads as unsent and the next sync + // adopts it; cut before it, the home answers "already stored" next time. Marked sent first, it + // would leave the chain from the old base and take the versions after it along. bundle = WithBase(bundle, stored); project.WriteBundle(bundle); + Step("push.base"); + Adopt(project, local, stored); pushed.Add(stored); continue; case PushOutcome.Diverged: @@ -240,6 +266,7 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName) var project = Existing(projectId); using (await project.LockAsync(LockWait, cancellationToken).ConfigureAwait(false)) { + await SettleAsync(project, home, cancellationToken).ConfigureAwait(false); IReadOnlyList pulled = []; if (project.ReadSync().Conflict is null) { @@ -267,6 +294,7 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName) var project = Existing(projectId); using (await project.LockAsync(LockWait, cancellationToken).ConfigureAwait(false)) { + await SettleAsync(project, home, cancellationToken).ConfigureAwait(false); var bundle = project.ReadBundle(); var sync = project.ReadSync(); if (sync.Conflict is null) return new WorkspaceSyncResult(await StatusLockedAsync(project, home, cancellationToken).ConfigureAwait(false), [], []); @@ -328,15 +356,19 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName) /// /// Pulls the home's versions after the base and replays the local ones and the working state on its head. With /// null a conflict is recorded and nothing else moves (the versions pulled are kept as - /// files); with a choice the conflicts are settled by it. + /// files; the base only moves onto local commits the home already holds); with a choice the conflicts are settled by it. + /// Written in steps a cut can stop at any point (): the journal, the replayed versions, the + /// base with the merged working state (the commit point), then the old versions set aside and the held ones adopted. /// private async Task<(ProjectBundle Bundle, WorkspaceConflict? Conflict, IReadOnlyList Pulled)> ReplayAsync(WorkspaceProject project, ProjectBundle bundle, IProjectHome home, IReadOnlySet? takeTheirs, CancellationToken cancellationToken) { var basis = await BaseVersionAsync(project, bundle, home, cancellationToken).ConfigureAwait(false); - // Read before the home's versions are written over them: a local commit the home stored although its answer was lost - // comes back among them, and the commits after it were made on it. + // A local commit the home stored although its answer was lost comes back among the home's versions, and the commits + // after it were made on it. Its file keeps reading as unsent until the base has moved onto it or past it: written + // over now, it would leave the chain from the base, and the commits after it with it. var locals = Unpushed(project, bundle); + var unsent = project.Versions().Where(v => v.StoredAt is null).Select(v => v.Id).ToHashSet(); var theirs = new List(); for (var after = basis?.Number ?? 0; ;) { @@ -344,13 +376,14 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName) foreach (var version in page) { Verify(version); - project.WriteVersion(version, replace: true); + if (!unsent.Contains(version.Id)) project.WriteVersion(version, replace: true); } theirs.AddRange(page); if (page.Count < PullPage) break; after = page[^1].Number; } if (theirs.Count == 0) return (bundle, null, theirs); + var held = theirs.Where(v => unsent.Contains(v.Id)).ToList(); var head = theirs[^1]; var atHome = theirs.Select(v => v.Id).ToHashSet(); var ours = locals.TakeWhile(v => atHome.Contains(v.Id)).ToList(); @@ -377,15 +410,28 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName) if (differences.Count > 0 && takeTheirs is null) { var conflict = new WorkspaceConflict(head.Id, head.Number, basis?.Id ?? Guid.Empty, [.. locals.Select(v => v.Id)], differences, DateTimeOffset.UtcNow); - project.WriteSync(project.ReadSync() with { Conflict = conflict }); + if (ours.Count > 0) + { + // The conflict is met past local commits the home holds: the base moves onto the last of them (their files + // adopted once it has), so the commits after them stay in the chain and the conflict is settled against it. + var onto = new WorkspaceReplay(basis!.Id, [], [], [.. held.Select(v => v.Id)]); + project.WriteSync(project.ReadSync() with { Replay = onto }); + Step("replay.journal"); + bundle = WithBase(bundle, basis); + project.WriteBundle(bundle); + Step("replay.base"); + await FinishReplayAsync(project, onto, held, home, cancellationToken).ConfigureAwait(false); + } + project.WriteSync(project.ReadSync() with { Conflict = conflict, Replay = null }); return (bundle, conflict, theirs); } - var parent = head; + var rebased = new List(locals.Count); + ProjectVersion parent = head; for (var i = 0; i < locals.Count; i++) { var local = locals[i]; var portable = ProjectContent.Canonical(replay.Commits[i].ToJson()); - var rebased = local with + parent = local with { Id = Guid.CreateVersion7(), ParentId = parent.Id, @@ -396,48 +442,162 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName) SettingsFormat = ProjectContent.FormatOf(portable), StoredAt = null, }; - project.SetAside(local); - project.WriteVersion(rebased); - parent = rebased; + rebased.Add(parent); } var placed = ProjectSettings.FromJson(ProjectContent.Place(ProjectContent.Canonical(merge.Merged.ToJson()), photos), allowNewer: true); var moved = WithBase(bundle, head) with { Project = placed, Saved = DateTimeOffset.UtcNow }; + var journal = new WorkspaceReplay(head.Id, [.. locals.Select(v => v.Id)], [.. rebased.Select(v => v.Id)], [.. held.Select(v => v.Id)]); + project.WriteSync(project.ReadSync() with { Replay = journal }); + Step("replay.journal"); + foreach (var version in rebased) + { + project.WriteVersion(version); + Step("replay.version"); + } + // The commit point: before it the old chain hangs from the old base, after it the replayed one from the head. project.WriteBundle(moved); + Step("replay.base"); + await FinishReplayAsync(project, journal, held, home, cancellationToken).ConfigureAwait(false); + project.WriteSync(project.ReadSync() with { Replay = null }); return (moved, null, theirs); } - /// Both sides kept: this workspace's versions to a new project at the home, this project onto the home's head. + /// + /// Finishes a replay whose base moved (): each local version the home holds takes + /// its copy (from , a file of the workspace, else the home), the replaced ones are set aside. + /// + private static async Task FinishReplayAsync(WorkspaceProject project, WorkspaceReplay replay, IReadOnlyList known, IProjectHome home, + CancellationToken cancellationToken) + { + foreach (var id in replay.Adopted) + { + var locals = project.Versions().Where(v => v.Id == id && v.StoredAt is null).ToList(); + if (locals.Count == 0) continue; + if ((known.FirstOrDefault(v => v.Id == id) ?? await StoredCopyAsync(project, id, home, cancellationToken).ConfigureAwait(false)) is not { } stored) continue; + foreach (var local in locals) Adopt(project, local, stored); + } + foreach (var id in replay.Replaced) SetAsideUnsent(project, id); + } + + /// + /// Finishes or undoes what a cut left (see the class remarks), before anything reads the chain: a replay's journal is + /// finished once the base is its target, else its replayed versions are set aside (the old chain still hangs from the + /// base); a save-both whose project moved onto the home's head sets its sent versions aside and clears its conflict; a + /// local file of the version the base names (a push cut after it moved the base) takes the home's copy. + /// + private static async Task SettleAsync(WorkspaceProject project, IProjectHome home, CancellationToken cancellationToken) + { + var sync = project.ReadSync(); + var basis = project.ReadBundle().Workspace!.Base; + if (sync.Replay is { } replay) + { + if (basis?.VersionId == replay.OntoBase) await FinishReplayAsync(project, replay, [], home, cancellationToken).ConfigureAwait(false); + else foreach (var id in replay.Replayed) SetAsideUnsent(project, id); + sync = sync with { Replay = null }; + project.WriteSync(sync); + Log.Info("workspace", $"A sync of project {project.ProjectId} cut short was {(basis?.VersionId == replay.OntoBase ? "finished" : "undone")}"); + } + if (sync.SaveBoth is { } both && basis?.VersionId != both.FromBase) + { + foreach (var move in both.Moves) SetAsideUnsent(project, move.LocalId); + var settled = sync.Conflict is { } conflict && conflict.BaseVersionId == (both.FromBase ?? Guid.Empty); + sync = sync with { SaveBoth = null, Conflict = settled ? null : sync.Conflict }; + project.WriteSync(sync); + Log.Info("workspace", $"A save-both of project {project.ProjectId} cut short was finished"); + } + if (basis is not null && project.Versions().Where(v => v.Id == basis.VersionId && v.StoredAt is null).ToList() is { Count: > 0 } lingering + && await StoredCopyAsync(project, basis.VersionId, home, cancellationToken).ConfigureAwait(false) is { } stored) + { + foreach (var local in lingering) Adopt(project, local, stored); + } + } + + /// The home's copy of version : a file of the workspace marked sent, else asked from the home (kept as a file); null when the home has none. + private static async Task StoredCopyAsync(WorkspaceProject project, Guid id, IProjectHome home, CancellationToken cancellationToken) + { + if (project.Versions().FirstOrDefault(v => v.Id == id && v.StoredAt is not null) is { } file) return file; + if (await home.GetVersionAsync(project.ProjectId, id, cancellationToken).ConfigureAwait(false) is not { StoredAt: not null } fetched) return null; + Verify(fetched); + return fetched; + } + + /// + /// The local file of a version the home holds takes the home's copy (sent, maybe under another number). Called only once + /// the base names the version or one after it: before, the copy would take it out of the chain of unsent versions. + /// + private static void Adopt(WorkspaceProject project, ProjectVersion local, ProjectVersion stored) + { + project.WriteVersion(stored, replace: true); + if (stored.Number != local.Number) project.SetAside(local); + } + + /// Sets aside the unsent local files of version . + private static void SetAsideUnsent(WorkspaceProject project, Guid id) + { + foreach (var version in project.Versions().Where(v => v.Id == id && v.StoredAt is null)) project.SetAside(version); + } + + /// + /// Both sides kept: this workspace's versions to a new project at the home, this project onto the home's head. The copy + /// and the ids its versions are sent under are journaled before the first is sent (): a + /// retry after a cut sends to the same copy what it lacks (the home answers "already stored" for the rest), whatever name + /// it is asked for, and the base moving onto the home's head is the commit point. + /// private async Task SaveBothAsync(WorkspaceProject project, ProjectBundle bundle, IProjectHome home, string? name, CancellationToken cancellationToken) { var workspace = bundle.Workspace!; if (!bundle.IsReadOnly) CommitLocked(project, bundle, null, null); bundle = project.ReadBundle(); var locals = Unpushed(project, bundle); - var basis = await BaseVersionAsync(project, bundle, home, cancellationToken).ConfigureAwait(false); + var sync = project.ReadSync(); + var pending = sync.SaveBoth; SequenceProject? copy = null; + if (pending is not null) + { + copy = (await home.ListAsync(cancellationToken).ConfigureAwait(false)).FirstOrDefault(p => p.Id == pending.CopyId); + // A copy deleted at the home meanwhile: a new one is made. + if (copy is null) pending = null; + } var pushed = new List(); if (locals.Count > 0) { - copy = await CreateNamedAsync(home, name, workspace.Name, basis?.Id, cancellationToken).ConfigureAwait(false); - var parent = await home.HeadAsync(copy.Id, cancellationToken).ConfigureAwait(false); + if (pending is null) + { + var basis = await BaseVersionAsync(project, bundle, home, cancellationToken).ConfigureAwait(false); + copy = await CreateNamedAsync(home, name, workspace.Name, basis?.Id, cancellationToken).ConfigureAwait(false); + var first = await home.HeadAsync(copy.Id, cancellationToken).ConfigureAwait(false); + pending = new WorkspaceSaveBoth(copy.Id, first.Id, first.Number, workspace.Base?.VersionId, []); + } + var known = pending.Moves.Select(m => m.LocalId).ToHashSet(); + if (locals.Any(l => !known.Contains(l.Id)) || !ReferenceEquals(pending, sync.SaveBoth)) + { + pending = pending with { Moves = [.. pending.Moves, .. locals.Where(l => !known.Contains(l.Id)).Select(l => new WorkspaceMove(l.Id, Guid.CreateVersion7()))] }; + project.WriteSync(sync with { SaveBoth = pending }); + } + Step("saveboth.journal"); + var ids = pending.Moves.ToDictionary(m => m.LocalId, m => m.CopyVersionId); + var (parentId, parentNumber) = (pending.CopyBaseId, pending.CopyBaseNumber); foreach (var local in locals) { - var moved = local with { Id = Guid.CreateVersion7(), ProjectId = copy.Id, ParentId = parent.Id, Number = parent.Number + 1, StoredAt = null }; + var moved = local with { Id = ids[local.Id], ProjectId = copy!.Id, ParentId = parentId, Number = parentNumber + 1, StoredAt = null }; var result = await home.PushAsync(copy.Id, moved, cancellationToken).ConfigureAwait(false); if (result is not { Outcome: PushOutcome.Stored or PushOutcome.AlreadyStored, Version: { } stored }) throw new WorkspaceSyncException(result.Outcome, $"The home refused version {local.Number} saved as project '{copy.Name}': {result.Outcome}."); pushed.Add(stored); - parent = stored; + (parentId, parentNumber) = (stored.Id, stored.Number); + Step("saveboth.pushed"); } } // This project takes the home's head: the local versions now live in the copy (and their files in versions/.replaced). var head = await home.HeadAsync(project.ProjectId, cancellationToken).ConfigureAwait(false); Verify(head); project.WriteVersion(head, replace: true); - foreach (var local in locals) project.SetAside(local); var media = project.ReadMedia(); var working = ProjectSettings.FromJson(ProjectContent.Place(head.SettingsJson, media?.Photos ?? NoFrames), allowNewer: true); project.WriteBundle(WithBase(bundle, head) with { Project = working, Saved = DateTimeOffset.UtcNow }); + Step("saveboth.base"); + foreach (var local in locals) project.SetAside(local); + foreach (var move in pending?.Moves ?? []) SetAsideUnsent(project, move.LocalId); project.WriteSync(WorkspaceSyncState.None); return new WorkspaceSyncResult(await StatusLockedAsync(project, home, cancellationToken).ConfigureAwait(false), pushed, [head], copy); } diff --git a/tests/Chronos.App.Tests/Workspace/WorkspaceEditorTests.cs b/tests/Chronos.App.Tests/Workspace/WorkspaceEditorTests.cs index 642fc007..099da7c1 100644 --- a/tests/Chronos.App.Tests/Workspace/WorkspaceEditorTests.cs +++ b/tests/Chronos.App.Tests/Workspace/WorkspaceEditorTests.cs @@ -139,7 +139,13 @@ public sealed class WorkspaceEditorTests(TestContext context) Assert.Null(portable.HeroFrame, "positions travel by photo"); Assert.Contains(portable.PendingFrames!, f => f.Hero); + // A version saved here and not sent yet may reach the home under another id (a replay): the render names none. var saved = (await host.SaveVersionAsync(projectId, null)).Version!; + var (unsent, _) = ProcessingJobs.ReadRender(session.RenderRequests(session.Project.Export)[0].ParametersJson); + Assert.Null(unsent.ProjectVersionId, "a version not sent names no version"); + Assert.Equal(projectId, unsent.ProjectId); + // Sent, the home holds it under its id: the render names it. + await host.SyncAsync(projectId); var (versioned, _) = ProcessingJobs.ReadRender(session.RenderRequests(session.Project.Export)[0].ParametersJson); Assert.Equal(saved.Id, versioned.ProjectVersionId); Assert.Equal(parameters.SettingsJson, versioned.SettingsJson); diff --git a/tests/Chronos.Core.Tests/Project/ProjectBundleTests.cs b/tests/Chronos.Core.Tests/Project/ProjectBundleTests.cs index d9f8f049..711b9123 100644 --- a/tests/Chronos.Core.Tests/Project/ProjectBundleTests.cs +++ b/tests/Chronos.Core.Tests/Project/ProjectBundleTests.cs @@ -1,3 +1,4 @@ +using Chronos.Core.Localization; using Chronos.Core.Project; using Chronos.Testing; @@ -107,6 +108,23 @@ public sealed class ProjectBundleTests(TestContext context) Assert.Throws(() => ProjectBundle.Load(path)); } + [Test] + public void AFileNoReadCouldTakeBackIsRefusedBeforeItIsWritten() + { + var folder = context.CreateTempDirectory("bundle"); + var path = Path.Combine(folder, "huge.chronos"); + var sample = Sample(); + var huge = sample with { Sequence = sample.Sequence with { Title = new string('x', ProjectBundle.MaxBytes) } }; + var refused = Assert.Throws(() => huge.Save(path)); + Assert.Equal(Loc.L("Errors.Project.TooLarge", ProjectBundle.MaxBytes >> 20), refused.Message); + Assert.False(File.Exists(path)); + Assert.Empty(Directory.GetFiles(folder)); + + // Within the limit, the file is written and read back. + sample.Save(path); + Assert.Equal(sample.Sequence.Title, ProjectBundle.Load(path).Sequence.Title); + } + [Test] public void FramePositionsApplyOnlyToTheFramesTheyWereSavedFor() { diff --git a/tests/Chronos.Data.Tests/Jobs/JobPriorityQueueTests.cs b/tests/Chronos.Data.Tests/Jobs/JobPriorityQueueTests.cs index ffa74cd8..9f87aa39 100644 --- a/tests/Chronos.Data.Tests/Jobs/JobPriorityQueueTests.cs +++ b/tests/Chronos.Data.Tests/Jobs/JobPriorityQueueTests.cs @@ -17,6 +17,11 @@ public sealed class JobPriorityQueueTests Assert.Equal(JobKindPriority.High, queue.LevelOf(JobKind.Render)); Assert.Equal(JobKindPriority.High, queue.LevelOf(JobKind.Import)); Assert.Equal(JobKindPriority.High, queue.LevelOf(JobKind.Download)); + // Edit waits for the copy of a project's media into the workspace: it goes before scans and analyses. + Assert.Equal(JobKindPriority.High, queue.LevelOf(JobKind.WorkspaceCopy)); + var on = queue with { Enabled = true }; + Assert.True(on.Interrupts(JobKind.WorkspaceCopy, JobKind.Scan)); + Assert.False(on.Interrupts(JobKind.WorkspaceCopy, JobKind.Render)); Assert.Equal(JobKindPriority.Low, queue.LevelOf(JobKind.Verify)); Assert.Equal(JobKindPriority.Low, queue.LevelOf(JobKind.Proxies)); foreach (var kind in new[] { JobKind.Scan, JobKind.Thumbs, JobKind.LightAnalyse, JobKind.DeepAnalyse, JobKind.Maintenance }) diff --git a/tests/Chronos.Data.Tests/Processing/RenderJobTests.cs b/tests/Chronos.Data.Tests/Processing/RenderJobTests.cs index f30c947f..c764a9f1 100644 --- a/tests/Chronos.Data.Tests/Processing/RenderJobTests.cs +++ b/tests/Chronos.Data.Tests/Processing/RenderJobTests.cs @@ -108,6 +108,18 @@ public sealed class RenderJobTests(TestContext context) var second = await fx.EnqueueAsync(requests[0] with { ParametersJson = System.Text.Json.JsonSerializer.Serialize(parameters, named) }); await fx.Archive.WaitForAsync(second.Id, JobState.Done, (int)TestTiming.Budget(60)); Assert.True((await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct)).All(r => r.ProjectVersionId == version.Id)); + + // A version the index does not hold (saved on a PC, sent later under another id), or one whose settings the job does + // not embed: the render records the project alone, never a version it did not render. + var embedded = System.Text.Json.JsonSerializer.Deserialize(requests[0].ParametersJson!, named)!; + var other = ProjectContent.ToPortable(settings with { ExcludedFrames = [3] }, photos); + foreach (var wrong in new[] { embedded with { ProjectVersionId = Guid.CreateVersion7() }, embedded with { SettingsJson = other } }) + { + var refused = await fx.EnqueueAsync(requests[0] with { ParametersJson = System.Text.Json.JsonSerializer.Serialize(wrong, named) }); + await fx.Archive.WaitForAsync(refused.Id, JobState.Done, (int)TestTiming.Budget(60)); + var recorded = (await fx.Db.Renders.GetAsync(Chronos.Data.Processing.Jobs.RenderJobHandler.RenderId(refused.Id, 0), fx.Ct))!; + Assert.Equal<(Guid?, Guid?)>((vertical.Id, null), (recorded.ProjectId, recorded.ProjectVersionId)); + } } [Test] diff --git a/tests/Chronos.Data.Tests/Workspaces/WorkspaceFixture.cs b/tests/Chronos.Data.Tests/Workspaces/WorkspaceFixture.cs index f6422baf..487d1332 100644 --- a/tests/Chronos.Data.Tests/Workspaces/WorkspaceFixture.cs +++ b/tests/Chronos.Data.Tests/Workspaces/WorkspaceFixture.cs @@ -44,6 +44,23 @@ internal sealed class WorkspaceFixture : IAsyncDisposable return (store, new WorkspaceSync(store, "Chronos on " + name) { LockWait = TimeSpan.FromSeconds(TestTiming.Budget(10)) }); } + /// + /// Another sync of that stops, as a crash would, at the -th time it + /// reaches step (), throwing . + /// + public static WorkspaceSync Cutting(WorkspaceStore store, string step, int occurrence = 1) + { + var seen = 0; + return new WorkspaceSync(store, "Chronos cut") + { + LockWait = TimeSpan.FromSeconds(TestTiming.Budget(10)), + Cut = name => + { + if (name == step && ++seen == occurrence) throw new CutException(step); + }, + }; + } + /// Main of the sequence checked out into 's workspace, its media copied. public async Task CheckoutAsync(WorkspaceSync sync, Guid? projectId = null) { @@ -81,6 +98,52 @@ internal sealed class WorkspaceFixture : IAsyncDisposable } } +/// A crash between two writes of a sync (). +internal sealed class CutException(string step) : Exception($"Cut at {step}."); + +/// +/// A home whose pushes can be cut: one refused as out of reach before the home sees it, or one stored whose answer is lost +/// on the way back (the program killed, the network dropped). +/// +internal sealed class CuttingHome(IProjectHome home) : IProjectHome +{ + public int Pushes { get; private set; } + + /// The push (counted from 1) refused as out of reach. + public int? Refuse { get; set; } + + /// The push (counted from 1) the home stores and whose answer is lost. + public int? LoseAnswer { get; set; } + + public async Task PushAsync(Guid projectId, ProjectVersion version, CancellationToken cancellationToken) + { + var push = ++Pushes; + if (push == Refuse) throw new HomeUnavailableException("The home cannot be reached."); + var result = await home.PushAsync(projectId, version, cancellationToken); + if (push == LoseAnswer) throw new HomeUnavailableException("The answer was lost."); + return result; + } + + public Task SequenceAsync(CancellationToken cancellationToken) => home.SequenceAsync(cancellationToken); + public Task> ListAsync(CancellationToken cancellationToken) => home.ListAsync(cancellationToken); + + public Task CreateAsync(string name, Guid? fromVersionId, string? author, CancellationToken cancellationToken) => + home.CreateAsync(name, fromVersionId, author, cancellationToken); + + public Task HeadAsync(Guid projectId, CancellationToken cancellationToken) => home.HeadAsync(projectId, cancellationToken); + + public Task GetVersionAsync(Guid projectId, Guid versionId, CancellationToken cancellationToken) => + home.GetVersionAsync(projectId, versionId, cancellationToken); + + public Task> VersionsAfterAsync(Guid projectId, int afterNumber, int limit, CancellationToken cancellationToken) => + home.VersionsAfterAsync(projectId, afterNumber, limit, cancellationToken); + + public Task> HistoryAsync(Guid projectId, int? beforeNumber, int limit, CancellationToken cancellationToken) => + home.HistoryAsync(projectId, beforeNumber, limit, cancellationToken); + + public Task FramesAsync(CancellationToken cancellationToken) => home.FramesAsync(cancellationToken); +} + /// A home that cannot be reached, standing for a server out of reach or an archive offline. internal sealed class UnreachableHome(IProjectHome home) : IProjectHome { diff --git a/tests/Chronos.Data.Tests/Workspaces/WorkspaceRecoveryTests.cs b/tests/Chronos.Data.Tests/Workspaces/WorkspaceRecoveryTests.cs new file mode 100644 index 00000000..774f215f --- /dev/null +++ b/tests/Chronos.Data.Tests/Workspaces/WorkspaceRecoveryTests.cs @@ -0,0 +1,334 @@ +using System.Security.Cryptography; +using Chronos.Core.Archiving; +using Chronos.Core.Localization; +using Chronos.Core.Model; +using Chronos.Core.Project; +using Chronos.Data.Archiving; +using Chronos.Data.Projects; +using Chronos.Data.Tests.Archiving; +using Chronos.Data.Tests.Infrastructure; +using Chronos.Data.Tests.Projects; +using Chronos.Data.Workspaces; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Workspaces; + +/// +/// The workspace survives a cut at every step (owner review 2, R8): a push, a replay and a save-both stopped between two +/// writes lose no version and make no false conflict, and the next sync finishes them; the files it writes are whole. +/// +[Category("workspace")] +public abstract class WorkspaceRecoveryTests(TestContext context) : DatabaseTest(context) +{ + private async Task FixtureAsync(int frames = 5, Func? configure = null) + { + var archive = Own(await ArchiveFixture.CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true), configure: configure)); + return Own(await WorkspaceFixture.CreateAsync(Context, archive, frames)); + } + + private static async Task EditAsync(WorkspaceSync sync, Guid projectId, Func edit, CancellationToken ct) + { + var session = await sync.OpenAsync(projectId, ct); + await sync.SaveAsync(projectId, edit(session.Working), ct); + } + + private static ProjectSettings Offset(ProjectSettings s, double ev) => s with { Exposure = s.Exposure with { ExposureOffsetEv = ev } }; + + private static ProjectSettings Settings(ProjectVersion version) => ProjectSettings.FromJson(version.SettingsJson); + + private static async Task> NotesAsync(WorkspaceFixture fx, Guid projectId) => + [.. (await fx.Home.HistoryAsync(projectId, null, 50, fx.Ct)).OrderBy(v => v.Number).Select(v => v.Note)]; + + [Test] + public async Task APushCutAfterTheBaseMovedLosesNoVersionAndMakesNoFalseConflict() + { + var fx = await FixtureAsync(); + var (store, sync) = fx.Workspace(); + var project = await fx.CheckoutAsync(sync); + var id = project.ProjectId; + await EditAsync(sync, id, s => Offset(s, 0.5), fx.Ct); + var first = (await sync.CommitAsync(id, "first", cancellationToken: fx.Ct))!; + await EditAsync(sync, id, s => Offset(s, 0.8), fx.Ct); + var second = (await sync.CommitAsync(id, "second", cancellationToken: fx.Ct))!; + + // Stored at the home, the base moved onto it, then the program died: 'second' still waits after it. + await Assert.ThrowsAsync(() => WorkspaceFixture.Cutting(store, "push.base").PushAsync(id, fx.Home, fx.Ct)); + var status = await sync.StatusAsync(id, fx.Home, fx.Ct); + Assert.Equal((WorkspaceState.Ahead, 1, 0, first.Id), (status.State, status.Ahead, status.Behind, status.Base!.VersionId)); + Assert.Equal(second.Id, sync.Unpushed(id).Single().Id); + + // The next sync, as the app runs it (pull, then push): no conflict, both versions at the home, the local file adopted. + Assert.Null((await sync.PullAsync(id, fx.Home, fx.Ct)).Status.Conflict); + Assert.Equal(WorkspaceState.InSync, (await sync.PushAsync(id, fx.Home, fx.Ct)).Status.State); + Assert.SequenceEqual(["first", "second"], (await NotesAsync(fx, id)).Skip(1)); + Assert.Equal(0.8, Settings(await fx.Home.HeadAsync(id, fx.Ct)).Exposure.ExposureOffsetEv); + Assert.True(project.Versions().Where(v => v.Id == first.Id).All(v => v.StoredAt is not null), "the local file reads as sent"); + Assert.Empty(project.Versions().Where(v => v.StoredAt is null)); + } + + [Test] + public async Task APushWhoseAnswerWasLostIsFinishedByTheNextSync() + { + var fx = await FixtureAsync(); + var (_, sync) = fx.Workspace(); + var project = await fx.CheckoutAsync(sync); + var id = project.ProjectId; + await EditAsync(sync, id, s => Offset(s, 0.5), fx.Ct); + await sync.CommitAsync(id, "first", cancellationToken: fx.Ct); + await EditAsync(sync, id, s => Offset(s, 0.8), fx.Ct); + await sync.CommitAsync(id, "second", cancellationToken: fx.Ct); + + await Assert.ThrowsAsync(() => sync.PushAsync(id, new CuttingHome(fx.Home) { LoseAnswer = 1 }, fx.Ct)); + Assert.Equal(2, sync.Unpushed(id).Count); + Assert.Null((await sync.PullAsync(id, fx.Home, fx.Ct)).Status.Conflict); + Assert.Equal(WorkspaceState.InSync, (await sync.PushAsync(id, fx.Home, fx.Ct)).Status.State); + Assert.SequenceEqual(["first", "second"], (await NotesAsync(fx, id)).Skip(1)); + Assert.Equal(0.8, Settings(await fx.Home.HeadAsync(id, fx.Ct)).Exposure.ExposureOffsetEv); + } + + [Test] + public async Task AReplayCutAtAnyStepLosesNoVersionEvenWhenTheHomeMovesAgain() + { + var fx = await FixtureAsync(); + var cuts = new[] { ("replay.journal", 1), ("replay.version", 1), ("replay.version", 2), ("replay.base", 1) }; + foreach (var (step, occurrence) in cuts) + { + var label = $"{step} #{occurrence}"; + var id = (await fx.Archive.Service.Projects.CreateAsync(fx.Sequence.Id, $"Cut {step} {occurrence}", cancellationToken: fx.Ct)).Id; + var (store, one) = fx.Workspace("one"); + var (_, two) = fx.Workspace("two"); + var project = await fx.CheckoutAsync(one, id); + await fx.CheckoutAsync(two, id); + await EditAsync(two, id, s => s with { Exposure = s.Exposure with { DeflickerStrength = new Auto().WithManual(0.6) } }, fx.Ct); + await two.CommitAsync(id, cancellationToken: fx.Ct); + await two.PushAsync(id, fx.Home, fx.Ct); + await EditAsync(one, id, s => Offset(s, 0.3), fx.Ct); + await one.CommitAsync(id, "offset", cancellationToken: fx.Ct); + await EditAsync(one, id, s => s with { ExcludedFrames = [3], Stack = new StackSettings { Mode = StackSetting.Mean } }, fx.Ct); + await one.CommitAsync(id, "stack", cancellationToken: fx.Ct); + + // The push meets the home moved: the replay is cut, then the home moves again before this PC syncs. + await Assert.ThrowsAsync(() => WorkspaceFixture.Cutting(store, step, occurrence).PushAsync(id, fx.Home, fx.Ct)); + Assert.Equal(2, one.Unpushed(id).Count, label); + await two.PullAsync(id, fx.Home, fx.Ct); + await EditAsync(two, id, s => s with { Exposure = s.Exposure with { DeflickerStrength = new Auto().WithManual(0.7) } }, fx.Ct); + await two.CommitAsync(id, "again", cancellationToken: fx.Ct); + await two.PushAsync(id, fx.Home, fx.Ct); + + Assert.Null((await one.PullAsync(id, fx.Home, fx.Ct)).Status.Conflict, label); + Assert.Equal(WorkspaceState.InSync, (await one.PushAsync(id, fx.Home, fx.Ct)).Status.State, label); + var notes = await NotesAsync(fx, id); + Assert.Equal((1, 1), (notes.Count(n => n == "offset"), notes.Count(n => n == "stack")), label); + var head = ProjectSettings.FromJson(ProjectContent.Place((await fx.Home.HeadAsync(id, fx.Ct)).SettingsJson, ProjectServiceTests.Photos(fx.Frames))); + Assert.Equal((0.7, 0.3, StackSetting.Mean), (head.Exposure.DeflickerStrength.Resolve(1), head.Exposure.ExposureOffsetEv, head.Stack.Mode), label); + Assert.SequenceEqual([3], head.ExcludedFrames, label); + Assert.Null(project.ReadSync().Replay, label); + Assert.Empty(project.Versions().Where(v => v.StoredAt is null)); + } + } + + [Test] + public async Task AConflictMetPastAVersionTheHomeHeldKeepsTheVersionsAfterIt() + { + var fx = await FixtureAsync(); + var (_, one) = fx.Workspace("one"); + var (_, two) = fx.Workspace("two"); + var id = (await fx.CheckoutAsync(one)).ProjectId; + await fx.CheckoutAsync(two, id); + await EditAsync(one, id, s => Offset(s, 0.5), fx.Ct); + var first = (await one.CommitAsync(id, "first", cancellationToken: fx.Ct))!; + await EditAsync(one, id, s => s with { Stack = new StackSettings { Mode = StackSetting.Mean } }, fx.Ct); + var second = (await one.CommitAsync(id, "second", cancellationToken: fx.Ct))!; + // 'first' is stored, its answer lost; another PC builds on it. + await Assert.ThrowsAsync(() => one.PushAsync(id, new CuttingHome(fx.Home) { LoseAnswer = 1 }, fx.Ct)); + await two.PullAsync(id, fx.Home, fx.Ct); + await EditAsync(two, id, s => Offset(s, 1.0), fx.Ct); + await two.CommitAsync(id, "other", cancellationToken: fx.Ct); + await two.PushAsync(id, fx.Home, fx.Ct); + + // This PC changed the offset too, without saving a version: a real conflict, met past 'first'. + await EditAsync(one, id, s => Offset(s, 2.0), fx.Ct); + var stopped = await one.PullAsync(id, fx.Home, fx.Ct); + Assert.Equal(WorkspaceState.Conflict, stopped.Status.State); + Assert.Equal((first.Id, 1), (stopped.Status.Base!.VersionId, stopped.Status.Ahead)); + Assert.Equal(second.Id, one.Unpushed(id).Single().Id); + Assert.Equal("exposure.exposureOffsetEv", stopped.Status.Conflict!.Differences.Single().Path); + + // The server's offset; 'second''s own change, which nobody else made, reaches the home. + await one.ResolveAsync(id, fx.Home, ConflictChoice.TakeServer, cancellationToken: fx.Ct); + Assert.Equal(WorkspaceState.InSync, (await one.PushAsync(id, fx.Home, fx.Ct)).Status.State); + var head = Settings(await fx.Home.HeadAsync(id, fx.Ct)); + Assert.Equal((1.0, StackSetting.Mean), (head.Exposure.ExposureOffsetEv, head.Stack.Mode)); + Assert.SequenceEqual(["first", "other", "second"], (await NotesAsync(fx, id)).Skip(1)); + } + + [Test] + public async Task ASaveBothCutShortIsFinishedIntoTheSameCopy() + { + var fx = await FixtureAsync(); + var (_, one) = fx.Workspace("one"); + var (store, three) = fx.Workspace("three"); + var id = (await fx.CheckoutAsync(one)).ProjectId; + var project = await fx.CheckoutAsync(three, id); + await EditAsync(one, id, s => Offset(s, 0.5), fx.Ct); + await one.CommitAsync(id, cancellationToken: fx.Ct); + await one.PushAsync(id, fx.Home, fx.Ct); + await EditAsync(three, id, s => Offset(s, 2), fx.Ct); + await three.CommitAsync(id, "bright", cancellationToken: fx.Ct); + await EditAsync(three, id, s => s with { Stack = new StackSettings { Mode = StackSetting.StarTrails } }, fx.Ct); + Assert.Equal(WorkspaceState.Conflict, (await three.PushAsync(id, fx.Home, fx.Ct)).Status.State); + + // The copy is made and the first version sent, then the home goes out of reach. + var cut = new CuttingHome(fx.Home) { Refuse = 2 }; + await Assert.ThrowsAsync(() => three.ResolveAsync(id, cut, ConflictChoice.SaveBoth, saveBothName: "Main (PC)", cancellationToken: fx.Ct)); + var partial = (await fx.Home.ListAsync(fx.Ct)).Single(p => p.Name == "Main (PC)"); + Assert.Equal(WorkspaceState.Conflict, (await three.StatusAsync(id, fx.Home, fx.Ct)).State); + + // The retry, asked another name as the app asks once the first is taken, sends the rest to the same copy. + var both = await three.ResolveAsync(id, fx.Home, ConflictChoice.SaveBoth, saveBothName: "Main (other)", cancellationToken: fx.Ct); + Assert.Equal(partial.Id, both.SavedAs!.Id); + Assert.Equal(2, (await fx.Home.ListAsync(fx.Ct)).Count); + var kept = Settings(await fx.Home.HeadAsync(partial.Id, fx.Ct)); + Assert.Equal((2, StackSetting.StarTrails), (kept.Exposure.ExposureOffsetEv, kept.Stack.Mode)); + Assert.Equal(3, (await fx.Home.HistoryAsync(partial.Id, null, 10, fx.Ct)).Count); + Assert.Equal(WorkspaceState.InSync, both.Status.State); + Assert.Equal(WorkspaceSyncState.None, project.ReadSync()); + + // Cut once this project moved onto the home's head: the next sync finishes it, the conflict settled. + await EditAsync(one, id, s => Offset(s, 0.7), fx.Ct); + await one.CommitAsync(id, cancellationToken: fx.Ct); + await one.PushAsync(id, fx.Home, fx.Ct); + await EditAsync(three, id, s => Offset(s, 3), fx.Ct); + await three.CommitAsync(id, "brighter", cancellationToken: fx.Ct); + Assert.Equal(WorkspaceState.Conflict, (await three.PushAsync(id, fx.Home, fx.Ct)).Status.State); + await Assert.ThrowsAsync(() => WorkspaceFixture.Cutting(store, "saveboth.base").ResolveAsync(id, fx.Home, ConflictChoice.SaveBoth, cancellationToken: fx.Ct)); + var settled = await three.PushAsync(id, fx.Home, fx.Ct); + Assert.Equal((WorkspaceState.InSync, 0), (settled.Status.State, settled.Status.Ahead)); + Assert.Equal(0.7, (await three.OpenAsync(id, fx.Ct)).Working.Exposure.ExposureOffsetEv); + var second = (await fx.Home.ListAsync(fx.Ct)).Single(p => p.Name == "Main (2)"); + Assert.Equal(3, Settings(await fx.Home.HeadAsync(second.Id, fx.Ct)).Exposure.ExposureOffsetEv); + Assert.Equal(WorkspaceSyncState.None, project.ReadSync()); + Assert.Empty(project.Versions().Where(v => v.StoredAt is null)); + } + + [Test] + public async Task AProjectWithTwentyThousandParkedExclusionsIsCheckedOutEditedAndSent() + { + var fx = await FixtureAsync(3); + var projects = fx.Archive.Service.Projects; + var large = await projects.CreateAsync(fx.Sequence.Id, "Large", cancellationToken: fx.Ct); + var head = await projects.HeadAsync(large.Id, fx.Ct); + // Exclusions of photos this sequence does not hold (taken from another copy): they wait, parked by photo. + var pending = Enumerable.Range(0, 20_000).Select(i => new PendingFrame + { + Sha256 = Convert.ToHexStringLower(SHA256.HashData(BitConverter.GetBytes(i))), + Size = 1_000_000 + i, + CapturedAt = new DateTime(2024, 1, 1).AddMinutes(i), + Excluded = true, + }).ToList(); + var portable = ProjectContent.Canonical((Settings(head) with { PendingFrames = pending }).ToJson()); + var draft = head with + { + Id = Guid.CreateVersion7(), + ParentId = head.Id, + Number = head.Number + 1, + Kind = ProjectVersionKind.Commit, + SettingsJson = portable, + ContentHash = ProjectContent.Hash(portable), + StoredAt = null, + }; + Assert.Equal(PushOutcome.Stored, (await projects.PushAsync(large.Id, draft, fx.Ct)).Outcome); + + var (_, sync) = fx.Workspace(); + var project = await fx.CheckoutAsync(sync, large.Id); + Assert.True(new FileInfo(project.BundlePath).Length > 4 << 20, "larger than a project file could be before"); + var session = await sync.OpenAsync(large.Id, fx.Ct); + Assert.Equal(20_000, session.Working.PendingFrames!.Count); + await sync.SaveAsync(large.Id, Offset(session.Working, 0.2), fx.Ct); + Assert.NotNull(await sync.CommitAsync(large.Id, cancellationToken: fx.Ct)); + Assert.Equal(WorkspaceState.InSync, (await sync.PushAsync(large.Id, fx.Home, fx.Ct)).Status.State); + Assert.Equal(20_000, Settings(await fx.Home.HeadAsync(large.Id, fx.Ct)).PendingFrames!.Count); + } + + [Test] + public async Task TheCopyStopsBeforeItFillsTheWorkspaceDrive() + { + var fx = await FixtureAsync(3, options => options with { FreeSpace = _ => 1 << 20 }); + var (store, sync) = fx.Workspace(); + var main = await fx.Archive.Service.Projects.MainAsync(fx.Sequence.Id, fx.Ct); + var project = await sync.CheckoutAsync(fx.Home, main.Id, cancellationToken: fx.Ct); + var job = await fx.CopyAsync(store, main.Id, JobState.Failed); + var parts = Loc.L("Errors.Workspace.DiskFull", "\u0001").Split('\u0001'); + Assert.True(job.Error!.StartsWith(parts[0], StringComparison.Ordinal) && job.Error.EndsWith(parts[1], StringComparison.Ordinal), job.Error); + Assert.Null(project.ReadMedia()); + Assert.Empty(Directory.GetFiles(project.ProxiesDirectory)); + } + + [Test] + public async Task ACopyResumedAfterTheSequenceMovedChecksEveryFileAgain() + { + var fx = await FixtureAsync(); + var (store, sync) = fx.Workspace(); + var main = await fx.Archive.Service.Projects.MainAsync(fx.Sequence.Id, fx.Ct); + var project = await sync.CheckoutAsync(fx.Home, main.Id, cancellationToken: fx.Ct); + await fx.Archive.Service.Images.BuildProxiesAsync(fx.Archive.Archive, fx.Sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct); + await fx.CopyAsync(store, main.Id); + var copied = project.ReadMedia()!; + var damaged = project.MediaFile(copied.Frames[0].File!); + + // A run of the same frames in the same folder keeps what it had copied. + File.WriteAllText(damaged, "not a photo"); + var job = await fx.CopyAsync(store, main.Id, checkpoint: $"copy:{copied.Frames.Count}:{fx.Sequence.FramesVersion}:{fx.Sequence.Folder}"); + Assert.Contains(Loc.L("Archive.Log.WorkspaceCopied", fx.Sequence.DisplayTitle, 0, 0, 5, 0), job.LogTail!); + // One read in another folder (the sequence moved since, another may hold its old one) checks every file again. + job = await fx.CopyAsync(store, main.Id, checkpoint: $"copy:{copied.Frames.Count}:{fx.Sequence.FramesVersion}:elsewhere/folder"); + Assert.Contains(Loc.L("Archive.Log.WorkspaceCopied", fx.Sequence.DisplayTitle, 1, 0, 4, 0), job.LogTail!); + Assert.Equal(copied.Frames[0].FileSha256, Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(damaged)))); + } + + [Test] + public async Task TheEditorReadsTheWorkingStateAndTheMediaUnderTheLease() + { + var fx = await FixtureAsync(2); + var (store, sync) = fx.Workspace(); + var project = await fx.CheckoutAsync(sync); + var impatient = new WorkspaceSync(store, "Chronos test") { LockWait = TimeSpan.FromMilliseconds(200) }; + using (await project.LockAsync(TimeSpan.FromSeconds(TestTiming.Budget(5)), fx.Ct)) + await Assert.ThrowsAsync(() => impatient.OpenAsync(project.ProjectId, fx.Ct)); + Assert.NotNull((await impatient.OpenAsync(project.ProjectId, fx.Ct)).Sequence); + } + + [Test] + public void AManifestLeftUnfinishedIsMadeAgainOnlyWhileTheFolderHoldsNoProject() + { + var root = Path.Combine(Context.CreateTempDirectory("ws"), "workspace"); + Directory.CreateDirectory(root); + var manifest = Path.Combine(root, WorkspaceStore.ManifestFile); + File.WriteAllBytes(manifest, []); + var store = new WorkspaceStore(root); + Assert.Equal(WorkspaceStore.FormatName, store.Manifest.Format); + Assert.Empty(Directory.GetFiles(root, "*.tmp")); + Assert.Equal(store.Id, new WorkspaceStore(root).Id); + + File.WriteAllText(manifest, """{ "format": "chronos-wor"""); + Assert.NotEqual(store.Id, new WorkspaceStore(root).Id); + + // With a project in the folder, a damaged manifest is never written over: it is reported. + var project = Path.Combine(root, Guid.CreateVersion7().ToString("N")); + Directory.CreateDirectory(project); + File.WriteAllText(Path.Combine(project, WorkspaceProject.BundleFile), "{}"); + File.WriteAllText(manifest, """{ "format": "chronos-wor"""); + Assert.Throws(() => new WorkspaceStore(root)); + } +} + +public sealed class SqliteWorkspaceRecoveryTests(TestContext context) : WorkspaceRecoveryTests(context) +{ + protected override Backend Backend => Backend.SqliteFile; +} + +[RequiresEnvironment(PgTestServer.Variable)] +public sealed class PostgresWorkspaceRecoveryTests(TestContext context) : WorkspaceRecoveryTests(context) +{ + protected override Backend Backend => Backend.Postgres; +}