From 7692ab402ece3dc4e4bccbbe8ca8c2b1727eadae Mon Sep 17 00:00:00 2001 From: Alberto Balbo Date: Tue, 3 Nov 2026 15:10:44 +0100 Subject: [PATCH] fix(workspace): a cancelled copy clears only media no other copy writes; a deletion keeps late versions Owner review 2, phase 5a, fix round 5. - A cancelled workspace copy clears its partial media only under the project's lease, after checking again that the job is still cancelled, that no other copy of the same root and project is queued, running or paused, and that no copy in this process is writing. The engine now waits for a paused job's run to end before calling the cancel handler, so the clean-up is never skipped. - The clean-up never recreates a project folder that was removed meanwhile (the lease no longer creates the folder). The next list removes a project folder that holds only lease files nobody holds, older than the lease. - A Main this Chronos cannot read (newer or damaged) refuses the open at once only when nothing else can be offered. Otherwise Open in workspace lists it disabled, with the reason as its tooltip. The refusal's action tooltip is now Remove copy. - The removal plan counts a conflict the same way the sync status does: a conflict that a settling replay took past its commit point is not work left. - A healed record is stamped after both the record's time and the index's, so the next rename made elsewhere is taken. - A trash folder holds back the rebuild of a lost record only when it is this sequence's deletion: its record names this sequence or cannot be read, or it has no record, is within a one-day grace and holds no version of another project. What was seen of folders that are gone or have a record again is forgotten. - When a deletion wins, version files pushed into the live folder before the deletion arrived are moved beside the deleted record in the trash, never deleted. Co-Authored-By: Claude Opus 5.5 --- src/Chronos.App/Sections/SequenceViewModel.cs | 33 +++-- src/Chronos.App/Workspace/WorkspaceDialogs.cs | 29 +++- .../Workspace/WorkspaceDialogs.xaml | 11 +- src/Chronos.App/Workspace/WorkspaceHost.cs | 5 +- src/Chronos.Core/Archiving/LeaseFile.cs | 20 ++- src/Chronos.Data/Jobs/JobEngine.cs | 16 +- src/Chronos.Data/Projects/ProjectIndexer.cs | 127 ++++++++++++++-- src/Chronos.Data/Projects/ProjectService.cs | 6 +- .../Repositories/JobRepository.cs | 7 + .../Workspaces/WorkspaceCopyJobHandler.cs | 102 +++++++++++-- src/Chronos.Data/Workspaces/WorkspaceStore.cs | 78 +++++++++- src/Chronos.Data/Workspaces/WorkspaceSync.cs | 2 +- .../Workspace/WorkspaceHostTests.cs | 28 ++++ .../Workspace/WorkspaceSectionTests.cs | 68 ++++++++- .../Projects/ProjectRecoveryTests.cs | 140 ++++++++++++++++++ .../Workspaces/WorkspaceSafetyTests.cs | 139 +++++++++++++++++ 16 files changed, 750 insertions(+), 61 deletions(-) diff --git a/src/Chronos.App/Sections/SequenceViewModel.cs b/src/Chronos.App/Sections/SequenceViewModel.cs index aa0cd014..f03e0cf0 100644 --- a/src/Chronos.App/Sections/SequenceViewModel.cs +++ b/src/Chronos.App/Sections/SequenceViewModel.cs @@ -607,13 +607,19 @@ public sealed class SequenceViewModel : SectionViewModel // This Chronos cannot read its working state (damaged, or a newer Chronos's): nothing is checked out over it. if (entry is { IsReadable: false } unreadable) throw WorkspaceHost.Unreadable(unreadable); // No project of the sequence here opens, and its Main here is such a copy (never among its projects): said at once, - // with Remove local copy for a damaged one, rather than asking which project and then refusing Main. + // with Remove local copy for a damaged one, when nothing else could be chosen (Main asked for, or the home has no + // other project); otherwise "Open in workspace" lists Main as it is, disabled, and offers the others. + WorkspaceEntry? unreadableMain = null; if (entry is null && request?.ProjectId is null && await UnreadableMainAsync(workspace, row.Id) is { } main) { - RefuseUnreadable(main); - _afterOpen = null; - Shell.KeepSequence(OpenSequenceId); - return null; + if (request is { Main: true } || (await workspace.HomeProjectsAsync(row.Id)).All(p => p.IsMain)) + { + RefuseUnreadable(main); + _afterOpen = null; + Shell.KeepSequence(OpenSequenceId); + return null; + } + unreadableMain = main; } if (entry is { HasMedia: true }) { @@ -631,7 +637,7 @@ public sealed class SequenceViewModel : SectionViewModel } else if (entry is null && projectId is null && request is not { Main: true }) { - if (await AskProjectAsync(workspace, row) is not { } choice) + if (await AskProjectAsync(workspace, row, unreadableMain) is not { } choice) { _afterOpen = null; Shell.KeepSequence(OpenSequenceId); @@ -679,14 +685,15 @@ public sealed class SequenceViewModel : SectionViewModel } var item = new WorkspaceProjectItem(copy, null); Shell.Notify(Loc.L("Editor.Workspace.OpenFailed"), Loc.L("Errors.Workspace.Damaged"), Loc.L("Editor.Project.RemoveCopy"), - Loc.L("Tooltips.Editor.Workspace.State.DamagedCopy"), () => _ = RemoveProjectCopyAsync(item), isError: true); + Loc.L("Tooltips.Editor.Project.RemoveCopy"), () => _ = RemoveProjectCopyAsync(item), isError: true); } /// /// "Open in workspace": the projects of the sequence's home, the media, what their copy writes and the room the - /// workspace's drive has; null when closed without a choice. + /// workspace's drive has; null when closed without a choice. , the copy of Main here + /// this Chronos cannot read, is listed disabled, why in its tooltip. /// - private async Task AskProjectAsync(WorkspaceHost workspace, Sequence row) + private async Task AskProjectAsync(WorkspaceHost workspace, Sequence row, WorkspaceEntry? unreadableMain = null) { var projects = await workspace.HomeProjectsAsync(row.Id); WorkspaceCopyFacts? copy = null; @@ -699,7 +706,13 @@ public sealed class SequenceViewModel : SectionViewModel // Only shown: the copy measures again before it writes. Log.Info("workspace", $"The copy of sequence {row.Id} could not be measured: {ex.Message}"); } - var dialog = new OpenInWorkspaceDialogViewModel(row.DisplayTitle, projects, d => CloseDialog(d), copy: copy); + var unreadable = unreadableMain is { } main + ? new Dictionary + { + [main.ProjectId] = Loc.L(main.Newer ? "Tooltips.Editor.Workspace.State.NewerCopy" : "Tooltips.Editor.Workspace.State.DamagedCopy"), + } + : null; + var dialog = new OpenInWorkspaceDialogViewModel(row.DisplayTitle, projects, d => CloseDialog(d), copy: copy, unreadable: unreadable); // The picker or the open sequence shows behind the dialog meanwhile, not the progress bar. IsLoading = false; OpenDialog(dialog); diff --git a/src/Chronos.App/Workspace/WorkspaceDialogs.cs b/src/Chronos.App/Workspace/WorkspaceDialogs.cs index 37ca4e06..d6050b0d 100644 --- a/src/Chronos.App/Workspace/WorkspaceDialogs.cs +++ b/src/Chronos.App/Workspace/WorkspaceDialogs.cs @@ -16,11 +16,19 @@ namespace Chronos.App.Workspace; /// What "Open in workspace" chose: an existing project of the home () or a new one (), and the media. public sealed record OpenChoice(Guid? ProjectId, string? NewName, string Media); -/// A project of the home offered by "Open in workspace" (Main in the UI language). -public sealed record ProjectOption(SequenceProject Project) +/// +/// A project of the home offered by "Open in workspace" (Main in the UI language). One whose copy here this Chronos cannot +/// read (: why) is shown, disabled, its reason in the tooltip. +/// +public sealed record ProjectOption(SequenceProject Project, string? Unreadable = null) { public string Label => WorkspaceText.ProjectName(Project.Name, Project.IsMain); + /// It can be chosen: its copy here, if any, is one this Chronos reads. + public bool IsAvailable => Unreadable is null; + + public string ToolTip => Unreadable ?? Loc.L("Tooltips.Editor.OpenWorkspace.Projects"); + public override string ToString() => Label; } @@ -50,8 +58,11 @@ public sealed class OpenInWorkspaceDialogViewModel : ArchiveDialogViewModel /// The names a new project cannot take (default: those of ). /// What the copy of the media writes and the free space of the workspace's drive; null when not measured (shown as nothing). + /// + /// The projects whose copy here this Chronos cannot read (damaged, or a newer Chronos's), with why: listed, never chosen. + /// public OpenInWorkspaceDialogViewModel(string sequenceTitle, IReadOnlyList projects, Action close, - IReadOnlyList? taken = null, WorkspaceCopyFacts? copy = null) + IReadOnlyList? taken = null, WorkspaceCopyFacts? copy = null, IReadOnlyDictionary? unreadable = null) : base(close) { SequenceTitle = sequenceTitle; @@ -63,9 +74,9 @@ public sealed class OpenInWorkspaceDialogViewModel : ArchiveDialogViewModel new DialogFact(Loc.L("Editor.OpenWorkspace.Free"), copy.Free is { } free ? Formatting.Bytes(free) : Loc.L("Server.Sync.None"), Reason: Loc.L(copy.TooLittleSpace ? "Errors.Workspace.NoRoom" : "Tooltips.Editor.OpenWorkspace.Free")), ]; - Projects = [.. projects.OrderByDescending(p => p.IsMain).Select(p => new ProjectOption(p))]; - _selected = Projects.FirstOrDefault(); - _isNew = Projects.Count == 0; + Projects = [.. projects.OrderByDescending(p => p.IsMain).Select(p => new ProjectOption(p, unreadable?.GetValueOrDefault(p.Id)))]; + _selected = Projects.FirstOrDefault(p => p.IsAvailable); + _isNew = !HasProjects; var names = taken ?? [.. projects.Select(p => p.Name)]; NewName = new EditableField(string.Empty, n => ProjectNames.Validate(n, names), _ => { }, () => Loc.L("Tooltips.Editor.OpenWorkspace.Name")); NewName.PropertyChanged += (_, _) => Refresh(); @@ -90,13 +101,15 @@ public sealed class OpenInWorkspaceDialogViewModel : ArchiveDialogViewModel public IReadOnlyList Projects { get; } - public bool HasProjects => Projects.Count > 0; + /// A project of the home can be chosen (one this Chronos cannot read here is only listed). + public bool HasProjects => Projects.Any(p => p.IsAvailable); public ProjectOption? SelectedProject { get => _selected; set { + if (value is { IsAvailable: false }) return; if (Set(ref _selected, value) && value is not null) IsExisting = true; } } @@ -133,7 +146,7 @@ public sealed class OpenInWorkspaceDialogViewModel : ArchiveDialogViewModel public ICommand OpenCommand { get; } - public bool CanOpen => _isNew ? !NewName.HasErrors : _selected is not null; + public bool CanOpen => _isNew ? !NewName.HasErrors : _selected is { IsAvailable: true }; public string OpenToolTip => _isNew && NewName.Error is { } error ? error : Loc.L("Tooltips.Editor.OpenWorkspace.Open"); diff --git a/src/Chronos.App/Workspace/WorkspaceDialogs.xaml b/src/Chronos.App/Workspace/WorkspaceDialogs.xaml index cefab681..82f9eeee 100644 --- a/src/Chronos.App/Workspace/WorkspaceDialogs.xaml +++ b/src/Chronos.App/Workspace/WorkspaceDialogs.xaml @@ -17,6 +17,15 @@ + + + @@ -38,7 +47,7 @@ + ItemContainerStyle="{StaticResource ComboBoxItem.Project}" ToolTip="{l:Loc Tooltips.Editor.OpenWorkspace.Projects}" AutomationProperties.Name="{l:Loc Editor.OpenWorkspace.Project}" /> Takes the lease when it is free or stale; null when someone else holds it. - public static LeaseFile? TryAcquire(string locksDirectory, string name, string owner, TimeSpan? duration = null) + public static LeaseFile? TryAcquire(string locksDirectory, string name, string owner, TimeSpan? duration = null) => + TryAcquire(locksDirectory, name, owner, duration, createDirectory: true); + + /// + /// Takes the lease when it is free or stale; null when someone else holds it. Without + /// a folder that is not there is never made: the lease is refused. + /// + /// is not there and is not to be made. + public static LeaseFile? TryAcquire(string locksDirectory, string name, string owner, TimeSpan? duration, bool createDirectory) { var length = duration ?? DefaultDuration; var path = PathFor(locksDirectory, name); - Directory.CreateDirectory(locksDirectory); + if (createDirectory) Directory.CreateDirectory(locksDirectory); + else if (!Directory.Exists(locksDirectory)) throw new DirectoryNotFoundException($"{locksDirectory} is not there."); for (var attempt = 0; attempt < 3; attempt++) { var now = DateTimeOffset.UtcNow; @@ -77,14 +86,17 @@ public sealed class LeaseFile : IDisposable /// Waits up to for the lease, polling with back-off. /// Still held by someone else after . + /// + /// is not there (or went meanwhile) and is false. + /// public static async Task AcquireAsync(string locksDirectory, string name, string owner, TimeSpan wait, - TimeSpan? duration = null, CancellationToken cancellationToken = default) + TimeSpan? duration = null, CancellationToken cancellationToken = default, bool createDirectory = true) { var deadline = DateTime.UtcNow + wait; var delay = TimeSpan.FromMilliseconds(20); while (true) { - if (TryAcquire(locksDirectory, name, owner, duration) is { } lease) return lease; + if (TryAcquire(locksDirectory, name, owner, duration, createDirectory) is { } lease) return lease; if (DateTime.UtcNow >= deadline) throw new LeaseUnavailableException(name, Read(PathFor(locksDirectory, name))); await Task.Delay(delay, cancellationToken).ConfigureAwait(false); delay = TimeSpan.FromMilliseconds(System.Math.Min(delay.TotalMilliseconds * 2, LongestPoll.TotalMilliseconds)); diff --git a/src/Chronos.Data/Jobs/JobEngine.cs b/src/Chronos.Data/Jobs/JobEngine.cs index 67a16749..73cf6232 100644 --- a/src/Chronos.Data/Jobs/JobEngine.cs +++ b/src/Chronos.Data/Jobs/JobEngine.cs @@ -130,16 +130,18 @@ public sealed class JobEngine : IAsyncDisposable /// /// Cancels a queued or running job; a running handler observes its token. A paused or failed one, which no handler runs, - /// leaves what it wrote to its handler to clear (). + /// leaves what it wrote to its handler to clear (), once a handler of this engine that + /// was still stopping it (paused a moment ago: what it cleared itself was for a pause) has let go. /// public async Task CancelAsync(Guid id, CancellationToken cancellationToken = default) { var before = await _db.Jobs.GetAsync(id, cancellationToken).ConfigureAwait(false); await TransitionAsync(id, JobState.Cancelled, cancellationToken).ConfigureAwait(false); // Neither claimable nor running here: no handler cleans up after it itself. - if (before is not { State: JobState.Paused or JobState.Failed } || _running.ContainsKey(id) - || !_handlers.TryGetValue(before.Kind, out var handler) || handler is not ICancelledJobHandler cancelled) + if (before is not { State: JobState.Paused or JobState.Failed } || !_handlers.TryGetValue(before.Kind, out var handler) + || handler is not ICancelledJobHandler cancelled) return; + if (_running.TryGetValue(id, out var stopping)) await stopping.Ended.WaitAsync(cancellationToken).ConfigureAwait(false); try { await cancelled.CancelledAsync(before with { State = JobState.Cancelled }, cancellationToken).ConfigureAwait(false); @@ -478,6 +480,7 @@ public sealed class JobEngine : IAsyncDisposable if (yieldsTo is not null) context.Log(Loc.L("Jobs.Line.Yielded", Loc.L("Jobs.Kind." + yieldsTo.Kind))); await PersistAsync(context).ConfigureAwait(false); _running.TryRemove(job.Id, out _); + run.End(); // Engine stopping: leave the job running so the next start requeues it from its checkpoint. if (shutdown.IsCancellationRequested) return; // Yielded to a higher job: back to the queue with its checkpoint, unless the user paused or cancelled it @@ -497,6 +500,7 @@ public sealed class JobEngine : IAsyncDisposable finally { _running.TryRemove(job.Id, out _); + run.End(); } await _db.Jobs.SetStateAsync(job.Id, final, error, CancellationToken.None).ConfigureAwait(false); await NotifyAsync(job.Id, CancellationToken.None).ConfigureAwait(false); @@ -506,9 +510,15 @@ public sealed class JobEngine : IAsyncDisposable private sealed class RunningJob(Job job, CancellationTokenSource cancellation) { private Job? _yieldsTo; + private readonly TaskCompletionSource _ended = new(TaskCreationOptions.RunContinuationsAsynchronously); public Job Job { get; } = job; + /// Completes once the handler has returned and the job left the engine's running jobs. + public Task Ended => _ended.Task; + + public void End() => _ended.TrySetResult(); + /// Jobs this one queued as follow-ups (): they never make it yield. public ConcurrentDictionary FollowUps { get; } = new(); diff --git a/src/Chronos.Data/Projects/ProjectIndexer.cs b/src/Chronos.Data/Projects/ProjectIndexer.cs index 3a69bab1..6a61b627 100644 --- a/src/Chronos.Data/Projects/ProjectIndexer.cs +++ b/src/Chronos.Data/Projects/ProjectIndexer.cs @@ -45,8 +45,17 @@ internal sealed class ProjectIndexer(ChronosDatabase db) /// Versions read at once when a project's sidecars are written again from the index. private const int WriteBackPage = 500; - /// The live folders with no record seen by a reconcile, by project: what they held then, and when (). - private readonly ConcurrentDictionary _unsettled = new(); + /// + /// The live folders with no record seen by a reconcile, by project: of which sequence, what they held then, and when + /// (). A reconcile of the sequence that no longer meets the folder so forgets it. + /// + private readonly ConcurrentDictionary _unsettled = new(); + + /// + /// How long a folder of the projects' trash with no record (a deletion on its way, the record not there yet) holds back the + /// rebuild of a lost record, counted from the newest thing written in it: past that it is a leftover, not a deletion. + /// + internal static readonly TimeSpan TrashRecordGrace = TimeSpan.FromDays(1); /// /// How long a live folder with no record must look the same before its record is written again from the index (a sync @@ -62,10 +71,21 @@ internal sealed class ProjectIndexer(ChronosDatabase db) var seen = new HashSet(); var heldBack = new HashSet(); var live = new Dictionary(); + // The projects of this sequence whose record in the projects' trash was read, with the newest such folder. + var deletions = new Dictionary(); + var unrecorded = new HashSet(); if (!Directory.Exists(folder.ProjectsDirectory)) return new ProjectReconcile(true, seen, heldBack, live); foreach (var (directory, trashed) in ProjectFolders(folder)) { cancellationToken.ThrowIfCancellationRequested(); + // What is left of the live folder of a project whose deletion was read (the trash is read first): skipped, the + // versions it holds beyond the trash's moved there, beside the deleted record (a push made before the deletion + // reached this PC). + if (!trashed && Guid.TryParseExact(Path.GetFileName(directory), "N", out var left) && deletions.TryGetValue(left, out var deletion)) + { + if (writable) KeepWithDeletion(directory, deletion, left); + continue; + } ProjectRecord? record = null; var recordPath = Path.Combine(directory, ArchiveLayout.ProjectFile); try @@ -90,8 +110,11 @@ internal sealed class ProjectIndexer(ChronosDatabase db) } // A record lost (a sync tool, a disk) of a live project the index holds: written again from the index, once the // folder has settled and no deletion of it is on its way (Rebuildable). - if (writable && row is { DeletedAt: null } && row.SequenceId == sequenceId && Rebuildable(folder, directory, named)) - record = await RecordFromIndexAsync(folder, row, recordPath, cancellationToken).ConfigureAwait(false); + if (writable && row is { DeletedAt: null } && row.SequenceId == sequenceId) + { + unrecorded.Add(named); + if (Rebuildable(folder, directory, named, sequenceId)) record = await RecordFromIndexAsync(folder, row, recordPath, cancellationToken).ConfigureAwait(false); + } } if (record is null) { @@ -103,9 +126,11 @@ internal sealed class ProjectIndexer(ChronosDatabase db) continue; } } - _unsettled.TryRemove(record.Id, out _); - // A sequence folder copied by hand carries the projects of its source, which stay that sequence's. - if (record.SequenceId != sequenceId || !seen.Add(record.Id)) continue; + // A sequence folder copied by hand carries the projects of its source, which stay that sequence's. (A live folder + // with a record now is no longer unsettled: forgotten after the loop.) + if (record.SequenceId != sequenceId) continue; + if (trashed) deletions[record.Id] = directory; + if (!seen.Add(record.Id)) continue; if (trashed && record.Deleted is null) record = record with { Deleted = Directory.GetLastWriteTimeUtc(directory) }; if (record.Deleted is null) { @@ -123,6 +148,11 @@ internal sealed class ProjectIndexer(ChronosDatabase db) if (record.Deleted is null) heldBack.Add(record.Id); } } + // A folder with no record this sequence no longer has (it got one, or went): what was seen of it is forgotten. + foreach (var (id, unsettled) in _unsettled) + { + if (unsettled.Sequence == sequenceId && !unrecorded.Contains(id)) _unsettled.TryRemove(id, out _); + } try { await SettleMainAsync(folder, sequenceId, live, heldBack, writable, cancellationToken).ConfigureAwait(false); @@ -310,18 +340,19 @@ internal sealed class ProjectIndexer(ChronosDatabase db) /// /// Whether the live folder of project , which has no record, may - /// have its record written again from the index now. Never while the projects' trash holds a folder of it (a deletion on - /// its way, its record maybe not there yet), and only once the folder has looked the same, file by file, on two - /// reconciles apart: a sync tool still bringing the real record (with a rename, a newer head) - /// has it win. Until then the project is held back. + /// have its record written again from the index now. Never while the projects' trash holds a folder of it that may be a + /// deletion on its way (), and only once the folder has looked the same, file by file, on + /// two reconciles apart: a sync tool still bringing the real record (with a rename, a newer + /// head) has it win. Until then the project is held back. /// - private bool Rebuildable(SequencePaths folder, string directory, Guid projectId) + private bool Rebuildable(SequencePaths folder, string directory, Guid projectId, Guid sequenceId) { var prefix = projectId.ToString("N") + "-"; try { if (Directory.Exists(folder.ProjectsTrashDirectory) - && Directory.EnumerateDirectories(folder.ProjectsTrashDirectory).Any(d => Path.GetFileName(d).StartsWith(prefix, StringComparison.OrdinalIgnoreCase))) + && Directory.EnumerateDirectories(folder.ProjectsTrashDirectory) + .Any(d => Path.GetFileName(d).StartsWith(prefix, StringComparison.OrdinalIgnoreCase) && DeletionOnItsWay(d, projectId, sequenceId, DateTime.UtcNow))) { Log.Info("projects", $"Project {projectId} has a folder in the projects' trash; its record is not written again"); return false; @@ -334,7 +365,7 @@ internal sealed class ProjectIndexer(ChronosDatabase db) _unsettled.TryRemove(projectId, out _); return true; } - _unsettled[projectId] = (look, now); + _unsettled[projectId] = (sequenceId, look, now); Log.Info("projects", $"Project {projectId} has no record in its folder; it is written again from the index once the folder settles"); return false; } @@ -345,6 +376,74 @@ internal sealed class ProjectIndexer(ChronosDatabase db) } } + /// + /// Whether the folder of the projects' trash, named for , may be + /// a deletion of this sequence's project on its way. Its record names the sequence (one of another, a sequence folder + /// copied by hand, is not), or cannot be read (nothing is guessed). Without a record (versions name no sequence) it is + /// one until has passed since anything was last written in it, unless a version in it is + /// another project's: a leftover then, which no longer holds a live project back. + /// + internal static bool DeletionOnItsWay(string directory, Guid projectId, Guid sequenceId, DateTime nowUtc) + { + var recordPath = Path.Combine(directory, ArchiveLayout.ProjectFile); + if (File.Exists(recordPath)) + { + try + { + return ProjectSidecars.ReadRecord(recordPath).SequenceId == sequenceId; + } + catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) + { + return true; + } + } + var info = new DirectoryInfo(directory); + var newest = info.LastWriteTimeUtc; + foreach (var entry in info.EnumerateFileSystemInfos("*", SearchOption.AllDirectories)) + { + if (entry.LastWriteTimeUtc > newest) newest = entry.LastWriteTimeUtc; + } + if (nowUtc - newest >= TrashRecordGrace) return false; + foreach (var (_, _, path) in ProjectSidecars.ListVersions(Path.Combine(directory, ArchiveLayout.ProjectVersionsFolder))) + { + try + { + if (ProjectSidecars.ReadVersion(path).ProjectId != projectId) return false; + } + catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) + { + // Not read: proves nothing either way. + } + } + return true; + } + + /// + /// The version files the live folder of a deleted project holds that its folder in the + /// projects' trash () lacks, moved there beside the deleted record: a version pushed into the + /// live folder before the deletion reached this PC is kept with the project, never deleted nor left where nothing reads + /// it. A file the trash has under the same name is left as it is. + /// + private static void KeepWithDeletion(string directory, string deletion, Guid projectId) + { + var target = Path.Combine(deletion, ArchiveLayout.ProjectVersionsFolder); + foreach (var (number, _, path) in ProjectSidecars.ListVersions(Path.Combine(directory, ArchiveLayout.ProjectVersionsFolder))) + { + var to = Path.Combine(target, Path.GetFileName(path)); + try + { + if (File.Exists(to) || ProjectSidecars.ReadVersion(path).ProjectId != projectId) continue; + Directory.CreateDirectory(target); + File.Move(path, to); + Log.Warning("projects", $"Version {number} of deleted project {projectId} was in its live folder; it is kept beside its deleted record in {deletion}"); + } + catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) + { + Log.Warning("projects", $"Version file {path} of deleted project {projectId} could not be moved beside its deleted record; tried again next time", ex); + } + } + } + /// What a project folder holds now, file by file (path, size, time written), as one hash. private static string Look(string directory) { diff --git a/src/Chronos.Data/Projects/ProjectService.cs b/src/Chronos.Data/Projects/ProjectService.cs index fdda901f..1f4033e2 100644 --- a/src/Chronos.Data/Projects/ProjectService.cs +++ b/src/Chronos.Data/Projects/ProjectService.cs @@ -836,12 +836,14 @@ public sealed class ProjectService /// /// A record behind the index made whole: the version files past its head written from the index (those there are left - /// as they are), then the record naming the index's head. + /// as they are), then the record naming the index's head, stamped past both the record's time and what the index knows of + /// the project (a rename taken from a clock ahead), so a rename made next anywhere is newer than what this index knows. /// private async Task HealAsync(SequencePaths folder, SequenceProject project, ProjectRecord record, CancellationToken cancellationToken) { await WriteVersionsAsync(folder, project.Id, record.Head?.Number ?? 0, cancellationToken).ConfigureAwait(false); - var healed = ProjectRecord.Of(project with { UpdatedAt = ProjectIndexer.UpdatedAfter(record.Updated) }, await MirroredHeadAsync(project, record.Mirrored, cancellationToken) + var known = record.Updated > project.UpdatedAt ? record.Updated : project.UpdatedAt; + var healed = ProjectRecord.Of(project with { UpdatedAt = ProjectIndexer.UpdatedAfter(known) }, await MirroredHeadAsync(project, record.Mirrored, cancellationToken) .ConfigureAwait(false)); ProjectSidecars.WriteRecord(folder.ProjectRecordPath(project.Id), healed); Log.Warning("projects", $"The record of project {project.Id} named version {record.Head?.Number} as its head, behind the index's {project.HeadNumber}; " diff --git a/src/Chronos.Data/Repositories/JobRepository.cs b/src/Chronos.Data/Repositories/JobRepository.cs index 8fbbe23e..f13ad4d3 100644 --- a/src/Chronos.Data/Repositories/JobRepository.cs +++ b/src/Chronos.Data/Repositories/JobRepository.cs @@ -195,6 +195,13 @@ public sealed class JobRepository return await _db.ScalarAsync(sql, p.ToArray(), cancellationToken).ConfigureAwait(false) > 0; } + /// The queued, running and paused jobs of that target , newest first (5000 at most). + public Task> ListPendingAsync(JobKind kind, Guid targetId, CancellationToken cancellationToken = default) => + // Target ids are stored comma-joined as lower-case text, which never holds a LIKE wildcard. + _db.ListAsync( + $"SELECT {Columns} FROM jobs WHERE kind = $1 AND state IN ({PendingStates}) AND target_ids LIKE $2 ESCAPE '\\' ORDER BY created DESC, id DESC LIMIT 5000", + [(int)kind, SqlDialect.ContainsPattern(targetId.ToString())], Map, cancellationToken); + /// The targets of the queued, running and paused jobs of (the newest 5000 such jobs). public Task> ListPendingTargetsAsync(JobKind kind, CancellationToken cancellationToken = default) => _db.RunAsync(async session => diff --git a/src/Chronos.Data/Workspaces/WorkspaceCopyJobHandler.cs b/src/Chronos.Data/Workspaces/WorkspaceCopyJobHandler.cs index 7cfc389a..98f4bf69 100644 --- a/src/Chronos.Data/Workspaces/WorkspaceCopyJobHandler.cs +++ b/src/Chronos.Data/Workspaces/WorkspaceCopyJobHandler.cs @@ -66,8 +66,9 @@ public static class WorkspaceJobs /// job stops saying how much is missing (the workspace shares this PC's drive with the index and the logs). /// Checkpoint: copy:<frame>:<frames version>:<folder>; a run resumed on other frames or in another folder /// checks every file again, and a pass started again stores that before it reads anything. A first copy the user cancels -/// removes the media it wrote itself, once it has stopped writing and under the project's lease (); -/// one cancelled while paused or failed, which no handler runs, has them removed the same way (). +/// removes the media it wrote itself, once it has stopped writing and under the project's lease, unless another copy of the +/// project goes on with them (); one cancelled while paused or failed, which no handler +/// runs, has them removed the same way (). /// Originals come in phase 5c: asked for, the job fails saying so. /// internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService archives) : ICancelledJobHandler @@ -110,7 +111,7 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService try { - await CopyAsync(context, project, sequenceId, archive, parameters, ct).ConfigureAwait(false); + using (WritingMedia(project)) await CopyAsync(context, project, sequenceId, archive, parameters, ct).ConfigureAwait(false); } catch (OperationCanceledException) when (ct.IsCancellationRequested) { @@ -158,27 +159,106 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService /// A first copy of the project's media the user cancelled (the job reads ), once it has /// stopped writing: the photos it wrote (copies the home can make again) are removed under the project's lease, so the /// workspace holds no half media. Media that a copy ever finished (media.json) stay, as do those of a copy paused, - /// yielded or stopped with the program, which goes on from its checkpoint. A failure is logged: the next copy checks - /// every file it finds. + /// yielded or stopped with the program, which goes on from its checkpoint, and those another copy of the project into + /// this workspace still writes or goes on with (queued, running or paused: ). The last + /// look is taken under the lease, and no copy of this program starts writing while the media go + /// (). A folder that is no project any more (its removal under way) is left as it is, and the + /// lease never makes it again. A failure is logged: the next copy checks every file it finds. /// private async Task RemovePartialMediaAsync(Job job, WorkspaceProject project) { try { - if (await db.Jobs.GetAsync(job.Id, CancellationToken.None).ConfigureAwait(false) is not { State: JobState.Cancelled }) return; - // Nothing half written (or the copy removed meanwhile): the lease, which makes the folder, is not taken. - if (File.Exists(project.MediaIndexPath) || !Directory.Exists(project.MediaDirectory)) return; - using (await project.LockAsync(LockWait, CancellationToken.None).ConfigureAwait(false)) + // Nothing half written (or the copy removed meanwhile): the lease is not taken. + if (!Clearable(project) || !await StillToClearAsync(job, project).ConfigureAwait(false)) return; + using (await project.LockExistingAsync(LockWait, CancellationToken.None).ConfigureAwait(false)) { - if (File.Exists(project.MediaIndexPath) || !Directory.Exists(project.MediaDirectory)) return; - Directory.Delete(project.MediaDirectory, recursive: true); + if (!Clearable(project) || !await StillToClearAsync(job, project).ConfigureAwait(false)) return; + lock (WritersGate) + { + // A copy of this program that started since the jobs were read: its files stay. + if (Writers.ContainsKey(project.Folder) || !Clearable(project)) return; + Directory.Delete(project.MediaDirectory, recursive: true); + } } Log.Info("workspace", $"The media a cancelled first copy of project {project.ProjectId} wrote were removed"); } + catch (DirectoryNotFoundException) + { + // The project's folder went meanwhile (its removal): nothing is left to clear. + } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or DatabaseException) { Log.Warning("workspace", $"The media of a cancelled copy of project {project.ProjectId} could not be removed", ex); } + + // A project (its working state there) with media no copy ever finished. + static bool Clearable(WorkspaceProject project) => project.Exists && !File.Exists(project.MediaIndexPath) && Directory.Exists(project.MediaDirectory); + } + + /// The cancelled copy still reads so, and no other copy of the project into this workspace waits, runs or is paused. + private async Task StillToClearAsync(Job job, WorkspaceProject project) + { + if (await db.Jobs.GetAsync(job.Id, CancellationToken.None).ConfigureAwait(false) is not { State: JobState.Cancelled }) return false; + if (await OtherCopyAsync(job, project).ConfigureAwait(false) is not { } other) return true; + Log.Info("workspace", $"The media of cancelled copy {job.Id} stay: copy {other.Id} of project {project.ProjectId} is {other.State}"); + return false; + } + + /// + /// Another copy of the project into the same workspace that is queued, running or paused: it writes the media folder, or + /// goes on with what is there. A job whose parameters cannot be read counts, so nothing is removed on a guess. + /// + private async Task OtherCopyAsync(Job job, WorkspaceProject project) + { + var sequenceId = job.TargetIds.FirstOrDefault(); + foreach (var other in await db.Jobs.ListPendingAsync(JobKind.WorkspaceCopy, sequenceId, CancellationToken.None).ConfigureAwait(false)) + { + if (other.Id == job.Id) continue; + try + { + var parameters = Parameters(other); + if (parameters.ProjectId != project.ProjectId) continue; + // Case aside: a match on a case-sensitive drive only keeps media that could have gone. + if (!string.Equals(WorkspaceStore.FullRoot(parameters.WorkspaceRoot), project.Store.Root, StringComparison.OrdinalIgnoreCase)) continue; + } + catch (Exception ex) when (ex is InvalidDataException or ArgumentException or IOException or NotSupportedException or System.Security.SecurityException) + { + Log.Debug("workspace", $"Copy {other.Id} cannot be read: it counts as one of project {project.ProjectId}: {ex.Message}"); + } + return other; + } + return null; + } + + /// The project folders copies of this program write media into now, with how many copies write each. + private static readonly Dictionary Writers = new(StringComparer.OrdinalIgnoreCase); + + private static readonly Lock WritersGate = new(); + + /// + /// Marks the copy into as writing its media until disposed: a clean-up after a cancelled copy + /// leaves them (), and one under way finishes before the copy writes anything. + /// + private static IDisposable WritingMedia(WorkspaceProject project) + { + lock (WritersGate) Writers[project.Folder] = Writers.GetValueOrDefault(project.Folder) + 1; + return new Writing(project.Folder); + } + + private sealed class Writing(string folder) : IDisposable + { + private int _done; + + public void Dispose() + { + if (Interlocked.Exchange(ref _done, 1) != 0) return; + lock (WritersGate) + { + if (Writers.GetValueOrDefault(folder) <= 1) Writers.Remove(folder); + else Writers[folder]--; + } + } } private async Task CopyAsync(JobContext context, WorkspaceProject project, Guid sequenceId, Archive archive, WorkspaceCopyParameters parameters, diff --git a/src/Chronos.Data/Workspaces/WorkspaceStore.cs b/src/Chronos.Data/Workspaces/WorkspaceStore.cs index 77fcac28..8e101450 100644 --- a/src/Chronos.Data/Workspaces/WorkspaceStore.cs +++ b/src/Chronos.Data/Workspaces/WorkspaceStore.cs @@ -128,7 +128,7 @@ public sealed class WorkspaceStore public WorkspaceStore(string root) { ArgumentException.ThrowIfNullOrWhiteSpace(root); - Root = LongPath.Normalize(Path.TrimEndingDirectorySeparator(Path.GetFullPath(root))); + Root = FullRoot(root); Directory.CreateDirectory(Root); var path = Path.Combine(Root, ManifestFile); if (!File.Exists(path)) Create(path, replace: false); @@ -152,18 +152,59 @@ public sealed class WorkspaceStore return project.Exists ? project : null; } - /// The projects of the workspace: every folder named by a project id that holds a project.chronos. + /// The folder a workspace at has as its (the store opened or not). + internal static string FullRoot(string root) => LongPath.Normalize(Path.TrimEndingDirectorySeparator(Path.GetFullPath(root))); + + /// + /// The projects of the workspace: every folder named by a project id that holds a project.chronos. A folder named + /// by a project id that holds nothing but what a lease no one holds left (a removal that met a lease taken meanwhile) + /// goes (). + /// public IReadOnlyList List() { var found = new List(); foreach (var directory in Directory.EnumerateDirectories(Root).Order(StringComparer.Ordinal)) { - if (Guid.TryParseExact(Path.GetFileName(directory), "N", out var id) && File.Exists(Path.Combine(directory, WorkspaceProject.BundleFile))) - found.Add(Project(id)); + if (!Guid.TryParseExact(Path.GetFileName(directory), "N", out var id)) continue; + if (File.Exists(Path.Combine(directory, WorkspaceProject.BundleFile))) found.Add(Project(id)); + else RemoveLeftover(directory, DateTime.UtcNow); } return found; } + /// + /// A project's folder that is no project and holds nothing a user or a writer has: no project.chronos, no folder, + /// only lease files (the lease and those set aside as stale), none of them held, and nothing written there for as long + /// as a lease lasts (a lease taken this moment makes its folder first). + /// + internal static bool IsLeftover(string folder, DateTime nowUtc) + { + var info = new DirectoryInfo(folder); + if (!info.Exists || File.Exists(Path.Combine(folder, WorkspaceProject.BundleFile)) || info.LastWriteTimeUtc + LeaseFile.DefaultDuration > nowUtc) return false; + foreach (var entry in info.EnumerateFileSystemInfos()) + { + if (entry is not FileInfo file || !WorkspaceProject.IsLeaseFile(file.Name) || file.LastWriteTimeUtc + LeaseFile.DefaultDuration > nowUtc) return false; + if (string.Equals(file.Name, WorkspaceProject.LeaseFileName, StringComparison.OrdinalIgnoreCase) && LeaseFile.Read(file.FullName) is { } held + && held.Expires >= nowUtc) + return false; + } + return true; + } + + private static void RemoveLeftover(string folder, DateTime nowUtc) + { + try + { + if (!IsLeftover(folder, nowUtc)) return; + Directory.Delete(folder, recursive: true); + Log.Info("workspace", $"{folder} held only an old lease and was removed"); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Debug("workspace", $"{folder} stays: {ex.Message}"); + } + } + /// The working states of the workspace's projects (one unreadable is left out, and logged). public IReadOnlyList<(WorkspaceProject Project, ProjectBundle Bundle)> ListBundles() { @@ -297,6 +338,12 @@ public sealed class WorkspaceProject private const string LeaseName = ".lock"; + /// The file of the project's lease () in . + internal const string LeaseFileName = LeaseName + ".lease"; + + /// The project's lease, or one set aside as stale (.lock.lease.<tag>.stale). + internal static bool IsLeaseFile(string name) => name.StartsWith(LeaseFileName, StringComparison.OrdinalIgnoreCase); + internal WorkspaceProject(WorkspaceStore store, Guid projectId) { Store = store; @@ -343,6 +390,16 @@ public sealed class WorkspaceProject return await LeaseFile.AcquireAsync(Folder, LeaseName, "workspace@" + Environment.MachineName, wait, cancellationToken: cancellationToken).ConfigureAwait(false); } + /// + /// Holds the project as does, never making its folder: taken by a writer that only clears what is + /// there, which a removal of the folder may race. + /// + /// The folder is not there, or went while the lease was awaited. + /// Another writer kept it for longer than . + internal async Task LockExistingAsync(TimeSpan wait, CancellationToken cancellationToken = default) => + await LeaseFile.AcquireAsync(Folder, LeaseName, "workspace@" + Environment.MachineName, wait, cancellationToken: cancellationToken, createDirectory: false) + .ConfigureAwait(false); + /// Not a project file of a workspace, damaged or newer. /// The file cannot be read. public ProjectBundle ReadBundle() @@ -582,6 +639,19 @@ public sealed class WorkspaceProject } } + /// + /// The conflict sync.json records that still waits for a choice, as reads + /// it: one a replay that settles it took past its commit point (its base moved onto the replay's) is over, the choice + /// made applied, even before the next sync clears it. + /// + /// sync.json or the working state cannot be read. + /// A file cannot be read. + public WorkspaceConflict? OpenConflict() => ReadSync() is { Conflict: not null } sync ? Unsettled(sync, ReadBundle()) : null; + + /// The conflict of that waits for a choice with the working state (). + internal static WorkspaceConflict? Unsettled(WorkspaceSyncState sync, ProjectBundle bundle) => + sync.Replay is { Settles: true } replay && bundle.Workspace?.Base?.VersionId == replay.OntoBase ? null : sync.Conflict; + internal void WriteSync(WorkspaceSyncState state) { ArgumentNullException.ThrowIfNull(state); diff --git a/src/Chronos.Data/Workspaces/WorkspaceSync.cs b/src/Chronos.Data/Workspaces/WorkspaceSync.cs index c03697df..ab84b05c 100644 --- a/src/Chronos.Data/Workspaces/WorkspaceSync.cs +++ b/src/Chronos.Data/Workspaces/WorkspaceSync.cs @@ -1122,7 +1122,7 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName) var workspace = bundle.Workspace!; // A take of the server's values (or any replay that settles) cut past its commit point applied the choice: the // conflict is over, as the next sync writes. - var conflict = sync.Replay is { Settles: true } replay && workspace.Base?.VersionId == replay.OntoBase ? null : sync.Conflict; + var conflict = WorkspaceProject.Unsettled(sync, bundle); ProjectVersion? head = null; HomeFrames? frames = null; var offline = false; diff --git a/tests/Chronos.App.Tests/Workspace/WorkspaceHostTests.cs b/tests/Chronos.App.Tests/Workspace/WorkspaceHostTests.cs index 069b0529..65cd8195 100644 --- a/tests/Chronos.App.Tests/Workspace/WorkspaceHostTests.cs +++ b/tests/Chronos.App.Tests/Workspace/WorkspaceHostTests.cs @@ -308,6 +308,34 @@ public sealed class WorkspaceHostTests(TestContext context) Assert.Empty(host.Store.List()); } + [Test] + public async Task AConflictASettlingReplayTookPastItsCommitPointIsNoWorkLeftForTheRemoval() + { + await using var bed = await WorkspaceTestBed.CreateAsync(context, frames: 4); + var host = bed.Host(new AppSettings { WorkspaceAutoSync = false }); + var mainId = await bed.CheckoutAsync(host); + var project = host.Store.Find(mainId)!; + var basis = project.ReadBundle().Workspace!.Base!; + var conflict = $$""" + "conflict": { "theirsVersionId": "{{basis.VersionId}}", "theirsNumber": {{basis.Number}}, "baseVersionId": "{{Guid.CreateVersion7()}}", + "mine": [], "differences": [], "found": "2026-11-01T10:00:00+00:00" } + """; + + // A take of the library's values cut after its base moved (before sync.json said so): the choice is applied. + await File.WriteAllTextAsync(project.SyncPath, + $$"""{ {{conflict}}, "replay": { "ontoBase": "{{basis.VersionId}}", "replaced": [], "replayed": [], "adopted": [], "settles": true } }"""); + Assert.NotNull(project.ReadSync().Conflict, "still in sync.json"); + Assert.Null(project.OpenConflict()); + Assert.Null((await host.StatusAsync(mainId)).Conflict); + var plan = await host.PlanRemovalAsync(mainId); + Assert.True(plan is { Conflict: false, IsSafe: true }, "counted as the status reads it"); + + // A conflict still waiting for a choice counts. + await File.WriteAllTextAsync(project.SyncPath, $$"""{ {{conflict}} }"""); + Assert.NotNull(project.OpenConflict()); + Assert.True((await host.PlanRemovalAsync(mainId)) is { Conflict: true, IsSafe: false }); + } + [Test] public async Task ACopyOfAProjectDeletedInTheLibraryReadsGoneAndGoesOnlyWhenDiscarded() { diff --git a/tests/Chronos.App.Tests/Workspace/WorkspaceSectionTests.cs b/tests/Chronos.App.Tests/Workspace/WorkspaceSectionTests.cs index 35a5e652..85d629a0 100644 --- a/tests/Chronos.App.Tests/Workspace/WorkspaceSectionTests.cs +++ b/tests/Chronos.App.Tests/Workspace/WorkspaceSectionTests.cs @@ -249,7 +249,7 @@ public sealed class WorkspaceSectionTests(TestContext context) shell.OpenSequence(sequenceId); await WorkspaceTestBed.UntilAsync(() => toasts.Any(t => t.Message == Loc.L("Errors.Workspace.Damaged")), "a damaged Main is refused"); var damaged = toasts.Single(t => t.Message == Loc.L("Errors.Workspace.Damaged")); - Assert.Equal((Loc.L("Editor.Workspace.OpenFailed"), true, Loc.L("Editor.Project.RemoveCopy"), Loc.L("Tooltips.Editor.Workspace.State.DamagedCopy")), + Assert.Equal((Loc.L("Editor.Workspace.OpenFailed"), true, Loc.L("Editor.Project.RemoveCopy"), Loc.L("Tooltips.Editor.Project.RemoveCopy")), (damaged.Title, damaged.IsError, damaged.ActionText, damaged.ActionToolTip)); Assert.True(section.Dialog is null && section.Editor is null, "nothing asked, nothing opened"); damaged.ActionCommand!.Execute(null); @@ -271,6 +271,72 @@ public sealed class WorkspaceSectionTests(TestContext context) } }); + [Test] + public Task AMainThisChronosCannotReadIsListedDisabledWhileAnotherProjectCanOpen() => ScreenHost.RunAsync(async () => + { + var directory = context.CreateTempDirectory("workspace-unreadable-main-others"); + var environment = AppEnvironment.From(AppPaths.Resolve(Directory.CreateDirectory(Path.Combine(directory, "app")).FullName, + Directory.CreateDirectory(Path.Combine(directory, "local")).FullName, [])); + UiLanguage.Apply("en"); + var settings = new AppSettings { Language = "en", FirstRunCompleted = true, CheckForUpdates = false }; + var services = new AppServices(environment, settings, () => { }, new ScriptedDialogs()); + await services.AttachAsync(await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(environment.DatabasePath)), startEngine: true); + var shell = new ShellViewModel(services); + var toasts = new List(); + shell.PendingToasts.CollectionChanged += (_, e) => toasts.AddRange(e.NewItems?.OfType() ?? []); + try + { + var sequenceId = await SequenceAsync(services, Path.Combine(directory, "archive")); + var section = shell.Sequence; + shell.OpenSequence(sequenceId); + await WorkspaceTestBed.UntilAsync(() => section.Dialog is OpenInWorkspaceDialogViewModel, "Open in workspace asks the first time"); + ((OpenInWorkspaceDialogViewModel)section.Dialog!).OpenCommand.Execute(null); + await WorkspaceTestBed.UntilAsync(() => section.Editor is not null, "Main opens"); + var mainId = section.OpenProjectId!.Value; + Assert.True(await section.ConfirmCloseAsync()); + await section.CloseAsync(); + shell.KeepSequence(null); + var vertical = (await services.Archives!.Projects.CreateAsync(sequenceId, "Vertical", cancellationToken: context.Cancellation)).Id; + var bundle = Path.Combine(services.Workspace.Root, mainId.ToString("N"), WorkspaceProject.BundleFile); + + // Damaged: the sequence has another project, so it is asked which; Main is listed, disabled, why in its tooltip. + await WriteWhenFreeAsync(bundle, "{ not a project"); + shell.OpenSequence(sequenceId); + await WorkspaceTestBed.UntilAsync(() => section.Dialog is OpenInWorkspaceDialogViewModel, "Open in workspace offers the other project"); + var asked = (OpenInWorkspaceDialogViewModel)section.Dialog!; + var main = asked.Projects.Single(p => p.Project.IsMain); + Assert.True(main is { IsAvailable: false } && main.ToolTip == Loc.L("Tooltips.Editor.Workspace.State.DamagedCopy"), "a damaged Main is listed as it is"); + Assert.Equal(vertical, asked.SelectedProject?.Project.Id); + asked.SelectedProject = main; + Assert.Equal(vertical, asked.SelectedProject?.Project.Id, "never chosen"); + Assert.True(asked.CanOpen && asked.HasProjects); + asked.CloseCommand.Execute(null); + await WorkspaceTestBed.UntilAsync(() => section.Dialog is null, "closed without a choice"); + Assert.False(toasts.Any(t => t.Message == Loc.L("Errors.Workspace.Damaged")), "nothing refused"); + shell.KeepSequence(null); + + // Saved by a newer Chronos: the same, and the other project opens; Main's copy is left as it is. + var newer = $$"""{ "format": "chronos-project", "formatVersion": 9, "sequence": { "title": "Synthetic" }, "workspace": { "projectId": "{{mainId}}", "name": "Main", "isMain": true } }"""; + await WriteWhenFreeAsync(bundle, newer); + shell.OpenSequence(sequenceId); + await WorkspaceTestBed.UntilAsync(() => section.Dialog is OpenInWorkspaceDialogViewModel, "Open in workspace offers the other project"); + asked = (OpenInWorkspaceDialogViewModel)section.Dialog!; + main = asked.Projects.Single(p => p.Project.IsMain); + Assert.True(main is { IsAvailable: false } && main.ToolTip == Loc.L("Tooltips.Editor.Workspace.State.NewerCopy"), "a newer Main is listed as it is"); + asked.OpenCommand.Execute(null); + await WorkspaceTestBed.UntilAsync(() => section.Editor is not null, "the other project opens"); + Assert.Equal(vertical, section.OpenProjectId); + Assert.False(toasts.Any(t => t.Message == Loc.L("Errors.Workspace.NewerCopy")), "nothing refused"); + Assert.Equal(newer, await File.ReadAllTextAsync(bundle)); + } + finally + { + await shell.Sequence.CloseAsync(); + shell.Dispose(); + await services.DisposeAsync(); + } + }); + /// Writes once the project's last background read (its state, after the close) lets go of it. private static async Task WriteWhenFreeAsync(string path, string text) { diff --git a/tests/Chronos.Data.Tests/Projects/ProjectRecoveryTests.cs b/tests/Chronos.Data.Tests/Projects/ProjectRecoveryTests.cs index d6a63e14..3273e38d 100644 --- a/tests/Chronos.Data.Tests/Projects/ProjectRecoveryTests.cs +++ b/tests/Chronos.Data.Tests/Projects/ProjectRecoveryTests.cs @@ -806,6 +806,146 @@ public abstract class ProjectRecoveryTests(TestContext context) : DatabaseTest(c Assert.SequenceEqual([main.Id], (await fx.Service.Projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); } + [Test] + public async Task APushMadeIntoTheLiveFolderBeforeADeletionArrivedIsKeptBesideTheDeletedRecord() + { + var fx = await FixtureAsync(); + var (sequence, frames) = await UploadedAsync(fx, 3); + var projects = fx.Service.Projects; + var main = await projects.MainAsync(sequence.Id, fx.Ct); + var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); + var first = await projects.HeadAsync(vertical.Id, fx.Ct); + var folder = fx.Paths.Sequence(sequence.Folder); + var (other, _) = await fx.OtherWriterAsync(); + var before = folder.ProjectsDirectory + "-before"; + CopyDirectory(folder.ProjectDirectory(vertical.Id), before); + await projects.DeleteAsync(vertical.Id, fx.Ct); + + // The other PC pushed version 2 into its live folder before the deletion reached it: a sync tool brings that folder + // back beside the trash. The deletion wins, and version 2 goes beside the deleted record, never left where nothing reads it. + CopyDirectory(before, folder.ProjectDirectory(vertical.Id)); + var theirs = Draft(first, new ProjectSettings { ExcludedFrames = [2] }, frames) with { StoredAt = DateTimeOffset.UtcNow }; + var pushed = folder.VersionPath(vertical.Id, 2, theirs.Id); + ProjectSidecars.WriteVersion(pushed, theirs); + var record = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)); + ProjectSidecars.WriteRecord(folder.ProjectRecordPath(vertical.Id), record with { Head = new ProjectHead(theirs.Id, 2), Updated = DateTimeOffset.UtcNow }); + Assert.SequenceEqual([main.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); + var trashed = Directory.GetDirectories(folder.ProjectsTrashDirectory).Single(); + var kept = Path.Combine(trashed, ArchiveLayout.ProjectVersionsFolder, Path.GetFileName(pushed)); + Assert.FileExists(kept); + Assert.FileMissing(pushed); + Assert.Equal(theirs.Id, ProjectSidecars.ReadVersion(kept).Id); + Assert.FileExists(Path.Combine(trashed, ArchiveLayout.ProjectVersionsFolder, Path.GetFileName(folder.VersionPath(vertical.Id, 1, first.Id)))); + Assert.NotNull(ProjectSidecars.ReadRecord(Path.Combine(trashed, ArchiveLayout.ProjectFile)).Deleted); + Assert.SequenceEqual([main.Id], (await other.Projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); + Assert.SequenceEqual([main.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); + } + + [Test] + public async Task ATrashFolderOfAnotherSequenceOrLongWithoutARecordNoLongerHoldsBackALostRecord() + { + var fx = await FixtureAsync(); + var (sequence, frames) = await UploadedAsync(fx, 3); + var projects = fx.Service.Projects; + var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); + var first = await projects.HeadAsync(vertical.Id, fx.Ct); + var folder = fx.Paths.Sequence(sequence.Folder); + projects.RecordSettle = TimeSpan.Zero; + var path = folder.ProjectRecordPath(vertical.Id); + var lost = ProjectSidecars.ReadRecord(path); + + // A folder of the projects' trash with no record, just made (a deletion on its way): the lost record is not written. + var leftover = Path.Combine(folder.ProjectsTrashDirectory, vertical.Id.ToString("N") + "-1"); + Directory.CreateDirectory(leftover); + File.Delete(path); + for (var i = 0; i < 2; i++) await projects.ListAsync(sequence.Id, fx.Ct); + Assert.False(File.Exists(path), "held back while a deletion may be on its way"); + await Assert.ThrowsAsync(() => projects.PushAsync(vertical.Id, Draft(first, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)); + + // Nothing written in it for longer than the grace: a leftover, which holds nothing back any more. + Directory.SetLastWriteTimeUtc(leftover, DateTime.UtcNow - ProjectIndexer.TrashRecordGrace - TimeSpan.FromHours(1)); + for (var i = 0; i < 2; i++) await projects.ListAsync(sequence.Id, fx.Ct); + Assert.Equal(("Vertical", first.Id), (ProjectSidecars.ReadRecord(path).Name, ProjectSidecars.ReadRecord(path).Head!.Id)); + Assert.Equal(PushOutcome.Stored, (await projects.PushAsync(vertical.Id, Draft(first, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)).Outcome); + Directory.Delete(leftover); + + // A trash folder whose record names another sequence (a sequence folder copied by hand): never this one's deletion. + var foreign = Path.Combine(folder.ProjectsTrashDirectory, vertical.Id.ToString("N") + "-2"); + Directory.CreateDirectory(foreign); + ProjectSidecars.WriteRecord(Path.Combine(foreign, ArchiveLayout.ProjectFile), lost with { SequenceId = Guid.CreateVersion7(), Deleted = DateTimeOffset.UtcNow }); + File.Delete(path); + for (var i = 0; i < 2; i++) await projects.ListAsync(sequence.Id, fx.Ct); + Assert.FileExists(path); + Assert.Contains(vertical.Id, (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); + + // One whose record names this sequence is its deletion. + Assert.True(ProjectIndexer.DeletionOnItsWay(foreign, vertical.Id, sequence.Id, DateTime.UtcNow) == false); + ProjectSidecars.WriteRecord(Path.Combine(foreign, ArchiveLayout.ProjectFile), lost with { Deleted = DateTimeOffset.UtcNow }); + Assert.True(ProjectIndexer.DeletionOnItsWay(foreign, vertical.Id, sequence.Id, DateTime.UtcNow)); + } + + [Test] + public async Task AFolderWithNoRecordThatWentAndCameBackSettlesAgainFromWhenItCameBack() + { + var fx = await FixtureAsync(); + var (sequence, _) = await UploadedAsync(fx, 3); + var vertical = await fx.Service.Projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); + var folder = fx.Paths.Sequence(sequence.Folder); + var settle = TimeSpan.FromMilliseconds(TestTiming.Budget(400)); + var indexer = new ProjectIndexer(fx.Db) { RecordSettle = settle }; + var path = folder.ProjectRecordPath(vertical.Id); + File.Delete(path); + Assert.Contains(vertical.Id, (await indexer.ReconcileAsync(folder, sequence.Id, fx.Ct)).HeldBack); + + // The folder goes for a while (a sync tool moving it): what was seen of it is forgotten, so once it is back it must + // look the same for a settling time again before its record is written from the index. + var away = folder.ProjectsDirectory + "-away"; + Directory.Move(folder.ProjectDirectory(vertical.Id), away); + Assert.False((await indexer.ReconcileAsync(folder, sequence.Id, fx.Ct)).Seen.Contains(vertical.Id)); + await Task.Delay(settle * 2, fx.Ct); + Directory.Move(away, folder.ProjectDirectory(vertical.Id)); + Assert.Contains(vertical.Id, (await indexer.ReconcileAsync(folder, sequence.Id, fx.Ct)).HeldBack); + Assert.False(File.Exists(path), "not written at once on what was seen before it went"); + await Task.Delay(settle * 2, fx.Ct); + Assert.False((await indexer.ReconcileAsync(folder, sequence.Id, fx.Ct)).HeldBack.Contains(vertical.Id)); + Assert.FileExists(path); + } + + [Test] + public async Task AHealedRecordIsStampedPastWhatTheIndexKnowsSoTheNextRenameIsTaken() + { + var fx = await FixtureAsync(); + var (sequence, frames) = await UploadedAsync(fx, 3); + var projects = fx.Service.Projects; + var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); + var folder = fx.Paths.Sequence(sequence.Folder); + var backup = folder.ProjectsDirectory + "-backup"; + CopyDirectory(folder.ProjectsDirectory, backup); + await projects.PushAsync(vertical.Id, Draft(await projects.HeadAsync(vertical.Id, fx.Ct), new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct); + + // A PC whose clock is ten minutes ahead renames it: this index takes the name and that time. + var ahead = DateTimeOffset.UtcNow.AddMinutes(10); + ProjectSidecars.WriteRecord(folder.ProjectRecordPath(vertical.Id), ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)) with { Name = "Square", Updated = ahead }); + Assert.Equal("Square", (await projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id).Name); + + // The backup is put back (behind): healed, and stamped past what the index knows. + Directory.Delete(folder.ProjectsDirectory, recursive: true); + CopyDirectory(backup, folder.ProjectsDirectory); + await projects.CreateAsync(sequence.Id, "Other", cancellationToken: fx.Ct); + var healed = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)); + Assert.Equal(("Square", 2), (healed.Name, healed.Head!.Number)); + Assert.True(healed.Updated > ahead, "past the time the index took"); + + // Another PC with a right clock renames it: every index takes it, and nothing writes the old name back. + var (other, _) = await fx.OtherWriterAsync(); + Assert.Equal("Square", (await other.Projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id).Name); + await other.Projects.RenameAsync(vertical.Id, "Wide", fx.Ct); + Assert.Equal("Wide", (await projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id).Name); + await projects.CreateAsync(sequence.Id, "Other2", cancellationToken: fx.Ct); + Assert.Equal("Wide", ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)).Name); + Assert.Equal("Wide", (await other.Projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id).Name); + } + [Test] public async Task ALostRecordIsWrittenAgainOnlyOnceItsFolderHasSettledSoAnIncomingOneWins() { diff --git a/tests/Chronos.Data.Tests/Workspaces/WorkspaceSafetyTests.cs b/tests/Chronos.Data.Tests/Workspaces/WorkspaceSafetyTests.cs index d7274f25..d5312999 100644 --- a/tests/Chronos.Data.Tests/Workspaces/WorkspaceSafetyTests.cs +++ b/tests/Chronos.Data.Tests/Workspaces/WorkspaceSafetyTests.cs @@ -86,6 +86,9 @@ public abstract class WorkspaceSafetyTests(TestContext context) : DatabaseTest(c var (store, two, project, shown) = await AskedAsync(fx, id); await Assert.ThrowsAsync(() => WorkspaceFixture.Cutting(store, "replay.base").ResolveAsync(id, fx.Home, ConflictChoice.TakeServer, shown: shown, cancellationToken: fx.Ct)); + // sync.json still records the conflict, but it waits for no choice: what counts it reads it as the status does. + Assert.NotNull(project.ReadSync().Conflict); + Assert.Null(project.OpenConflict(), "settled past its commit point"); // The working state is on the library's head with its value: the choice was made, nothing asks it again. var status = await two.StatusAsync(id, fx.Home, fx.Ct); @@ -543,6 +546,142 @@ public abstract class WorkspaceSafetyTests(TestContext context) : DatabaseTest(c foreach (var frame in media.Frames) Assert.FileExists(project.MediaFile(frame.File!)); } + [Test] + public async Task APausedFirstCopyCancelledWhileASecondCopyOfTheProjectRunsLeavesItsMedia() + { + var fx = await FixtureAsync(); + var (store, sync) = fx.Workspace(); + var id = (await fx.Archive.Service.Projects.MainAsync(fx.Sequence.Id, fx.Ct)).Id; + var project = await sync.CheckoutAsync(fx.Home, id, cancellationToken: fx.Ct); + Job second; + // Held so that neither copy commits before the first is cancelled. + var lease = await project.LockAsync(Wait, fx.Ct); + try + { + var first = await fx.StartCopyAsync(store, id); + await UntilAsync(() => Directory.Exists(project.ProxiesDirectory) && Directory.GetFiles(project.ProxiesDirectory, "*.jpg").Length == fx.Frames.Count, + "the first copy's files", fx.Ct); + await fx.PauseAsync(first.Id); + await fx.Archive.WaitForAsync(first.Id, JobState.Paused, (int)TestTiming.Budget(30)); + // The sequence opened again queues another copy (the first is paused), then the first is cancelled in Jobs. + second = await fx.StartCopyAsync(store, id); + await fx.Archive.WaitForAsync(second.Id, JobState.Running, (int)TestTiming.Budget(30)); + await fx.CancelAsync(first.Id); + await fx.Archive.WaitForAsync(first.Id, JobState.Cancelled, (int)TestTiming.Budget(30)); + } + finally + { + lease.Dispose(); + } + await fx.Archive.WaitForAsync(second.Id, JobState.Done, (int)TestTiming.Budget(60)); + // The clean-up had its chance at the lease: it found the second copy and left the media. + await Task.Delay(500, fx.Ct); + using (await project.LockAsync(Wait, fx.Ct)) + { + var media = Assert.NotNull(project.ReadMedia()); + Assert.Equal(fx.Frames.Count, media.Frames.Count); + foreach (var frame in media.Frames) Assert.FileExists(project.MediaFile(frame.File!)); + } + var status = await sync.StatusAsync(id, fx.Home, fx.Ct); + Assert.True(status is { State: WorkspaceState.InSync, MediaRemoved: false }); + } + + [Test] + public async Task ACancelledCopysCleanUpLeavesAFolderNoLongerAProjectAndNeverMakesOneAgain() + { + var fx = await FixtureAsync(); + var (store, sync) = fx.Workspace(); + var main = (await fx.Archive.Service.Projects.MainAsync(fx.Sequence.Id, fx.Ct)).Id; + var side = (await fx.Archive.Service.Projects.CreateAsync(fx.Sequence.Id, "Side", cancellationToken: fx.Ct)).Id; + + // The working state moved aside while the clean-up waited for the lease (Remove local copy): its folder is left as it is. + var moving = await PausedFirstCopyAsync(fx, store, sync, main); + var lease = await moving.Project.LockAsync(Wait, fx.Ct); + var aside = Path.Combine(Context.CreateTempDirectory("aside"), WorkspaceProject.BundleFile); + try + { + await fx.CancelAsync(moving.Copy.Id); + // The clean-up waits for the lease meanwhile. + await Task.Delay(300, fx.Ct); + File.Move(moving.Project.BundlePath, aside); + } + finally + { + lease.Dispose(); + } + await Task.Delay(1000, fx.Ct); + Assert.True(Directory.Exists(moving.Project.ProxiesDirectory), "a folder that is no project is not the clean-up's"); + + // The folder removed while the clean-up waited: it is never made again. + var removed = await PausedFirstCopyAsync(fx, store, sync, side); + lease = await removed.Project.LockAsync(Wait, fx.Ct); + try + { + await fx.CancelAsync(removed.Copy.Id); + await Task.Delay(300, fx.Ct); + Directory.Delete(removed.Project.Folder, recursive: true); + } + finally + { + lease.Dispose(); + } + await Task.Delay(1000, fx.Ct); + Assert.False(Directory.Exists(removed.Project.Folder), "no folder made again for a lease"); + } + + /// A first copy of , checked out, that wrote every file and is paused before it committed. + private async Task<(WorkspaceProject Project, Job Copy)> PausedFirstCopyAsync(WorkspaceFixture fx, WorkspaceStore store, WorkspaceSync sync, Guid projectId) + { + var project = await sync.CheckoutAsync(fx.Home, projectId, cancellationToken: fx.Ct); + using (await project.LockAsync(Wait, fx.Ct)) + { + var copy = await fx.StartCopyAsync(store, projectId); + await UntilAsync(() => Directory.Exists(project.ProxiesDirectory) && Directory.GetFiles(project.ProxiesDirectory, "*.jpg").Length == fx.Frames.Count, + "the copy's files", fx.Ct); + await fx.PauseAsync(copy.Id); + await fx.Archive.WaitForAsync(copy.Id, JobState.Paused, (int)TestTiming.Budget(30)); + return (project, copy); + } + } + + [Test] + public async Task AProjectFolderHoldingOnlyALeaseNoOneHoldsGoesOnTheNextList() + { + var store = new WorkspaceStore(Path.Combine(Context.CreateTempDirectory("leftovers"), "workspace")); + var old = DateTime.UtcNow - TimeSpan.FromHours(1); + string Folder(params string[] files) + { + var folder = Path.Combine(store.Root, Guid.CreateVersion7().ToString("N")); + Directory.CreateDirectory(folder); + foreach (var file in files) + { + var path = Path.Combine(folder, file); + if (!File.Exists(path)) File.WriteAllText(path, string.Empty); + File.SetLastWriteTimeUtc(path, old); + } + Directory.SetLastWriteTimeUtc(folder, old); + return folder; + } + + var leftover = Folder(".lock.lease", ".lock.lease.0a1b2c3d.stale"); + var empty = Folder(); + var other = Folder(".lock.lease", "notes.txt"); + var fresh = Path.Combine(store.Root, Guid.CreateVersion7().ToString("N")); + Directory.CreateDirectory(fresh); + File.WriteAllText(Path.Combine(fresh, ".lock.lease"), string.Empty); + var heldId = Guid.CreateVersion7(); + using var held = await store.Project(heldId).LockAsync(TimeSpan.Zero); + foreach (var file in Directory.GetFiles(store.Project(heldId).Folder)) File.SetLastWriteTimeUtc(file, old); + Directory.SetLastWriteTimeUtc(store.Project(heldId).Folder, old); + + Assert.Empty(store.List()); + Assert.False(Directory.Exists(leftover), "only an old lease and its stale copy: removed"); + Assert.False(Directory.Exists(empty), "nothing in it for a while: removed"); + Assert.True(Directory.Exists(other), "a file that is no lease stays"); + Assert.True(Directory.Exists(fresh), "a lease just written stays"); + Assert.True(Directory.Exists(store.Project(heldId).Folder), "a lease held stays"); + } + [Test] public void TheSpaceACopyNeedsCountsAPhotoHeldUnderEitherName() {