diff --git a/src/Chronos.App/Sections/SequenceViewModel.cs b/src/Chronos.App/Sections/SequenceViewModel.cs index aa0cd014..f03e0cf0 100644 --- a/src/Chronos.App/Sections/SequenceViewModel.cs +++ b/src/Chronos.App/Sections/SequenceViewModel.cs @@ -607,13 +607,19 @@ public sealed class SequenceViewModel : SectionViewModel // This Chronos cannot read its working state (damaged, or a newer Chronos's): nothing is checked out over it. if (entry is { IsReadable: false } unreadable) throw WorkspaceHost.Unreadable(unreadable); // No project of the sequence here opens, and its Main here is such a copy (never among its projects): said at once, - // with Remove local copy for a damaged one, rather than asking which project and then refusing Main. + // with Remove local copy for a damaged one, when nothing else could be chosen (Main asked for, or the home has no + // other project); otherwise "Open in workspace" lists Main as it is, disabled, and offers the others. + WorkspaceEntry? unreadableMain = null; if (entry is null && request?.ProjectId is null && await UnreadableMainAsync(workspace, row.Id) is { } main) { - RefuseUnreadable(main); - _afterOpen = null; - Shell.KeepSequence(OpenSequenceId); - return null; + if (request is { Main: true } || (await workspace.HomeProjectsAsync(row.Id)).All(p => p.IsMain)) + { + RefuseUnreadable(main); + _afterOpen = null; + Shell.KeepSequence(OpenSequenceId); + return null; + } + unreadableMain = main; } if (entry is { HasMedia: true }) { @@ -631,7 +637,7 @@ public sealed class SequenceViewModel : SectionViewModel } else if (entry is null && projectId is null && request is not { Main: true }) { - if (await AskProjectAsync(workspace, row) is not { } choice) + if (await AskProjectAsync(workspace, row, unreadableMain) is not { } choice) { _afterOpen = null; Shell.KeepSequence(OpenSequenceId); @@ -679,14 +685,15 @@ public sealed class SequenceViewModel : SectionViewModel } var item = new WorkspaceProjectItem(copy, null); Shell.Notify(Loc.L("Editor.Workspace.OpenFailed"), Loc.L("Errors.Workspace.Damaged"), Loc.L("Editor.Project.RemoveCopy"), - Loc.L("Tooltips.Editor.Workspace.State.DamagedCopy"), () => _ = RemoveProjectCopyAsync(item), isError: true); + Loc.L("Tooltips.Editor.Project.RemoveCopy"), () => _ = RemoveProjectCopyAsync(item), isError: true); } /// /// "Open in workspace": the projects of the sequence's home, the media, what their copy writes and the room the - /// workspace's drive has; null when closed without a choice. + /// workspace's drive has; null when closed without a choice. , the copy of Main here + /// this Chronos cannot read, is listed disabled, why in its tooltip. /// - private async Task AskProjectAsync(WorkspaceHost workspace, Sequence row) + private async Task AskProjectAsync(WorkspaceHost workspace, Sequence row, WorkspaceEntry? unreadableMain = null) { var projects = await workspace.HomeProjectsAsync(row.Id); WorkspaceCopyFacts? copy = null; @@ -699,7 +706,13 @@ public sealed class SequenceViewModel : SectionViewModel // Only shown: the copy measures again before it writes. Log.Info("workspace", $"The copy of sequence {row.Id} could not be measured: {ex.Message}"); } - var dialog = new OpenInWorkspaceDialogViewModel(row.DisplayTitle, projects, d => CloseDialog(d), copy: copy); + var unreadable = unreadableMain is { } main + ? new Dictionary + { + [main.ProjectId] = Loc.L(main.Newer ? "Tooltips.Editor.Workspace.State.NewerCopy" : "Tooltips.Editor.Workspace.State.DamagedCopy"), + } + : null; + var dialog = new OpenInWorkspaceDialogViewModel(row.DisplayTitle, projects, d => CloseDialog(d), copy: copy, unreadable: unreadable); // The picker or the open sequence shows behind the dialog meanwhile, not the progress bar. IsLoading = false; OpenDialog(dialog); diff --git a/src/Chronos.App/Workspace/WorkspaceDialogs.cs b/src/Chronos.App/Workspace/WorkspaceDialogs.cs index 37ca4e06..d6050b0d 100644 --- a/src/Chronos.App/Workspace/WorkspaceDialogs.cs +++ b/src/Chronos.App/Workspace/WorkspaceDialogs.cs @@ -16,11 +16,19 @@ namespace Chronos.App.Workspace; /// What "Open in workspace" chose: an existing project of the home () or a new one (), and the media. public sealed record OpenChoice(Guid? ProjectId, string? NewName, string Media); -/// A project of the home offered by "Open in workspace" (Main in the UI language). -public sealed record ProjectOption(SequenceProject Project) +/// +/// A project of the home offered by "Open in workspace" (Main in the UI language). One whose copy here this Chronos cannot +/// read (: why) is shown, disabled, its reason in the tooltip. +/// +public sealed record ProjectOption(SequenceProject Project, string? Unreadable = null) { public string Label => WorkspaceText.ProjectName(Project.Name, Project.IsMain); + /// It can be chosen: its copy here, if any, is one this Chronos reads. + public bool IsAvailable => Unreadable is null; + + public string ToolTip => Unreadable ?? Loc.L("Tooltips.Editor.OpenWorkspace.Projects"); + public override string ToString() => Label; } @@ -50,8 +58,11 @@ public sealed class OpenInWorkspaceDialogViewModel : ArchiveDialogViewModel /// The names a new project cannot take (default: those of ). /// What the copy of the media writes and the free space of the workspace's drive; null when not measured (shown as nothing). + /// + /// The projects whose copy here this Chronos cannot read (damaged, or a newer Chronos's), with why: listed, never chosen. + /// public OpenInWorkspaceDialogViewModel(string sequenceTitle, IReadOnlyList projects, Action close, - IReadOnlyList? taken = null, WorkspaceCopyFacts? copy = null) + IReadOnlyList? taken = null, WorkspaceCopyFacts? copy = null, IReadOnlyDictionary? unreadable = null) : base(close) { SequenceTitle = sequenceTitle; @@ -63,9 +74,9 @@ public sealed class OpenInWorkspaceDialogViewModel : ArchiveDialogViewModel new DialogFact(Loc.L("Editor.OpenWorkspace.Free"), copy.Free is { } free ? Formatting.Bytes(free) : Loc.L("Server.Sync.None"), Reason: Loc.L(copy.TooLittleSpace ? "Errors.Workspace.NoRoom" : "Tooltips.Editor.OpenWorkspace.Free")), ]; - Projects = [.. projects.OrderByDescending(p => p.IsMain).Select(p => new ProjectOption(p))]; - _selected = Projects.FirstOrDefault(); - _isNew = Projects.Count == 0; + Projects = [.. projects.OrderByDescending(p => p.IsMain).Select(p => new ProjectOption(p, unreadable?.GetValueOrDefault(p.Id)))]; + _selected = Projects.FirstOrDefault(p => p.IsAvailable); + _isNew = !HasProjects; var names = taken ?? [.. projects.Select(p => p.Name)]; NewName = new EditableField(string.Empty, n => ProjectNames.Validate(n, names), _ => { }, () => Loc.L("Tooltips.Editor.OpenWorkspace.Name")); NewName.PropertyChanged += (_, _) => Refresh(); @@ -90,13 +101,15 @@ public sealed class OpenInWorkspaceDialogViewModel : ArchiveDialogViewModel public IReadOnlyList Projects { get; } - public bool HasProjects => Projects.Count > 0; + /// A project of the home can be chosen (one this Chronos cannot read here is only listed). + public bool HasProjects => Projects.Any(p => p.IsAvailable); public ProjectOption? SelectedProject { get => _selected; set { + if (value is { IsAvailable: false }) return; if (Set(ref _selected, value) && value is not null) IsExisting = true; } } @@ -133,7 +146,7 @@ public sealed class OpenInWorkspaceDialogViewModel : ArchiveDialogViewModel public ICommand OpenCommand { get; } - public bool CanOpen => _isNew ? !NewName.HasErrors : _selected is not null; + public bool CanOpen => _isNew ? !NewName.HasErrors : _selected is { IsAvailable: true }; public string OpenToolTip => _isNew && NewName.Error is { } error ? error : Loc.L("Tooltips.Editor.OpenWorkspace.Open"); diff --git a/src/Chronos.App/Workspace/WorkspaceDialogs.xaml b/src/Chronos.App/Workspace/WorkspaceDialogs.xaml index cefab681..82f9eeee 100644 --- a/src/Chronos.App/Workspace/WorkspaceDialogs.xaml +++ b/src/Chronos.App/Workspace/WorkspaceDialogs.xaml @@ -17,6 +17,15 @@ + + + @@ -38,7 +47,7 @@ + ItemContainerStyle="{StaticResource ComboBoxItem.Project}" ToolTip="{l:Loc Tooltips.Editor.OpenWorkspace.Projects}" AutomationProperties.Name="{l:Loc Editor.OpenWorkspace.Project}" /> Takes the lease when it is free or stale; null when someone else holds it. - public static LeaseFile? TryAcquire(string locksDirectory, string name, string owner, TimeSpan? duration = null) + public static LeaseFile? TryAcquire(string locksDirectory, string name, string owner, TimeSpan? duration = null) => + TryAcquire(locksDirectory, name, owner, duration, createDirectory: true); + + /// + /// Takes the lease when it is free or stale; null when someone else holds it. Without + /// a folder that is not there is never made: the lease is refused. + /// + /// is not there and is not to be made. + public static LeaseFile? TryAcquire(string locksDirectory, string name, string owner, TimeSpan? duration, bool createDirectory) { var length = duration ?? DefaultDuration; var path = PathFor(locksDirectory, name); - Directory.CreateDirectory(locksDirectory); + if (createDirectory) Directory.CreateDirectory(locksDirectory); + else if (!Directory.Exists(locksDirectory)) throw new DirectoryNotFoundException($"{locksDirectory} is not there."); for (var attempt = 0; attempt < 3; attempt++) { var now = DateTimeOffset.UtcNow; @@ -77,14 +86,17 @@ public sealed class LeaseFile : IDisposable /// Waits up to for the lease, polling with back-off. /// Still held by someone else after . + /// + /// is not there (or went meanwhile) and is false. + /// public static async Task AcquireAsync(string locksDirectory, string name, string owner, TimeSpan wait, - TimeSpan? duration = null, CancellationToken cancellationToken = default) + TimeSpan? duration = null, CancellationToken cancellationToken = default, bool createDirectory = true) { var deadline = DateTime.UtcNow + wait; var delay = TimeSpan.FromMilliseconds(20); while (true) { - if (TryAcquire(locksDirectory, name, owner, duration) is { } lease) return lease; + if (TryAcquire(locksDirectory, name, owner, duration, createDirectory) is { } lease) return lease; if (DateTime.UtcNow >= deadline) throw new LeaseUnavailableException(name, Read(PathFor(locksDirectory, name))); await Task.Delay(delay, cancellationToken).ConfigureAwait(false); delay = TimeSpan.FromMilliseconds(System.Math.Min(delay.TotalMilliseconds * 2, LongestPoll.TotalMilliseconds)); diff --git a/src/Chronos.Data/Jobs/JobEngine.cs b/src/Chronos.Data/Jobs/JobEngine.cs index 67a16749..73cf6232 100644 --- a/src/Chronos.Data/Jobs/JobEngine.cs +++ b/src/Chronos.Data/Jobs/JobEngine.cs @@ -130,16 +130,18 @@ public sealed class JobEngine : IAsyncDisposable /// /// Cancels a queued or running job; a running handler observes its token. A paused or failed one, which no handler runs, - /// leaves what it wrote to its handler to clear (). + /// leaves what it wrote to its handler to clear (), once a handler of this engine that + /// was still stopping it (paused a moment ago: what it cleared itself was for a pause) has let go. /// public async Task CancelAsync(Guid id, CancellationToken cancellationToken = default) { var before = await _db.Jobs.GetAsync(id, cancellationToken).ConfigureAwait(false); await TransitionAsync(id, JobState.Cancelled, cancellationToken).ConfigureAwait(false); // Neither claimable nor running here: no handler cleans up after it itself. - if (before is not { State: JobState.Paused or JobState.Failed } || _running.ContainsKey(id) - || !_handlers.TryGetValue(before.Kind, out var handler) || handler is not ICancelledJobHandler cancelled) + if (before is not { State: JobState.Paused or JobState.Failed } || !_handlers.TryGetValue(before.Kind, out var handler) + || handler is not ICancelledJobHandler cancelled) return; + if (_running.TryGetValue(id, out var stopping)) await stopping.Ended.WaitAsync(cancellationToken).ConfigureAwait(false); try { await cancelled.CancelledAsync(before with { State = JobState.Cancelled }, cancellationToken).ConfigureAwait(false); @@ -478,6 +480,7 @@ public sealed class JobEngine : IAsyncDisposable if (yieldsTo is not null) context.Log(Loc.L("Jobs.Line.Yielded", Loc.L("Jobs.Kind." + yieldsTo.Kind))); await PersistAsync(context).ConfigureAwait(false); _running.TryRemove(job.Id, out _); + run.End(); // Engine stopping: leave the job running so the next start requeues it from its checkpoint. if (shutdown.IsCancellationRequested) return; // Yielded to a higher job: back to the queue with its checkpoint, unless the user paused or cancelled it @@ -497,6 +500,7 @@ public sealed class JobEngine : IAsyncDisposable finally { _running.TryRemove(job.Id, out _); + run.End(); } await _db.Jobs.SetStateAsync(job.Id, final, error, CancellationToken.None).ConfigureAwait(false); await NotifyAsync(job.Id, CancellationToken.None).ConfigureAwait(false); @@ -506,9 +510,15 @@ public sealed class JobEngine : IAsyncDisposable private sealed class RunningJob(Job job, CancellationTokenSource cancellation) { private Job? _yieldsTo; + private readonly TaskCompletionSource _ended = new(TaskCreationOptions.RunContinuationsAsynchronously); public Job Job { get; } = job; + /// Completes once the handler has returned and the job left the engine's running jobs. + public Task Ended => _ended.Task; + + public void End() => _ended.TrySetResult(); + /// Jobs this one queued as follow-ups (): they never make it yield. public ConcurrentDictionary FollowUps { get; } = new(); diff --git a/src/Chronos.Data/Projects/ProjectIndexer.cs b/src/Chronos.Data/Projects/ProjectIndexer.cs index 3a69bab1..6a61b627 100644 --- a/src/Chronos.Data/Projects/ProjectIndexer.cs +++ b/src/Chronos.Data/Projects/ProjectIndexer.cs @@ -45,8 +45,17 @@ internal sealed class ProjectIndexer(ChronosDatabase db) /// Versions read at once when a project's sidecars are written again from the index. private const int WriteBackPage = 500; - /// The live folders with no record seen by a reconcile, by project: what they held then, and when (). - private readonly ConcurrentDictionary _unsettled = new(); + /// + /// The live folders with no record seen by a reconcile, by project: of which sequence, what they held then, and when + /// (). A reconcile of the sequence that no longer meets the folder so forgets it. + /// + private readonly ConcurrentDictionary _unsettled = new(); + + /// + /// How long a folder of the projects' trash with no record (a deletion on its way, the record not there yet) holds back the + /// rebuild of a lost record, counted from the newest thing written in it: past that it is a leftover, not a deletion. + /// + internal static readonly TimeSpan TrashRecordGrace = TimeSpan.FromDays(1); /// /// How long a live folder with no record must look the same before its record is written again from the index (a sync @@ -62,10 +71,21 @@ internal sealed class ProjectIndexer(ChronosDatabase db) var seen = new HashSet(); var heldBack = new HashSet(); var live = new Dictionary(); + // The projects of this sequence whose record in the projects' trash was read, with the newest such folder. + var deletions = new Dictionary(); + var unrecorded = new HashSet(); if (!Directory.Exists(folder.ProjectsDirectory)) return new ProjectReconcile(true, seen, heldBack, live); foreach (var (directory, trashed) in ProjectFolders(folder)) { cancellationToken.ThrowIfCancellationRequested(); + // What is left of the live folder of a project whose deletion was read (the trash is read first): skipped, the + // versions it holds beyond the trash's moved there, beside the deleted record (a push made before the deletion + // reached this PC). + if (!trashed && Guid.TryParseExact(Path.GetFileName(directory), "N", out var left) && deletions.TryGetValue(left, out var deletion)) + { + if (writable) KeepWithDeletion(directory, deletion, left); + continue; + } ProjectRecord? record = null; var recordPath = Path.Combine(directory, ArchiveLayout.ProjectFile); try @@ -90,8 +110,11 @@ internal sealed class ProjectIndexer(ChronosDatabase db) } // A record lost (a sync tool, a disk) of a live project the index holds: written again from the index, once the // folder has settled and no deletion of it is on its way (Rebuildable). - if (writable && row is { DeletedAt: null } && row.SequenceId == sequenceId && Rebuildable(folder, directory, named)) - record = await RecordFromIndexAsync(folder, row, recordPath, cancellationToken).ConfigureAwait(false); + if (writable && row is { DeletedAt: null } && row.SequenceId == sequenceId) + { + unrecorded.Add(named); + if (Rebuildable(folder, directory, named, sequenceId)) record = await RecordFromIndexAsync(folder, row, recordPath, cancellationToken).ConfigureAwait(false); + } } if (record is null) { @@ -103,9 +126,11 @@ internal sealed class ProjectIndexer(ChronosDatabase db) continue; } } - _unsettled.TryRemove(record.Id, out _); - // A sequence folder copied by hand carries the projects of its source, which stay that sequence's. - if (record.SequenceId != sequenceId || !seen.Add(record.Id)) continue; + // A sequence folder copied by hand carries the projects of its source, which stay that sequence's. (A live folder + // with a record now is no longer unsettled: forgotten after the loop.) + if (record.SequenceId != sequenceId) continue; + if (trashed) deletions[record.Id] = directory; + if (!seen.Add(record.Id)) continue; if (trashed && record.Deleted is null) record = record with { Deleted = Directory.GetLastWriteTimeUtc(directory) }; if (record.Deleted is null) { @@ -123,6 +148,11 @@ internal sealed class ProjectIndexer(ChronosDatabase db) if (record.Deleted is null) heldBack.Add(record.Id); } } + // A folder with no record this sequence no longer has (it got one, or went): what was seen of it is forgotten. + foreach (var (id, unsettled) in _unsettled) + { + if (unsettled.Sequence == sequenceId && !unrecorded.Contains(id)) _unsettled.TryRemove(id, out _); + } try { await SettleMainAsync(folder, sequenceId, live, heldBack, writable, cancellationToken).ConfigureAwait(false); @@ -310,18 +340,19 @@ internal sealed class ProjectIndexer(ChronosDatabase db) /// /// Whether the live folder of project , which has no record, may - /// have its record written again from the index now. Never while the projects' trash holds a folder of it (a deletion on - /// its way, its record maybe not there yet), and only once the folder has looked the same, file by file, on two - /// reconciles apart: a sync tool still bringing the real record (with a rename, a newer head) - /// has it win. Until then the project is held back. + /// have its record written again from the index now. Never while the projects' trash holds a folder of it that may be a + /// deletion on its way (), and only once the folder has looked the same, file by file, on + /// two reconciles apart: a sync tool still bringing the real record (with a rename, a newer + /// head) has it win. Until then the project is held back. /// - private bool Rebuildable(SequencePaths folder, string directory, Guid projectId) + private bool Rebuildable(SequencePaths folder, string directory, Guid projectId, Guid sequenceId) { var prefix = projectId.ToString("N") + "-"; try { if (Directory.Exists(folder.ProjectsTrashDirectory) - && Directory.EnumerateDirectories(folder.ProjectsTrashDirectory).Any(d => Path.GetFileName(d).StartsWith(prefix, StringComparison.OrdinalIgnoreCase))) + && Directory.EnumerateDirectories(folder.ProjectsTrashDirectory) + .Any(d => Path.GetFileName(d).StartsWith(prefix, StringComparison.OrdinalIgnoreCase) && DeletionOnItsWay(d, projectId, sequenceId, DateTime.UtcNow))) { Log.Info("projects", $"Project {projectId} has a folder in the projects' trash; its record is not written again"); return false; @@ -334,7 +365,7 @@ internal sealed class ProjectIndexer(ChronosDatabase db) _unsettled.TryRemove(projectId, out _); return true; } - _unsettled[projectId] = (look, now); + _unsettled[projectId] = (sequenceId, look, now); Log.Info("projects", $"Project {projectId} has no record in its folder; it is written again from the index once the folder settles"); return false; } @@ -345,6 +376,74 @@ internal sealed class ProjectIndexer(ChronosDatabase db) } } + /// + /// Whether the folder of the projects' trash, named for , may be + /// a deletion of this sequence's project on its way. Its record names the sequence (one of another, a sequence folder + /// copied by hand, is not), or cannot be read (nothing is guessed). Without a record (versions name no sequence) it is + /// one until has passed since anything was last written in it, unless a version in it is + /// another project's: a leftover then, which no longer holds a live project back. + /// + internal static bool DeletionOnItsWay(string directory, Guid projectId, Guid sequenceId, DateTime nowUtc) + { + var recordPath = Path.Combine(directory, ArchiveLayout.ProjectFile); + if (File.Exists(recordPath)) + { + try + { + return ProjectSidecars.ReadRecord(recordPath).SequenceId == sequenceId; + } + catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) + { + return true; + } + } + var info = new DirectoryInfo(directory); + var newest = info.LastWriteTimeUtc; + foreach (var entry in info.EnumerateFileSystemInfos("*", SearchOption.AllDirectories)) + { + if (entry.LastWriteTimeUtc > newest) newest = entry.LastWriteTimeUtc; + } + if (nowUtc - newest >= TrashRecordGrace) return false; + foreach (var (_, _, path) in ProjectSidecars.ListVersions(Path.Combine(directory, ArchiveLayout.ProjectVersionsFolder))) + { + try + { + if (ProjectSidecars.ReadVersion(path).ProjectId != projectId) return false; + } + catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) + { + // Not read: proves nothing either way. + } + } + return true; + } + + /// + /// The version files the live folder of a deleted project holds that its folder in the + /// projects' trash () lacks, moved there beside the deleted record: a version pushed into the + /// live folder before the deletion reached this PC is kept with the project, never deleted nor left where nothing reads + /// it. A file the trash has under the same name is left as it is. + /// + private static void KeepWithDeletion(string directory, string deletion, Guid projectId) + { + var target = Path.Combine(deletion, ArchiveLayout.ProjectVersionsFolder); + foreach (var (number, _, path) in ProjectSidecars.ListVersions(Path.Combine(directory, ArchiveLayout.ProjectVersionsFolder))) + { + var to = Path.Combine(target, Path.GetFileName(path)); + try + { + if (File.Exists(to) || ProjectSidecars.ReadVersion(path).ProjectId != projectId) continue; + Directory.CreateDirectory(target); + File.Move(path, to); + Log.Warning("projects", $"Version {number} of deleted project {projectId} was in its live folder; it is kept beside its deleted record in {deletion}"); + } + catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) + { + Log.Warning("projects", $"Version file {path} of deleted project {projectId} could not be moved beside its deleted record; tried again next time", ex); + } + } + } + /// What a project folder holds now, file by file (path, size, time written), as one hash. private static string Look(string directory) { diff --git a/src/Chronos.Data/Projects/ProjectService.cs b/src/Chronos.Data/Projects/ProjectService.cs index fdda901f..1f4033e2 100644 --- a/src/Chronos.Data/Projects/ProjectService.cs +++ b/src/Chronos.Data/Projects/ProjectService.cs @@ -836,12 +836,14 @@ public sealed class ProjectService /// /// A record behind the index made whole: the version files past its head written from the index (those there are left - /// as they are), then the record naming the index's head. + /// as they are), then the record naming the index's head, stamped past both the record's time and what the index knows of + /// the project (a rename taken from a clock ahead), so a rename made next anywhere is newer than what this index knows. /// private async Task HealAsync(SequencePaths folder, SequenceProject project, ProjectRecord record, CancellationToken cancellationToken) { await WriteVersionsAsync(folder, project.Id, record.Head?.Number ?? 0, cancellationToken).ConfigureAwait(false); - var healed = ProjectRecord.Of(project with { UpdatedAt = ProjectIndexer.UpdatedAfter(record.Updated) }, await MirroredHeadAsync(project, record.Mirrored, cancellationToken) + var known = record.Updated > project.UpdatedAt ? record.Updated : project.UpdatedAt; + var healed = ProjectRecord.Of(project with { UpdatedAt = ProjectIndexer.UpdatedAfter(known) }, await MirroredHeadAsync(project, record.Mirrored, cancellationToken) .ConfigureAwait(false)); ProjectSidecars.WriteRecord(folder.ProjectRecordPath(project.Id), healed); Log.Warning("projects", $"The record of project {project.Id} named version {record.Head?.Number} as its head, behind the index's {project.HeadNumber}; " diff --git a/src/Chronos.Data/Repositories/JobRepository.cs b/src/Chronos.Data/Repositories/JobRepository.cs index 8fbbe23e..f13ad4d3 100644 --- a/src/Chronos.Data/Repositories/JobRepository.cs +++ b/src/Chronos.Data/Repositories/JobRepository.cs @@ -195,6 +195,13 @@ public sealed class JobRepository return await _db.ScalarAsync(sql, p.ToArray(), cancellationToken).ConfigureAwait(false) > 0; } + /// The queued, running and paused jobs of that target , newest first (5000 at most). + public Task> ListPendingAsync(JobKind kind, Guid targetId, CancellationToken cancellationToken = default) => + // Target ids are stored comma-joined as lower-case text, which never holds a LIKE wildcard. + _db.ListAsync( + $"SELECT {Columns} FROM jobs WHERE kind = $1 AND state IN ({PendingStates}) AND target_ids LIKE $2 ESCAPE '\\' ORDER BY created DESC, id DESC LIMIT 5000", + [(int)kind, SqlDialect.ContainsPattern(targetId.ToString())], Map, cancellationToken); + /// The targets of the queued, running and paused jobs of (the newest 5000 such jobs). public Task> ListPendingTargetsAsync(JobKind kind, CancellationToken cancellationToken = default) => _db.RunAsync(async session => diff --git a/src/Chronos.Data/Workspaces/WorkspaceCopyJobHandler.cs b/src/Chronos.Data/Workspaces/WorkspaceCopyJobHandler.cs index 7cfc389a..98f4bf69 100644 --- a/src/Chronos.Data/Workspaces/WorkspaceCopyJobHandler.cs +++ b/src/Chronos.Data/Workspaces/WorkspaceCopyJobHandler.cs @@ -66,8 +66,9 @@ public static class WorkspaceJobs /// job stops saying how much is missing (the workspace shares this PC's drive with the index and the logs). /// Checkpoint: copy:<frame>:<frames version>:<folder>; a run resumed on other frames or in another folder /// checks every file again, and a pass started again stores that before it reads anything. A first copy the user cancels -/// removes the media it wrote itself, once it has stopped writing and under the project's lease (); -/// one cancelled while paused or failed, which no handler runs, has them removed the same way (). +/// removes the media it wrote itself, once it has stopped writing and under the project's lease, unless another copy of the +/// project goes on with them (); one cancelled while paused or failed, which no handler +/// runs, has them removed the same way (). /// Originals come in phase 5c: asked for, the job fails saying so. /// internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService archives) : ICancelledJobHandler @@ -110,7 +111,7 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService try { - await CopyAsync(context, project, sequenceId, archive, parameters, ct).ConfigureAwait(false); + using (WritingMedia(project)) await CopyAsync(context, project, sequenceId, archive, parameters, ct).ConfigureAwait(false); } catch (OperationCanceledException) when (ct.IsCancellationRequested) { @@ -158,27 +159,106 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService /// A first copy of the project's media the user cancelled (the job reads ), once it has /// stopped writing: the photos it wrote (copies the home can make again) are removed under the project's lease, so the /// workspace holds no half media. Media that a copy ever finished (media.json) stay, as do those of a copy paused, - /// yielded or stopped with the program, which goes on from its checkpoint. A failure is logged: the next copy checks - /// every file it finds. + /// yielded or stopped with the program, which goes on from its checkpoint, and those another copy of the project into + /// this workspace still writes or goes on with (queued, running or paused: ). The last + /// look is taken under the lease, and no copy of this program starts writing while the media go + /// (). A folder that is no project any more (its removal under way) is left as it is, and the + /// lease never makes it again. A failure is logged: the next copy checks every file it finds. /// private async Task RemovePartialMediaAsync(Job job, WorkspaceProject project) { try { - if (await db.Jobs.GetAsync(job.Id, CancellationToken.None).ConfigureAwait(false) is not { State: JobState.Cancelled }) return; - // Nothing half written (or the copy removed meanwhile): the lease, which makes the folder, is not taken. - if (File.Exists(project.MediaIndexPath) || !Directory.Exists(project.MediaDirectory)) return; - using (await project.LockAsync(LockWait, CancellationToken.None).ConfigureAwait(false)) + // Nothing half written (or the copy removed meanwhile): the lease is not taken. + if (!Clearable(project) || !await StillToClearAsync(job, project).ConfigureAwait(false)) return; + using (await project.LockExistingAsync(LockWait, CancellationToken.None).ConfigureAwait(false)) { - if (File.Exists(project.MediaIndexPath) || !Directory.Exists(project.MediaDirectory)) return; - Directory.Delete(project.MediaDirectory, recursive: true); + if (!Clearable(project) || !await StillToClearAsync(job, project).ConfigureAwait(false)) return; + lock (WritersGate) + { + // A copy of this program that started since the jobs were read: its files stay. + if (Writers.ContainsKey(project.Folder) || !Clearable(project)) return; + Directory.Delete(project.MediaDirectory, recursive: true); + } } Log.Info("workspace", $"The media a cancelled first copy of project {project.ProjectId} wrote were removed"); } + catch (DirectoryNotFoundException) + { + // The project's folder went meanwhile (its removal): nothing is left to clear. + } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or DatabaseException) { Log.Warning("workspace", $"The media of a cancelled copy of project {project.ProjectId} could not be removed", ex); } + + // A project (its working state there) with media no copy ever finished. + static bool Clearable(WorkspaceProject project) => project.Exists && !File.Exists(project.MediaIndexPath) && Directory.Exists(project.MediaDirectory); + } + + /// The cancelled copy still reads so, and no other copy of the project into this workspace waits, runs or is paused. + private async Task StillToClearAsync(Job job, WorkspaceProject project) + { + if (await db.Jobs.GetAsync(job.Id, CancellationToken.None).ConfigureAwait(false) is not { State: JobState.Cancelled }) return false; + if (await OtherCopyAsync(job, project).ConfigureAwait(false) is not { } other) return true; + Log.Info("workspace", $"The media of cancelled copy {job.Id} stay: copy {other.Id} of project {project.ProjectId} is {other.State}"); + return false; + } + + /// + /// Another copy of the project into the same workspace that is queued, running or paused: it writes the media folder, or + /// goes on with what is there. A job whose parameters cannot be read counts, so nothing is removed on a guess. + /// + private async Task OtherCopyAsync(Job job, WorkspaceProject project) + { + var sequenceId = job.TargetIds.FirstOrDefault(); + foreach (var other in await db.Jobs.ListPendingAsync(JobKind.WorkspaceCopy, sequenceId, CancellationToken.None).ConfigureAwait(false)) + { + if (other.Id == job.Id) continue; + try + { + var parameters = Parameters(other); + if (parameters.ProjectId != project.ProjectId) continue; + // Case aside: a match on a case-sensitive drive only keeps media that could have gone. + if (!string.Equals(WorkspaceStore.FullRoot(parameters.WorkspaceRoot), project.Store.Root, StringComparison.OrdinalIgnoreCase)) continue; + } + catch (Exception ex) when (ex is InvalidDataException or ArgumentException or IOException or NotSupportedException or System.Security.SecurityException) + { + Log.Debug("workspace", $"Copy {other.Id} cannot be read: it counts as one of project {project.ProjectId}: {ex.Message}"); + } + return other; + } + return null; + } + + /// The project folders copies of this program write media into now, with how many copies write each. + private static readonly Dictionary Writers = new(StringComparer.OrdinalIgnoreCase); + + private static readonly Lock WritersGate = new(); + + /// + /// Marks the copy into as writing its media until disposed: a clean-up after a cancelled copy + /// leaves them (), and one under way finishes before the copy writes anything. + /// + private static IDisposable WritingMedia(WorkspaceProject project) + { + lock (WritersGate) Writers[project.Folder] = Writers.GetValueOrDefault(project.Folder) + 1; + return new Writing(project.Folder); + } + + private sealed class Writing(string folder) : IDisposable + { + private int _done; + + public void Dispose() + { + if (Interlocked.Exchange(ref _done, 1) != 0) return; + lock (WritersGate) + { + if (Writers.GetValueOrDefault(folder) <= 1) Writers.Remove(folder); + else Writers[folder]--; + } + } } private async Task CopyAsync(JobContext context, WorkspaceProject project, Guid sequenceId, Archive archive, WorkspaceCopyParameters parameters, diff --git a/src/Chronos.Data/Workspaces/WorkspaceStore.cs b/src/Chronos.Data/Workspaces/WorkspaceStore.cs index 77fcac28..8e101450 100644 --- a/src/Chronos.Data/Workspaces/WorkspaceStore.cs +++ b/src/Chronos.Data/Workspaces/WorkspaceStore.cs @@ -128,7 +128,7 @@ public sealed class WorkspaceStore public WorkspaceStore(string root) { ArgumentException.ThrowIfNullOrWhiteSpace(root); - Root = LongPath.Normalize(Path.TrimEndingDirectorySeparator(Path.GetFullPath(root))); + Root = FullRoot(root); Directory.CreateDirectory(Root); var path = Path.Combine(Root, ManifestFile); if (!File.Exists(path)) Create(path, replace: false); @@ -152,18 +152,59 @@ public sealed class WorkspaceStore return project.Exists ? project : null; } - /// The projects of the workspace: every folder named by a project id that holds a project.chronos. + /// The folder a workspace at has as its (the store opened or not). + internal static string FullRoot(string root) => LongPath.Normalize(Path.TrimEndingDirectorySeparator(Path.GetFullPath(root))); + + /// + /// The projects of the workspace: every folder named by a project id that holds a project.chronos. A folder named + /// by a project id that holds nothing but what a lease no one holds left (a removal that met a lease taken meanwhile) + /// goes (). + /// public IReadOnlyList List() { var found = new List(); foreach (var directory in Directory.EnumerateDirectories(Root).Order(StringComparer.Ordinal)) { - if (Guid.TryParseExact(Path.GetFileName(directory), "N", out var id) && File.Exists(Path.Combine(directory, WorkspaceProject.BundleFile))) - found.Add(Project(id)); + if (!Guid.TryParseExact(Path.GetFileName(directory), "N", out var id)) continue; + if (File.Exists(Path.Combine(directory, WorkspaceProject.BundleFile))) found.Add(Project(id)); + else RemoveLeftover(directory, DateTime.UtcNow); } return found; } + /// + /// A project's folder that is no project and holds nothing a user or a writer has: no project.chronos, no folder, + /// only lease files (the lease and those set aside as stale), none of them held, and nothing written there for as long + /// as a lease lasts (a lease taken this moment makes its folder first). + /// + internal static bool IsLeftover(string folder, DateTime nowUtc) + { + var info = new DirectoryInfo(folder); + if (!info.Exists || File.Exists(Path.Combine(folder, WorkspaceProject.BundleFile)) || info.LastWriteTimeUtc + LeaseFile.DefaultDuration > nowUtc) return false; + foreach (var entry in info.EnumerateFileSystemInfos()) + { + if (entry is not FileInfo file || !WorkspaceProject.IsLeaseFile(file.Name) || file.LastWriteTimeUtc + LeaseFile.DefaultDuration > nowUtc) return false; + if (string.Equals(file.Name, WorkspaceProject.LeaseFileName, StringComparison.OrdinalIgnoreCase) && LeaseFile.Read(file.FullName) is { } held + && held.Expires >= nowUtc) + return false; + } + return true; + } + + private static void RemoveLeftover(string folder, DateTime nowUtc) + { + try + { + if (!IsLeftover(folder, nowUtc)) return; + Directory.Delete(folder, recursive: true); + Log.Info("workspace", $"{folder} held only an old lease and was removed"); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Debug("workspace", $"{folder} stays: {ex.Message}"); + } + } + /// The working states of the workspace's projects (one unreadable is left out, and logged). public IReadOnlyList<(WorkspaceProject Project, ProjectBundle Bundle)> ListBundles() { @@ -297,6 +338,12 @@ public sealed class WorkspaceProject private const string LeaseName = ".lock"; + /// The file of the project's lease () in . + internal const string LeaseFileName = LeaseName + ".lease"; + + /// The project's lease, or one set aside as stale (.lock.lease.<tag>.stale). + internal static bool IsLeaseFile(string name) => name.StartsWith(LeaseFileName, StringComparison.OrdinalIgnoreCase); + internal WorkspaceProject(WorkspaceStore store, Guid projectId) { Store = store; @@ -343,6 +390,16 @@ public sealed class WorkspaceProject return await LeaseFile.AcquireAsync(Folder, LeaseName, "workspace@" + Environment.MachineName, wait, cancellationToken: cancellationToken).ConfigureAwait(false); } + /// + /// Holds the project as does, never making its folder: taken by a writer that only clears what is + /// there, which a removal of the folder may race. + /// + /// The folder is not there, or went while the lease was awaited. + /// Another writer kept it for longer than . + internal async Task LockExistingAsync(TimeSpan wait, CancellationToken cancellationToken = default) => + await LeaseFile.AcquireAsync(Folder, LeaseName, "workspace@" + Environment.MachineName, wait, cancellationToken: cancellationToken, createDirectory: false) + .ConfigureAwait(false); + /// Not a project file of a workspace, damaged or newer. /// The file cannot be read. public ProjectBundle ReadBundle() @@ -582,6 +639,19 @@ public sealed class WorkspaceProject } } + /// + /// The conflict sync.json records that still waits for a choice, as reads + /// it: one a replay that settles it took past its commit point (its base moved onto the replay's) is over, the choice + /// made applied, even before the next sync clears it. + /// + /// sync.json or the working state cannot be read. + /// A file cannot be read. + public WorkspaceConflict? OpenConflict() => ReadSync() is { Conflict: not null } sync ? Unsettled(sync, ReadBundle()) : null; + + /// The conflict of that waits for a choice with the working state (). + internal static WorkspaceConflict? Unsettled(WorkspaceSyncState sync, ProjectBundle bundle) => + sync.Replay is { Settles: true } replay && bundle.Workspace?.Base?.VersionId == replay.OntoBase ? null : sync.Conflict; + internal void WriteSync(WorkspaceSyncState state) { ArgumentNullException.ThrowIfNull(state); diff --git a/src/Chronos.Data/Workspaces/WorkspaceSync.cs b/src/Chronos.Data/Workspaces/WorkspaceSync.cs index c03697df..ab84b05c 100644 --- a/src/Chronos.Data/Workspaces/WorkspaceSync.cs +++ b/src/Chronos.Data/Workspaces/WorkspaceSync.cs @@ -1122,7 +1122,7 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName) var workspace = bundle.Workspace!; // A take of the server's values (or any replay that settles) cut past its commit point applied the choice: the // conflict is over, as the next sync writes. - var conflict = sync.Replay is { Settles: true } replay && workspace.Base?.VersionId == replay.OntoBase ? null : sync.Conflict; + var conflict = WorkspaceProject.Unsettled(sync, bundle); ProjectVersion? head = null; HomeFrames? frames = null; var offline = false; diff --git a/tests/Chronos.App.Tests/Workspace/WorkspaceHostTests.cs b/tests/Chronos.App.Tests/Workspace/WorkspaceHostTests.cs index 069b0529..65cd8195 100644 --- a/tests/Chronos.App.Tests/Workspace/WorkspaceHostTests.cs +++ b/tests/Chronos.App.Tests/Workspace/WorkspaceHostTests.cs @@ -308,6 +308,34 @@ public sealed class WorkspaceHostTests(TestContext context) Assert.Empty(host.Store.List()); } + [Test] + public async Task AConflictASettlingReplayTookPastItsCommitPointIsNoWorkLeftForTheRemoval() + { + await using var bed = await WorkspaceTestBed.CreateAsync(context, frames: 4); + var host = bed.Host(new AppSettings { WorkspaceAutoSync = false }); + var mainId = await bed.CheckoutAsync(host); + var project = host.Store.Find(mainId)!; + var basis = project.ReadBundle().Workspace!.Base!; + var conflict = $$""" + "conflict": { "theirsVersionId": "{{basis.VersionId}}", "theirsNumber": {{basis.Number}}, "baseVersionId": "{{Guid.CreateVersion7()}}", + "mine": [], "differences": [], "found": "2026-11-01T10:00:00+00:00" } + """; + + // A take of the library's values cut after its base moved (before sync.json said so): the choice is applied. + await File.WriteAllTextAsync(project.SyncPath, + $$"""{ {{conflict}}, "replay": { "ontoBase": "{{basis.VersionId}}", "replaced": [], "replayed": [], "adopted": [], "settles": true } }"""); + Assert.NotNull(project.ReadSync().Conflict, "still in sync.json"); + Assert.Null(project.OpenConflict()); + Assert.Null((await host.StatusAsync(mainId)).Conflict); + var plan = await host.PlanRemovalAsync(mainId); + Assert.True(plan is { Conflict: false, IsSafe: true }, "counted as the status reads it"); + + // A conflict still waiting for a choice counts. + await File.WriteAllTextAsync(project.SyncPath, $$"""{ {{conflict}} }"""); + Assert.NotNull(project.OpenConflict()); + Assert.True((await host.PlanRemovalAsync(mainId)) is { Conflict: true, IsSafe: false }); + } + [Test] public async Task ACopyOfAProjectDeletedInTheLibraryReadsGoneAndGoesOnlyWhenDiscarded() { diff --git a/tests/Chronos.App.Tests/Workspace/WorkspaceSectionTests.cs b/tests/Chronos.App.Tests/Workspace/WorkspaceSectionTests.cs index 35a5e652..85d629a0 100644 --- a/tests/Chronos.App.Tests/Workspace/WorkspaceSectionTests.cs +++ b/tests/Chronos.App.Tests/Workspace/WorkspaceSectionTests.cs @@ -249,7 +249,7 @@ public sealed class WorkspaceSectionTests(TestContext context) shell.OpenSequence(sequenceId); await WorkspaceTestBed.UntilAsync(() => toasts.Any(t => t.Message == Loc.L("Errors.Workspace.Damaged")), "a damaged Main is refused"); var damaged = toasts.Single(t => t.Message == Loc.L("Errors.Workspace.Damaged")); - Assert.Equal((Loc.L("Editor.Workspace.OpenFailed"), true, Loc.L("Editor.Project.RemoveCopy"), Loc.L("Tooltips.Editor.Workspace.State.DamagedCopy")), + Assert.Equal((Loc.L("Editor.Workspace.OpenFailed"), true, Loc.L("Editor.Project.RemoveCopy"), Loc.L("Tooltips.Editor.Project.RemoveCopy")), (damaged.Title, damaged.IsError, damaged.ActionText, damaged.ActionToolTip)); Assert.True(section.Dialog is null && section.Editor is null, "nothing asked, nothing opened"); damaged.ActionCommand!.Execute(null); @@ -271,6 +271,72 @@ public sealed class WorkspaceSectionTests(TestContext context) } }); + [Test] + public Task AMainThisChronosCannotReadIsListedDisabledWhileAnotherProjectCanOpen() => ScreenHost.RunAsync(async () => + { + var directory = context.CreateTempDirectory("workspace-unreadable-main-others"); + var environment = AppEnvironment.From(AppPaths.Resolve(Directory.CreateDirectory(Path.Combine(directory, "app")).FullName, + Directory.CreateDirectory(Path.Combine(directory, "local")).FullName, [])); + UiLanguage.Apply("en"); + var settings = new AppSettings { Language = "en", FirstRunCompleted = true, CheckForUpdates = false }; + var services = new AppServices(environment, settings, () => { }, new ScriptedDialogs()); + await services.AttachAsync(await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(environment.DatabasePath)), startEngine: true); + var shell = new ShellViewModel(services); + var toasts = new List(); + shell.PendingToasts.CollectionChanged += (_, e) => toasts.AddRange(e.NewItems?.OfType() ?? []); + try + { + var sequenceId = await SequenceAsync(services, Path.Combine(directory, "archive")); + var section = shell.Sequence; + shell.OpenSequence(sequenceId); + await WorkspaceTestBed.UntilAsync(() => section.Dialog is OpenInWorkspaceDialogViewModel, "Open in workspace asks the first time"); + ((OpenInWorkspaceDialogViewModel)section.Dialog!).OpenCommand.Execute(null); + await WorkspaceTestBed.UntilAsync(() => section.Editor is not null, "Main opens"); + var mainId = section.OpenProjectId!.Value; + Assert.True(await section.ConfirmCloseAsync()); + await section.CloseAsync(); + shell.KeepSequence(null); + var vertical = (await services.Archives!.Projects.CreateAsync(sequenceId, "Vertical", cancellationToken: context.Cancellation)).Id; + var bundle = Path.Combine(services.Workspace.Root, mainId.ToString("N"), WorkspaceProject.BundleFile); + + // Damaged: the sequence has another project, so it is asked which; Main is listed, disabled, why in its tooltip. + await WriteWhenFreeAsync(bundle, "{ not a project"); + shell.OpenSequence(sequenceId); + await WorkspaceTestBed.UntilAsync(() => section.Dialog is OpenInWorkspaceDialogViewModel, "Open in workspace offers the other project"); + var asked = (OpenInWorkspaceDialogViewModel)section.Dialog!; + var main = asked.Projects.Single(p => p.Project.IsMain); + Assert.True(main is { IsAvailable: false } && main.ToolTip == Loc.L("Tooltips.Editor.Workspace.State.DamagedCopy"), "a damaged Main is listed as it is"); + Assert.Equal(vertical, asked.SelectedProject?.Project.Id); + asked.SelectedProject = main; + Assert.Equal(vertical, asked.SelectedProject?.Project.Id, "never chosen"); + Assert.True(asked.CanOpen && asked.HasProjects); + asked.CloseCommand.Execute(null); + await WorkspaceTestBed.UntilAsync(() => section.Dialog is null, "closed without a choice"); + Assert.False(toasts.Any(t => t.Message == Loc.L("Errors.Workspace.Damaged")), "nothing refused"); + shell.KeepSequence(null); + + // Saved by a newer Chronos: the same, and the other project opens; Main's copy is left as it is. + var newer = $$"""{ "format": "chronos-project", "formatVersion": 9, "sequence": { "title": "Synthetic" }, "workspace": { "projectId": "{{mainId}}", "name": "Main", "isMain": true } }"""; + await WriteWhenFreeAsync(bundle, newer); + shell.OpenSequence(sequenceId); + await WorkspaceTestBed.UntilAsync(() => section.Dialog is OpenInWorkspaceDialogViewModel, "Open in workspace offers the other project"); + asked = (OpenInWorkspaceDialogViewModel)section.Dialog!; + main = asked.Projects.Single(p => p.Project.IsMain); + Assert.True(main is { IsAvailable: false } && main.ToolTip == Loc.L("Tooltips.Editor.Workspace.State.NewerCopy"), "a newer Main is listed as it is"); + asked.OpenCommand.Execute(null); + await WorkspaceTestBed.UntilAsync(() => section.Editor is not null, "the other project opens"); + Assert.Equal(vertical, section.OpenProjectId); + Assert.False(toasts.Any(t => t.Message == Loc.L("Errors.Workspace.NewerCopy")), "nothing refused"); + Assert.Equal(newer, await File.ReadAllTextAsync(bundle)); + } + finally + { + await shell.Sequence.CloseAsync(); + shell.Dispose(); + await services.DisposeAsync(); + } + }); + /// Writes once the project's last background read (its state, after the close) lets go of it. private static async Task WriteWhenFreeAsync(string path, string text) { diff --git a/tests/Chronos.Data.Tests/Projects/ProjectRecoveryTests.cs b/tests/Chronos.Data.Tests/Projects/ProjectRecoveryTests.cs index d6a63e14..3273e38d 100644 --- a/tests/Chronos.Data.Tests/Projects/ProjectRecoveryTests.cs +++ b/tests/Chronos.Data.Tests/Projects/ProjectRecoveryTests.cs @@ -806,6 +806,146 @@ public abstract class ProjectRecoveryTests(TestContext context) : DatabaseTest(c Assert.SequenceEqual([main.Id], (await fx.Service.Projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); } + [Test] + public async Task APushMadeIntoTheLiveFolderBeforeADeletionArrivedIsKeptBesideTheDeletedRecord() + { + var fx = await FixtureAsync(); + var (sequence, frames) = await UploadedAsync(fx, 3); + var projects = fx.Service.Projects; + var main = await projects.MainAsync(sequence.Id, fx.Ct); + var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); + var first = await projects.HeadAsync(vertical.Id, fx.Ct); + var folder = fx.Paths.Sequence(sequence.Folder); + var (other, _) = await fx.OtherWriterAsync(); + var before = folder.ProjectsDirectory + "-before"; + CopyDirectory(folder.ProjectDirectory(vertical.Id), before); + await projects.DeleteAsync(vertical.Id, fx.Ct); + + // The other PC pushed version 2 into its live folder before the deletion reached it: a sync tool brings that folder + // back beside the trash. The deletion wins, and version 2 goes beside the deleted record, never left where nothing reads it. + CopyDirectory(before, folder.ProjectDirectory(vertical.Id)); + var theirs = Draft(first, new ProjectSettings { ExcludedFrames = [2] }, frames) with { StoredAt = DateTimeOffset.UtcNow }; + var pushed = folder.VersionPath(vertical.Id, 2, theirs.Id); + ProjectSidecars.WriteVersion(pushed, theirs); + var record = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)); + ProjectSidecars.WriteRecord(folder.ProjectRecordPath(vertical.Id), record with { Head = new ProjectHead(theirs.Id, 2), Updated = DateTimeOffset.UtcNow }); + Assert.SequenceEqual([main.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); + var trashed = Directory.GetDirectories(folder.ProjectsTrashDirectory).Single(); + var kept = Path.Combine(trashed, ArchiveLayout.ProjectVersionsFolder, Path.GetFileName(pushed)); + Assert.FileExists(kept); + Assert.FileMissing(pushed); + Assert.Equal(theirs.Id, ProjectSidecars.ReadVersion(kept).Id); + Assert.FileExists(Path.Combine(trashed, ArchiveLayout.ProjectVersionsFolder, Path.GetFileName(folder.VersionPath(vertical.Id, 1, first.Id)))); + Assert.NotNull(ProjectSidecars.ReadRecord(Path.Combine(trashed, ArchiveLayout.ProjectFile)).Deleted); + Assert.SequenceEqual([main.Id], (await other.Projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); + Assert.SequenceEqual([main.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); + } + + [Test] + public async Task ATrashFolderOfAnotherSequenceOrLongWithoutARecordNoLongerHoldsBackALostRecord() + { + var fx = await FixtureAsync(); + var (sequence, frames) = await UploadedAsync(fx, 3); + var projects = fx.Service.Projects; + var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); + var first = await projects.HeadAsync(vertical.Id, fx.Ct); + var folder = fx.Paths.Sequence(sequence.Folder); + projects.RecordSettle = TimeSpan.Zero; + var path = folder.ProjectRecordPath(vertical.Id); + var lost = ProjectSidecars.ReadRecord(path); + + // A folder of the projects' trash with no record, just made (a deletion on its way): the lost record is not written. + var leftover = Path.Combine(folder.ProjectsTrashDirectory, vertical.Id.ToString("N") + "-1"); + Directory.CreateDirectory(leftover); + File.Delete(path); + for (var i = 0; i < 2; i++) await projects.ListAsync(sequence.Id, fx.Ct); + Assert.False(File.Exists(path), "held back while a deletion may be on its way"); + await Assert.ThrowsAsync(() => projects.PushAsync(vertical.Id, Draft(first, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)); + + // Nothing written in it for longer than the grace: a leftover, which holds nothing back any more. + Directory.SetLastWriteTimeUtc(leftover, DateTime.UtcNow - ProjectIndexer.TrashRecordGrace - TimeSpan.FromHours(1)); + for (var i = 0; i < 2; i++) await projects.ListAsync(sequence.Id, fx.Ct); + Assert.Equal(("Vertical", first.Id), (ProjectSidecars.ReadRecord(path).Name, ProjectSidecars.ReadRecord(path).Head!.Id)); + Assert.Equal(PushOutcome.Stored, (await projects.PushAsync(vertical.Id, Draft(first, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)).Outcome); + Directory.Delete(leftover); + + // A trash folder whose record names another sequence (a sequence folder copied by hand): never this one's deletion. + var foreign = Path.Combine(folder.ProjectsTrashDirectory, vertical.Id.ToString("N") + "-2"); + Directory.CreateDirectory(foreign); + ProjectSidecars.WriteRecord(Path.Combine(foreign, ArchiveLayout.ProjectFile), lost with { SequenceId = Guid.CreateVersion7(), Deleted = DateTimeOffset.UtcNow }); + File.Delete(path); + for (var i = 0; i < 2; i++) await projects.ListAsync(sequence.Id, fx.Ct); + Assert.FileExists(path); + Assert.Contains(vertical.Id, (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); + + // One whose record names this sequence is its deletion. + Assert.True(ProjectIndexer.DeletionOnItsWay(foreign, vertical.Id, sequence.Id, DateTime.UtcNow) == false); + ProjectSidecars.WriteRecord(Path.Combine(foreign, ArchiveLayout.ProjectFile), lost with { Deleted = DateTimeOffset.UtcNow }); + Assert.True(ProjectIndexer.DeletionOnItsWay(foreign, vertical.Id, sequence.Id, DateTime.UtcNow)); + } + + [Test] + public async Task AFolderWithNoRecordThatWentAndCameBackSettlesAgainFromWhenItCameBack() + { + var fx = await FixtureAsync(); + var (sequence, _) = await UploadedAsync(fx, 3); + var vertical = await fx.Service.Projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); + var folder = fx.Paths.Sequence(sequence.Folder); + var settle = TimeSpan.FromMilliseconds(TestTiming.Budget(400)); + var indexer = new ProjectIndexer(fx.Db) { RecordSettle = settle }; + var path = folder.ProjectRecordPath(vertical.Id); + File.Delete(path); + Assert.Contains(vertical.Id, (await indexer.ReconcileAsync(folder, sequence.Id, fx.Ct)).HeldBack); + + // The folder goes for a while (a sync tool moving it): what was seen of it is forgotten, so once it is back it must + // look the same for a settling time again before its record is written from the index. + var away = folder.ProjectsDirectory + "-away"; + Directory.Move(folder.ProjectDirectory(vertical.Id), away); + Assert.False((await indexer.ReconcileAsync(folder, sequence.Id, fx.Ct)).Seen.Contains(vertical.Id)); + await Task.Delay(settle * 2, fx.Ct); + Directory.Move(away, folder.ProjectDirectory(vertical.Id)); + Assert.Contains(vertical.Id, (await indexer.ReconcileAsync(folder, sequence.Id, fx.Ct)).HeldBack); + Assert.False(File.Exists(path), "not written at once on what was seen before it went"); + await Task.Delay(settle * 2, fx.Ct); + Assert.False((await indexer.ReconcileAsync(folder, sequence.Id, fx.Ct)).HeldBack.Contains(vertical.Id)); + Assert.FileExists(path); + } + + [Test] + public async Task AHealedRecordIsStampedPastWhatTheIndexKnowsSoTheNextRenameIsTaken() + { + var fx = await FixtureAsync(); + var (sequence, frames) = await UploadedAsync(fx, 3); + var projects = fx.Service.Projects; + var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); + var folder = fx.Paths.Sequence(sequence.Folder); + var backup = folder.ProjectsDirectory + "-backup"; + CopyDirectory(folder.ProjectsDirectory, backup); + await projects.PushAsync(vertical.Id, Draft(await projects.HeadAsync(vertical.Id, fx.Ct), new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct); + + // A PC whose clock is ten minutes ahead renames it: this index takes the name and that time. + var ahead = DateTimeOffset.UtcNow.AddMinutes(10); + ProjectSidecars.WriteRecord(folder.ProjectRecordPath(vertical.Id), ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)) with { Name = "Square", Updated = ahead }); + Assert.Equal("Square", (await projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id).Name); + + // The backup is put back (behind): healed, and stamped past what the index knows. + Directory.Delete(folder.ProjectsDirectory, recursive: true); + CopyDirectory(backup, folder.ProjectsDirectory); + await projects.CreateAsync(sequence.Id, "Other", cancellationToken: fx.Ct); + var healed = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)); + Assert.Equal(("Square", 2), (healed.Name, healed.Head!.Number)); + Assert.True(healed.Updated > ahead, "past the time the index took"); + + // Another PC with a right clock renames it: every index takes it, and nothing writes the old name back. + var (other, _) = await fx.OtherWriterAsync(); + Assert.Equal("Square", (await other.Projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id).Name); + await other.Projects.RenameAsync(vertical.Id, "Wide", fx.Ct); + Assert.Equal("Wide", (await projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id).Name); + await projects.CreateAsync(sequence.Id, "Other2", cancellationToken: fx.Ct); + Assert.Equal("Wide", ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)).Name); + Assert.Equal("Wide", (await other.Projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id).Name); + } + [Test] public async Task ALostRecordIsWrittenAgainOnlyOnceItsFolderHasSettledSoAnIncomingOneWins() { diff --git a/tests/Chronos.Data.Tests/Workspaces/WorkspaceSafetyTests.cs b/tests/Chronos.Data.Tests/Workspaces/WorkspaceSafetyTests.cs index d7274f25..d5312999 100644 --- a/tests/Chronos.Data.Tests/Workspaces/WorkspaceSafetyTests.cs +++ b/tests/Chronos.Data.Tests/Workspaces/WorkspaceSafetyTests.cs @@ -86,6 +86,9 @@ public abstract class WorkspaceSafetyTests(TestContext context) : DatabaseTest(c var (store, two, project, shown) = await AskedAsync(fx, id); await Assert.ThrowsAsync(() => WorkspaceFixture.Cutting(store, "replay.base").ResolveAsync(id, fx.Home, ConflictChoice.TakeServer, shown: shown, cancellationToken: fx.Ct)); + // sync.json still records the conflict, but it waits for no choice: what counts it reads it as the status does. + Assert.NotNull(project.ReadSync().Conflict); + Assert.Null(project.OpenConflict(), "settled past its commit point"); // The working state is on the library's head with its value: the choice was made, nothing asks it again. var status = await two.StatusAsync(id, fx.Home, fx.Ct); @@ -543,6 +546,142 @@ public abstract class WorkspaceSafetyTests(TestContext context) : DatabaseTest(c foreach (var frame in media.Frames) Assert.FileExists(project.MediaFile(frame.File!)); } + [Test] + public async Task APausedFirstCopyCancelledWhileASecondCopyOfTheProjectRunsLeavesItsMedia() + { + var fx = await FixtureAsync(); + var (store, sync) = fx.Workspace(); + var id = (await fx.Archive.Service.Projects.MainAsync(fx.Sequence.Id, fx.Ct)).Id; + var project = await sync.CheckoutAsync(fx.Home, id, cancellationToken: fx.Ct); + Job second; + // Held so that neither copy commits before the first is cancelled. + var lease = await project.LockAsync(Wait, fx.Ct); + try + { + var first = await fx.StartCopyAsync(store, id); + await UntilAsync(() => Directory.Exists(project.ProxiesDirectory) && Directory.GetFiles(project.ProxiesDirectory, "*.jpg").Length == fx.Frames.Count, + "the first copy's files", fx.Ct); + await fx.PauseAsync(first.Id); + await fx.Archive.WaitForAsync(first.Id, JobState.Paused, (int)TestTiming.Budget(30)); + // The sequence opened again queues another copy (the first is paused), then the first is cancelled in Jobs. + second = await fx.StartCopyAsync(store, id); + await fx.Archive.WaitForAsync(second.Id, JobState.Running, (int)TestTiming.Budget(30)); + await fx.CancelAsync(first.Id); + await fx.Archive.WaitForAsync(first.Id, JobState.Cancelled, (int)TestTiming.Budget(30)); + } + finally + { + lease.Dispose(); + } + await fx.Archive.WaitForAsync(second.Id, JobState.Done, (int)TestTiming.Budget(60)); + // The clean-up had its chance at the lease: it found the second copy and left the media. + await Task.Delay(500, fx.Ct); + using (await project.LockAsync(Wait, fx.Ct)) + { + var media = Assert.NotNull(project.ReadMedia()); + Assert.Equal(fx.Frames.Count, media.Frames.Count); + foreach (var frame in media.Frames) Assert.FileExists(project.MediaFile(frame.File!)); + } + var status = await sync.StatusAsync(id, fx.Home, fx.Ct); + Assert.True(status is { State: WorkspaceState.InSync, MediaRemoved: false }); + } + + [Test] + public async Task ACancelledCopysCleanUpLeavesAFolderNoLongerAProjectAndNeverMakesOneAgain() + { + var fx = await FixtureAsync(); + var (store, sync) = fx.Workspace(); + var main = (await fx.Archive.Service.Projects.MainAsync(fx.Sequence.Id, fx.Ct)).Id; + var side = (await fx.Archive.Service.Projects.CreateAsync(fx.Sequence.Id, "Side", cancellationToken: fx.Ct)).Id; + + // The working state moved aside while the clean-up waited for the lease (Remove local copy): its folder is left as it is. + var moving = await PausedFirstCopyAsync(fx, store, sync, main); + var lease = await moving.Project.LockAsync(Wait, fx.Ct); + var aside = Path.Combine(Context.CreateTempDirectory("aside"), WorkspaceProject.BundleFile); + try + { + await fx.CancelAsync(moving.Copy.Id); + // The clean-up waits for the lease meanwhile. + await Task.Delay(300, fx.Ct); + File.Move(moving.Project.BundlePath, aside); + } + finally + { + lease.Dispose(); + } + await Task.Delay(1000, fx.Ct); + Assert.True(Directory.Exists(moving.Project.ProxiesDirectory), "a folder that is no project is not the clean-up's"); + + // The folder removed while the clean-up waited: it is never made again. + var removed = await PausedFirstCopyAsync(fx, store, sync, side); + lease = await removed.Project.LockAsync(Wait, fx.Ct); + try + { + await fx.CancelAsync(removed.Copy.Id); + await Task.Delay(300, fx.Ct); + Directory.Delete(removed.Project.Folder, recursive: true); + } + finally + { + lease.Dispose(); + } + await Task.Delay(1000, fx.Ct); + Assert.False(Directory.Exists(removed.Project.Folder), "no folder made again for a lease"); + } + + /// A first copy of , checked out, that wrote every file and is paused before it committed. + private async Task<(WorkspaceProject Project, Job Copy)> PausedFirstCopyAsync(WorkspaceFixture fx, WorkspaceStore store, WorkspaceSync sync, Guid projectId) + { + var project = await sync.CheckoutAsync(fx.Home, projectId, cancellationToken: fx.Ct); + using (await project.LockAsync(Wait, fx.Ct)) + { + var copy = await fx.StartCopyAsync(store, projectId); + await UntilAsync(() => Directory.Exists(project.ProxiesDirectory) && Directory.GetFiles(project.ProxiesDirectory, "*.jpg").Length == fx.Frames.Count, + "the copy's files", fx.Ct); + await fx.PauseAsync(copy.Id); + await fx.Archive.WaitForAsync(copy.Id, JobState.Paused, (int)TestTiming.Budget(30)); + return (project, copy); + } + } + + [Test] + public async Task AProjectFolderHoldingOnlyALeaseNoOneHoldsGoesOnTheNextList() + { + var store = new WorkspaceStore(Path.Combine(Context.CreateTempDirectory("leftovers"), "workspace")); + var old = DateTime.UtcNow - TimeSpan.FromHours(1); + string Folder(params string[] files) + { + var folder = Path.Combine(store.Root, Guid.CreateVersion7().ToString("N")); + Directory.CreateDirectory(folder); + foreach (var file in files) + { + var path = Path.Combine(folder, file); + if (!File.Exists(path)) File.WriteAllText(path, string.Empty); + File.SetLastWriteTimeUtc(path, old); + } + Directory.SetLastWriteTimeUtc(folder, old); + return folder; + } + + var leftover = Folder(".lock.lease", ".lock.lease.0a1b2c3d.stale"); + var empty = Folder(); + var other = Folder(".lock.lease", "notes.txt"); + var fresh = Path.Combine(store.Root, Guid.CreateVersion7().ToString("N")); + Directory.CreateDirectory(fresh); + File.WriteAllText(Path.Combine(fresh, ".lock.lease"), string.Empty); + var heldId = Guid.CreateVersion7(); + using var held = await store.Project(heldId).LockAsync(TimeSpan.Zero); + foreach (var file in Directory.GetFiles(store.Project(heldId).Folder)) File.SetLastWriteTimeUtc(file, old); + Directory.SetLastWriteTimeUtc(store.Project(heldId).Folder, old); + + Assert.Empty(store.List()); + Assert.False(Directory.Exists(leftover), "only an old lease and its stale copy: removed"); + Assert.False(Directory.Exists(empty), "nothing in it for a while: removed"); + Assert.True(Directory.Exists(other), "a file that is no lease stays"); + Assert.True(Directory.Exists(fresh), "a lease just written stays"); + Assert.True(Directory.Exists(store.Project(heldId).Folder), "a lease held stays"); + } + [Test] public void TheSpaceACopyNeedsCountsAPhotoHeldUnderEitherName() {