fix(workspace): a cancelled copy clears only media no other copy writes; a deletion keeps late versions

Owner review 2, phase 5a, fix round 5.

- A cancelled workspace copy clears its partial media only under the project's
  lease, after checking again that the job is still cancelled, that no other
  copy of the same root and project is queued, running or paused, and that no
  copy in this process is writing. The engine now waits for a paused job's run
  to end before calling the cancel handler, so the clean-up is never skipped.
- The clean-up never recreates a project folder that was removed meanwhile
  (the lease no longer creates the folder). The next list removes a project
  folder that holds only lease files nobody holds, older than the lease.
- A Main this Chronos cannot read (newer or damaged) refuses the open at once
  only when nothing else can be offered. Otherwise Open in workspace lists it
  disabled, with the reason as its tooltip. The refusal's action tooltip is
  now Remove copy.
- The removal plan counts a conflict the same way the sync status does: a
  conflict that a settling replay took past its commit point is not work
  left.
- A healed record is stamped after both the record's time and the index's, so
  the next rename made elsewhere is taken.
- A trash folder holds back the rebuild of a lost record only when it is this
  sequence's deletion: its record names this sequence or cannot be read, or it
  has no record, is within a one-day grace and holds no version of another
  project. What was seen of folders that are gone or have a record again is
  forgotten.
- When a deletion wins, version files pushed into the live folder before the
  deletion arrived are moved beside the deleted record in the trash, never
  deleted.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Alby96andClaude Opus 5.5 committed 2026-11-03 15:10:44 +01:00
1 parent 93afe71639
commit 7692ab402e
16 files changed
+750 -61

No files matched your search

@@ -806,6 +806,146 @@ public abstract class ProjectRecoveryTests(TestContext context) : DatabaseTest(c
Assert.SequenceEqual([main.Id], (await fx.Service.Projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id));
}
[Test]
public async Task APushMadeIntoTheLiveFolderBeforeADeletionArrivedIsKeptBesideTheDeletedRecord()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var projects = fx.Service.Projects;
var main = await projects.MainAsync(sequence.Id, fx.Ct);
var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct);
var first = await projects.HeadAsync(vertical.Id, fx.Ct);
var folder = fx.Paths.Sequence(sequence.Folder);
var (other, _) = await fx.OtherWriterAsync();
var before = folder.ProjectsDirectory + "-before";
CopyDirectory(folder.ProjectDirectory(vertical.Id), before);
await projects.DeleteAsync(vertical.Id, fx.Ct);
// The other PC pushed version 2 into its live folder before the deletion reached it: a sync tool brings that folder
// back beside the trash. The deletion wins, and version 2 goes beside the deleted record, never left where nothing reads it.
CopyDirectory(before, folder.ProjectDirectory(vertical.Id));
var theirs = Draft(first, new ProjectSettings { ExcludedFrames = [2] }, frames) with { StoredAt = DateTimeOffset.UtcNow };
var pushed = folder.VersionPath(vertical.Id, 2, theirs.Id);
ProjectSidecars.WriteVersion(pushed, theirs);
var record = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id));
ProjectSidecars.WriteRecord(folder.ProjectRecordPath(vertical.Id), record with { Head = new ProjectHead(theirs.Id, 2), Updated = DateTimeOffset.UtcNow });
Assert.SequenceEqual([main.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id));
var trashed = Directory.GetDirectories(folder.ProjectsTrashDirectory).Single();
var kept = Path.Combine(trashed, ArchiveLayout.ProjectVersionsFolder, Path.GetFileName(pushed));
Assert.FileExists(kept);
Assert.FileMissing(pushed);
Assert.Equal(theirs.Id, ProjectSidecars.ReadVersion(kept).Id);
Assert.FileExists(Path.Combine(trashed, ArchiveLayout.ProjectVersionsFolder, Path.GetFileName(folder.VersionPath(vertical.Id, 1, first.Id))));
Assert.NotNull(ProjectSidecars.ReadRecord(Path.Combine(trashed, ArchiveLayout.ProjectFile)).Deleted);
Assert.SequenceEqual([main.Id], (await other.Projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id));
Assert.SequenceEqual([main.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id));
}
[Test]
public async Task ATrashFolderOfAnotherSequenceOrLongWithoutARecordNoLongerHoldsBackALostRecord()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var projects = fx.Service.Projects;
var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct);
var first = await projects.HeadAsync(vertical.Id, fx.Ct);
var folder = fx.Paths.Sequence(sequence.Folder);
projects.RecordSettle = TimeSpan.Zero;
var path = folder.ProjectRecordPath(vertical.Id);
var lost = ProjectSidecars.ReadRecord(path);
// A folder of the projects' trash with no record, just made (a deletion on its way): the lost record is not written.
var leftover = Path.Combine(folder.ProjectsTrashDirectory, vertical.Id.ToString("N") + "-1");
Directory.CreateDirectory(leftover);
File.Delete(path);
for (var i = 0; i < 2; i++) await projects.ListAsync(sequence.Id, fx.Ct);
Assert.False(File.Exists(path), "held back while a deletion may be on its way");
await Assert.ThrowsAsync<ProjectsUnreadableException>(() => projects.PushAsync(vertical.Id, Draft(first, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct));
// Nothing written in it for longer than the grace: a leftover, which holds nothing back any more.
Directory.SetLastWriteTimeUtc(leftover, DateTime.UtcNow - ProjectIndexer.TrashRecordGrace - TimeSpan.FromHours(1));
for (var i = 0; i < 2; i++) await projects.ListAsync(sequence.Id, fx.Ct);
Assert.Equal(("Vertical", first.Id), (ProjectSidecars.ReadRecord(path).Name, ProjectSidecars.ReadRecord(path).Head!.Id));
Assert.Equal(PushOutcome.Stored, (await projects.PushAsync(vertical.Id, Draft(first, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)).Outcome);
Directory.Delete(leftover);
// A trash folder whose record names another sequence (a sequence folder copied by hand): never this one's deletion.
var foreign = Path.Combine(folder.ProjectsTrashDirectory, vertical.Id.ToString("N") + "-2");
Directory.CreateDirectory(foreign);
ProjectSidecars.WriteRecord(Path.Combine(foreign, ArchiveLayout.ProjectFile), lost with { SequenceId = Guid.CreateVersion7(), Deleted = DateTimeOffset.UtcNow });
File.Delete(path);
for (var i = 0; i < 2; i++) await projects.ListAsync(sequence.Id, fx.Ct);
Assert.FileExists(path);
Assert.Contains(vertical.Id, (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id));
// One whose record names this sequence is its deletion.
Assert.True(ProjectIndexer.DeletionOnItsWay(foreign, vertical.Id, sequence.Id, DateTime.UtcNow) == false);
ProjectSidecars.WriteRecord(Path.Combine(foreign, ArchiveLayout.ProjectFile), lost with { Deleted = DateTimeOffset.UtcNow });
Assert.True(ProjectIndexer.DeletionOnItsWay(foreign, vertical.Id, sequence.Id, DateTime.UtcNow));
}
[Test]
public async Task AFolderWithNoRecordThatWentAndCameBackSettlesAgainFromWhenItCameBack()
{
var fx = await FixtureAsync();
var (sequence, _) = await UploadedAsync(fx, 3);
var vertical = await fx.Service.Projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct);
var folder = fx.Paths.Sequence(sequence.Folder);
var settle = TimeSpan.FromMilliseconds(TestTiming.Budget(400));
var indexer = new ProjectIndexer(fx.Db) { RecordSettle = settle };
var path = folder.ProjectRecordPath(vertical.Id);
File.Delete(path);
Assert.Contains(vertical.Id, (await indexer.ReconcileAsync(folder, sequence.Id, fx.Ct)).HeldBack);
// The folder goes for a while (a sync tool moving it): what was seen of it is forgotten, so once it is back it must
// look the same for a settling time again before its record is written from the index.
var away = folder.ProjectsDirectory + "-away";
Directory.Move(folder.ProjectDirectory(vertical.Id), away);
Assert.False((await indexer.ReconcileAsync(folder, sequence.Id, fx.Ct)).Seen.Contains(vertical.Id));
await Task.Delay(settle * 2, fx.Ct);
Directory.Move(away, folder.ProjectDirectory(vertical.Id));
Assert.Contains(vertical.Id, (await indexer.ReconcileAsync(folder, sequence.Id, fx.Ct)).HeldBack);
Assert.False(File.Exists(path), "not written at once on what was seen before it went");
await Task.Delay(settle * 2, fx.Ct);
Assert.False((await indexer.ReconcileAsync(folder, sequence.Id, fx.Ct)).HeldBack.Contains(vertical.Id));
Assert.FileExists(path);
}
[Test]
public async Task AHealedRecordIsStampedPastWhatTheIndexKnowsSoTheNextRenameIsTaken()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var projects = fx.Service.Projects;
var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct);
var folder = fx.Paths.Sequence(sequence.Folder);
var backup = folder.ProjectsDirectory + "-backup";
CopyDirectory(folder.ProjectsDirectory, backup);
await projects.PushAsync(vertical.Id, Draft(await projects.HeadAsync(vertical.Id, fx.Ct), new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct);
// A PC whose clock is ten minutes ahead renames it: this index takes the name and that time.
var ahead = DateTimeOffset.UtcNow.AddMinutes(10);
ProjectSidecars.WriteRecord(folder.ProjectRecordPath(vertical.Id), ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)) with { Name = "Square", Updated = ahead });
Assert.Equal("Square", (await projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id).Name);
// The backup is put back (behind): healed, and stamped past what the index knows.
Directory.Delete(folder.ProjectsDirectory, recursive: true);
CopyDirectory(backup, folder.ProjectsDirectory);
await projects.CreateAsync(sequence.Id, "Other", cancellationToken: fx.Ct);
var healed = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id));
Assert.Equal(("Square", 2), (healed.Name, healed.Head!.Number));
Assert.True(healed.Updated > ahead, "past the time the index took");
// Another PC with a right clock renames it: every index takes it, and nothing writes the old name back.
var (other, _) = await fx.OtherWriterAsync();
Assert.Equal("Square", (await other.Projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id).Name);
await other.Projects.RenameAsync(vertical.Id, "Wide", fx.Ct);
Assert.Equal("Wide", (await projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id).Name);
await projects.CreateAsync(sequence.Id, "Other2", cancellationToken: fx.Ct);
Assert.Equal("Wide", ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)).Name);
Assert.Equal("Wide", (await other.Projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id).Name);
}
[Test]
public async Task ALostRecordIsWrittenAgainOnlyOnceItsFolderHasSettledSoAnIncomingOneWins()
{
@@ -86,6 +86,9 @@ public abstract class WorkspaceSafetyTests(TestContext context) : DatabaseTest(c
var (store, two, project, shown) = await AskedAsync(fx, id);
await Assert.ThrowsAsync<CutException>(() =>
WorkspaceFixture.Cutting(store, "replay.base").ResolveAsync(id, fx.Home, ConflictChoice.TakeServer, shown: shown, cancellationToken: fx.Ct));
// sync.json still records the conflict, but it waits for no choice: what counts it reads it as the status does.
Assert.NotNull(project.ReadSync().Conflict);
Assert.Null(project.OpenConflict(), "settled past its commit point");
// The working state is on the library's head with its value: the choice was made, nothing asks it again.
var status = await two.StatusAsync(id, fx.Home, fx.Ct);
@@ -543,6 +546,142 @@ public abstract class WorkspaceSafetyTests(TestContext context) : DatabaseTest(c
foreach (var frame in media.Frames) Assert.FileExists(project.MediaFile(frame.File!));
}
[Test]
public async Task APausedFirstCopyCancelledWhileASecondCopyOfTheProjectRunsLeavesItsMedia()
{
var fx = await FixtureAsync();
var (store, sync) = fx.Workspace();
var id = (await fx.Archive.Service.Projects.MainAsync(fx.Sequence.Id, fx.Ct)).Id;
var project = await sync.CheckoutAsync(fx.Home, id, cancellationToken: fx.Ct);
Job second;
// Held so that neither copy commits before the first is cancelled.
var lease = await project.LockAsync(Wait, fx.Ct);
try
{
var first = await fx.StartCopyAsync(store, id);
await UntilAsync(() => Directory.Exists(project.ProxiesDirectory) && Directory.GetFiles(project.ProxiesDirectory, "*.jpg").Length == fx.Frames.Count,
"the first copy's files", fx.Ct);
await fx.PauseAsync(first.Id);
await fx.Archive.WaitForAsync(first.Id, JobState.Paused, (int)TestTiming.Budget(30));
// The sequence opened again queues another copy (the first is paused), then the first is cancelled in Jobs.
second = await fx.StartCopyAsync(store, id);
await fx.Archive.WaitForAsync(second.Id, JobState.Running, (int)TestTiming.Budget(30));
await fx.CancelAsync(first.Id);
await fx.Archive.WaitForAsync(first.Id, JobState.Cancelled, (int)TestTiming.Budget(30));
}
finally
{
lease.Dispose();
}
await fx.Archive.WaitForAsync(second.Id, JobState.Done, (int)TestTiming.Budget(60));
// The clean-up had its chance at the lease: it found the second copy and left the media.
await Task.Delay(500, fx.Ct);
using (await project.LockAsync(Wait, fx.Ct))
{
var media = Assert.NotNull(project.ReadMedia());
Assert.Equal(fx.Frames.Count, media.Frames.Count);
foreach (var frame in media.Frames) Assert.FileExists(project.MediaFile(frame.File!));
}
var status = await sync.StatusAsync(id, fx.Home, fx.Ct);
Assert.True(status is { State: WorkspaceState.InSync, MediaRemoved: false });
}
[Test]
public async Task ACancelledCopysCleanUpLeavesAFolderNoLongerAProjectAndNeverMakesOneAgain()
{
var fx = await FixtureAsync();
var (store, sync) = fx.Workspace();
var main = (await fx.Archive.Service.Projects.MainAsync(fx.Sequence.Id, fx.Ct)).Id;
var side = (await fx.Archive.Service.Projects.CreateAsync(fx.Sequence.Id, "Side", cancellationToken: fx.Ct)).Id;
// The working state moved aside while the clean-up waited for the lease (Remove local copy): its folder is left as it is.
var moving = await PausedFirstCopyAsync(fx, store, sync, main);
var lease = await moving.Project.LockAsync(Wait, fx.Ct);
var aside = Path.Combine(Context.CreateTempDirectory("aside"), WorkspaceProject.BundleFile);
try
{
await fx.CancelAsync(moving.Copy.Id);
// The clean-up waits for the lease meanwhile.
await Task.Delay(300, fx.Ct);
File.Move(moving.Project.BundlePath, aside);
}
finally
{
lease.Dispose();
}
await Task.Delay(1000, fx.Ct);
Assert.True(Directory.Exists(moving.Project.ProxiesDirectory), "a folder that is no project is not the clean-up's");
// The folder removed while the clean-up waited: it is never made again.
var removed = await PausedFirstCopyAsync(fx, store, sync, side);
lease = await removed.Project.LockAsync(Wait, fx.Ct);
try
{
await fx.CancelAsync(removed.Copy.Id);
await Task.Delay(300, fx.Ct);
Directory.Delete(removed.Project.Folder, recursive: true);
}
finally
{
lease.Dispose();
}
await Task.Delay(1000, fx.Ct);
Assert.False(Directory.Exists(removed.Project.Folder), "no folder made again for a lease");
}
/// <summary>A first copy of <paramref name="projectId"/>, checked out, that wrote every file and is paused before it committed.</summary>
private async Task<(WorkspaceProject Project, Job Copy)> PausedFirstCopyAsync(WorkspaceFixture fx, WorkspaceStore store, WorkspaceSync sync, Guid projectId)
{
var project = await sync.CheckoutAsync(fx.Home, projectId, cancellationToken: fx.Ct);
using (await project.LockAsync(Wait, fx.Ct))
{
var copy = await fx.StartCopyAsync(store, projectId);
await UntilAsync(() => Directory.Exists(project.ProxiesDirectory) && Directory.GetFiles(project.ProxiesDirectory, "*.jpg").Length == fx.Frames.Count,
"the copy's files", fx.Ct);
await fx.PauseAsync(copy.Id);
await fx.Archive.WaitForAsync(copy.Id, JobState.Paused, (int)TestTiming.Budget(30));
return (project, copy);
}
}
[Test]
public async Task AProjectFolderHoldingOnlyALeaseNoOneHoldsGoesOnTheNextList()
{
var store = new WorkspaceStore(Path.Combine(Context.CreateTempDirectory("leftovers"), "workspace"));
var old = DateTime.UtcNow - TimeSpan.FromHours(1);
string Folder(params string[] files)
{
var folder = Path.Combine(store.Root, Guid.CreateVersion7().ToString("N"));
Directory.CreateDirectory(folder);
foreach (var file in files)
{
var path = Path.Combine(folder, file);
if (!File.Exists(path)) File.WriteAllText(path, string.Empty);
File.SetLastWriteTimeUtc(path, old);
}
Directory.SetLastWriteTimeUtc(folder, old);
return folder;
}
var leftover = Folder(".lock.lease", ".lock.lease.0a1b2c3d.stale");
var empty = Folder();
var other = Folder(".lock.lease", "notes.txt");
var fresh = Path.Combine(store.Root, Guid.CreateVersion7().ToString("N"));
Directory.CreateDirectory(fresh);
File.WriteAllText(Path.Combine(fresh, ".lock.lease"), string.Empty);
var heldId = Guid.CreateVersion7();
using var held = await store.Project(heldId).LockAsync(TimeSpan.Zero);
foreach (var file in Directory.GetFiles(store.Project(heldId).Folder)) File.SetLastWriteTimeUtc(file, old);
Directory.SetLastWriteTimeUtc(store.Project(heldId).Folder, old);
Assert.Empty(store.List());
Assert.False(Directory.Exists(leftover), "only an old lease and its stale copy: removed");
Assert.False(Directory.Exists(empty), "nothing in it for a while: removed");
Assert.True(Directory.Exists(other), "a file that is no lease stays");
Assert.True(Directory.Exists(fresh), "a lease just written stays");
Assert.True(Directory.Exists(store.Project(heldId).Folder), "a lease held stays");
}
[Test]
public void TheSpaceACopyNeedsCountsAPhotoHeldUnderEitherName()
{