fix(workspace): a cancelled copy clears only media no other copy writes; a deletion keeps late versions

Owner review 2, phase 5a, fix round 5.

- A cancelled workspace copy clears its partial media only under the project's
  lease, after checking again that the job is still cancelled, that no other
  copy of the same root and project is queued, running or paused, and that no
  copy in this process is writing. The engine now waits for a paused job's run
  to end before calling the cancel handler, so the clean-up is never skipped.
- The clean-up never recreates a project folder that was removed meanwhile
  (the lease no longer creates the folder). The next list removes a project
  folder that holds only lease files nobody holds, older than the lease.
- A Main this Chronos cannot read (newer or damaged) refuses the open at once
  only when nothing else can be offered. Otherwise Open in workspace lists it
  disabled, with the reason as its tooltip. The refusal's action tooltip is
  now Remove copy.
- The removal plan counts a conflict the same way the sync status does: a
  conflict that a settling replay took past its commit point is not work
  left.
- A healed record is stamped after both the record's time and the index's, so
  the next rename made elsewhere is taken.
- A trash folder holds back the rebuild of a lost record only when it is this
  sequence's deletion: its record names this sequence or cannot be read, or it
  has no record, is within a one-day grace and holds no version of another
  project. What was seen of folders that are gone or have a record again is
  forgotten.
- When a deletion wins, version files pushed into the live folder before the
  deletion arrived are moved beside the deleted record in the trash, never
  deleted.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Alby96andClaude Opus 5.5 committed 2026-11-03 15:10:44 +01:00
1 parent 93afe71639
commit 7692ab402e
16 files changed
+750 -61

No files matched your search

@@ -806,6 +806,146 @@ public abstract class ProjectRecoveryTests(TestContext context) : DatabaseTest(c
Assert.SequenceEqual([main.Id], (await fx.Service.Projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id));
}
[Test]
public async Task APushMadeIntoTheLiveFolderBeforeADeletionArrivedIsKeptBesideTheDeletedRecord()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var projects = fx.Service.Projects;
var main = await projects.MainAsync(sequence.Id, fx.Ct);
var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct);
var first = await projects.HeadAsync(vertical.Id, fx.Ct);
var folder = fx.Paths.Sequence(sequence.Folder);
var (other, _) = await fx.OtherWriterAsync();
var before = folder.ProjectsDirectory + "-before";
CopyDirectory(folder.ProjectDirectory(vertical.Id), before);
await projects.DeleteAsync(vertical.Id, fx.Ct);
// The other PC pushed version 2 into its live folder before the deletion reached it: a sync tool brings that folder
// back beside the trash. The deletion wins, and version 2 goes beside the deleted record, never left where nothing reads it.
CopyDirectory(before, folder.ProjectDirectory(vertical.Id));
var theirs = Draft(first, new ProjectSettings { ExcludedFrames = [2] }, frames) with { StoredAt = DateTimeOffset.UtcNow };
var pushed = folder.VersionPath(vertical.Id, 2, theirs.Id);
ProjectSidecars.WriteVersion(pushed, theirs);
var record = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id));
ProjectSidecars.WriteRecord(folder.ProjectRecordPath(vertical.Id), record with { Head = new ProjectHead(theirs.Id, 2), Updated = DateTimeOffset.UtcNow });
Assert.SequenceEqual([main.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id));
var trashed = Directory.GetDirectories(folder.ProjectsTrashDirectory).Single();
var kept = Path.Combine(trashed, ArchiveLayout.ProjectVersionsFolder, Path.GetFileName(pushed));
Assert.FileExists(kept);
Assert.FileMissing(pushed);
Assert.Equal(theirs.Id, ProjectSidecars.ReadVersion(kept).Id);
Assert.FileExists(Path.Combine(trashed, ArchiveLayout.ProjectVersionsFolder, Path.GetFileName(folder.VersionPath(vertical.Id, 1, first.Id))));
Assert.NotNull(ProjectSidecars.ReadRecord(Path.Combine(trashed, ArchiveLayout.ProjectFile)).Deleted);
Assert.SequenceEqual([main.Id], (await other.Projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id));
Assert.SequenceEqual([main.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id));
}
[Test]
public async Task ATrashFolderOfAnotherSequenceOrLongWithoutARecordNoLongerHoldsBackALostRecord()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var projects = fx.Service.Projects;
var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct);
var first = await projects.HeadAsync(vertical.Id, fx.Ct);
var folder = fx.Paths.Sequence(sequence.Folder);
projects.RecordSettle = TimeSpan.Zero;
var path = folder.ProjectRecordPath(vertical.Id);
var lost = ProjectSidecars.ReadRecord(path);
// A folder of the projects' trash with no record, just made (a deletion on its way): the lost record is not written.
var leftover = Path.Combine(folder.ProjectsTrashDirectory, vertical.Id.ToString("N") + "-1");
Directory.CreateDirectory(leftover);
File.Delete(path);
for (var i = 0; i < 2; i++) await projects.ListAsync(sequence.Id, fx.Ct);
Assert.False(File.Exists(path), "held back while a deletion may be on its way");
await Assert.ThrowsAsync<ProjectsUnreadableException>(() => projects.PushAsync(vertical.Id, Draft(first, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct));
// Nothing written in it for longer than the grace: a leftover, which holds nothing back any more.
Directory.SetLastWriteTimeUtc(leftover, DateTime.UtcNow - ProjectIndexer.TrashRecordGrace - TimeSpan.FromHours(1));
for (var i = 0; i < 2; i++) await projects.ListAsync(sequence.Id, fx.Ct);
Assert.Equal(("Vertical", first.Id), (ProjectSidecars.ReadRecord(path).Name, ProjectSidecars.ReadRecord(path).Head!.Id));
Assert.Equal(PushOutcome.Stored, (await projects.PushAsync(vertical.Id, Draft(first, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)).Outcome);
Directory.Delete(leftover);
// A trash folder whose record names another sequence (a sequence folder copied by hand): never this one's deletion.
var foreign = Path.Combine(folder.ProjectsTrashDirectory, vertical.Id.ToString("N") + "-2");
Directory.CreateDirectory(foreign);
ProjectSidecars.WriteRecord(Path.Combine(foreign, ArchiveLayout.ProjectFile), lost with { SequenceId = Guid.CreateVersion7(), Deleted = DateTimeOffset.UtcNow });
File.Delete(path);
for (var i = 0; i < 2; i++) await projects.ListAsync(sequence.Id, fx.Ct);
Assert.FileExists(path);
Assert.Contains(vertical.Id, (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id));
// One whose record names this sequence is its deletion.
Assert.True(ProjectIndexer.DeletionOnItsWay(foreign, vertical.Id, sequence.Id, DateTime.UtcNow) == false);
ProjectSidecars.WriteRecord(Path.Combine(foreign, ArchiveLayout.ProjectFile), lost with { Deleted = DateTimeOffset.UtcNow });
Assert.True(ProjectIndexer.DeletionOnItsWay(foreign, vertical.Id, sequence.Id, DateTime.UtcNow));
}
[Test]
public async Task AFolderWithNoRecordThatWentAndCameBackSettlesAgainFromWhenItCameBack()
{
var fx = await FixtureAsync();
var (sequence, _) = await UploadedAsync(fx, 3);
var vertical = await fx.Service.Projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct);
var folder = fx.Paths.Sequence(sequence.Folder);
var settle = TimeSpan.FromMilliseconds(TestTiming.Budget(400));
var indexer = new ProjectIndexer(fx.Db) { RecordSettle = settle };
var path = folder.ProjectRecordPath(vertical.Id);
File.Delete(path);
Assert.Contains(vertical.Id, (await indexer.ReconcileAsync(folder, sequence.Id, fx.Ct)).HeldBack);
// The folder goes for a while (a sync tool moving it): what was seen of it is forgotten, so once it is back it must
// look the same for a settling time again before its record is written from the index.
var away = folder.ProjectsDirectory + "-away";
Directory.Move(folder.ProjectDirectory(vertical.Id), away);
Assert.False((await indexer.ReconcileAsync(folder, sequence.Id, fx.Ct)).Seen.Contains(vertical.Id));
await Task.Delay(settle * 2, fx.Ct);
Directory.Move(away, folder.ProjectDirectory(vertical.Id));
Assert.Contains(vertical.Id, (await indexer.ReconcileAsync(folder, sequence.Id, fx.Ct)).HeldBack);
Assert.False(File.Exists(path), "not written at once on what was seen before it went");
await Task.Delay(settle * 2, fx.Ct);
Assert.False((await indexer.ReconcileAsync(folder, sequence.Id, fx.Ct)).HeldBack.Contains(vertical.Id));
Assert.FileExists(path);
}
[Test]
public async Task AHealedRecordIsStampedPastWhatTheIndexKnowsSoTheNextRenameIsTaken()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var projects = fx.Service.Projects;
var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct);
var folder = fx.Paths.Sequence(sequence.Folder);
var backup = folder.ProjectsDirectory + "-backup";
CopyDirectory(folder.ProjectsDirectory, backup);
await projects.PushAsync(vertical.Id, Draft(await projects.HeadAsync(vertical.Id, fx.Ct), new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct);
// A PC whose clock is ten minutes ahead renames it: this index takes the name and that time.
var ahead = DateTimeOffset.UtcNow.AddMinutes(10);
ProjectSidecars.WriteRecord(folder.ProjectRecordPath(vertical.Id), ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)) with { Name = "Square", Updated = ahead });
Assert.Equal("Square", (await projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id).Name);
// The backup is put back (behind): healed, and stamped past what the index knows.
Directory.Delete(folder.ProjectsDirectory, recursive: true);
CopyDirectory(backup, folder.ProjectsDirectory);
await projects.CreateAsync(sequence.Id, "Other", cancellationToken: fx.Ct);
var healed = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id));
Assert.Equal(("Square", 2), (healed.Name, healed.Head!.Number));
Assert.True(healed.Updated > ahead, "past the time the index took");
// Another PC with a right clock renames it: every index takes it, and nothing writes the old name back.
var (other, _) = await fx.OtherWriterAsync();
Assert.Equal("Square", (await other.Projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id).Name);
await other.Projects.RenameAsync(vertical.Id, "Wide", fx.Ct);
Assert.Equal("Wide", (await projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id).Name);
await projects.CreateAsync(sequence.Id, "Other2", cancellationToken: fx.Ct);
Assert.Equal("Wide", ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)).Name);
Assert.Equal("Wide", (await other.Projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id).Name);
}
[Test]
public async Task ALostRecordIsWrittenAgainOnlyOnceItsFolderHasSettledSoAnIncomingOneWins()
{