Originals are now judged by their bytes, never by the media index: a removal or clean-up hashes every file under media/originals and deletes it only when the home holds that SHA-256; every other file, named or not, goes to the workspace trash. Trash entries are filled under a staging name that Empty never lists. A ledger can only name files inside exports/. Jobs follow the workspace: a copy, render or send resolves the folder that holds its project now (a moved workspace, a retry after a move) and waits while a move has not switched yet. A clean-up failure after the switch is a warning that keeps the journal for a later resume; entries that changed since they were copied are kept, and a same-volume rename refused across volumes falls back to copy and verify. Sends map a replayed version by content hash, or go with the project only, and note an old server that drops provenance. The media index (format 2) and project file (format 3) refuse older readers while they hold originals. Metered connections also hold sync downloads and server copies. The editor gains Send again, the Settings page can resume a stopped move, the disabled tooltips say why, and unused strings are pruned. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
310 lines
18 KiB
C#
310 lines
18 KiB
C#
using System.Text.Json;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Core.Project;
|
|
using Chronos.Data.Projects;
|
|
using Chronos.Data.Tests.Archiving;
|
|
using Chronos.Data.Tests.Infrastructure;
|
|
using Chronos.Data.Workspaces;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Data.Tests.Workspaces;
|
|
|
|
/// <summary>
|
|
/// The workspace's clean-up and trash (phase 5c, P4): media go only from projects whose home holds everything (each other case
|
|
/// kept with its reason), an original the home lost goes to the trash, Size limit takes the least recently used first, Clean
|
|
/// after counts from the last opening (else the save), and the trash is never touched by the clean-up and emptied only as
|
|
/// planned.
|
|
/// </summary>
|
|
[Category("workspace")]
|
|
public abstract class WorkspaceCleanupTests(TestContext context) : DatabaseTest(context)
|
|
{
|
|
private static readonly TimeSpan Day = TimeSpan.FromDays(1);
|
|
|
|
private async Task<WorkspaceFixture> FixtureAsync(int frames = 4)
|
|
{
|
|
var archive = Own(await ArchiveFixture.CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true)));
|
|
return Own(await WorkspaceFixture.CreateAsync(Context, archive, frames));
|
|
}
|
|
|
|
private static WorkspaceCleaner Cleaner(WorkspaceFixture fx, WorkspaceSync sync, Func<BundleHome, IProjectHome?>? homes = null) =>
|
|
new(sync, fx.Db, homes ?? (_ => fx.Home)) { LockWait = TimeSpan.FromSeconds(TestTiming.Budget(10)) };
|
|
|
|
private static T One<T>(IReadOnlyList<T> items, [System.Runtime.CompilerServices.CallerArgumentExpression(nameof(items))] string? expression = null)
|
|
{
|
|
Assert.Count(1, items, expression);
|
|
return items[0];
|
|
}
|
|
|
|
/// <summary>Clean after 30 days, the project last opened 40 days ago.</summary>
|
|
private static WorkspaceCleanOptions Aged(Guid projectId, Func<Guid, bool>? isOpen = null, Guid? justCopied = null) => new()
|
|
{
|
|
CleanAfterDays = 30,
|
|
Opened = new Dictionary<Guid, DateTimeOffset> { [projectId] = DateTimeOffset.UtcNow - 40 * Day },
|
|
IsOpen = isOpen ?? (_ => false),
|
|
JustCopied = justCopied,
|
|
};
|
|
|
|
/// <summary>A render of the project's ledger with its file in <c>exports/</c>.</summary>
|
|
private static WorkspaceRenderEntry Render(WorkspaceProject project, string name, string state)
|
|
{
|
|
Directory.CreateDirectory(project.ExportsDirectory);
|
|
File.WriteAllText(project.ExportFile(name), "rendered " + name);
|
|
return new WorkspaceRenderEntry(Guid.CreateVersion7(), Guid.CreateVersion7(), name, new string('a', 64), 12, "preset", "h264", 1920, 1080, 25, 4,
|
|
project.ProjectId, null, null, DateTimeOffset.UtcNow, state, state == WorkspaceRenderEntry.SentState ? DateTimeOffset.UtcNow : null);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AProjectInStepWithItsHomeLosesItsMediaAndKeepsItsWork()
|
|
{
|
|
var fx = await FixtureAsync();
|
|
var (store, sync) = fx.Workspace();
|
|
var project = await fx.CheckoutAsync(sync);
|
|
var id = project.ProjectId;
|
|
var sent = Render(project, "sent.mp4", WorkspaceRenderEntry.SentState);
|
|
project.WriteRenders(WorkspaceRenders.Empty.With(sent));
|
|
var bundle = File.ReadAllBytes(project.BundlePath);
|
|
var proxies = WorkspaceCleaner.Measure(project.ProxiesDirectory).Bytes;
|
|
Assert.True(WorkspaceCleaner.HoldsMedia(project), "the copy brought the media");
|
|
|
|
var report = await Cleaner(fx, sync).CleanAsync(Aged(id), fx.Ct);
|
|
|
|
var cleaned = One(report.Cleaned);
|
|
Assert.Equal((id, 0), (cleaned.ProjectId, cleaned.Trashed));
|
|
Assert.Equal(proxies + "rendered sent.mp4".Length, cleaned.Freed);
|
|
Assert.Empty(report.Kept);
|
|
Assert.False(Directory.Exists(project.ProxiesDirectory), "the proxies went");
|
|
Assert.FileMissing(project.ExportFile("sent.mp4"));
|
|
Assert.Empty(project.ReadRenders().Entries);
|
|
// The frames stay, naming no file; the working state, the deep report and the sync state are as they were.
|
|
var media = Assert.NotNull(project.ReadMedia());
|
|
Assert.True(media.IsCleaned, "the index says the clean-up emptied it");
|
|
Assert.Equal(fx.Frames.Count, media.Frames.Count);
|
|
Assert.True(media.Frames.All(f => f.File is null && f.Original is null && f.FileBytes == 0), "no frame names a file");
|
|
Assert.BytesEqual(bundle, File.ReadAllBytes(project.BundlePath));
|
|
Assert.False(WorkspaceCleaner.HoldsMedia(project), "nothing to edit until a copy");
|
|
Assert.Equal(WorkspaceState.MediaRemoved, (await sync.StatusAsync(id, fx.Home, fx.Ct)).State);
|
|
|
|
// A second run finds nothing to do; a copy brings the media back, in step as before.
|
|
Assert.Empty((await Cleaner(fx, sync).CleanAsync(Aged(id), fx.Ct)).Cleaned);
|
|
await fx.CopyAsync(store, id);
|
|
Assert.True(WorkspaceCleaner.HoldsMedia(project), "copied again");
|
|
Assert.False(project.ReadMedia()!.IsCleaned, "a copy writes an index of its own");
|
|
Assert.Equal(WorkspaceState.InSync, (await sync.StatusAsync(id, fx.Home, fx.Ct)).State);
|
|
}
|
|
|
|
[Test]
|
|
public async Task EachReasonToKeepAProjectKeepsItsMedia()
|
|
{
|
|
var fx = await FixtureAsync();
|
|
var (_, sync) = fx.Workspace();
|
|
var project = await fx.CheckoutAsync(sync);
|
|
var id = project.ProjectId;
|
|
var cleaner = Cleaner(fx, sync);
|
|
|
|
async Task KeptAsync(WorkspaceKeep reason, WorkspaceCleanOptions options, WorkspaceCleaner? by = null)
|
|
{
|
|
var report = await (by ?? cleaner).CleanAsync(options, fx.Ct);
|
|
Assert.Empty(report.Cleaned);
|
|
Assert.Equal((id, reason), (One(report.Kept).ProjectId, report.Kept[0].Reason));
|
|
Assert.True(Directory.Exists(project.ProxiesDirectory) && !project.ReadMedia()!.IsCleaned, $"kept ({reason})");
|
|
}
|
|
|
|
await KeptAsync(WorkspaceKeep.JustCopied, Aged(id, justCopied: id));
|
|
await KeptAsync(WorkspaceKeep.Open, Aged(id, isOpen: p => p == id));
|
|
|
|
// A copy of this project waiting (paused) in the queue; another project's does not count.
|
|
var copy = TestData.Job(JobKind.WorkspaceCopy) with
|
|
{
|
|
TargetIds = [fx.Sequence.Id],
|
|
State = JobState.Paused,
|
|
ParametersJson = JsonSerializer.Serialize(new { workspaceRoot = sync.Store.Root, projectId = id, media = BundleMedia.Proxies }),
|
|
};
|
|
await fx.Db.Jobs.EnqueueAsync(copy, fx.Ct);
|
|
await KeptAsync(WorkspaceKeep.Jobs, Aged(id));
|
|
await fx.Db.Jobs.SetStateAsync(copy.Id, JobState.Cancelled, cancellationToken: fx.Ct);
|
|
|
|
var renders = WorkspaceRenders.Empty.With(Render(project, "pending.mp4", WorkspaceRenderEntry.Pending));
|
|
project.WriteRenders(renders);
|
|
await KeptAsync(WorkspaceKeep.UnsentRenders, Aged(id));
|
|
project.WriteRenders(renders.With(renders.Entries[0] with { State = WorkspaceRenderEntry.Failed, Error = "refused" }));
|
|
await KeptAsync(WorkspaceKeep.UnsentRenders, Aged(id));
|
|
File.WriteAllText(project.RendersPath, "{ not a ledger");
|
|
await KeptAsync(WorkspaceKeep.Unreadable, Aged(id));
|
|
File.Delete(project.RendersPath);
|
|
File.Delete(project.ExportFile("pending.mp4"));
|
|
|
|
await KeptAsync(WorkspaceKeep.Offline, Aged(id), Cleaner(fx, sync, _ => null));
|
|
await KeptAsync(WorkspaceKeep.Offline, Aged(id), Cleaner(fx, sync, _ => new UnreachableHome(fx.Home)));
|
|
|
|
var session = await sync.OpenAsync(id, fx.Ct);
|
|
await sync.SaveAsync(id, session.Working with { Exposure = session.Working.Exposure with { ExposureOffsetEv = 0.7 } }, fx.Ct);
|
|
await KeptAsync(WorkspaceKeep.NotInSync, Aged(id));
|
|
await sync.CommitAsync(id, null, null, fx.Ct);
|
|
await KeptAsync(WorkspaceKeep.NotInSync, Aged(id));
|
|
await sync.PushAsync(id, fx.Home, fx.Ct);
|
|
|
|
// In step again: it goes. Media whose files are gone already hold nothing to remove.
|
|
var other = await fx.Archive.Service.Projects.CreateAsync(fx.Sequence.Id, "Second", cancellationToken: fx.Ct);
|
|
var second = await fx.CheckoutAsync(sync, other.Id);
|
|
Directory.Delete(second.ProxiesDirectory, recursive: true);
|
|
var options = Aged(id) with { Opened = new Dictionary<Guid, DateTimeOffset> { [id] = DateTimeOffset.UtcNow - 40 * Day, [other.Id] = DateTimeOffset.UtcNow - 50 * Day } };
|
|
var report = await cleaner.CleanAsync(options, fx.Ct);
|
|
Assert.Equal(id, One(report.Cleaned).ProjectId);
|
|
Assert.Equal((other.Id, WorkspaceKeep.NoMedia), (One(report.Kept).ProjectId, report.Kept[0].Reason));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnOriginalTheHomeNoLongerHoldsGoesToTheTrashNeverAway()
|
|
{
|
|
var fx = await FixtureAsync();
|
|
var (store, sync) = fx.Workspace();
|
|
var project = await fx.CheckoutAsync(sync);
|
|
// Originals of three frames: the first as the home holds it; the second of bytes the home no longer has; the third
|
|
// recorded with the home's hash, its bytes changed since (the index is no proof). And a file the index never named.
|
|
var media = project.ReadMedia()!;
|
|
var held = WorkspaceProject.OriginalName(media.Frames[0].Id, fx.Frames[0].Sha256!, ".jpg");
|
|
var lost = WorkspaceProject.OriginalName(media.Frames[1].Id, new string('f', 64), ".jpg");
|
|
var changed = WorkspaceProject.OriginalName(media.Frames[2].Id, fx.Frames[2].Sha256!, ".jpg");
|
|
Directory.CreateDirectory(project.OriginalsDirectory);
|
|
var heldBytes = File.ReadAllBytes(fx.Archive.Paths.FromRelative(fx.Frames[0].RelativePath));
|
|
File.WriteAllBytes(project.MediaFile(held), heldBytes);
|
|
File.WriteAllText(project.MediaFile(lost), "last original");
|
|
File.WriteAllText(project.MediaFile(changed), "edited since");
|
|
File.WriteAllText(Path.Combine(project.OriginalsDirectory, "stray.jpg"), "named by no index");
|
|
project.WriteMedia(media with
|
|
{
|
|
Kind = BundleMedia.Originals,
|
|
Frames =
|
|
[
|
|
media.Frames[0] with { Original = held, OriginalSha256 = fx.Frames[0].Sha256, OriginalBytes = heldBytes.Length },
|
|
media.Frames[1] with { Original = lost, OriginalSha256 = new string('f', 64), OriginalBytes = 13 },
|
|
media.Frames[2] with { Original = changed, OriginalSha256 = fx.Frames[2].Sha256, OriginalBytes = 12 },
|
|
.. media.Frames.Skip(3),
|
|
],
|
|
});
|
|
// The index alone names only the second as the last one.
|
|
Assert.SequenceEqual([media.Frames[1].Id], WorkspaceCleaner.LastOriginals(project.ReadMedia(), await fx.Home.FramesAsync(fx.Ct)).Select(f => f.Id));
|
|
|
|
var report = await Cleaner(fx, sync).CleanAsync(Aged(project.ProjectId), fx.Ct);
|
|
|
|
// Each file hashed now: only the one the home holds byte for byte went away.
|
|
Assert.Equal(3, One(report.Cleaned).Trashed);
|
|
Assert.False(Directory.Exists(project.OriginalsDirectory), "the originals folder went");
|
|
var trash = WorkspaceTrash.List(store.Root, fx.Ct);
|
|
var entry = One(trash.Entries);
|
|
Assert.True(entry.Name.StartsWith(project.ProjectId.ToString("N") + "-", StringComparison.Ordinal), entry.Name);
|
|
var kept = Path.Combine(trash.Folder, entry.Name, WorkspaceProject.OriginalsFolder);
|
|
Assert.Equal("last original", File.ReadAllText(Path.Combine(kept, Path.GetFileName(lost))));
|
|
Assert.Equal("edited since", File.ReadAllText(Path.Combine(kept, Path.GetFileName(changed))));
|
|
Assert.Equal("named by no index", File.ReadAllText(Path.Combine(kept, "stray.jpg")));
|
|
Assert.False(File.Exists(Path.Combine(kept, Path.GetFileName(held))), "the home holds that one");
|
|
Assert.Equal((3, (long)("last original".Length + "edited since".Length + "named by no index".Length)), (entry.Files, entry.Bytes));
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheSizeLimitTakesTheLeastRecentlyUsedFirstAndNeverTheProjectJustCopied()
|
|
{
|
|
var fx = await FixtureAsync();
|
|
var (_, sync) = fx.Workspace();
|
|
var main = await fx.CheckoutAsync(sync);
|
|
var b = await fx.CheckoutAsync(sync, (await fx.Archive.Service.Projects.CreateAsync(fx.Sequence.Id, "B", cancellationToken: fx.Ct)).Id);
|
|
var c = await fx.CheckoutAsync(sync, (await fx.Archive.Service.Projects.CreateAsync(fx.Sequence.Id, "C", cancellationToken: fx.Ct)).Id);
|
|
var now = DateTimeOffset.UtcNow;
|
|
var opened = new Dictionary<Guid, DateTimeOffset> { [main.ProjectId] = now - 3 * Day, [b.ProjectId] = now - Day, [c.ProjectId] = now - 2 * Day };
|
|
var total = new[] { main, b, c }.Sum(p => WorkspaceCleaner.Measure(p.Folder).Bytes);
|
|
var media = WorkspaceCleaner.Measure(main.ProxiesDirectory).Bytes;
|
|
// Two projects' media must go to get under the limit (each holds the same proxies).
|
|
var limit = total - media - 1;
|
|
|
|
// After a copy of Main, the oldest: Main stays, the next two oldest go while the folders stay above the limit.
|
|
var report = await Cleaner(fx, sync).CleanAsync(new WorkspaceCleanOptions { SizeLimitBytes = limit, Opened = opened, JustCopied = main.ProjectId, Now = now }, fx.Ct);
|
|
Assert.SequenceEqual([(main.ProjectId, WorkspaceKeep.JustCopied)], report.Kept.Select(k => (k.ProjectId, k.Reason)));
|
|
Assert.SequenceEqual([c.ProjectId, b.ProjectId], report.Cleaned.Select(p => p.ProjectId));
|
|
Assert.Equal(total, report.UsedBefore);
|
|
Assert.Equal(total - report.Freed, report.UsedAfter);
|
|
|
|
// Once copied again: the oldest goes first, and one is enough this time.
|
|
await fx.CopyAsync(sync.Store, b.ProjectId);
|
|
await fx.CopyAsync(sync.Store, c.ProjectId);
|
|
// Measured again: a copy's timestamps may be a byte shorter or longer than the first copy's.
|
|
var again = new[] { main, b, c }.Sum(p => WorkspaceCleaner.Measure(p.Folder).Bytes);
|
|
report = await Cleaner(fx, sync).CleanAsync(new WorkspaceCleanOptions { SizeLimitBytes = again - 1, Opened = opened, Now = now }, fx.Ct);
|
|
Assert.SequenceEqual([main.ProjectId], report.Cleaned.Select(p => p.ProjectId));
|
|
Assert.Empty(report.Kept);
|
|
Assert.True(Directory.Exists(b.ProxiesDirectory) && Directory.Exists(c.ProxiesDirectory), "the others stay");
|
|
|
|
// Under the limit, or with none: nothing goes.
|
|
Assert.Empty((await Cleaner(fx, sync).CleanAsync(new WorkspaceCleanOptions { SizeLimitBytes = total * 2, Opened = opened, Now = now }, fx.Ct)).Cleaned);
|
|
Assert.Empty((await Cleaner(fx, sync).CleanAsync(new WorkspaceCleanOptions { Opened = opened, Now = now }, fx.Ct)).Cleaned);
|
|
}
|
|
|
|
[Test]
|
|
public async Task CleanAfterCountsFromTheLastOpeningElseFromTheSave()
|
|
{
|
|
var fx = await FixtureAsync();
|
|
var (_, sync) = fx.Workspace();
|
|
var main = await fx.CheckoutAsync(sync);
|
|
var other = await fx.CheckoutAsync(sync, (await fx.Archive.Service.Projects.CreateAsync(fx.Sequence.Id, "Other", cancellationToken: fx.Ct)).Id);
|
|
var saved = main.ReadBundle().Saved;
|
|
// Main was opened yesterday; the other never here (its save counts), a year after the save past Clean after.
|
|
var opened = new Dictionary<Guid, DateTimeOffset> { [main.ProjectId] = saved + 364 * Day };
|
|
var options = new WorkspaceCleanOptions { CleanAfterDays = 30, Opened = opened, Now = saved + 365 * Day };
|
|
|
|
var report = await Cleaner(fx, sync).CleanAsync(options, fx.Ct);
|
|
Assert.SequenceEqual([other.ProjectId], report.Cleaned.Select(p => p.ProjectId));
|
|
Assert.Empty(report.Kept);
|
|
Assert.True(Directory.Exists(main.ProxiesDirectory), "opened yesterday");
|
|
|
|
// Within Clean after of the save, or with Clean after off: kept.
|
|
await fx.CopyAsync(sync.Store, other.ProjectId);
|
|
Assert.Empty((await Cleaner(fx, sync).CleanAsync(options with { Now = saved + 29 * Day, Opened = new Dictionary<Guid, DateTimeOffset>() }, fx.Ct)).Cleaned);
|
|
Assert.Empty((await Cleaner(fx, sync).CleanAsync(options with { CleanAfterDays = 0 }, fx.Ct)).Cleaned);
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheCleanUpNeverTouchesTheTrashWhichEmptiesOnlyAsPlanned()
|
|
{
|
|
var fx = await FixtureAsync();
|
|
var (store, sync) = fx.Workspace();
|
|
var project = await fx.CheckoutAsync(sync);
|
|
var folder = WorkspaceTrash.FolderOf(store.Root);
|
|
var kept = Directory.CreateDirectory(Path.Combine(folder, Guid.CreateVersion7().ToString("N") + "-" + DateTime.UtcNow.AddDays(-90).Ticks)).FullName;
|
|
File.WriteAllText(Path.Combine(kept, "project.chronos"), "a removed copy's work");
|
|
var before = WorkspaceTrash.List(store.Root, fx.Ct);
|
|
Assert.Equal((1, 1, (long)"a removed copy's work".Length), (before.Entries.Count, before.Entries[0].Files, before.Bytes));
|
|
Assert.InRange((before.Entries[0].Trashed - DateTimeOffset.UtcNow.AddDays(-90)).TotalMinutes, -1, 1);
|
|
|
|
// Even a size limit of one byte leaves the trash as it was.
|
|
await Cleaner(fx, sync).CleanAsync(new WorkspaceCleanOptions { CleanAfterDays = 1, SizeLimitBytes = 1, Now = DateTimeOffset.UtcNow + 10 * Day }, fx.Ct);
|
|
Assert.False(Directory.Exists(project.ProxiesDirectory), "the media went");
|
|
Assert.Equal(before.Fingerprint, WorkspaceTrash.List(store.Root, fx.Ct).Fingerprint);
|
|
|
|
// An entry added after the plan: nothing is erased.
|
|
var added = Directory.CreateDirectory(Path.Combine(folder, Guid.CreateVersion7().ToString("N") + "-" + DateTime.UtcNow.Ticks)).FullName;
|
|
File.WriteAllText(Path.Combine(added, "late.txt"), "late");
|
|
Assert.Throws<WorkspaceTrashChangedException>(() => WorkspaceTrash.Empty(store.Root, before, fx.Ct));
|
|
Assert.True(Directory.Exists(kept) && Directory.Exists(added), "nothing was erased");
|
|
// A file changed inside an entry changes the fingerprint too.
|
|
var plan = WorkspaceTrash.List(store.Root, fx.Ct);
|
|
File.AppendAllText(Path.Combine(added, "late.txt"), " and longer");
|
|
Assert.Throws<WorkspaceTrashChangedException>(() => WorkspaceTrash.Empty(store.Root, plan, fx.Ct));
|
|
|
|
plan = WorkspaceTrash.List(store.Root, fx.Ct);
|
|
File.SetAttributes(Path.Combine(kept, "project.chronos"), FileAttributes.ReadOnly);
|
|
Assert.Equal((2, plan.Bytes), WorkspaceTrash.Empty(store.Root, plan, fx.Ct));
|
|
Assert.True(WorkspaceTrash.List(store.Root, fx.Ct).IsEmpty, "the trash is empty");
|
|
Assert.True(Directory.Exists(folder), "the trash folder itself stays");
|
|
}
|
|
}
|
|
|
|
public sealed class SqliteWorkspaceCleanupTests(TestContext context) : WorkspaceCleanupTests(context)
|
|
{
|
|
protected override Backend Backend => Backend.SqliteFile;
|
|
}
|
|
|
|
[RequiresEnvironment(PgTestServer.Variable)]
|
|
public sealed class PostgresWorkspaceCleanupTests(TestContext context) : WorkspaceCleanupTests(context)
|
|
{
|
|
protected override Backend Backend => Backend.Postgres;
|
|
}
|