CI / Build and test (push) Successful in 9m26s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 8s
Release / Build, test and publish (push) Successful in 14m31s
CI / PostgreSQL 16 integration (push) Successful in 9m53s
CI / PostgreSQL 17 integration (push) Successful in 9m22s
CI / Docker image (push) Successful in 44s
Release / Server image (push) Successful in 1m36s
Originals in the workspace, local render and send home with provenance, the workspace clean-up, size limit and trash, moving the workspace folder, and the complete Settings › Workspace page (owner review 2, R8, phase 5c). One adversarial review round (35 findings, 32 fixed), a copy-cancel race fixed, 184 strings in all 38 languages. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
310 lines
18 KiB
C#
310 lines
18 KiB
C#
using System.Text.Json;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Core.Project;
|
|
using Chronos.Data.Projects;
|
|
using Chronos.Data.Tests.Archiving;
|
|
using Chronos.Data.Tests.Infrastructure;
|
|
using Chronos.Data.Workspaces;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Data.Tests.Workspaces;
|
|
|
|
/// <summary>
|
|
/// The workspace's clean-up and trash (phase 5c, P4): media go only from projects whose home holds everything (each other case
|
|
/// kept with its reason), an original the home lost goes to the trash, Size limit takes the least recently used first, Clean
|
|
/// after counts from the last opening (else the save), and the trash is never touched by the clean-up and emptied only as
|
|
/// planned.
|
|
/// </summary>
|
|
[Category("workspace")]
|
|
public abstract class WorkspaceCleanupTests(TestContext context) : DatabaseTest(context)
|
|
{
|
|
private static readonly TimeSpan Day = TimeSpan.FromDays(1);
|
|
|
|
private async Task<WorkspaceFixture> FixtureAsync(int frames = 4)
|
|
{
|
|
var archive = Own(await ArchiveFixture.CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true)));
|
|
return Own(await WorkspaceFixture.CreateAsync(Context, archive, frames));
|
|
}
|
|
|
|
private static WorkspaceCleaner Cleaner(WorkspaceFixture fx, WorkspaceSync sync, Func<BundleHome, IProjectHome?>? homes = null) =>
|
|
new(sync, fx.Db, homes ?? (_ => fx.Home)) { LockWait = TimeSpan.FromSeconds(TestTiming.Budget(10)) };
|
|
|
|
private static T One<T>(IReadOnlyList<T> items, [System.Runtime.CompilerServices.CallerArgumentExpression(nameof(items))] string? expression = null)
|
|
{
|
|
Assert.Count(1, items, expression);
|
|
return items[0];
|
|
}
|
|
|
|
/// <summary>Clean after 30 days, the project last opened 40 days ago.</summary>
|
|
private static WorkspaceCleanOptions Aged(Guid projectId, Func<Guid, bool>? isOpen = null, Guid? justCopied = null) => new()
|
|
{
|
|
CleanAfterDays = 30,
|
|
Opened = new Dictionary<Guid, DateTimeOffset> { [projectId] = DateTimeOffset.UtcNow - 40 * Day },
|
|
IsOpen = isOpen ?? (_ => false),
|
|
JustCopied = justCopied,
|
|
};
|
|
|
|
/// <summary>A render of the project's ledger with its file in <c>exports/</c>.</summary>
|
|
private static WorkspaceRenderEntry Render(WorkspaceProject project, string name, string state)
|
|
{
|
|
Directory.CreateDirectory(project.ExportsDirectory);
|
|
File.WriteAllText(project.ExportFile(name), "rendered " + name);
|
|
return new WorkspaceRenderEntry(Guid.CreateVersion7(), Guid.CreateVersion7(), name, new string('a', 64), 12, "preset", "h264", 1920, 1080, 25, 4,
|
|
project.ProjectId, null, null, DateTimeOffset.UtcNow, state, state == WorkspaceRenderEntry.SentState ? DateTimeOffset.UtcNow : null);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AProjectInStepWithItsHomeLosesItsMediaAndKeepsItsWork()
|
|
{
|
|
var fx = await FixtureAsync();
|
|
var (store, sync) = fx.Workspace();
|
|
var project = await fx.CheckoutAsync(sync);
|
|
var id = project.ProjectId;
|
|
var sent = Render(project, "sent.mp4", WorkspaceRenderEntry.SentState);
|
|
project.WriteRenders(WorkspaceRenders.Empty.With(sent));
|
|
var bundle = File.ReadAllBytes(project.BundlePath);
|
|
var proxies = WorkspaceCleaner.Measure(project.ProxiesDirectory).Bytes;
|
|
Assert.True(WorkspaceCleaner.HoldsMedia(project), "the copy brought the media");
|
|
|
|
var report = await Cleaner(fx, sync).CleanAsync(Aged(id), fx.Ct);
|
|
|
|
var cleaned = One(report.Cleaned);
|
|
Assert.Equal((id, 0), (cleaned.ProjectId, cleaned.Trashed));
|
|
Assert.Equal(proxies + "rendered sent.mp4".Length, cleaned.Freed);
|
|
Assert.Empty(report.Kept);
|
|
Assert.False(Directory.Exists(project.ProxiesDirectory), "the proxies went");
|
|
Assert.FileMissing(project.ExportFile("sent.mp4"));
|
|
Assert.Empty(project.ReadRenders().Entries);
|
|
// The frames stay, naming no file; the working state, the deep report and the sync state are as they were.
|
|
var media = Assert.NotNull(project.ReadMedia());
|
|
Assert.True(media.IsCleaned, "the index says the clean-up emptied it");
|
|
Assert.Equal(fx.Frames.Count, media.Frames.Count);
|
|
Assert.True(media.Frames.All(f => f.File is null && f.Original is null && f.FileBytes == 0), "no frame names a file");
|
|
Assert.BytesEqual(bundle, File.ReadAllBytes(project.BundlePath));
|
|
Assert.False(WorkspaceCleaner.HoldsMedia(project), "nothing to edit until a copy");
|
|
Assert.Equal(WorkspaceState.MediaRemoved, (await sync.StatusAsync(id, fx.Home, fx.Ct)).State);
|
|
|
|
// A second run finds nothing to do; a copy brings the media back, in step as before.
|
|
Assert.Empty((await Cleaner(fx, sync).CleanAsync(Aged(id), fx.Ct)).Cleaned);
|
|
await fx.CopyAsync(store, id);
|
|
Assert.True(WorkspaceCleaner.HoldsMedia(project), "copied again");
|
|
Assert.False(project.ReadMedia()!.IsCleaned, "a copy writes an index of its own");
|
|
Assert.Equal(WorkspaceState.InSync, (await sync.StatusAsync(id, fx.Home, fx.Ct)).State);
|
|
}
|
|
|
|
[Test]
|
|
public async Task EachReasonToKeepAProjectKeepsItsMedia()
|
|
{
|
|
var fx = await FixtureAsync();
|
|
var (_, sync) = fx.Workspace();
|
|
var project = await fx.CheckoutAsync(sync);
|
|
var id = project.ProjectId;
|
|
var cleaner = Cleaner(fx, sync);
|
|
|
|
async Task KeptAsync(WorkspaceKeep reason, WorkspaceCleanOptions options, WorkspaceCleaner? by = null)
|
|
{
|
|
var report = await (by ?? cleaner).CleanAsync(options, fx.Ct);
|
|
Assert.Empty(report.Cleaned);
|
|
Assert.Equal((id, reason), (One(report.Kept).ProjectId, report.Kept[0].Reason));
|
|
Assert.True(Directory.Exists(project.ProxiesDirectory) && !project.ReadMedia()!.IsCleaned, $"kept ({reason})");
|
|
}
|
|
|
|
await KeptAsync(WorkspaceKeep.JustCopied, Aged(id, justCopied: id));
|
|
await KeptAsync(WorkspaceKeep.Open, Aged(id, isOpen: p => p == id));
|
|
|
|
// A copy of this project waiting (paused) in the queue; another project's does not count.
|
|
var copy = TestData.Job(JobKind.WorkspaceCopy) with
|
|
{
|
|
TargetIds = [fx.Sequence.Id],
|
|
State = JobState.Paused,
|
|
ParametersJson = JsonSerializer.Serialize(new { workspaceRoot = sync.Store.Root, projectId = id, media = BundleMedia.Proxies }),
|
|
};
|
|
await fx.Db.Jobs.EnqueueAsync(copy, fx.Ct);
|
|
await KeptAsync(WorkspaceKeep.Jobs, Aged(id));
|
|
await fx.Db.Jobs.SetStateAsync(copy.Id, JobState.Cancelled, cancellationToken: fx.Ct);
|
|
|
|
var renders = WorkspaceRenders.Empty.With(Render(project, "pending.mp4", WorkspaceRenderEntry.Pending));
|
|
project.WriteRenders(renders);
|
|
await KeptAsync(WorkspaceKeep.UnsentRenders, Aged(id));
|
|
project.WriteRenders(renders.With(renders.Entries[0] with { State = WorkspaceRenderEntry.Failed, Error = "refused" }));
|
|
await KeptAsync(WorkspaceKeep.UnsentRenders, Aged(id));
|
|
File.WriteAllText(project.RendersPath, "{ not a ledger");
|
|
await KeptAsync(WorkspaceKeep.Unreadable, Aged(id));
|
|
File.Delete(project.RendersPath);
|
|
File.Delete(project.ExportFile("pending.mp4"));
|
|
|
|
await KeptAsync(WorkspaceKeep.Offline, Aged(id), Cleaner(fx, sync, _ => null));
|
|
await KeptAsync(WorkspaceKeep.Offline, Aged(id), Cleaner(fx, sync, _ => new UnreachableHome(fx.Home)));
|
|
|
|
var session = await sync.OpenAsync(id, fx.Ct);
|
|
await sync.SaveAsync(id, session.Working with { Exposure = session.Working.Exposure with { ExposureOffsetEv = 0.7 } }, fx.Ct);
|
|
await KeptAsync(WorkspaceKeep.NotInSync, Aged(id));
|
|
await sync.CommitAsync(id, null, null, fx.Ct);
|
|
await KeptAsync(WorkspaceKeep.NotInSync, Aged(id));
|
|
await sync.PushAsync(id, fx.Home, fx.Ct);
|
|
|
|
// In step again: it goes. Media whose files are gone already hold nothing to remove.
|
|
var other = await fx.Archive.Service.Projects.CreateAsync(fx.Sequence.Id, "Second", cancellationToken: fx.Ct);
|
|
var second = await fx.CheckoutAsync(sync, other.Id);
|
|
Directory.Delete(second.ProxiesDirectory, recursive: true);
|
|
var options = Aged(id) with { Opened = new Dictionary<Guid, DateTimeOffset> { [id] = DateTimeOffset.UtcNow - 40 * Day, [other.Id] = DateTimeOffset.UtcNow - 50 * Day } };
|
|
var report = await cleaner.CleanAsync(options, fx.Ct);
|
|
Assert.Equal(id, One(report.Cleaned).ProjectId);
|
|
Assert.Equal((other.Id, WorkspaceKeep.NoMedia), (One(report.Kept).ProjectId, report.Kept[0].Reason));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnOriginalTheHomeNoLongerHoldsGoesToTheTrashNeverAway()
|
|
{
|
|
var fx = await FixtureAsync();
|
|
var (store, sync) = fx.Workspace();
|
|
var project = await fx.CheckoutAsync(sync);
|
|
// Originals of three frames: the first as the home holds it; the second of bytes the home no longer has; the third
|
|
// recorded with the home's hash, its bytes changed since (the index is no proof). And a file the index never named.
|
|
var media = project.ReadMedia()!;
|
|
var held = WorkspaceProject.OriginalName(media.Frames[0].Id, fx.Frames[0].Sha256!, ".jpg");
|
|
var lost = WorkspaceProject.OriginalName(media.Frames[1].Id, new string('f', 64), ".jpg");
|
|
var changed = WorkspaceProject.OriginalName(media.Frames[2].Id, fx.Frames[2].Sha256!, ".jpg");
|
|
Directory.CreateDirectory(project.OriginalsDirectory);
|
|
var heldBytes = File.ReadAllBytes(fx.Archive.Paths.FromRelative(fx.Frames[0].RelativePath));
|
|
File.WriteAllBytes(project.MediaFile(held), heldBytes);
|
|
File.WriteAllText(project.MediaFile(lost), "last original");
|
|
File.WriteAllText(project.MediaFile(changed), "edited since");
|
|
File.WriteAllText(Path.Combine(project.OriginalsDirectory, "stray.jpg"), "named by no index");
|
|
project.WriteMedia(media with
|
|
{
|
|
Kind = BundleMedia.Originals,
|
|
Frames =
|
|
[
|
|
media.Frames[0] with { Original = held, OriginalSha256 = fx.Frames[0].Sha256, OriginalBytes = heldBytes.Length },
|
|
media.Frames[1] with { Original = lost, OriginalSha256 = new string('f', 64), OriginalBytes = 13 },
|
|
media.Frames[2] with { Original = changed, OriginalSha256 = fx.Frames[2].Sha256, OriginalBytes = 12 },
|
|
.. media.Frames.Skip(3),
|
|
],
|
|
});
|
|
// The index alone names only the second as the last one.
|
|
Assert.SequenceEqual([media.Frames[1].Id], WorkspaceCleaner.LastOriginals(project.ReadMedia(), await fx.Home.FramesAsync(fx.Ct)).Select(f => f.Id));
|
|
|
|
var report = await Cleaner(fx, sync).CleanAsync(Aged(project.ProjectId), fx.Ct);
|
|
|
|
// Each file hashed now: only the one the home holds byte for byte went away.
|
|
Assert.Equal(3, One(report.Cleaned).Trashed);
|
|
Assert.False(Directory.Exists(project.OriginalsDirectory), "the originals folder went");
|
|
var trash = WorkspaceTrash.List(store.Root, fx.Ct);
|
|
var entry = One(trash.Entries);
|
|
Assert.True(entry.Name.StartsWith(project.ProjectId.ToString("N") + "-", StringComparison.Ordinal), entry.Name);
|
|
var kept = Path.Combine(trash.Folder, entry.Name, WorkspaceProject.OriginalsFolder);
|
|
Assert.Equal("last original", File.ReadAllText(Path.Combine(kept, Path.GetFileName(lost))));
|
|
Assert.Equal("edited since", File.ReadAllText(Path.Combine(kept, Path.GetFileName(changed))));
|
|
Assert.Equal("named by no index", File.ReadAllText(Path.Combine(kept, "stray.jpg")));
|
|
Assert.False(File.Exists(Path.Combine(kept, Path.GetFileName(held))), "the home holds that one");
|
|
Assert.Equal((3, (long)("last original".Length + "edited since".Length + "named by no index".Length)), (entry.Files, entry.Bytes));
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheSizeLimitTakesTheLeastRecentlyUsedFirstAndNeverTheProjectJustCopied()
|
|
{
|
|
var fx = await FixtureAsync();
|
|
var (_, sync) = fx.Workspace();
|
|
var main = await fx.CheckoutAsync(sync);
|
|
var b = await fx.CheckoutAsync(sync, (await fx.Archive.Service.Projects.CreateAsync(fx.Sequence.Id, "B", cancellationToken: fx.Ct)).Id);
|
|
var c = await fx.CheckoutAsync(sync, (await fx.Archive.Service.Projects.CreateAsync(fx.Sequence.Id, "C", cancellationToken: fx.Ct)).Id);
|
|
var now = DateTimeOffset.UtcNow;
|
|
var opened = new Dictionary<Guid, DateTimeOffset> { [main.ProjectId] = now - 3 * Day, [b.ProjectId] = now - Day, [c.ProjectId] = now - 2 * Day };
|
|
var total = new[] { main, b, c }.Sum(p => WorkspaceCleaner.Measure(p.Folder).Bytes);
|
|
var media = WorkspaceCleaner.Measure(main.ProxiesDirectory).Bytes;
|
|
// Two projects' media must go to get under the limit (each holds the same proxies).
|
|
var limit = total - media - 1;
|
|
|
|
// After a copy of Main, the oldest: Main stays, the next two oldest go while the folders stay above the limit.
|
|
var report = await Cleaner(fx, sync).CleanAsync(new WorkspaceCleanOptions { SizeLimitBytes = limit, Opened = opened, JustCopied = main.ProjectId, Now = now }, fx.Ct);
|
|
Assert.SequenceEqual([(main.ProjectId, WorkspaceKeep.JustCopied)], report.Kept.Select(k => (k.ProjectId, k.Reason)));
|
|
Assert.SequenceEqual([c.ProjectId, b.ProjectId], report.Cleaned.Select(p => p.ProjectId));
|
|
Assert.Equal(total, report.UsedBefore);
|
|
Assert.Equal(total - report.Freed, report.UsedAfter);
|
|
|
|
// Once copied again: the oldest goes first, and one is enough this time.
|
|
await fx.CopyAsync(sync.Store, b.ProjectId);
|
|
await fx.CopyAsync(sync.Store, c.ProjectId);
|
|
// Measured again: a copy's timestamps may be a byte shorter or longer than the first copy's.
|
|
var again = new[] { main, b, c }.Sum(p => WorkspaceCleaner.Measure(p.Folder).Bytes);
|
|
report = await Cleaner(fx, sync).CleanAsync(new WorkspaceCleanOptions { SizeLimitBytes = again - 1, Opened = opened, Now = now }, fx.Ct);
|
|
Assert.SequenceEqual([main.ProjectId], report.Cleaned.Select(p => p.ProjectId));
|
|
Assert.Empty(report.Kept);
|
|
Assert.True(Directory.Exists(b.ProxiesDirectory) && Directory.Exists(c.ProxiesDirectory), "the others stay");
|
|
|
|
// Under the limit, or with none: nothing goes.
|
|
Assert.Empty((await Cleaner(fx, sync).CleanAsync(new WorkspaceCleanOptions { SizeLimitBytes = total * 2, Opened = opened, Now = now }, fx.Ct)).Cleaned);
|
|
Assert.Empty((await Cleaner(fx, sync).CleanAsync(new WorkspaceCleanOptions { Opened = opened, Now = now }, fx.Ct)).Cleaned);
|
|
}
|
|
|
|
[Test]
|
|
public async Task CleanAfterCountsFromTheLastOpeningElseFromTheSave()
|
|
{
|
|
var fx = await FixtureAsync();
|
|
var (_, sync) = fx.Workspace();
|
|
var main = await fx.CheckoutAsync(sync);
|
|
var other = await fx.CheckoutAsync(sync, (await fx.Archive.Service.Projects.CreateAsync(fx.Sequence.Id, "Other", cancellationToken: fx.Ct)).Id);
|
|
var saved = main.ReadBundle().Saved;
|
|
// Main was opened yesterday; the other never here (its save counts), a year after the save past Clean after.
|
|
var opened = new Dictionary<Guid, DateTimeOffset> { [main.ProjectId] = saved + 364 * Day };
|
|
var options = new WorkspaceCleanOptions { CleanAfterDays = 30, Opened = opened, Now = saved + 365 * Day };
|
|
|
|
var report = await Cleaner(fx, sync).CleanAsync(options, fx.Ct);
|
|
Assert.SequenceEqual([other.ProjectId], report.Cleaned.Select(p => p.ProjectId));
|
|
Assert.Empty(report.Kept);
|
|
Assert.True(Directory.Exists(main.ProxiesDirectory), "opened yesterday");
|
|
|
|
// Within Clean after of the save, or with Clean after off: kept.
|
|
await fx.CopyAsync(sync.Store, other.ProjectId);
|
|
Assert.Empty((await Cleaner(fx, sync).CleanAsync(options with { Now = saved + 29 * Day, Opened = new Dictionary<Guid, DateTimeOffset>() }, fx.Ct)).Cleaned);
|
|
Assert.Empty((await Cleaner(fx, sync).CleanAsync(options with { CleanAfterDays = 0 }, fx.Ct)).Cleaned);
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheCleanUpNeverTouchesTheTrashWhichEmptiesOnlyAsPlanned()
|
|
{
|
|
var fx = await FixtureAsync();
|
|
var (store, sync) = fx.Workspace();
|
|
var project = await fx.CheckoutAsync(sync);
|
|
var folder = WorkspaceTrash.FolderOf(store.Root);
|
|
var kept = Directory.CreateDirectory(Path.Combine(folder, Guid.CreateVersion7().ToString("N") + "-" + DateTime.UtcNow.AddDays(-90).Ticks)).FullName;
|
|
File.WriteAllText(Path.Combine(kept, "project.chronos"), "a removed copy's work");
|
|
var before = WorkspaceTrash.List(store.Root, fx.Ct);
|
|
Assert.Equal((1, 1, (long)"a removed copy's work".Length), (before.Entries.Count, before.Entries[0].Files, before.Bytes));
|
|
Assert.InRange((before.Entries[0].Trashed - DateTimeOffset.UtcNow.AddDays(-90)).TotalMinutes, -1, 1);
|
|
|
|
// Even a size limit of one byte leaves the trash as it was.
|
|
await Cleaner(fx, sync).CleanAsync(new WorkspaceCleanOptions { CleanAfterDays = 1, SizeLimitBytes = 1, Now = DateTimeOffset.UtcNow + 10 * Day }, fx.Ct);
|
|
Assert.False(Directory.Exists(project.ProxiesDirectory), "the media went");
|
|
Assert.Equal(before.Fingerprint, WorkspaceTrash.List(store.Root, fx.Ct).Fingerprint);
|
|
|
|
// An entry added after the plan: nothing is erased.
|
|
var added = Directory.CreateDirectory(Path.Combine(folder, Guid.CreateVersion7().ToString("N") + "-" + DateTime.UtcNow.Ticks)).FullName;
|
|
File.WriteAllText(Path.Combine(added, "late.txt"), "late");
|
|
Assert.Throws<WorkspaceTrashChangedException>(() => WorkspaceTrash.Empty(store.Root, before, fx.Ct));
|
|
Assert.True(Directory.Exists(kept) && Directory.Exists(added), "nothing was erased");
|
|
// A file changed inside an entry changes the fingerprint too.
|
|
var plan = WorkspaceTrash.List(store.Root, fx.Ct);
|
|
File.AppendAllText(Path.Combine(added, "late.txt"), " and longer");
|
|
Assert.Throws<WorkspaceTrashChangedException>(() => WorkspaceTrash.Empty(store.Root, plan, fx.Ct));
|
|
|
|
plan = WorkspaceTrash.List(store.Root, fx.Ct);
|
|
File.SetAttributes(Path.Combine(kept, "project.chronos"), FileAttributes.ReadOnly);
|
|
Assert.Equal((2, plan.Bytes), WorkspaceTrash.Empty(store.Root, plan, fx.Ct));
|
|
Assert.True(WorkspaceTrash.List(store.Root, fx.Ct).IsEmpty, "the trash is empty");
|
|
Assert.True(Directory.Exists(folder), "the trash folder itself stays");
|
|
}
|
|
}
|
|
|
|
public sealed class SqliteWorkspaceCleanupTests(TestContext context) : WorkspaceCleanupTests(context)
|
|
{
|
|
protected override Backend Backend => Backend.SqliteFile;
|
|
}
|
|
|
|
[RequiresEnvironment(PgTestServer.Variable)]
|
|
public sealed class PostgresWorkspaceCleanupTests(TestContext context) : WorkspaceCleanupTests(context)
|
|
{
|
|
protected override Backend Backend => Backend.Postgres;
|
|
}
|