Phase 5c P4. The workspace no longer grows without bound: a daily workspace-cleanup system schedule (04:30, catch-up) removes the media of projects in step with their home that were not opened for Clean after, and the size limit removes the least recently used after each copy. Each project kept is logged with its reason (open in Edit, not in step, home offline, jobs pending, renders not sent, unreadable, just copied). - WorkspaceCleaner: cleans under the project's lease, re-checking every condition; the originals the home no longer holds go to the workspace trash, never away. The media index is marked Cleaned, so the project reads as MediaRemoved and opening it copies the media again. - WorkspaceTrash: lists entries with a fingerprint, and empties only the plan the user confirmed (refused when the trash changed since). - Trash dialog view model and view: Erase is enabled only after the user types the number of entries. - Removing a copy counts the unsent renders and the last originals. Send first queues the renders' sending and refuses the removal until they are sent. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
296 lines
17 KiB
C#
296 lines
17 KiB
C#
using System.Text.Json;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Core.Project;
|
|
using Chronos.Data.Projects;
|
|
using Chronos.Data.Tests.Archiving;
|
|
using Chronos.Data.Tests.Infrastructure;
|
|
using Chronos.Data.Workspaces;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Data.Tests.Workspaces;
|
|
|
|
/// <summary>
|
|
/// The workspace's clean-up and trash (phase 5c, P4): media go only from projects whose home holds everything (each other case
|
|
/// kept with its reason), an original the home lost goes to the trash, Size limit takes the least recently used first, Clean
|
|
/// after counts from the last opening (else the save), and the trash is never touched by the clean-up and emptied only as
|
|
/// planned.
|
|
/// </summary>
|
|
[Category("workspace")]
|
|
public abstract class WorkspaceCleanupTests(TestContext context) : DatabaseTest(context)
|
|
{
|
|
private static readonly TimeSpan Day = TimeSpan.FromDays(1);
|
|
|
|
private async Task<WorkspaceFixture> FixtureAsync(int frames = 4)
|
|
{
|
|
var archive = Own(await ArchiveFixture.CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true)));
|
|
return Own(await WorkspaceFixture.CreateAsync(Context, archive, frames));
|
|
}
|
|
|
|
private static WorkspaceCleaner Cleaner(WorkspaceFixture fx, WorkspaceSync sync, Func<BundleHome, IProjectHome?>? homes = null) =>
|
|
new(sync, fx.Db, homes ?? (_ => fx.Home)) { LockWait = TimeSpan.FromSeconds(TestTiming.Budget(10)) };
|
|
|
|
private static T One<T>(IReadOnlyList<T> items, [System.Runtime.CompilerServices.CallerArgumentExpression(nameof(items))] string? expression = null)
|
|
{
|
|
Assert.Count(1, items, expression);
|
|
return items[0];
|
|
}
|
|
|
|
/// <summary>Clean after 30 days, the project last opened 40 days ago.</summary>
|
|
private static WorkspaceCleanOptions Aged(Guid projectId, Func<Guid, bool>? isOpen = null, Guid? justCopied = null) => new()
|
|
{
|
|
CleanAfterDays = 30,
|
|
Opened = new Dictionary<Guid, DateTimeOffset> { [projectId] = DateTimeOffset.UtcNow - 40 * Day },
|
|
IsOpen = isOpen ?? (_ => false),
|
|
JustCopied = justCopied,
|
|
};
|
|
|
|
/// <summary>A render of the project's ledger with its file in <c>exports/</c>.</summary>
|
|
private static WorkspaceRenderEntry Render(WorkspaceProject project, string name, string state)
|
|
{
|
|
Directory.CreateDirectory(project.ExportsDirectory);
|
|
File.WriteAllText(project.ExportFile(name), "rendered " + name);
|
|
return new WorkspaceRenderEntry(Guid.CreateVersion7(), Guid.CreateVersion7(), name, new string('a', 64), 12, "preset", "h264", 1920, 1080, 25, 4,
|
|
project.ProjectId, null, null, DateTimeOffset.UtcNow, state, state == WorkspaceRenderEntry.SentState ? DateTimeOffset.UtcNow : null);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AProjectInStepWithItsHomeLosesItsMediaAndKeepsItsWork()
|
|
{
|
|
var fx = await FixtureAsync();
|
|
var (store, sync) = fx.Workspace();
|
|
var project = await fx.CheckoutAsync(sync);
|
|
var id = project.ProjectId;
|
|
var sent = Render(project, "sent.mp4", WorkspaceRenderEntry.SentState);
|
|
project.WriteRenders(WorkspaceRenders.Empty.With(sent));
|
|
var bundle = File.ReadAllBytes(project.BundlePath);
|
|
var proxies = WorkspaceCleaner.Measure(project.ProxiesDirectory).Bytes;
|
|
Assert.True(WorkspaceCleaner.HoldsMedia(project), "the copy brought the media");
|
|
|
|
var report = await Cleaner(fx, sync).CleanAsync(Aged(id), fx.Ct);
|
|
|
|
var cleaned = One(report.Cleaned);
|
|
Assert.Equal((id, 0), (cleaned.ProjectId, cleaned.Trashed));
|
|
Assert.Equal(proxies + "rendered sent.mp4".Length, cleaned.Freed);
|
|
Assert.Empty(report.Kept);
|
|
Assert.False(Directory.Exists(project.ProxiesDirectory), "the proxies went");
|
|
Assert.FileMissing(project.ExportFile("sent.mp4"));
|
|
Assert.Empty(project.ReadRenders().Entries);
|
|
// The frames stay, naming no file; the working state, the deep report and the sync state are as they were.
|
|
var media = Assert.NotNull(project.ReadMedia());
|
|
Assert.True(media.IsCleaned, "the index says the clean-up emptied it");
|
|
Assert.Equal(fx.Frames.Count, media.Frames.Count);
|
|
Assert.True(media.Frames.All(f => f.File is null && f.Original is null && f.FileBytes == 0), "no frame names a file");
|
|
Assert.BytesEqual(bundle, File.ReadAllBytes(project.BundlePath));
|
|
Assert.False(WorkspaceCleaner.HoldsMedia(project), "nothing to edit until a copy");
|
|
Assert.Equal(WorkspaceState.MediaRemoved, (await sync.StatusAsync(id, fx.Home, fx.Ct)).State);
|
|
|
|
// A second run finds nothing to do; a copy brings the media back, in step as before.
|
|
Assert.Empty((await Cleaner(fx, sync).CleanAsync(Aged(id), fx.Ct)).Cleaned);
|
|
await fx.CopyAsync(store, id);
|
|
Assert.True(WorkspaceCleaner.HoldsMedia(project), "copied again");
|
|
Assert.False(project.ReadMedia()!.IsCleaned, "a copy writes an index of its own");
|
|
Assert.Equal(WorkspaceState.InSync, (await sync.StatusAsync(id, fx.Home, fx.Ct)).State);
|
|
}
|
|
|
|
[Test]
|
|
public async Task EachReasonToKeepAProjectKeepsItsMedia()
|
|
{
|
|
var fx = await FixtureAsync();
|
|
var (_, sync) = fx.Workspace();
|
|
var project = await fx.CheckoutAsync(sync);
|
|
var id = project.ProjectId;
|
|
var cleaner = Cleaner(fx, sync);
|
|
|
|
async Task KeptAsync(WorkspaceKeep reason, WorkspaceCleanOptions options, WorkspaceCleaner? by = null)
|
|
{
|
|
var report = await (by ?? cleaner).CleanAsync(options, fx.Ct);
|
|
Assert.Empty(report.Cleaned);
|
|
Assert.Equal((id, reason), (One(report.Kept).ProjectId, report.Kept[0].Reason));
|
|
Assert.True(Directory.Exists(project.ProxiesDirectory) && !project.ReadMedia()!.IsCleaned, $"kept ({reason})");
|
|
}
|
|
|
|
await KeptAsync(WorkspaceKeep.JustCopied, Aged(id, justCopied: id));
|
|
await KeptAsync(WorkspaceKeep.Open, Aged(id, isOpen: p => p == id));
|
|
|
|
// A copy of this project waiting (paused) in the queue; another project's does not count.
|
|
var copy = TestData.Job(JobKind.WorkspaceCopy) with
|
|
{
|
|
TargetIds = [fx.Sequence.Id],
|
|
State = JobState.Paused,
|
|
ParametersJson = JsonSerializer.Serialize(new { workspaceRoot = sync.Store.Root, projectId = id, media = BundleMedia.Proxies }),
|
|
};
|
|
await fx.Db.Jobs.EnqueueAsync(copy, fx.Ct);
|
|
await KeptAsync(WorkspaceKeep.Jobs, Aged(id));
|
|
await fx.Db.Jobs.SetStateAsync(copy.Id, JobState.Cancelled, cancellationToken: fx.Ct);
|
|
|
|
var renders = WorkspaceRenders.Empty.With(Render(project, "pending.mp4", WorkspaceRenderEntry.Pending));
|
|
project.WriteRenders(renders);
|
|
await KeptAsync(WorkspaceKeep.UnsentRenders, Aged(id));
|
|
project.WriteRenders(renders.With(renders.Entries[0] with { State = WorkspaceRenderEntry.Failed, Error = "refused" }));
|
|
await KeptAsync(WorkspaceKeep.UnsentRenders, Aged(id));
|
|
File.WriteAllText(project.RendersPath, "{ not a ledger");
|
|
await KeptAsync(WorkspaceKeep.Unreadable, Aged(id));
|
|
File.Delete(project.RendersPath);
|
|
File.Delete(project.ExportFile("pending.mp4"));
|
|
|
|
await KeptAsync(WorkspaceKeep.Offline, Aged(id), Cleaner(fx, sync, _ => null));
|
|
await KeptAsync(WorkspaceKeep.Offline, Aged(id), Cleaner(fx, sync, _ => new UnreachableHome(fx.Home)));
|
|
|
|
var session = await sync.OpenAsync(id, fx.Ct);
|
|
await sync.SaveAsync(id, session.Working with { Exposure = session.Working.Exposure with { ExposureOffsetEv = 0.7 } }, fx.Ct);
|
|
await KeptAsync(WorkspaceKeep.NotInSync, Aged(id));
|
|
await sync.CommitAsync(id, null, null, fx.Ct);
|
|
await KeptAsync(WorkspaceKeep.NotInSync, Aged(id));
|
|
await sync.PushAsync(id, fx.Home, fx.Ct);
|
|
|
|
// In step again: it goes. Media whose files are gone already hold nothing to remove.
|
|
var other = await fx.Archive.Service.Projects.CreateAsync(fx.Sequence.Id, "Second", cancellationToken: fx.Ct);
|
|
var second = await fx.CheckoutAsync(sync, other.Id);
|
|
Directory.Delete(second.ProxiesDirectory, recursive: true);
|
|
var options = Aged(id) with { Opened = new Dictionary<Guid, DateTimeOffset> { [id] = DateTimeOffset.UtcNow - 40 * Day, [other.Id] = DateTimeOffset.UtcNow - 50 * Day } };
|
|
var report = await cleaner.CleanAsync(options, fx.Ct);
|
|
Assert.Equal(id, One(report.Cleaned).ProjectId);
|
|
Assert.Equal((other.Id, WorkspaceKeep.NoMedia), (One(report.Kept).ProjectId, report.Kept[0].Reason));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnOriginalTheHomeNoLongerHoldsGoesToTheTrashNeverAway()
|
|
{
|
|
var fx = await FixtureAsync();
|
|
var (store, sync) = fx.Workspace();
|
|
var project = await fx.CheckoutAsync(sync);
|
|
// Originals of two frames: the first as the home holds it, the second of bytes the home no longer has.
|
|
var media = project.ReadMedia()!;
|
|
var held = WorkspaceProject.OriginalName(media.Frames[0].Id, fx.Frames[0].Sha256!, ".jpg");
|
|
var lost = WorkspaceProject.OriginalName(media.Frames[1].Id, new string('f', 64), ".jpg");
|
|
Directory.CreateDirectory(project.OriginalsDirectory);
|
|
File.WriteAllText(project.MediaFile(held), "held original");
|
|
File.WriteAllText(project.MediaFile(lost), "last original");
|
|
project.WriteMedia(media with
|
|
{
|
|
Kind = BundleMedia.Originals,
|
|
Frames =
|
|
[
|
|
media.Frames[0] with { Original = held, OriginalSha256 = fx.Frames[0].Sha256, OriginalBytes = 13 },
|
|
media.Frames[1] with { Original = lost, OriginalSha256 = new string('f', 64), OriginalBytes = 13 },
|
|
.. media.Frames.Skip(2),
|
|
],
|
|
});
|
|
Assert.SequenceEqual([media.Frames[1].Id], WorkspaceCleaner.LastOriginals(project.ReadMedia(), await fx.Home.FramesAsync(fx.Ct)).Select(f => f.Id));
|
|
|
|
var report = await Cleaner(fx, sync).CleanAsync(Aged(project.ProjectId), fx.Ct);
|
|
|
|
Assert.Equal(1, One(report.Cleaned).Trashed);
|
|
Assert.False(Directory.Exists(project.OriginalsDirectory), "the originals folder went");
|
|
var trash = WorkspaceTrash.List(store.Root, fx.Ct);
|
|
var entry = One(trash.Entries);
|
|
Assert.True(entry.Name.StartsWith(project.ProjectId.ToString("N") + "-", StringComparison.Ordinal), entry.Name);
|
|
Assert.Equal("last original", File.ReadAllText(Path.Combine(trash.Folder, entry.Name, WorkspaceProject.OriginalsFolder, Path.GetFileName(lost))));
|
|
Assert.Equal((1, "last original".Length), (entry.Files, entry.Bytes));
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheSizeLimitTakesTheLeastRecentlyUsedFirstAndNeverTheProjectJustCopied()
|
|
{
|
|
var fx = await FixtureAsync();
|
|
var (_, sync) = fx.Workspace();
|
|
var main = await fx.CheckoutAsync(sync);
|
|
var b = await fx.CheckoutAsync(sync, (await fx.Archive.Service.Projects.CreateAsync(fx.Sequence.Id, "B", cancellationToken: fx.Ct)).Id);
|
|
var c = await fx.CheckoutAsync(sync, (await fx.Archive.Service.Projects.CreateAsync(fx.Sequence.Id, "C", cancellationToken: fx.Ct)).Id);
|
|
var now = DateTimeOffset.UtcNow;
|
|
var opened = new Dictionary<Guid, DateTimeOffset> { [main.ProjectId] = now - 3 * Day, [b.ProjectId] = now - Day, [c.ProjectId] = now - 2 * Day };
|
|
var total = new[] { main, b, c }.Sum(p => WorkspaceCleaner.Measure(p.Folder).Bytes);
|
|
var media = WorkspaceCleaner.Measure(main.ProxiesDirectory).Bytes;
|
|
// Two projects' media must go to get under the limit (each holds the same proxies).
|
|
var limit = total - media - 1;
|
|
|
|
// After a copy of Main, the oldest: Main stays, the next two oldest go while the folders stay above the limit.
|
|
var report = await Cleaner(fx, sync).CleanAsync(new WorkspaceCleanOptions { SizeLimitBytes = limit, Opened = opened, JustCopied = main.ProjectId, Now = now }, fx.Ct);
|
|
Assert.SequenceEqual([(main.ProjectId, WorkspaceKeep.JustCopied)], report.Kept.Select(k => (k.ProjectId, k.Reason)));
|
|
Assert.SequenceEqual([c.ProjectId, b.ProjectId], report.Cleaned.Select(p => p.ProjectId));
|
|
Assert.Equal(total, report.UsedBefore);
|
|
Assert.Equal(total - report.Freed, report.UsedAfter);
|
|
|
|
// Once copied again: the oldest goes first, and one is enough this time.
|
|
await fx.CopyAsync(sync.Store, b.ProjectId);
|
|
await fx.CopyAsync(sync.Store, c.ProjectId);
|
|
report = await Cleaner(fx, sync).CleanAsync(new WorkspaceCleanOptions { SizeLimitBytes = total - 1, Opened = opened, Now = now }, fx.Ct);
|
|
Assert.SequenceEqual([main.ProjectId], report.Cleaned.Select(p => p.ProjectId));
|
|
Assert.Empty(report.Kept);
|
|
Assert.True(Directory.Exists(b.ProxiesDirectory) && Directory.Exists(c.ProxiesDirectory), "the others stay");
|
|
|
|
// Under the limit, or with none: nothing goes.
|
|
Assert.Empty((await Cleaner(fx, sync).CleanAsync(new WorkspaceCleanOptions { SizeLimitBytes = total * 2, Opened = opened, Now = now }, fx.Ct)).Cleaned);
|
|
Assert.Empty((await Cleaner(fx, sync).CleanAsync(new WorkspaceCleanOptions { Opened = opened, Now = now }, fx.Ct)).Cleaned);
|
|
}
|
|
|
|
[Test]
|
|
public async Task CleanAfterCountsFromTheLastOpeningElseFromTheSave()
|
|
{
|
|
var fx = await FixtureAsync();
|
|
var (_, sync) = fx.Workspace();
|
|
var main = await fx.CheckoutAsync(sync);
|
|
var other = await fx.CheckoutAsync(sync, (await fx.Archive.Service.Projects.CreateAsync(fx.Sequence.Id, "Other", cancellationToken: fx.Ct)).Id);
|
|
var saved = main.ReadBundle().Saved;
|
|
// Main was opened yesterday; the other never here (its save counts), a year after the save past Clean after.
|
|
var opened = new Dictionary<Guid, DateTimeOffset> { [main.ProjectId] = saved + 364 * Day };
|
|
var options = new WorkspaceCleanOptions { CleanAfterDays = 30, Opened = opened, Now = saved + 365 * Day };
|
|
|
|
var report = await Cleaner(fx, sync).CleanAsync(options, fx.Ct);
|
|
Assert.SequenceEqual([other.ProjectId], report.Cleaned.Select(p => p.ProjectId));
|
|
Assert.Empty(report.Kept);
|
|
Assert.True(Directory.Exists(main.ProxiesDirectory), "opened yesterday");
|
|
|
|
// Within Clean after of the save, or with Clean after off: kept.
|
|
await fx.CopyAsync(sync.Store, other.ProjectId);
|
|
Assert.Empty((await Cleaner(fx, sync).CleanAsync(options with { Now = saved + 29 * Day, Opened = new Dictionary<Guid, DateTimeOffset>() }, fx.Ct)).Cleaned);
|
|
Assert.Empty((await Cleaner(fx, sync).CleanAsync(options with { CleanAfterDays = 0 }, fx.Ct)).Cleaned);
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheCleanUpNeverTouchesTheTrashWhichEmptiesOnlyAsPlanned()
|
|
{
|
|
var fx = await FixtureAsync();
|
|
var (store, sync) = fx.Workspace();
|
|
var project = await fx.CheckoutAsync(sync);
|
|
var folder = WorkspaceTrash.FolderOf(store.Root);
|
|
var kept = Directory.CreateDirectory(Path.Combine(folder, Guid.CreateVersion7().ToString("N") + "-" + DateTime.UtcNow.AddDays(-90).Ticks)).FullName;
|
|
File.WriteAllText(Path.Combine(kept, "project.chronos"), "a removed copy's work");
|
|
var before = WorkspaceTrash.List(store.Root, fx.Ct);
|
|
Assert.Equal((1, 1, (long)"a removed copy's work".Length), (before.Entries.Count, before.Entries[0].Files, before.Bytes));
|
|
Assert.InRange((before.Entries[0].Trashed - DateTimeOffset.UtcNow.AddDays(-90)).TotalMinutes, -1, 1);
|
|
|
|
// Even a size limit of one byte leaves the trash as it was.
|
|
await Cleaner(fx, sync).CleanAsync(new WorkspaceCleanOptions { CleanAfterDays = 1, SizeLimitBytes = 1, Now = DateTimeOffset.UtcNow + 10 * Day }, fx.Ct);
|
|
Assert.False(Directory.Exists(project.ProxiesDirectory), "the media went");
|
|
Assert.Equal(before.Fingerprint, WorkspaceTrash.List(store.Root, fx.Ct).Fingerprint);
|
|
|
|
// An entry added after the plan: nothing is erased.
|
|
var added = Directory.CreateDirectory(Path.Combine(folder, Guid.CreateVersion7().ToString("N") + "-" + DateTime.UtcNow.Ticks)).FullName;
|
|
File.WriteAllText(Path.Combine(added, "late.txt"), "late");
|
|
Assert.Throws<WorkspaceTrashChangedException>(() => WorkspaceTrash.Empty(store.Root, before, fx.Ct));
|
|
Assert.True(Directory.Exists(kept) && Directory.Exists(added), "nothing was erased");
|
|
// A file changed inside an entry changes the fingerprint too.
|
|
var plan = WorkspaceTrash.List(store.Root, fx.Ct);
|
|
File.AppendAllText(Path.Combine(added, "late.txt"), " and longer");
|
|
Assert.Throws<WorkspaceTrashChangedException>(() => WorkspaceTrash.Empty(store.Root, plan, fx.Ct));
|
|
|
|
plan = WorkspaceTrash.List(store.Root, fx.Ct);
|
|
File.SetAttributes(Path.Combine(kept, "project.chronos"), FileAttributes.ReadOnly);
|
|
Assert.Equal((2, plan.Bytes), WorkspaceTrash.Empty(store.Root, plan, fx.Ct));
|
|
Assert.True(WorkspaceTrash.List(store.Root, fx.Ct).IsEmpty, "the trash is empty");
|
|
Assert.True(Directory.Exists(folder), "the trash folder itself stays");
|
|
}
|
|
}
|
|
|
|
public sealed class SqliteWorkspaceCleanupTests(TestContext context) : WorkspaceCleanupTests(context)
|
|
{
|
|
protected override Backend Backend => Backend.SqliteFile;
|
|
}
|
|
|
|
[RequiresEnvironment(PgTestServer.Variable)]
|
|
public sealed class PostgresWorkspaceCleanupTests(TestContext context) : WorkspaceCleanupTests(context)
|
|
{
|
|
protected override Backend Backend => Backend.Postgres;
|
|
}
|