using System.Text.Json;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Data.Projects;
using Chronos.Data.Tests.Archiving;
using Chronos.Data.Tests.Infrastructure;
using Chronos.Data.Workspaces;
using Chronos.Testing;
namespace Chronos.Data.Tests.Workspaces;
///
/// The workspace's clean-up and trash (phase 5c, P4): media go only from projects whose home holds everything (each other case
/// kept with its reason), an original the home lost goes to the trash, Size limit takes the least recently used first, Clean
/// after counts from the last opening (else the save), and the trash is never touched by the clean-up and emptied only as
/// planned.
///
[Category("workspace")]
public abstract class WorkspaceCleanupTests(TestContext context) : DatabaseTest(context)
{
private static readonly TimeSpan Day = TimeSpan.FromDays(1);
private async Task FixtureAsync(int frames = 4)
{
var archive = Own(await ArchiveFixture.CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true)));
return Own(await WorkspaceFixture.CreateAsync(Context, archive, frames));
}
private static WorkspaceCleaner Cleaner(WorkspaceFixture fx, WorkspaceSync sync, Func? homes = null) =>
new(sync, fx.Db, homes ?? (_ => fx.Home)) { LockWait = TimeSpan.FromSeconds(TestTiming.Budget(10)) };
private static T One(IReadOnlyList items, [System.Runtime.CompilerServices.CallerArgumentExpression(nameof(items))] string? expression = null)
{
Assert.Count(1, items, expression);
return items[0];
}
/// Clean after 30 days, the project last opened 40 days ago.
private static WorkspaceCleanOptions Aged(Guid projectId, Func? isOpen = null, Guid? justCopied = null) => new()
{
CleanAfterDays = 30,
Opened = new Dictionary { [projectId] = DateTimeOffset.UtcNow - 40 * Day },
IsOpen = isOpen ?? (_ => false),
JustCopied = justCopied,
};
/// A render of the project's ledger with its file in exports/.
private static WorkspaceRenderEntry Render(WorkspaceProject project, string name, string state)
{
Directory.CreateDirectory(project.ExportsDirectory);
File.WriteAllText(project.ExportFile(name), "rendered " + name);
return new WorkspaceRenderEntry(Guid.CreateVersion7(), Guid.CreateVersion7(), name, new string('a', 64), 12, "preset", "h264", 1920, 1080, 25, 4,
project.ProjectId, null, null, DateTimeOffset.UtcNow, state, state == WorkspaceRenderEntry.SentState ? DateTimeOffset.UtcNow : null);
}
[Test]
public async Task AProjectInStepWithItsHomeLosesItsMediaAndKeepsItsWork()
{
var fx = await FixtureAsync();
var (store, sync) = fx.Workspace();
var project = await fx.CheckoutAsync(sync);
var id = project.ProjectId;
var sent = Render(project, "sent.mp4", WorkspaceRenderEntry.SentState);
project.WriteRenders(WorkspaceRenders.Empty.With(sent));
var bundle = File.ReadAllBytes(project.BundlePath);
var proxies = WorkspaceCleaner.Measure(project.ProxiesDirectory).Bytes;
Assert.True(WorkspaceCleaner.HoldsMedia(project), "the copy brought the media");
var report = await Cleaner(fx, sync).CleanAsync(Aged(id), fx.Ct);
var cleaned = One(report.Cleaned);
Assert.Equal((id, 0), (cleaned.ProjectId, cleaned.Trashed));
Assert.Equal(proxies + "rendered sent.mp4".Length, cleaned.Freed);
Assert.Empty(report.Kept);
Assert.False(Directory.Exists(project.ProxiesDirectory), "the proxies went");
Assert.FileMissing(project.ExportFile("sent.mp4"));
Assert.Empty(project.ReadRenders().Entries);
// The frames stay, naming no file; the working state, the deep report and the sync state are as they were.
var media = Assert.NotNull(project.ReadMedia());
Assert.True(media.IsCleaned, "the index says the clean-up emptied it");
Assert.Equal(fx.Frames.Count, media.Frames.Count);
Assert.True(media.Frames.All(f => f.File is null && f.Original is null && f.FileBytes == 0), "no frame names a file");
Assert.BytesEqual(bundle, File.ReadAllBytes(project.BundlePath));
Assert.False(WorkspaceCleaner.HoldsMedia(project), "nothing to edit until a copy");
Assert.Equal(WorkspaceState.MediaRemoved, (await sync.StatusAsync(id, fx.Home, fx.Ct)).State);
// A second run finds nothing to do; a copy brings the media back, in step as before.
Assert.Empty((await Cleaner(fx, sync).CleanAsync(Aged(id), fx.Ct)).Cleaned);
await fx.CopyAsync(store, id);
Assert.True(WorkspaceCleaner.HoldsMedia(project), "copied again");
Assert.False(project.ReadMedia()!.IsCleaned, "a copy writes an index of its own");
Assert.Equal(WorkspaceState.InSync, (await sync.StatusAsync(id, fx.Home, fx.Ct)).State);
}
[Test]
public async Task EachReasonToKeepAProjectKeepsItsMedia()
{
var fx = await FixtureAsync();
var (_, sync) = fx.Workspace();
var project = await fx.CheckoutAsync(sync);
var id = project.ProjectId;
var cleaner = Cleaner(fx, sync);
async Task KeptAsync(WorkspaceKeep reason, WorkspaceCleanOptions options, WorkspaceCleaner? by = null)
{
var report = await (by ?? cleaner).CleanAsync(options, fx.Ct);
Assert.Empty(report.Cleaned);
Assert.Equal((id, reason), (One(report.Kept).ProjectId, report.Kept[0].Reason));
Assert.True(Directory.Exists(project.ProxiesDirectory) && !project.ReadMedia()!.IsCleaned, $"kept ({reason})");
}
await KeptAsync(WorkspaceKeep.JustCopied, Aged(id, justCopied: id));
await KeptAsync(WorkspaceKeep.Open, Aged(id, isOpen: p => p == id));
// A copy of this project waiting (paused) in the queue; another project's does not count.
var copy = TestData.Job(JobKind.WorkspaceCopy) with
{
TargetIds = [fx.Sequence.Id],
State = JobState.Paused,
ParametersJson = JsonSerializer.Serialize(new { workspaceRoot = sync.Store.Root, projectId = id, media = BundleMedia.Proxies }),
};
await fx.Db.Jobs.EnqueueAsync(copy, fx.Ct);
await KeptAsync(WorkspaceKeep.Jobs, Aged(id));
await fx.Db.Jobs.SetStateAsync(copy.Id, JobState.Cancelled, cancellationToken: fx.Ct);
var renders = WorkspaceRenders.Empty.With(Render(project, "pending.mp4", WorkspaceRenderEntry.Pending));
project.WriteRenders(renders);
await KeptAsync(WorkspaceKeep.UnsentRenders, Aged(id));
project.WriteRenders(renders.With(renders.Entries[0] with { State = WorkspaceRenderEntry.Failed, Error = "refused" }));
await KeptAsync(WorkspaceKeep.UnsentRenders, Aged(id));
File.WriteAllText(project.RendersPath, "{ not a ledger");
await KeptAsync(WorkspaceKeep.Unreadable, Aged(id));
File.Delete(project.RendersPath);
File.Delete(project.ExportFile("pending.mp4"));
await KeptAsync(WorkspaceKeep.Offline, Aged(id), Cleaner(fx, sync, _ => null));
await KeptAsync(WorkspaceKeep.Offline, Aged(id), Cleaner(fx, sync, _ => new UnreachableHome(fx.Home)));
var session = await sync.OpenAsync(id, fx.Ct);
await sync.SaveAsync(id, session.Working with { Exposure = session.Working.Exposure with { ExposureOffsetEv = 0.7 } }, fx.Ct);
await KeptAsync(WorkspaceKeep.NotInSync, Aged(id));
await sync.CommitAsync(id, null, null, fx.Ct);
await KeptAsync(WorkspaceKeep.NotInSync, Aged(id));
await sync.PushAsync(id, fx.Home, fx.Ct);
// In step again: it goes. Media whose files are gone already hold nothing to remove.
var other = await fx.Archive.Service.Projects.CreateAsync(fx.Sequence.Id, "Second", cancellationToken: fx.Ct);
var second = await fx.CheckoutAsync(sync, other.Id);
Directory.Delete(second.ProxiesDirectory, recursive: true);
var options = Aged(id) with { Opened = new Dictionary { [id] = DateTimeOffset.UtcNow - 40 * Day, [other.Id] = DateTimeOffset.UtcNow - 50 * Day } };
var report = await cleaner.CleanAsync(options, fx.Ct);
Assert.Equal(id, One(report.Cleaned).ProjectId);
Assert.Equal((other.Id, WorkspaceKeep.NoMedia), (One(report.Kept).ProjectId, report.Kept[0].Reason));
}
[Test]
public async Task AnOriginalTheHomeNoLongerHoldsGoesToTheTrashNeverAway()
{
var fx = await FixtureAsync();
var (store, sync) = fx.Workspace();
var project = await fx.CheckoutAsync(sync);
// Originals of two frames: the first as the home holds it, the second of bytes the home no longer has.
var media = project.ReadMedia()!;
var held = WorkspaceProject.OriginalName(media.Frames[0].Id, fx.Frames[0].Sha256!, ".jpg");
var lost = WorkspaceProject.OriginalName(media.Frames[1].Id, new string('f', 64), ".jpg");
Directory.CreateDirectory(project.OriginalsDirectory);
File.WriteAllText(project.MediaFile(held), "held original");
File.WriteAllText(project.MediaFile(lost), "last original");
project.WriteMedia(media with
{
Kind = BundleMedia.Originals,
Frames =
[
media.Frames[0] with { Original = held, OriginalSha256 = fx.Frames[0].Sha256, OriginalBytes = 13 },
media.Frames[1] with { Original = lost, OriginalSha256 = new string('f', 64), OriginalBytes = 13 },
.. media.Frames.Skip(2),
],
});
Assert.SequenceEqual([media.Frames[1].Id], WorkspaceCleaner.LastOriginals(project.ReadMedia(), await fx.Home.FramesAsync(fx.Ct)).Select(f => f.Id));
var report = await Cleaner(fx, sync).CleanAsync(Aged(project.ProjectId), fx.Ct);
Assert.Equal(1, One(report.Cleaned).Trashed);
Assert.False(Directory.Exists(project.OriginalsDirectory), "the originals folder went");
var trash = WorkspaceTrash.List(store.Root, fx.Ct);
var entry = One(trash.Entries);
Assert.True(entry.Name.StartsWith(project.ProjectId.ToString("N") + "-", StringComparison.Ordinal), entry.Name);
Assert.Equal("last original", File.ReadAllText(Path.Combine(trash.Folder, entry.Name, WorkspaceProject.OriginalsFolder, Path.GetFileName(lost))));
Assert.Equal((1, "last original".Length), (entry.Files, entry.Bytes));
}
[Test]
public async Task TheSizeLimitTakesTheLeastRecentlyUsedFirstAndNeverTheProjectJustCopied()
{
var fx = await FixtureAsync();
var (_, sync) = fx.Workspace();
var main = await fx.CheckoutAsync(sync);
var b = await fx.CheckoutAsync(sync, (await fx.Archive.Service.Projects.CreateAsync(fx.Sequence.Id, "B", cancellationToken: fx.Ct)).Id);
var c = await fx.CheckoutAsync(sync, (await fx.Archive.Service.Projects.CreateAsync(fx.Sequence.Id, "C", cancellationToken: fx.Ct)).Id);
var now = DateTimeOffset.UtcNow;
var opened = new Dictionary { [main.ProjectId] = now - 3 * Day, [b.ProjectId] = now - Day, [c.ProjectId] = now - 2 * Day };
var total = new[] { main, b, c }.Sum(p => WorkspaceCleaner.Measure(p.Folder).Bytes);
var media = WorkspaceCleaner.Measure(main.ProxiesDirectory).Bytes;
// Two projects' media must go to get under the limit (each holds the same proxies).
var limit = total - media - 1;
// After a copy of Main, the oldest: Main stays, the next two oldest go while the folders stay above the limit.
var report = await Cleaner(fx, sync).CleanAsync(new WorkspaceCleanOptions { SizeLimitBytes = limit, Opened = opened, JustCopied = main.ProjectId, Now = now }, fx.Ct);
Assert.SequenceEqual([(main.ProjectId, WorkspaceKeep.JustCopied)], report.Kept.Select(k => (k.ProjectId, k.Reason)));
Assert.SequenceEqual([c.ProjectId, b.ProjectId], report.Cleaned.Select(p => p.ProjectId));
Assert.Equal(total, report.UsedBefore);
Assert.Equal(total - report.Freed, report.UsedAfter);
// Once copied again: the oldest goes first, and one is enough this time.
await fx.CopyAsync(sync.Store, b.ProjectId);
await fx.CopyAsync(sync.Store, c.ProjectId);
report = await Cleaner(fx, sync).CleanAsync(new WorkspaceCleanOptions { SizeLimitBytes = total - 1, Opened = opened, Now = now }, fx.Ct);
Assert.SequenceEqual([main.ProjectId], report.Cleaned.Select(p => p.ProjectId));
Assert.Empty(report.Kept);
Assert.True(Directory.Exists(b.ProxiesDirectory) && Directory.Exists(c.ProxiesDirectory), "the others stay");
// Under the limit, or with none: nothing goes.
Assert.Empty((await Cleaner(fx, sync).CleanAsync(new WorkspaceCleanOptions { SizeLimitBytes = total * 2, Opened = opened, Now = now }, fx.Ct)).Cleaned);
Assert.Empty((await Cleaner(fx, sync).CleanAsync(new WorkspaceCleanOptions { Opened = opened, Now = now }, fx.Ct)).Cleaned);
}
[Test]
public async Task CleanAfterCountsFromTheLastOpeningElseFromTheSave()
{
var fx = await FixtureAsync();
var (_, sync) = fx.Workspace();
var main = await fx.CheckoutAsync(sync);
var other = await fx.CheckoutAsync(sync, (await fx.Archive.Service.Projects.CreateAsync(fx.Sequence.Id, "Other", cancellationToken: fx.Ct)).Id);
var saved = main.ReadBundle().Saved;
// Main was opened yesterday; the other never here (its save counts), a year after the save past Clean after.
var opened = new Dictionary { [main.ProjectId] = saved + 364 * Day };
var options = new WorkspaceCleanOptions { CleanAfterDays = 30, Opened = opened, Now = saved + 365 * Day };
var report = await Cleaner(fx, sync).CleanAsync(options, fx.Ct);
Assert.SequenceEqual([other.ProjectId], report.Cleaned.Select(p => p.ProjectId));
Assert.Empty(report.Kept);
Assert.True(Directory.Exists(main.ProxiesDirectory), "opened yesterday");
// Within Clean after of the save, or with Clean after off: kept.
await fx.CopyAsync(sync.Store, other.ProjectId);
Assert.Empty((await Cleaner(fx, sync).CleanAsync(options with { Now = saved + 29 * Day, Opened = new Dictionary() }, fx.Ct)).Cleaned);
Assert.Empty((await Cleaner(fx, sync).CleanAsync(options with { CleanAfterDays = 0 }, fx.Ct)).Cleaned);
}
[Test]
public async Task TheCleanUpNeverTouchesTheTrashWhichEmptiesOnlyAsPlanned()
{
var fx = await FixtureAsync();
var (store, sync) = fx.Workspace();
var project = await fx.CheckoutAsync(sync);
var folder = WorkspaceTrash.FolderOf(store.Root);
var kept = Directory.CreateDirectory(Path.Combine(folder, Guid.CreateVersion7().ToString("N") + "-" + DateTime.UtcNow.AddDays(-90).Ticks)).FullName;
File.WriteAllText(Path.Combine(kept, "project.chronos"), "a removed copy's work");
var before = WorkspaceTrash.List(store.Root, fx.Ct);
Assert.Equal((1, 1, (long)"a removed copy's work".Length), (before.Entries.Count, before.Entries[0].Files, before.Bytes));
Assert.InRange((before.Entries[0].Trashed - DateTimeOffset.UtcNow.AddDays(-90)).TotalMinutes, -1, 1);
// Even a size limit of one byte leaves the trash as it was.
await Cleaner(fx, sync).CleanAsync(new WorkspaceCleanOptions { CleanAfterDays = 1, SizeLimitBytes = 1, Now = DateTimeOffset.UtcNow + 10 * Day }, fx.Ct);
Assert.False(Directory.Exists(project.ProxiesDirectory), "the media went");
Assert.Equal(before.Fingerprint, WorkspaceTrash.List(store.Root, fx.Ct).Fingerprint);
// An entry added after the plan: nothing is erased.
var added = Directory.CreateDirectory(Path.Combine(folder, Guid.CreateVersion7().ToString("N") + "-" + DateTime.UtcNow.Ticks)).FullName;
File.WriteAllText(Path.Combine(added, "late.txt"), "late");
Assert.Throws(() => WorkspaceTrash.Empty(store.Root, before, fx.Ct));
Assert.True(Directory.Exists(kept) && Directory.Exists(added), "nothing was erased");
// A file changed inside an entry changes the fingerprint too.
var plan = WorkspaceTrash.List(store.Root, fx.Ct);
File.AppendAllText(Path.Combine(added, "late.txt"), " and longer");
Assert.Throws(() => WorkspaceTrash.Empty(store.Root, plan, fx.Ct));
plan = WorkspaceTrash.List(store.Root, fx.Ct);
File.SetAttributes(Path.Combine(kept, "project.chronos"), FileAttributes.ReadOnly);
Assert.Equal((2, plan.Bytes), WorkspaceTrash.Empty(store.Root, plan, fx.Ct));
Assert.True(WorkspaceTrash.List(store.Root, fx.Ct).IsEmpty, "the trash is empty");
Assert.True(Directory.Exists(folder), "the trash folder itself stays");
}
}
public sealed class SqliteWorkspaceCleanupTests(TestContext context) : WorkspaceCleanupTests(context)
{
protected override Backend Backend => Backend.SqliteFile;
}
[RequiresEnvironment(PgTestServer.Variable)]
public sealed class PostgresWorkspaceCleanupTests(TestContext context) : WorkspaceCleanupTests(context)
{
protected override Backend Backend => Backend.Postgres;
}