using System.Security.Cryptography; using Chronos.Core.Localization; using Chronos.Core.Project; using Chronos.Data.Archiving; using Chronos.Core.Model; using Chronos.Data.Tests.Archiving; using Chronos.Data.Tests.Infrastructure; using Chronos.Data.Workspaces; using Chronos.Testing; namespace Chronos.Data.Tests.Workspaces; /// A workspace project's originals (phase 5c): their copy from the archive, checked and resumable, and "Proxies only". [Category("workspace")] public abstract class WorkspaceOriginalsTests(TestContext context) : DatabaseTest(context) { private async Task FixtureAsync(int frames = 3, Func? configure = null) { var archive = Own(await ArchiveFixture.CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true), configure: configure)); var fx = Own(await WorkspaceFixture.CreateAsync(Context, archive, frames)); // The archive's own proxies: the copy takes those, so a damaged original never changes a proxy. await archive.Service.Images.BuildProxiesAsync(archive.Archive, fx.Sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct); return fx; } private static string Sha(string path) => Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(path))); private static async Task<(WorkspaceStore Store, WorkspaceProject Project)> CheckoutOriginalsAsync(WorkspaceFixture fx) { var (store, sync) = fx.Workspace(); var main = await fx.Archive.Service.Projects.MainAsync(fx.Sequence.Id, fx.Ct); var project = await sync.CheckoutAsync(fx.Home, main.Id, BundleMedia.Originals, fx.Ct); return (store, project); } private string Source(WorkspaceFixture fx, int index) => fx.Archive.Paths.FromRelative(fx.Frames.Single(f => f.Index == index).RelativePath); [Test] public async Task ACopyOfTheOriginalsHoldsEachPhotoCheckedAgainstTheHome() { var fx = await FixtureAsync(); var (store, project) = await CheckoutOriginalsAsync(fx); Assert.Equal(fx.Frames.Sum(f => f.Size), await WorkspaceOriginals.ArchiveOriginalsBytesAsync(fx.Db, fx.Archive.Service, fx.Sequence.Id, cancellationToken: fx.Ct)); await fx.CopyAsync(store, project.ProjectId, media: BundleMedia.Originals); var media = project.ReadMedia()!; Assert.True(media.HoldsOriginals); Assert.Equal(BundleMedia.Originals, project.ReadBundle().Workspace!.Media.Kind); foreach (var frame in fx.Frames) { var entry = media.Frames.Single(f => f.Id == frame.Id); Assert.NotNull(entry.File); var original = project.MediaFile(Assert.NotNull(entry.Original)); Assert.Equal(frame.Sha256!.ToLowerInvariant(), entry.OriginalSha256); Assert.Equal(entry.OriginalSha256, Sha(original)); Assert.Equal(frame.Size, entry.OriginalBytes); Assert.Equal(WorkspaceProject.OriginalName(frame.Id, frame.Sha256, Path.GetExtension(frame.RelativePath)), entry.Original); } Assert.Empty(Directory.GetFiles(project.OriginalsDirectory, "*.part")); Assert.Equal(0L, await WorkspaceOriginals.ArchiveOriginalsBytesAsync(fx.Db, fx.Archive.Service, fx.Sequence.Id, project, fx.Ct)); // The frames read by a local render are the originals. Assert.SequenceEqual(media.Frames.Select(f => f.Original), media.ToOriginalFrames().Select(f => f.RelativePath)); } [Test] public async Task AnOriginalThatIsGoneOrDamagedAtTheHomeLeavesItsFrameProxied() { var fx = await FixtureAsync(); var (store, project) = await CheckoutOriginalsAsync(fx); File.Delete(Source(fx, 0)); var damaged = Source(fx, 1); var bytes = File.ReadAllBytes(damaged); bytes[^3] ^= 0x5A; File.WriteAllBytes(damaged, bytes); await fx.CopyAsync(store, project.ProjectId, media: BundleMedia.Originals); var media = project.ReadMedia()!; Assert.True(media.HoldsOriginals); var gone = media.Frames.Single(f => f.Index == 0); var mismatched = media.Frames.Single(f => f.Index == 1); Assert.Null(gone.Original); Assert.Null(mismatched.Original); Assert.NotNull(mismatched.File); Assert.NotNull(media.Frames.Single(f => f.Index == 2).Original); Assert.Equal(1, Directory.GetFiles(project.OriginalsDirectory).Length); Assert.True(media.ToOriginalFrames().Count(f => (f.Flags & FrameFlags.Missing) != 0) == 2, "a render counts the two frames without an original"); } [Test] public async Task ACopyStoppedAmongTheOriginalsResumesWithWhatItHad() { var fx = await FixtureAsync(); var (store, project) = await CheckoutOriginalsAsync(fx); await fx.CopyAsync(store, project.ProjectId, media: BundleMedia.Originals); var media = project.ReadMedia()!; var kept = project.MediaFile(media.Frames[0].Original!); var written = File.GetLastWriteTimeUtc(kept); var lost = project.MediaFile(media.Frames[2].Original!); File.Delete(lost); File.WriteAllBytes(Path.Combine(project.OriginalsDirectory, media.Frames[2].Id.ToString("N") + ".part"), [1, 2, 3]); var row = (await fx.Db.Sequences.GetAsync(fx.Sequence.Id))!; await fx.CopyAsync(store, project.ProjectId, checkpoint: $"originals:2:{row.FramesVersion}:{row.Folder}", media: BundleMedia.Originals); Assert.Equal(written, File.GetLastWriteTimeUtc(kept)); Assert.Equal(media.Frames[2].OriginalSha256, Sha(lost)); Assert.Empty(Directory.GetFiles(project.OriginalsDirectory, "*.part")); Assert.Equal(3, project.ReadMedia()!.Frames.Count(f => f.HasOriginal)); } [Test] public async Task TheOriginalsAreNotCopiedWhenTheDriveLacksTheirRoom() { long? free = null; var fx = await FixtureAsync(configure: options => options with { FreeSpace = _ => free }); var (store, project) = await CheckoutOriginalsAsync(fx); await fx.CopyAsync(store, project.ProjectId); free = WorkspaceJobs.CopySpaceReserve + fx.Frames.Sum(f => f.Size) - 1; var job = await fx.CopyAsync(store, project.ProjectId, JobState.Failed, media: BundleMedia.Originals); var parts = Loc.L("Errors.Workspace.DiskFull", "\u0001").Split('\u0001'); Assert.True(job.Error!.StartsWith(parts[0], StringComparison.Ordinal), job.Error); Assert.False(project.ReadMedia()!.HoldsOriginals); Assert.False(Directory.Exists(project.OriginalsDirectory) && Directory.EnumerateFiles(project.OriginalsDirectory).Any()); } [Test] public async Task ProxiesOnlyDeletesWhatTheHomeHoldsAndTrashesTheRest() { var fx = await FixtureAsync(); var (store, project) = await CheckoutOriginalsAsync(fx); await fx.CopyAsync(store, project.ProjectId, media: BundleMedia.Originals); var media = project.ReadMedia()!; // The home lost frame 1's photo: its only copy is the workspace's, which goes to the trash, not away. var lostAtHome = project.MediaFile(media.Frames[1].Original!); var home = await fx.Home.FramesAsync(fx.Ct); home = home with { Frames = [.. home.Frames.Select(f => f.Index == 1 ? f with { Flags = f.Flags | FrameFlags.Missing } : f)] }; var plan = WorkspaceOriginals.PlanProxiesOnly(project, home); Assert.Equal((2, 1), (plan.Originals, plan.ToTrash)); Assert.Equal(fx.Frames.Where(f => f.Index != 1).Sum(f => f.Size), plan.Bytes); var trashed = await WorkspaceOriginals.ApplyProxiesOnlyAsync(project, plan, home, TimeSpan.FromSeconds(10), fx.Ct); Assert.Equal(1, trashed); Assert.False(Directory.Exists(project.OriginalsDirectory)); var after = project.ReadMedia()!; Assert.False(after.HoldsOriginals); Assert.True(after.Frames.All(f => f.Original is null && f.File is not null)); Assert.Equal(BundleMedia.Proxies, project.ReadBundle().Workspace!.Media.Kind); var inTrash = Directory.GetFiles(Path.Combine(store.Root, WorkspaceOriginals.TrashFolder), Path.GetFileName(lostAtHome), SearchOption.AllDirectories); Assert.Equal(1, inTrash.Length); Assert.Equal(media.Frames[1].OriginalSha256, Sha(inTrash[0])); } [Test] public async Task ACancelledCopyOfTheOriginalsRemovesWhatItWroteAndTrashesWhatTheHomeLacks() { var fx = await FixtureAsync(); var (store, sync) = fx.Workspace(); var project = await fx.CheckoutAsync(sync); Assert.False(project.ReadMedia()!.HoldsOriginals); var until = DateTime.UtcNow + TimeSpan.FromSeconds(TestTiming.Budget(30)); // Held so that the copy writes its originals and waits to record them; cancelled there. var lease = await project.LockAsync(TimeSpan.FromSeconds(10), fx.Ct); Job job; try { job = await fx.StartCopyAsync(store, project.ProjectId, BundleMedia.Originals); while (!(Directory.Exists(project.OriginalsDirectory) && Directory.GetFiles(project.OriginalsDirectory).Count(f => !f.EndsWith(".part", StringComparison.Ordinal)) == fx.Frames.Count)) { if (DateTime.UtcNow > until) Assert.Fail("the copy's originals were not written"); await Task.Delay(20, fx.Ct); } // A file of bytes the home does not hold, and a download cut short. File.WriteAllText(Path.Combine(project.OriginalsDirectory, "edited.jpg"), "not the home's photo"); File.WriteAllText(Path.Combine(project.OriginalsDirectory, Guid.NewGuid().ToString("N") + ".part"), "half"); await fx.CancelAsync(job.Id); } finally { lease.Dispose(); } await fx.Archive.WaitForAsync(job.Id, JobState.Cancelled, (int)TestTiming.Budget(30)); while (Directory.Exists(project.OriginalsDirectory)) { if (DateTime.UtcNow > until) Assert.Fail("the cancelled copy's originals stayed: " + string.Join(", ", Directory.GetFiles(project.OriginalsDirectory).Select(Path.GetFileName))); await Task.Delay(20, fx.Ct); } // The media a copy finished before stay; the one file the home lacks is in the trash, never away. var media = project.ReadMedia()!; Assert.False(media.HoldsOriginals); foreach (var frame in media.Frames) Assert.FileExists(project.MediaFile(frame.File!)); var inTrash = Directory.GetFiles(Path.Combine(store.Root, WorkspaceOriginals.TrashFolder), "*", SearchOption.AllDirectories); Assert.SequenceEqual(["edited.jpg"], inTrash.Select(Path.GetFileName)); } [Test] public async Task ProxiesOnlyRefusesAPlanTheFilesChangedSince() { var fx = await FixtureAsync(); var (store, project) = await CheckoutOriginalsAsync(fx); await fx.CopyAsync(store, project.ProjectId, media: BundleMedia.Originals); var home = await fx.Home.FramesAsync(fx.Ct); var plan = WorkspaceOriginals.PlanProxiesOnly(project, home); Assert.Equal((3, 0), (plan.Originals, plan.ToTrash)); var changed = project.MediaFile(project.ReadMedia()!.Frames[0].Original!); File.AppendAllText(changed, "x"); var error = await Assert.ThrowsAsync(() => WorkspaceOriginals.ApplyProxiesOnlyAsync(project, plan, home, TimeSpan.FromSeconds(10), fx.Ct)); Assert.Equal(Loc.L("Errors.Workspace.MediaChanged"), error.Message); Assert.True(project.ReadMedia()!.HoldsOriginals); Assert.Equal(3, Directory.GetFiles(project.OriginalsDirectory).Length); // Planned again, the changed file is no photo the home holds: it goes to the trash. var again = WorkspaceOriginals.PlanProxiesOnly(project, home); Assert.Equal((2, 1), (again.Originals, again.ToTrash)); Assert.Equal(1, await WorkspaceOriginals.ApplyProxiesOnlyAsync(project, again, home, TimeSpan.FromSeconds(10), fx.Ct)); } } public sealed class SqliteWorkspaceOriginalsTests(TestContext context) : WorkspaceOriginalsTests(context) { protected override Backend Backend => Backend.SqliteFile; } [RequiresEnvironment(PgTestServer.Variable)] public sealed class PostgresWorkspaceOriginalsTests(TestContext context) : WorkspaceOriginalsTests(context) { protected override Backend Backend => Backend.Postgres; }