Phase 5c P1. A workspace copy of kind "originals" now copies each frame's original beside its proxy, so a later local render has full-quality input: streamed into a .part file, hashed as written, checked against the home's SHA-256, renamed and read back. Archive and server homes both work, a stopped copy resumes (originals: checkpoint), and the space check counts the originals. A frame whose original is gone, damaged or refused keeps its proxy and the job says how many; the copy does not fail. Stale originals are deleted only when the home holds the same SHA-256, otherwise moved to the workspace's trash. Proxies only... plans first (every original hashed against the home's frames), asks for an explicit confirmation, rewrites the media to proxies before it touches a file, refuses a plan that changed since, and trashes what the home no longer holds instead of deleting it. Edit: Open in workspace offers Originals (the settings' default, disabled when the drive lacks room) with the originals' size for archive and server homes; Settings > Workspace > Open project picks Last used, Main or asks which; each open is recorded for the clean-up. The project menu gains Copy originals... and Proxies only..., refused while changes are unsaved or jobs of the sequence run. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
188 lines
9.9 KiB
C#
188 lines
9.9 KiB
C#
using System.Security.Cryptography;
|
|
using Chronos.Core.Localization;
|
|
using Chronos.Core.Project;
|
|
using Chronos.Data.Archiving;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Data.Tests.Archiving;
|
|
using Chronos.Data.Tests.Infrastructure;
|
|
using Chronos.Data.Workspaces;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Data.Tests.Workspaces;
|
|
|
|
/// <summary>A workspace project's originals (phase 5c): their copy from the archive, checked and resumable, and "Proxies only".</summary>
|
|
[Category("workspace")]
|
|
public abstract class WorkspaceOriginalsTests(TestContext context) : DatabaseTest(context)
|
|
{
|
|
private async Task<WorkspaceFixture> FixtureAsync(int frames = 3, Func<ArchiveServiceOptions, ArchiveServiceOptions>? configure = null)
|
|
{
|
|
var archive = Own(await ArchiveFixture.CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true), configure: configure));
|
|
var fx = Own(await WorkspaceFixture.CreateAsync(Context, archive, frames));
|
|
// The archive's own proxies: the copy takes those, so a damaged original never changes a proxy.
|
|
await archive.Service.Images.BuildProxiesAsync(archive.Archive, fx.Sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
|
return fx;
|
|
}
|
|
|
|
private static string Sha(string path) => Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(path)));
|
|
|
|
private static async Task<(WorkspaceStore Store, WorkspaceProject Project)> CheckoutOriginalsAsync(WorkspaceFixture fx)
|
|
{
|
|
var (store, sync) = fx.Workspace();
|
|
var main = await fx.Archive.Service.Projects.MainAsync(fx.Sequence.Id, fx.Ct);
|
|
var project = await sync.CheckoutAsync(fx.Home, main.Id, BundleMedia.Originals, fx.Ct);
|
|
return (store, project);
|
|
}
|
|
|
|
private string Source(WorkspaceFixture fx, int index) => fx.Archive.Paths.FromRelative(fx.Frames.Single(f => f.Index == index).RelativePath);
|
|
|
|
[Test]
|
|
public async Task ACopyOfTheOriginalsHoldsEachPhotoCheckedAgainstTheHome()
|
|
{
|
|
var fx = await FixtureAsync();
|
|
var (store, project) = await CheckoutOriginalsAsync(fx);
|
|
Assert.Equal(fx.Frames.Sum(f => f.Size), await WorkspaceOriginals.ArchiveOriginalsBytesAsync(fx.Db, fx.Archive.Service, fx.Sequence.Id, cancellationToken: fx.Ct));
|
|
|
|
await fx.CopyAsync(store, project.ProjectId, media: BundleMedia.Originals);
|
|
var media = project.ReadMedia()!;
|
|
Assert.True(media.HoldsOriginals);
|
|
Assert.Equal(BundleMedia.Originals, project.ReadBundle().Workspace!.Media.Kind);
|
|
foreach (var frame in fx.Frames)
|
|
{
|
|
var entry = media.Frames.Single(f => f.Id == frame.Id);
|
|
Assert.NotNull(entry.File);
|
|
var original = project.MediaFile(Assert.NotNull(entry.Original));
|
|
Assert.Equal(frame.Sha256!.ToLowerInvariant(), entry.OriginalSha256);
|
|
Assert.Equal(entry.OriginalSha256, Sha(original));
|
|
Assert.Equal(frame.Size, entry.OriginalBytes);
|
|
Assert.Equal(WorkspaceProject.OriginalName(frame.Id, frame.Sha256, Path.GetExtension(frame.RelativePath)), entry.Original);
|
|
}
|
|
Assert.Empty(Directory.GetFiles(project.OriginalsDirectory, "*.part"));
|
|
Assert.Equal(0L, await WorkspaceOriginals.ArchiveOriginalsBytesAsync(fx.Db, fx.Archive.Service, fx.Sequence.Id, project, fx.Ct));
|
|
|
|
// The frames read by a local render are the originals.
|
|
Assert.SequenceEqual(media.Frames.Select(f => f.Original), media.ToOriginalFrames().Select(f => f.RelativePath));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnOriginalThatIsGoneOrDamagedAtTheHomeLeavesItsFrameProxied()
|
|
{
|
|
var fx = await FixtureAsync();
|
|
var (store, project) = await CheckoutOriginalsAsync(fx);
|
|
File.Delete(Source(fx, 0));
|
|
var damaged = Source(fx, 1);
|
|
var bytes = File.ReadAllBytes(damaged);
|
|
bytes[^3] ^= 0x5A;
|
|
File.WriteAllBytes(damaged, bytes);
|
|
|
|
await fx.CopyAsync(store, project.ProjectId, media: BundleMedia.Originals);
|
|
var media = project.ReadMedia()!;
|
|
Assert.True(media.HoldsOriginals);
|
|
var gone = media.Frames.Single(f => f.Index == 0);
|
|
var mismatched = media.Frames.Single(f => f.Index == 1);
|
|
Assert.Null(gone.Original);
|
|
Assert.Null(mismatched.Original);
|
|
Assert.NotNull(mismatched.File);
|
|
Assert.NotNull(media.Frames.Single(f => f.Index == 2).Original);
|
|
Assert.Equal(1, Directory.GetFiles(project.OriginalsDirectory).Length);
|
|
Assert.True(media.ToOriginalFrames().Count(f => (f.Flags & FrameFlags.Missing) != 0) == 2, "a render counts the two frames without an original");
|
|
}
|
|
|
|
[Test]
|
|
public async Task ACopyStoppedAmongTheOriginalsResumesWithWhatItHad()
|
|
{
|
|
var fx = await FixtureAsync();
|
|
var (store, project) = await CheckoutOriginalsAsync(fx);
|
|
await fx.CopyAsync(store, project.ProjectId, media: BundleMedia.Originals);
|
|
var media = project.ReadMedia()!;
|
|
var kept = project.MediaFile(media.Frames[0].Original!);
|
|
var written = File.GetLastWriteTimeUtc(kept);
|
|
var lost = project.MediaFile(media.Frames[2].Original!);
|
|
File.Delete(lost);
|
|
File.WriteAllBytes(Path.Combine(project.OriginalsDirectory, media.Frames[2].Id.ToString("N") + ".part"), [1, 2, 3]);
|
|
|
|
var row = (await fx.Db.Sequences.GetAsync(fx.Sequence.Id))!;
|
|
await fx.CopyAsync(store, project.ProjectId, checkpoint: $"originals:2:{row.FramesVersion}:{row.Folder}", media: BundleMedia.Originals);
|
|
Assert.Equal(written, File.GetLastWriteTimeUtc(kept));
|
|
Assert.Equal(media.Frames[2].OriginalSha256, Sha(lost));
|
|
Assert.Empty(Directory.GetFiles(project.OriginalsDirectory, "*.part"));
|
|
Assert.Equal(3, project.ReadMedia()!.Frames.Count(f => f.HasOriginal));
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheOriginalsAreNotCopiedWhenTheDriveLacksTheirRoom()
|
|
{
|
|
long? free = null;
|
|
var fx = await FixtureAsync(configure: options => options with { FreeSpace = _ => free });
|
|
var (store, project) = await CheckoutOriginalsAsync(fx);
|
|
await fx.CopyAsync(store, project.ProjectId);
|
|
free = WorkspaceJobs.CopySpaceReserve + fx.Frames.Sum(f => f.Size) - 1;
|
|
|
|
var job = await fx.CopyAsync(store, project.ProjectId, JobState.Failed, media: BundleMedia.Originals);
|
|
var parts = Loc.L("Errors.Workspace.DiskFull", "\u0001").Split('\u0001');
|
|
Assert.True(job.Error!.StartsWith(parts[0], StringComparison.Ordinal), job.Error);
|
|
Assert.False(project.ReadMedia()!.HoldsOriginals);
|
|
Assert.False(Directory.Exists(project.OriginalsDirectory) && Directory.EnumerateFiles(project.OriginalsDirectory).Any());
|
|
}
|
|
|
|
[Test]
|
|
public async Task ProxiesOnlyDeletesWhatTheHomeHoldsAndTrashesTheRest()
|
|
{
|
|
var fx = await FixtureAsync();
|
|
var (store, project) = await CheckoutOriginalsAsync(fx);
|
|
await fx.CopyAsync(store, project.ProjectId, media: BundleMedia.Originals);
|
|
var media = project.ReadMedia()!;
|
|
// The home lost frame 1's photo: its only copy is the workspace's, which goes to the trash, not away.
|
|
var lostAtHome = project.MediaFile(media.Frames[1].Original!);
|
|
var home = await fx.Home.FramesAsync(fx.Ct);
|
|
home = home with { Frames = [.. home.Frames.Select(f => f.Index == 1 ? f with { Flags = f.Flags | FrameFlags.Missing } : f)] };
|
|
|
|
var plan = WorkspaceOriginals.PlanProxiesOnly(project, home);
|
|
Assert.Equal((2, 1), (plan.Originals, plan.ToTrash));
|
|
Assert.Equal(fx.Frames.Where(f => f.Index != 1).Sum(f => f.Size), plan.Bytes);
|
|
var trashed = await WorkspaceOriginals.ApplyProxiesOnlyAsync(project, plan, home, TimeSpan.FromSeconds(10), fx.Ct);
|
|
Assert.Equal(1, trashed);
|
|
Assert.False(Directory.Exists(project.OriginalsDirectory));
|
|
var after = project.ReadMedia()!;
|
|
Assert.False(after.HoldsOriginals);
|
|
Assert.True(after.Frames.All(f => f.Original is null && f.File is not null));
|
|
Assert.Equal(BundleMedia.Proxies, project.ReadBundle().Workspace!.Media.Kind);
|
|
var inTrash = Directory.GetFiles(Path.Combine(store.Root, WorkspaceOriginals.TrashFolder), Path.GetFileName(lostAtHome), SearchOption.AllDirectories);
|
|
Assert.Equal(1, inTrash.Length);
|
|
Assert.Equal(media.Frames[1].OriginalSha256, Sha(inTrash[0]));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ProxiesOnlyRefusesAPlanTheFilesChangedSince()
|
|
{
|
|
var fx = await FixtureAsync();
|
|
var (store, project) = await CheckoutOriginalsAsync(fx);
|
|
await fx.CopyAsync(store, project.ProjectId, media: BundleMedia.Originals);
|
|
var home = await fx.Home.FramesAsync(fx.Ct);
|
|
var plan = WorkspaceOriginals.PlanProxiesOnly(project, home);
|
|
Assert.Equal((3, 0), (plan.Originals, plan.ToTrash));
|
|
|
|
var changed = project.MediaFile(project.ReadMedia()!.Frames[0].Original!);
|
|
File.AppendAllText(changed, "x");
|
|
var error = await Assert.ThrowsAsync<InvalidOperationException>(() => WorkspaceOriginals.ApplyProxiesOnlyAsync(project, plan, home, TimeSpan.FromSeconds(10), fx.Ct));
|
|
Assert.Equal(Loc.L("Errors.Workspace.MediaChanged"), error.Message);
|
|
Assert.True(project.ReadMedia()!.HoldsOriginals);
|
|
Assert.Equal(3, Directory.GetFiles(project.OriginalsDirectory).Length);
|
|
|
|
// Planned again, the changed file is no photo the home holds: it goes to the trash.
|
|
var again = WorkspaceOriginals.PlanProxiesOnly(project, home);
|
|
Assert.Equal((2, 1), (again.Originals, again.ToTrash));
|
|
Assert.Equal(1, await WorkspaceOriginals.ApplyProxiesOnlyAsync(project, again, home, TimeSpan.FromSeconds(10), fx.Ct));
|
|
}
|
|
}
|
|
|
|
public sealed class SqliteWorkspaceOriginalsTests(TestContext context) : WorkspaceOriginalsTests(context)
|
|
{
|
|
protected override Backend Backend => Backend.SqliteFile;
|
|
}
|
|
|
|
[RequiresEnvironment(PgTestServer.Variable)]
|
|
public sealed class PostgresWorkspaceOriginalsTests(TestContext context) : WorkspaceOriginalsTests(context)
|
|
{
|
|
protected override Backend Backend => Backend.Postgres;
|
|
}
|