Files
Chronos/tests/Chronos.Data.Tests/Archiving/ConvertTests.cs
T
Alby96 0e52929274 feat(archive): convert sequences to JPEG, TIFF-16 or linear DNG after import
Owner review 2, R7 asks for mass conversion after import on the web panel and the desktop,
with a choice at every run between Replace and New sequence. This is the engine and the
server API both UIs build on.

- Core/Conversion: ConvertOptions (JPEG 4:4:4 at quality 1-100, default 95; TIFF-16 sRGB
  Deflate with the predictor; linear DNG, the archival choice; converted names keep the
  stem and take the target's extension in the old one's case). FrameConverter decodes at
  the native tier only, encodes with the in-house writers, writes Exif from the sidecar
  (capture time to the tick, offset, exposure) and the original (camera, serial, lens,
  modes, GPS), hashes the bytes as made, writes atomically and verifies: read back, the
  metadata again, a decode by the server's decoders. ExifWriter now writes the exposure
  program, metering mode, flash and white balance. ConvertPlanner: convertible frames,
  skips with reasons, size estimates per pixel, the plan's space and fingerprint.
- FrameTrash.ReplaceAsync swaps the files of frames in place: same ids and indices, the old
  files and their XMP companions into one Replaced trash entry (records, replacement
  hashes, a copy of analysis.json), tombstones aliasing old content to new, proxies of the
  swapped frames dropped, under .chronos/replace.pending.json, which the next writer undoes
  or completes. Restoring a Replaced entry swaps back while every converted frame is
  unchanged (the converted files become a Replaced entry in turn), else the originals come
  back as a new sequence.
- MassConverter and JobKind.Convert (19): the dry run (busy, offline, unreadable, space
  against ArchiveServiceOptions.FreeSpace), stage A converting into
  staging/convert-<job>/ in parallel with each source re-hashed and each staged file
  appended to journal/convert-<job>.staged, stage B per sequence all or nothing under the
  lease at the planned frames version: Replace, or a new sequence through the ingest with
  the source's tags, rating, notes and project. Resumes keep verified staged files; a
  sequence changed, busy or with a failed photo is left as it was and fails the job at the
  end, the journal kept for a retry.
- POST /api/v1/archive/convert: dry run with the plan, its space line and fingerprint, or a
  202 convert job; formats the server cannot read at full resolution are skipped as
  convert_on_pc; validated, audited (archive_converted), sequenceChanged converted/created.
  The server measures the space volume for the check.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit 809210d924cfc3f597ecdea26106fb3ff62d3762)
(cherry picked from commit e13d4c0bb1bd9441933b6f3e5b6220e865a7228c)
2026-10-27 22:01:23 +01:00

581 lines
35 KiB
C#

using System.Security.Cryptography;
using System.Text.Json;
using Chronos.Core.Archiving;
using Chronos.Core.Conversion;
using Chronos.Core.Import;
using Chronos.Core.Imaging;
using Chronos.Core.Imaging.Decoders;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Data.Archiving;
using Chronos.Data.Archiving.Jobs;
using Chronos.Data.Jobs;
using Chronos.Data.Repositories;
using Chronos.Testing;
using static Chronos.Data.Tests.Archiving.ArchiveFixture;
namespace Chronos.Data.Tests.Archiving;
/// <summary>
/// Mass conversion after import (owner review 2, R7): the dry run, the <c>convert</c> job in both modes, its undo through the
/// trash, all or nothing per sequence, resumes in stage A and B, plans gone stale, the space check, and the gates that must
/// keep holding (the card erase, the emptying of a replaced entry).
/// </summary>
[Category("archive")]
public sealed class ConvertTests(TestContext context)
{
private Task<ArchiveFixture> FixtureAsync(Func<ArchiveServiceOptions, ArchiveServiceOptions>? more = null) =>
CreateAsync(context, configure: o =>
{
var options = o with { Decoder = DecoderFactory.CreateServer(), Parallelism = () => 2 };
return more is null ? options : more(options);
});
/// <summary>A sequence of <paramref name="count"/> uploaded photos (hashed, under <c>originals/</c>), ten seconds apart.</summary>
private static async Task<Guid> UploadedAsync(ArchiveFixture fx, int count, string session = "upload-1", int seed = 100)
{
Guid? id = null;
for (var i = 0; i < count; i++)
{
var (path, size, sha) = fx.Stage(Start.AddSeconds(10 * i).AddTicks(1_234_567), seed + i);
var result = await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, id is null ? session : null, path, $"IMG_{i + 1:0000}.JPG",
size, sha, null, null), fx.Ct);
id = result.SequenceId;
}
return id!.Value;
}
private static Task<ConvertPlan> PlanAsync(ArchiveFixture fx, ConvertTarget target, ConvertMode mode, params Guid[] ids) =>
fx.Service.PlanConvertAsync(fx.Archive.Id, ids, new ConvertOptions(target), mode, fx.Ct);
private static async Task<Job> RunAsync(ArchiveFixture fx, JobEngine engine, ConvertPlan plan, JobState state = JobState.Done)
{
var job = await engine.EnqueueAsync(ArchiveJobs.Convert(fx.Archive, plan), cancellationToken: fx.Ct);
return await fx.WaitForAsync(job.Id, state, (int)TestTiming.Budget(60));
}
private static SequenceManifest Sidecar(ArchiveFixture fx, Sequence sequence) => ManifestStore.ReadSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath);
private static string Sha(string path) => Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(path)));
/// <summary>The index rows of a sequence, after checking that each file holds its row's content and that the sidecar agrees.</summary>
private static async Task<List<Frame>> FramesAsync(ArchiveFixture fx, Guid id)
{
var sequence = Assert.NotNull(await fx.Db.Sequences.GetAsync(id));
var rows = (await fx.Db.Frames.ListBySequenceAsync(id)).ToList();
var sidecar = Sidecar(fx, sequence);
Assert.Equal(sidecar.Versions.Frames, sequence.FramesVersion);
Assert.SequenceEqual(sidecar.Frames.Select(f => (f.Id, f.I, f.Sha256)), rows.Select(r => (r.Id, r.Index, r.Sha256)));
foreach (var row in rows) Assert.Equal(row.Sha256, Sha(fx.Paths.FromRelative(row.RelativePath)));
Assert.FileMissing(fx.Paths.Sequence(sequence.Folder).ReplaceJournalPath);
return rows;
}
private static string Staging(ArchiveFixture fx) => fx.Paths.StagingDirectory;
private static bool StagedAnything(ArchiveFixture fx) =>
Directory.Exists(Staging(fx)) && Directory.EnumerateDirectories(Staging(fx), "convert-*").Any();
[Test]
public async Task TheDryRunCountsWhatConvertsAndChangesNothing()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 3);
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id);
var step = plan.Sequences.Single();
Assert.Equal((3, 0, true), (step.Frames.Count, step.SkippedFrames, step.Converts));
Assert.Equal((await fx.Db.Sequences.GetAsync(id))!.FramesVersion, step.FramesVersion);
Assert.Equal(step.Frames.Sum(f => f.Size), plan.BytesBefore);
Assert.Equal(3 * ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.LinearDng), 48, 32), plan.BytesAfter);
Assert.Equal(ConvertPlanner.Required(plan.BytesAfter), plan.Space.Required);
Assert.True(plan.Space.Enough);
Assert.Equal(plan.Fingerprint(), (await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id)).Fingerprint());
Assert.NotEqual(plan.Fingerprint(), (await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.NewSequence, id)).Fingerprint());
// JPEG to JPEG converts nothing.
var same = (await PlanAsync(fx, ConvertTarget.Jpeg, ConvertMode.Replace, id)).Sequences.Single();
Assert.Equal((0, ConvertSkip.SameFormat, 3), (same.Frames.Count, same.Skipped.Single().Reason, same.Skipped.Single().Frames));
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
Assert.False(StagedAnything(fx));
await Assert.ThrowsAsync<ArgumentException>(() => PlanAsync(fx, ConvertTarget.Jpeg, ConvertMode.Replace, Guid.NewGuid()));
}
[Test]
public async Task ASequenceBusyOrUnreadableIsBlocked()
{
await using var fx = await FixtureAsync();
var busy = await UploadedAsync(fx, 2);
var unreadable = await UploadedAsync(fx, 2, "upload-2", seed: 200);
await fx.Db.Jobs.EnqueueAsync(new Job
{
Id = Guid.CreateVersion7(),
Kind = JobKind.Render,
TargetIds = [busy],
State = JobState.Running,
NodeId = "test#0",
Created = DateTimeOffset.UtcNow,
Started = DateTimeOffset.UtcNow,
});
File.WriteAllText(fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(unreadable))!.Folder).ManifestPath, "{ not json");
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, busy, unreadable);
Assert.SequenceEqual([ConvertBlock.Busy], plan.Sequences.Single(s => s.SequenceId == busy).Blocked);
Assert.SequenceEqual([ConvertBlock.Unreadable], plan.Sequences.Single(s => s.SequenceId == unreadable).Blocked);
Assert.Equal(0, plan.Frames);
Assert.Throws<ArgumentException>(() => ArchiveJobs.Convert(fx.Archive, plan));
}
[Test]
public async Task AReplaceKeepsTheFramesAndTrashesTheOriginalsWithTheirReplacements()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 3);
var engine = await fx.StartEngineAsync();
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
var originals = await FramesAsync(fx, id);
var project = new ProjectSettings { ExcludedFrames = [1], HeroFrame = 2 }.ToJson();
await fx.Db.Sequences.EditAsync(id, new SequenceEdit { SettingsJson = project, HeroFrameId = originals[2].Id }, null, fx.Ct);
var folder = fx.Paths.Sequence(sequence.Folder);
Directory.CreateDirectory(folder.MetaDirectory);
File.WriteAllText(folder.AnalysisPath, "{\"frames\":3}");
var before = Sidecar(fx, sequence);
var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id);
var job = await RunAsync(fx, engine, plan);
Assert.Contains(Loc.L("Archive.Log.Converted", sequence.DisplayTitle, 3), job.LogTail);
var row = (await fx.Db.Sequences.GetAsync(id))!;
var frames = await FramesAsync(fx, id);
Assert.Equal(before.Versions.Frames + 1, row.FramesVersion);
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Index)), frames.Select(f => (f.Id, f.Index)));
var after = Sidecar(fx, row);
var options = new ConvertOptions(ConvertTarget.LinearDng);
foreach (var (was, now) in before.Frames.Zip(after.Frames))
{
Assert.Equal((options.NameFor(was.File), ImageFileFormat.Dng, DecodeTier.Native, 1), (now.File, now.Format, now.Tier, now.Orientation));
Assert.Equal((was.CapturedAt, was.Original, was.Flags), (now.CapturedAt, now.Original, now.Flags));
var source = Assert.NotNull(now.Source);
Assert.Equal((was.Sha256, was.Format, was.File, was.Size), (source.Sha256, source.Format, source.File, source.Size));
Assert.FileMissing(folder.File(was.File));
}
// The project's positions stay on their frames.
var settings = ProjectSettings.FromJson(row.SettingsJson!);
Assert.SequenceEqual([1], settings.ExcludedFrames);
Assert.Equal((2, originals[2].Id), (settings.HeroFrame, row.HeroFrameId));
// One replaced entry with the originals, each naming its replacement, and the analysis as it was.
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
Assert.Equal((TrashKind.Replaced, id, 3), (entry.Kind, entry.SequenceId, entry.Frames));
var record = Assert.NotNull(FrameTrash.ReadEntry(fx.Paths, entry.Name));
var entryFolder = FrameTrash.Location(fx.Paths, entry.Name).Folder;
foreach (var photo in record.Photos)
{
var now = after.Frames.Single(f => f.Id == photo.Frame.Id);
Assert.Equal(now.Sha256, photo.Replacement);
Assert.Equal(photo.Frame.Sha256, Sha(Path.Combine(entryFolder, photo.Files[0].Path)));
}
Assert.Equal("{\"frames\":3}", File.ReadAllText(Path.Combine(entryFolder, FrameTrash.AnalysisCopy)));
// Tombstones alias the old content to the new, in the sidecar and the index.
var tombstones = Assert.NotNull(after.Removed);
Assert.SequenceEqual(before.Frames.Select(f => (f.Sha256, after.Frames.Single(n => n.Id == f.Id).Sha256, RemovalReason.Replaced)).Order(),
tombstones.Select(t => ((string?)t.Sha256, t.Replacement, t.Reason)).Order());
Assert.SequenceEqual(tombstones.Select(t => (t.Sha256, t.Replacement)).Order(),
(await fx.Db.FrameRemovals.ListBySequenceAsync(id)).Select(r => (r.Sha256, r.ReplacementSha256)).Order());
Assert.Equal((SequenceStatus)0, row.Status & (SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt));
Assert.False(StagedAnything(fx));
Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, job.Id));
Assert.FileMissing(ConvertJournal.StagedPathFor(fx.Paths, job.Id));
// Converted again, nothing is left to convert.
Assert.Equal(0, (await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id)).Frames);
}
[Test]
public async Task RestoringTheReplacedEntrySwapsTheOriginalsBack()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 3);
var engine = await fx.StartEngineAsync();
var originals = await FramesAsync(fx, id);
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
var before = Sidecar(fx, sequence);
var analysis = fx.Paths.Sequence(sequence.Folder).AnalysisPath;
Directory.CreateDirectory(Path.GetDirectoryName(analysis)!);
File.WriteAllText(analysis, "of the originals");
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id));
var converted = Sidecar(fx, (await fx.Db.Sequences.GetAsync(id))!);
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
// The light analysis of the converted frames, as it would be written again.
File.WriteAllText(analysis, "of the conversions");
var result = await fx.Service.RestoreEntryAsync(fx.Archive.Id, entry.Name, null, fx.Ct);
Assert.Equal((TrashKind.Replaced, 3, false, id), (result.Kind, result.Photos, result.NewSequence, result.Sequence.Id));
var frames = await FramesAsync(fx, id);
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Index, f.Sha256, f.RelativePath, f.Format)), frames.Select(f => (f.Id, f.Index, f.Sha256, f.RelativePath, f.Format)));
var back = Sidecar(fx, (await fx.Db.Sequences.GetAsync(id))!);
Assert.True(back.Frames.All(f => f.Source is null));
// The converted files are in the trash in turn, naming the originals as their replacements; the old entry is gone.
var turned = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
Assert.NotEqual(entry.Name, turned.Name);
Assert.Equal(TrashKind.Replaced, turned.Kind);
var record = FrameTrash.ReadEntry(fx.Paths, turned.Name)!;
Assert.SequenceEqual(converted.Frames.Select(f => (f.Sha256, before.Frames.Single(o => o.Id == f.Id).Sha256)).Order(),
record.Photos.Select(p => (p.Frame.Sha256, p.Replacement)).Order());
// Each analysis goes with its frames: the originals' back in the sequence, the conversions' into the new entry.
Assert.Equal("of the originals", File.ReadAllText(analysis));
Assert.Equal("of the conversions", File.ReadAllText(Path.Combine(FrameTrash.Location(fx.Paths, turned.Name).Folder, FrameTrash.AnalysisCopy)));
Assert.False(Directory.Exists(FrameTrash.Location(fx.Paths, entry.Name).Folder));
Assert.SequenceEqual(converted.Frames.Select(f => (f.Sha256, before.Frames.Single(o => o.Id == f.Id).Sha256)).Order(),
back.Removed!.Select(t => ((string?)t.Sha256, t.Replacement)).Order());
// That entry erases: its replacements (the originals) are where it says, re-hashed now.
var empty = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
var job = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, empty), cancellationToken: fx.Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
await FramesAsync(fx, id);
}
[Test]
public async Task AnUndoAfterTheFramesChangedBringsTheOriginalsBackAsANewSequence()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 3);
var engine = await fx.StartEngineAsync();
var originals = await FramesAsync(fx, id);
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id));
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
var converted = await FramesAsync(fx, id);
// A converted frame deleted since: the frames are no longer the ones the conversion left.
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
var delete = await fx.Service.FrameTrash.PlanDeleteAsync(id, [converted[1].Id], sequence.FramesVersion, fx.Ct);
await fx.Service.FrameTrash.DeleteAsync(delete, "alberto", null, fx.Ct);
var result = await fx.Service.RestoreEntryAsync(fx.Archive.Id, entry.Name, null, fx.Ct);
Assert.Equal((TrashKind.Replaced, 3, true), (result.Kind, result.Photos, result.NewSequence));
Assert.NotEqual(id, result.Sequence.Id);
Assert.SequenceEqual(originals.Select(f => f.Sha256), (await FramesAsync(fx, result.Sequence.Id)).Select(f => f.Sha256));
Assert.Count(2, await FramesAsync(fx, id));
// The replaced entry is gone with its last photo (and its copy of the analysis); the deleted frame's entry stays.
Assert.DoesNotContain(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name));
}
[Test]
public async Task ANewSequenceTakesTheCopiesAndLeavesTheSourceAsItIs()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 3);
var engine = await fx.StartEngineAsync();
await fx.Service.SetSequenceInfoAsync(id, "Roof", ["sunset", "milan"], 4, "From the roof", fx.Ct);
var originals = await FramesAsync(fx, id);
await fx.Db.Sequences.EditAsync(id, new SequenceEdit { SettingsJson = new ProjectSettings { ExcludedFrames = [1], HeroFrame = 2 }.ToJson() }, null, fx.Ct);
var source = (await fx.Db.Sequences.GetAsync(id))!;
var sidecar = File.ReadAllText(fx.Paths.Sequence(source.Folder).ManifestPath);
var job = await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id));
var made = Assert.NotNull(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, id)));
var copy = (await fx.Db.Sequences.GetAsync(made))!;
Assert.Equal(("Roof (TIFF)", 4, "From the roof"), (copy.Title, copy.Rating, copy.Notes));
Assert.SequenceEqual(["milan", "sunset"], copy.Tags.Order());
var frames = await FramesAsync(fx, made);
Assert.SequenceEqual(originals.Select(f => f.CapturedAt), frames.Select(f => f.CapturedAt));
Assert.True(frames.All(f => f.Format == ImageFileFormat.Tiff));
Assert.SequenceEqual(originals.Select(f => Path.ChangeExtension(f.FileName, ".tif")), frames.Select(f => Path.ChangeExtension(f.FileName, ".tif")));
var settings = ProjectSettings.FromJson(copy.SettingsJson!);
Assert.SequenceEqual([1], settings.ExcludedFrames);
Assert.Equal((2, frames[2].Id), (settings.HeroFrame, copy.HeroFrameId));
// The source: not a byte of its sidecar, nor a row, changed; nothing went to the trash.
Assert.Equal(sidecar, File.ReadAllText(fx.Paths.Sequence(source.Folder).ManifestPath));
var now = (await fx.Db.Sequences.GetAsync(id))!;
Assert.Equal((source.FramesVersion, source.SettingsVersion, source.Title, source.Folder), (now.FramesVersion, now.SettingsVersion, now.Title, now.Folder));
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256)), (await FramesAsync(fx, id)).Select(f => (f.Id, f.Sha256)));
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
Assert.False(StagedAnything(fx));
}
[Test]
public async Task APhotoThatFailsLeavesItsWholeSequenceAsItWasAndTheOthersConvert()
{
await using var fx = await FixtureAsync();
var bad = await UploadedAsync(fx, 3);
var good = await UploadedAsync(fx, 2, "upload-2", seed: 300);
var engine = await fx.StartEngineAsync();
var damaged = (await FramesAsync(fx, bad))[1];
var originals = await FramesAsync(fx, bad);
// Rot after the plan: the photo no longer matches the hash it was archived with.
var path = fx.Paths.FromRelative(damaged.RelativePath);
var bytes = File.ReadAllBytes(path);
bytes[^20] ^= 0x10;
File.WriteAllBytes(path, bytes);
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, bad, good);
Assert.Equal(5, plan.Frames);
var job = await RunAsync(fx, engine, plan, JobState.Failed);
Assert.Equal(Loc.Plural("Archive.Error.ConvertKept", 1), job.Error);
Assert.Contains(damaged.FileName, job.LogTail);
// All or nothing: not one frame of that sequence changed, and nothing of it is in the trash.
var rows = await fx.Db.Frames.ListBySequenceAsync(bad);
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256, f.RelativePath)), rows.Select(f => (f.Id, f.Sha256, f.RelativePath)));
Assert.True((await FramesAsync(fx, good)).All(f => f.Format == ImageFileFormat.Tiff));
Assert.SequenceEqual([good], (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.SequenceId));
Assert.False(StagedAnything(fx));
// The journal stays for a retry, which tries the failed sequence again and leaves the converted one.
var journal = Assert.NotNull(ConvertJournal.Read(fx.Paths, job.Id));
Assert.Equal(ConvertOutcome.Converted, journal.Of(good)!.Outcome);
Assert.Equal(ConvertOutcome.Failed, journal.Of(bad)!.Outcome);
}
[Test]
public async Task AResumedJobKeepsTheStagedFilesThatStillVerify()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 3);
var engine = await fx.StartEngineAsync();
// To JPEG needs other originals than the JPEGs uploaded: they become TIFF first.
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id));
var plan = await PlanAsync(fx, ConvertTarget.Jpeg, ConvertMode.Replace, id);
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
var folder = fx.Paths.Sequence(sequence.Folder);
var manifest = Sidecar(fx, sequence);
var jobId = Guid.CreateVersion7();
var staging = Path.Combine(Staging(fx), $"convert-{jobId:N}", $"{id:N}");
var log = StagedLog.Open(ConvertJournal.StagedPathFor(fx.Paths, jobId));
// An earlier attempt staged the first photo (at another quality, so its use shows) and the second, damaged since.
StagedConversion Stage(SidecarFrame frame, int quality)
{
var target = Path.Combine(staging, frame.Id.ToString("N") + ".jpg");
var file = FrameConverter.Convert(new ConvertSource(folder.File(frame.File), frame, manifest.Camera, manifest.Lens), new ConvertOptions(ConvertTarget.Jpeg, quality),
DecoderFactory.CreateServer(), target, cancellationToken: fx.Ct);
var conversion = new StagedConversion(frame.Id, frame.Sha256!, Path.GetFileName(target), file.Size, file.Sha256, file.Width, file.Height);
log.Append(id, manifest.Versions.Frames, conversion);
return conversion;
}
var kept = Stage(manifest.Frames[0], 40);
var torn = Stage(manifest.Frames[1], 95);
File.WriteAllBytes(Path.Combine(staging, torn.File), [1, 2, 3]);
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Convert(fx.Archive, plan), "convert", jobId);
await fx.WaitForAsync(job.Id, JobState.Done);
var frames = await FramesAsync(fx, id);
Assert.Equal(kept.Sha256, frames[0].Sha256);
Assert.Equal(torn.Sha256, frames[1].Sha256);
Assert.True(frames.All(f => f.Format == ImageFileFormat.Jpeg));
}
[Test]
public async Task AJobThatStoppedAfterItsReplacementReportsTheSequenceConverted()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
var engine = await fx.StartEngineAsync();
var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id);
var request = ArchiveJobs.Convert(fx.Archive, plan);
await RunAsync(fx, engine, plan);
// As a crash between the replacement and the journal's note leaves it: the staged log names the conversions done.
var sidecar = Sidecar(fx, (await fx.Db.Sequences.GetAsync(id))!);
var jobId = Guid.CreateVersion7();
var log = StagedLog.Open(ConvertJournal.StagedPathFor(fx.Paths, jobId));
foreach (var frame in sidecar.Frames)
log.Append(id, plan.Sequences.Single().FramesVersion, new StagedConversion(frame.Id, frame.Source!.Sha256!, "gone.dng", frame.Size, frame.Sha256!, frame.W, frame.H));
var job = await fx.EnqueueWithCheckpointAsync(engine, request, "convert", jobId);
var done = await fx.WaitForAsync(job.Id, JobState.Done);
Assert.Contains(Loc.L("Archive.Log.Converted", (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle, 2), done.LogTail);
Assert.Count(1, await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
}
[Test]
public async Task ASequenceThatChangedAfterThePlanIsLeftAsItWas()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
var engine = await fx.StartEngineAsync();
var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id);
// Another photo arrives before the job runs.
var (path, size, sha) = fx.Stage(Start.AddSeconds(60), 999);
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, null, path, "IMG_0099.JPG", size, sha, null, null), fx.Ct);
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
var job = await RunAsync(fx, engine, plan, JobState.Failed);
Assert.Equal(Loc.Plural("Archive.Error.ConvertKept", 1), job.Error);
Assert.Contains(Loc.L("Archive.Log.ConvertChanged", (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle), job.LogTail);
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
}
[Test]
public async Task AVolumeWithoutTheSpaceConvertsNothing()
{
await using var fx = await FixtureAsync(o => o with { FreeSpace = _ => 1_000 });
var id = await UploadedAsync(fx, 2);
var engine = await fx.StartEngineAsync();
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
Assert.False(plan.Space.Enough);
Assert.Equal(1_000, plan.Space.Free);
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
var job = await RunAsync(fx, engine, plan, JobState.Failed);
Assert.True(job.Error!.StartsWith("The conversion needs", StringComparison.Ordinal), job.Error);
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
Assert.False(StagedAnything(fx));
}
[Test]
public async Task ACardKeepsTheCopyOfAPhotoThatWasConverted()
{
await using var fx = await FixtureAsync();
var card = context.CreateTempDirectory("card");
Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, 3);
var media = await fx.Service.Imports.RegisterCardAsync(new CardVolume("3F2A9C11-EOS_DIGITAL-127865454592", 0x3F2A9C11, "EOS_DIGITAL", 127_865_454_592), fx.Ct);
var import = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
await fx.Service.Imports.ExecuteAsync(new ImportJobParameters(fx.Archive.Id, import, media.Id, true, null), Guid.NewGuid(), 0,
_ => ValueTask.CompletedTask, null, fx.Ct);
var id = (await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, import.Sessions.Single().SessionKey))!.Value;
Assert.Equal(3, (await fx.Service.PlanCardEraseAsync(fx.Archive.Id, media.Id, card, null, fx.Ct)).ToDelete.Count());
var engine = await fx.StartEngineAsync();
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id));
// The frames now hold converted files: nothing in the archive matches the card's photos byte for byte, so they stay.
var erase = await fx.Service.PlanCardEraseAsync(fx.Archive.Id, media.Id, card, null, fx.Ct);
Assert.Empty(erase.ToDelete);
Assert.True(erase.Items.All(i => i.Verdict == EraseVerdict.CopyMismatch));
Assert.Equal(0, CardEraser.Execute(erase).Deleted);
Assert.Count(3, Directory.EnumerateFiles(card, "*.JPG", SearchOption.AllDirectories));
}
[Test]
public async Task AReplacedEntryIsErasedOnlyWhileTheConvertedFilesStillHashToTheirRecord()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
var engine = await fx.StartEngineAsync();
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id));
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
var converted = (await FramesAsync(fx, id))[0];
var file = fx.Paths.FromRelative(converted.RelativePath);
var bytes = File.ReadAllBytes(file);
var damaged = (byte[])bytes.Clone();
damaged[^10] ^= 0x01;
File.WriteAllBytes(file, damaged);
var plan = await PlanEmptyAsync(fx);
var kept = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, plan), cancellationToken: fx.Ct);
await fx.WaitForAsync(kept.Id, JobState.Failed);
Assert.Contains(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name));
File.WriteAllBytes(file, bytes);
var erased = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, await PlanEmptyAsync(fx)), cancellationToken: fx.Ct);
await fx.WaitForAsync(erased.Id, JobState.Done);
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
static Task<TrashEmptyPlan> PlanEmptyAsync(ArchiveFixture fx) => fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
}
[Test]
public async Task AReplacementACrashInterruptedBeforeItsSidecarIsUndoneByTheNextWriter()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
var folder = fx.Paths.Sequence(sequence.Folder);
var manifest = Sidecar(fx, sequence);
var frame = manifest.Frames[0];
// Half a replacement: the original in its entry, the converted file in its place, the record pending, the sidecar as it was.
var staged = Path.Combine(Staging(fx), "convert-test", frame.Id.ToString("N") + ".tif");
FrameConverter.Convert(new ConvertSource(folder.File(frame.File), frame), new ConvertOptions(ConvertTarget.Tiff16), DecoderFactory.CreateServer(), staged,
cancellationToken: fx.Ct);
var entryFolder = Path.Combine(FrameTrash.Directory(fx.Paths), $"{id:N}-20261027120000");
var record = entryFolder + ".json";
var trashed = Path.Combine(entryFolder, "originals", Path.GetFileName(frame.File));
var placed = Path.ChangeExtension(frame.File, ".tif");
var journal = new FrameReplaceJournal(manifest.Versions.Frames, fx.Paths.ToRelative(record), fx.Paths.ToRelative(entryFolder),
[new SwapMove(frame.File, fx.Paths.ToRelative(trashed))], [new SwapMove(placed, fx.Paths.ToRelative(staged))], false, null);
journal.Write(folder);
Directory.CreateDirectory(FrameTrash.Directory(fx.Paths));
File.WriteAllText(FrameRemovalJournal.PendingRecord(record), "{}");
Directory.CreateDirectory(Path.GetDirectoryName(trashed)!);
File.Move(folder.File(frame.File), trashed);
File.Move(staged, folder.File(placed));
await fx.Service.EditSequenceInfoAsync(id, info => info with { Rating = 2 }, fx.Ct);
Assert.FileExists(folder.File(frame.File));
Assert.FileMissing(folder.File(placed));
Assert.FileExists(staged);
Assert.False(Directory.Exists(entryFolder));
Assert.FileMissing(FrameRemovalJournal.PendingRecord(record));
Assert.FileMissing(folder.ReplaceJournalPath);
Assert.Equal(frame.Sha256, Sha(folder.File(frame.File)));
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
}
[Test]
public async Task AReplacementACrashInterruptedAfterItsSidecarIsCompletedByTheNextWriter()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
var engine = await fx.StartEngineAsync();
var version = (await fx.Db.Sequences.GetAsync(id))!.FramesVersion;
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id));
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
var (entryFolder, record) = FrameTrash.Location(fx.Paths, entry.Name);
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
var folder = fx.Paths.Sequence(sequence.Folder);
// As a crash right after the sidecar was written leaves it: the record still pending, the journal there.
File.Move(record, FrameRemovalJournal.PendingRecord(record));
new FrameReplaceJournal(version, fx.Paths.ToRelative(record), fx.Paths.ToRelative(entryFolder), [], [], true, null).Write(folder);
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
await fx.Service.EditSequenceInfoAsync(id, info => info with { Rating = 2 }, fx.Ct);
Assert.FileMissing(folder.ReplaceJournalPath);
Assert.Equal(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name);
Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Tiff));
}
[Test]
public async Task APhotoTheServerCannotReadAtFullResolutionIsSkipped()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
// The sidecar says the second photo is a HEIC (a desktop with WIC filed it): the managed decoders have only its preview.
var manifest = Sidecar(fx, sequence);
ManifestStore.WriteSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath,
manifest with { Frames = [manifest.Frames[0], manifest.Frames[1] with { Format = ImageFileFormat.Heif }] });
var step = (await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.NewSequence, id)).Sequences.Single();
Assert.Equal(1, step.Frames.Count);
Assert.Equal((ConvertSkip.NoNativeDecoder, 1), (step.Skipped.Single().Reason, step.Skipped.Single().Frames));
}
[Test]
public async Task TheJournalsSurviveARoundTrip()
{
await using var fx = await FixtureAsync();
var journal = new ConvertJournal { JobId = Guid.NewGuid() };
var first = Guid.NewGuid();
journal.Note(new ConvertSequenceResult(first, "Roof", ConvertOutcome.Created, 3, Guid.NewGuid()));
journal.Note(new ConvertSequenceResult(Guid.NewGuid(), "Harbour", ConvertOutcome.Failed, 0, Error: "no"));
journal.Write(fx.Paths);
var read = Assert.NotNull(ConvertJournal.Read(fx.Paths, journal.JobId));
Assert.SequenceEqual(journal.Sequences.Select(s => (s.Id, s.Outcome, s.Frames, s.NewSequenceId, s.Final)),
read.Sequences.Select(s => (s.Id, s.Outcome, s.Frames, s.NewSequenceId, s.Final)));
Assert.True(read.Of(first)!.Final);
// A line torn by a crash is skipped; a later line for a frame replaces an earlier one.
var path = ConvertJournal.StagedPathFor(fx.Paths, journal.JobId);
var log = StagedLog.Open(path);
var frame = Guid.NewGuid();
log.Append(first, 4, new StagedConversion(frame, "a", "x.dng", 1, "b", 2, 3));
log.Append(first, 4, new StagedConversion(frame, "a", "x.dng", 1, "c", 2, 3));
File.AppendAllText(path, "{\"sequence\":");
var reread = StagedLog.Open(path);
Assert.Equal("c", reread.Of(first, 4).Single().Sha256);
Assert.Empty(reread.Of(first, 5));
ConvertJournal.Delete(fx.Paths, journal.JobId);
Assert.FileMissing(path);
Assert.Null(ConvertJournal.Read(fx.Paths, journal.JobId));
_ = JsonSerializer.Serialize(new ConvertJobParameters(fx.Archive.Id, ConvertTarget.Jpeg, 80, ConvertMode.NewSequence, [new ConvertSequence(first, 2)], "f"),
ArchiveJobJson.Default.ConvertJobParameters);
}
}