Files
Chronos/tests/Chronos.Data.Tests/Archiving/ConvertTests.cs
T
Alby96 341fb77507 fix: conversions name, lock and settle as phase 4a's writers do
Phase 4b was built on phase 4a before its review fixes. Integrated on the reviewed 4a, the
conversion now follows the same rules as filing and the other writers.

- Names: a converted file is named as filing names a photo of its sequence (FrameNaming: the
  archive's file template, the frame's context in the sequence with its camera, lens, number
  and title, {name} the card's, {ext} the target's, a long name keeping its extension).
  Replace through ConvertOptions.FileName; New sequence through the ingest itself
  (StagedFrame.NamedAs), which numbers and names the copies in the new sequence. Names stay
  unique against every file present, those leaving included, so a Replace never moves onto a
  file; a test pins a stray file that keeps its bytes.
- Leases: stage A starts from the sidecar settled under the sequence's lease where the index
  has it then (SequenceWriter.ReadLocked), and FrameTrash.ReplaceAsync reads the row again
  under the lease, so a folder a Reorganize moved meanwhile is followed, never made again.
  A Reorganize the archive waits for keeps the conversion off its sequences (pinned).
- Journals: the replace journal is settled with the others. MassRenamer steps, restores into
  a sequence and sequence deletions wait for an unsettled replacement as they wait for a
  restore, and so do photo deletions, whose frames version the recovery reads
  (FrameTrash.EnsureReplaced); a scan settles it and reconciles nothing meanwhile; a restore
  settles its sequence's replace journal first, at the folder the index has under the lease.
- The convert job tells each sequence converted or made as it completes, as a Reorganize
  tells each step, so an editor open on it reads it again while the rest of the job runs.
- Desktop: JobKind.Convert gets its row in the settings' job priority list. A click on Apply
  that commits the quality no longer starts the same dry run again under the click (as 4a
  fixed for Reorganize).
- Panel: a dry run answered during the confirmation with the same plan no longer drops the
  confirmed Apply; a different plan is shown with Admin.Convert.PlanChanged (en, it).
- Tests: TrashDialogTests.EmptyQueuesTheJobOfTheConfirmedPlanOnly waits for the command to
  end before running it again (a running AsyncRelayCommand ignored the second Execute).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit 2e0d1d6dcc10575734970bb9c77749f00777c859)
2026-10-27 22:03:07 +01:00

675 lines
41 KiB
C#

using System.Security.Cryptography;
using System.Text.Json;
using Chronos.Core.Archiving;
using Chronos.Core.Conversion;
using Chronos.Core.Import;
using Chronos.Core.Imaging;
using Chronos.Core.Imaging.Decoders;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Data.Archiving;
using Chronos.Data.Archiving.Jobs;
using Chronos.Data.Jobs;
using Chronos.Data.Repositories;
using Chronos.Testing;
using static Chronos.Data.Tests.Archiving.ArchiveFixture;
namespace Chronos.Data.Tests.Archiving;
/// <summary>
/// Mass conversion after import (owner review 2, R7): the dry run, the <c>convert</c> job in both modes, its undo through the
/// trash, all or nothing per sequence, resumes in stage A and B, plans gone stale, the space check, and the gates that must
/// keep holding (the card erase, the emptying of a replaced entry).
/// </summary>
[Category("archive")]
public sealed class ConvertTests(TestContext context)
{
private Task<ArchiveFixture> FixtureAsync(Func<ArchiveServiceOptions, ArchiveServiceOptions>? more = null) =>
CreateAsync(context, configure: o =>
{
var options = o with { Decoder = DecoderFactory.CreateServer(), Parallelism = () => 2 };
return more is null ? options : more(options);
});
/// <summary>A sequence of <paramref name="count"/> uploaded photos (hashed, under <c>originals/</c>), ten seconds apart.</summary>
private static async Task<Guid> UploadedAsync(ArchiveFixture fx, int count, string session = "upload-1", int seed = 100)
{
Guid? id = null;
for (var i = 0; i < count; i++)
{
var (path, size, sha) = fx.Stage(Start.AddSeconds(10 * i).AddTicks(1_234_567), seed + i);
var result = await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, id is null ? session : null, path, $"IMG_{i + 1:0000}.JPG",
size, sha, null, null), fx.Ct);
id = result.SequenceId;
}
return id!.Value;
}
private static Task<ConvertPlan> PlanAsync(ArchiveFixture fx, ConvertTarget target, ConvertMode mode, params Guid[] ids) =>
fx.Service.PlanConvertAsync(fx.Archive.Id, ids, new ConvertOptions(target), mode, fx.Ct);
private static async Task<Job> RunAsync(ArchiveFixture fx, JobEngine engine, ConvertPlan plan, JobState state = JobState.Done)
{
var job = await engine.EnqueueAsync(ArchiveJobs.Convert(fx.Archive, plan), cancellationToken: fx.Ct);
return await fx.WaitForAsync(job.Id, state, (int)TestTiming.Budget(60));
}
private static SequenceManifest Sidecar(ArchiveFixture fx, Sequence sequence) => ManifestStore.ReadSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath);
private static string Sha(string path) => Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(path)));
/// <summary>The index rows of a sequence, after checking that each file holds its row's content and that the sidecar agrees.</summary>
private static async Task<List<Frame>> FramesAsync(ArchiveFixture fx, Guid id)
{
var sequence = Assert.NotNull(await fx.Db.Sequences.GetAsync(id));
var rows = (await fx.Db.Frames.ListBySequenceAsync(id)).ToList();
var sidecar = Sidecar(fx, sequence);
Assert.Equal(sidecar.Versions.Frames, sequence.FramesVersion);
Assert.SequenceEqual(sidecar.Frames.Select(f => (f.Id, f.I, f.Sha256)), rows.Select(r => (r.Id, r.Index, r.Sha256)));
foreach (var row in rows) Assert.Equal(row.Sha256, Sha(fx.Paths.FromRelative(row.RelativePath)));
Assert.FileMissing(fx.Paths.Sequence(sequence.Folder).ReplaceJournalPath);
return rows;
}
private static string Staging(ArchiveFixture fx) => fx.Paths.StagingDirectory;
private static bool StagedAnything(ArchiveFixture fx) =>
Directory.Exists(Staging(fx)) && Directory.EnumerateDirectories(Staging(fx), "convert-*").Any();
[Test]
public async Task TheDryRunCountsWhatConvertsAndChangesNothing()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 3);
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id);
var step = plan.Sequences.Single();
Assert.Equal((3, 0, true), (step.Frames.Count, step.SkippedFrames, step.Converts));
Assert.Equal((await fx.Db.Sequences.GetAsync(id))!.FramesVersion, step.FramesVersion);
Assert.Equal(step.Frames.Sum(f => f.Size), plan.BytesBefore);
Assert.Equal(3 * ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.LinearDng), 48, 32), plan.BytesAfter);
Assert.Equal(ConvertPlanner.Required(plan.BytesAfter), plan.Space.Required);
Assert.True(plan.Space.Enough);
Assert.Equal(plan.Fingerprint(), (await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id)).Fingerprint());
Assert.NotEqual(plan.Fingerprint(), (await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.NewSequence, id)).Fingerprint());
// JPEG to JPEG converts nothing.
var same = (await PlanAsync(fx, ConvertTarget.Jpeg, ConvertMode.Replace, id)).Sequences.Single();
Assert.Equal((0, ConvertSkip.SameFormat, 3), (same.Frames.Count, same.Skipped.Single().Reason, same.Skipped.Single().Frames));
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
Assert.False(StagedAnything(fx));
await Assert.ThrowsAsync<ArgumentException>(() => PlanAsync(fx, ConvertTarget.Jpeg, ConvertMode.Replace, Guid.NewGuid()));
}
[Test]
public async Task ASequenceBusyOrUnreadableIsBlocked()
{
await using var fx = await FixtureAsync();
var busy = await UploadedAsync(fx, 2);
var unreadable = await UploadedAsync(fx, 2, "upload-2", seed: 200);
await fx.Db.Jobs.EnqueueAsync(new Job
{
Id = Guid.CreateVersion7(),
Kind = JobKind.Render,
TargetIds = [busy],
State = JobState.Running,
NodeId = "test#0",
Created = DateTimeOffset.UtcNow,
Started = DateTimeOffset.UtcNow,
});
File.WriteAllText(fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(unreadable))!.Folder).ManifestPath, "{ not json");
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, busy, unreadable);
Assert.SequenceEqual([ConvertBlock.Busy], plan.Sequences.Single(s => s.SequenceId == busy).Blocked);
Assert.SequenceEqual([ConvertBlock.Unreadable], plan.Sequences.Single(s => s.SequenceId == unreadable).Blocked);
Assert.Equal(0, plan.Frames);
Assert.Throws<ArgumentException>(() => ArchiveJobs.Convert(fx.Archive, plan));
}
[Test]
public async Task AReplaceKeepsTheFramesAndTrashesTheOriginalsWithTheirReplacements()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 3);
var engine = await fx.StartEngineAsync();
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
var originals = await FramesAsync(fx, id);
var project = new ProjectSettings { ExcludedFrames = [1], HeroFrame = 2 }.ToJson();
await fx.Db.Sequences.EditAsync(id, new SequenceEdit { SettingsJson = project, HeroFrameId = originals[2].Id }, null, fx.Ct);
var folder = fx.Paths.Sequence(sequence.Folder);
Directory.CreateDirectory(folder.MetaDirectory);
File.WriteAllText(folder.AnalysisPath, "{\"frames\":3}");
var before = Sidecar(fx, sequence);
var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id);
var job = await RunAsync(fx, engine, plan);
Assert.Contains(Loc.L("Archive.Log.Converted", sequence.DisplayTitle, 3), job.LogTail);
var row = (await fx.Db.Sequences.GetAsync(id))!;
var frames = await FramesAsync(fx, id);
Assert.Equal(before.Versions.Frames + 1, row.FramesVersion);
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Index)), frames.Select(f => (f.Id, f.Index)));
var after = Sidecar(fx, row);
foreach (var (was, now) in before.Frames.Zip(after.Frames))
{
// Named as filing named the original, with the target's extension: the same template, frame and sequence.
Assert.Equal((Path.ChangeExtension(was.File, ".dng"), ImageFileFormat.Dng, DecodeTier.Native, 1), (now.File, now.Format, now.Tier, now.Orientation));
Assert.Equal((was.CapturedAt, was.Original, was.Flags), (now.CapturedAt, now.Original, now.Flags));
var source = Assert.NotNull(now.Source);
Assert.Equal((was.Sha256, was.Format, was.File, was.Size), (source.Sha256, source.Format, source.File, source.Size));
Assert.FileMissing(folder.File(was.File));
}
// The project's positions stay on their frames.
var settings = ProjectSettings.FromJson(row.SettingsJson!);
Assert.SequenceEqual([1], settings.ExcludedFrames);
Assert.Equal((2, originals[2].Id), (settings.HeroFrame, row.HeroFrameId));
// One replaced entry with the originals, each naming its replacement, and the analysis as it was.
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
Assert.Equal((TrashKind.Replaced, id, 3), (entry.Kind, entry.SequenceId, entry.Frames));
var record = Assert.NotNull(FrameTrash.ReadEntry(fx.Paths, entry.Name));
var entryFolder = FrameTrash.Location(fx.Paths, entry.Name).Folder;
foreach (var photo in record.Photos)
{
var now = after.Frames.Single(f => f.Id == photo.Frame.Id);
Assert.Equal(now.Sha256, photo.Replacement);
Assert.Equal(photo.Frame.Sha256, Sha(Path.Combine(entryFolder, photo.Files[0].Path)));
}
Assert.Equal("{\"frames\":3}", File.ReadAllText(Path.Combine(entryFolder, FrameTrash.AnalysisCopy)));
// Tombstones alias the old content to the new, in the sidecar and the index.
var tombstones = Assert.NotNull(after.Removed);
Assert.SequenceEqual(before.Frames.Select(f => (f.Sha256, after.Frames.Single(n => n.Id == f.Id).Sha256, RemovalReason.Replaced)).Order(),
tombstones.Select(t => ((string?)t.Sha256, t.Replacement, t.Reason)).Order());
Assert.SequenceEqual(tombstones.Select(t => (t.Sha256, t.Replacement)).Order(),
(await fx.Db.FrameRemovals.ListBySequenceAsync(id)).Select(r => (r.Sha256, r.ReplacementSha256)).Order());
Assert.Equal((SequenceStatus)0, row.Status & (SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt));
Assert.False(StagedAnything(fx));
Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, job.Id));
Assert.FileMissing(ConvertJournal.StagedPathFor(fx.Paths, job.Id));
// Converted again, nothing is left to convert.
Assert.Equal(0, (await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id)).Frames);
}
[Test]
public async Task RestoringTheReplacedEntrySwapsTheOriginalsBack()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 3);
var engine = await fx.StartEngineAsync();
var originals = await FramesAsync(fx, id);
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
var before = Sidecar(fx, sequence);
var analysis = fx.Paths.Sequence(sequence.Folder).AnalysisPath;
Directory.CreateDirectory(Path.GetDirectoryName(analysis)!);
File.WriteAllText(analysis, "of the originals");
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id));
var converted = Sidecar(fx, (await fx.Db.Sequences.GetAsync(id))!);
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
// The light analysis of the converted frames, as it would be written again.
File.WriteAllText(analysis, "of the conversions");
var result = await fx.Service.RestoreEntryAsync(fx.Archive.Id, entry.Name, null, fx.Ct);
Assert.Equal((TrashKind.Replaced, 3, false, id), (result.Kind, result.Photos, result.NewSequence, result.Sequence.Id));
var frames = await FramesAsync(fx, id);
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Index, f.Sha256, f.RelativePath, f.Format)), frames.Select(f => (f.Id, f.Index, f.Sha256, f.RelativePath, f.Format)));
var back = Sidecar(fx, (await fx.Db.Sequences.GetAsync(id))!);
Assert.True(back.Frames.All(f => f.Source is null));
// The converted files are in the trash in turn, naming the originals as their replacements; the old entry is gone.
var turned = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
Assert.NotEqual(entry.Name, turned.Name);
Assert.Equal(TrashKind.Replaced, turned.Kind);
var record = FrameTrash.ReadEntry(fx.Paths, turned.Name)!;
Assert.SequenceEqual(converted.Frames.Select(f => (f.Sha256, before.Frames.Single(o => o.Id == f.Id).Sha256)).Order(),
record.Photos.Select(p => (p.Frame.Sha256, p.Replacement)).Order());
// Each analysis goes with its frames: the originals' back in the sequence, the conversions' into the new entry.
Assert.Equal("of the originals", File.ReadAllText(analysis));
Assert.Equal("of the conversions", File.ReadAllText(Path.Combine(FrameTrash.Location(fx.Paths, turned.Name).Folder, FrameTrash.AnalysisCopy)));
Assert.False(Directory.Exists(FrameTrash.Location(fx.Paths, entry.Name).Folder));
Assert.SequenceEqual(converted.Frames.Select(f => (f.Sha256, before.Frames.Single(o => o.Id == f.Id).Sha256)).Order(),
back.Removed!.Select(t => ((string?)t.Sha256, t.Replacement)).Order());
// That entry erases: its replacements (the originals) are where it says, re-hashed now.
var empty = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
var job = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, empty), cancellationToken: fx.Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
await FramesAsync(fx, id);
}
[Test]
public async Task AnUndoAfterTheFramesChangedBringsTheOriginalsBackAsANewSequence()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 3);
var engine = await fx.StartEngineAsync();
var originals = await FramesAsync(fx, id);
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id));
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
var converted = await FramesAsync(fx, id);
// A converted frame deleted since: the frames are no longer the ones the conversion left.
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
var delete = await fx.Service.FrameTrash.PlanDeleteAsync(id, [converted[1].Id], sequence.FramesVersion, fx.Ct);
await fx.Service.FrameTrash.DeleteAsync(delete, "alberto", null, fx.Ct);
var result = await fx.Service.RestoreEntryAsync(fx.Archive.Id, entry.Name, null, fx.Ct);
Assert.Equal((TrashKind.Replaced, 3, true), (result.Kind, result.Photos, result.NewSequence));
Assert.NotEqual(id, result.Sequence.Id);
Assert.SequenceEqual(originals.Select(f => f.Sha256), (await FramesAsync(fx, result.Sequence.Id)).Select(f => f.Sha256));
Assert.Count(2, await FramesAsync(fx, id));
// The replaced entry is gone with its last photo (and its copy of the analysis); the deleted frame's entry stays.
Assert.DoesNotContain(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name));
}
[Test]
public async Task ANewSequenceTakesTheCopiesAndLeavesTheSourceAsItIs()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 3);
var engine = await fx.StartEngineAsync();
await fx.Service.SetSequenceInfoAsync(id, "Roof", ["sunset", "milan"], 4, "From the roof", fx.Ct);
var originals = await FramesAsync(fx, id);
await fx.Db.Sequences.EditAsync(id, new SequenceEdit { SettingsJson = new ProjectSettings { ExcludedFrames = [1], HeroFrame = 2 }.ToJson() }, null, fx.Ct);
var source = (await fx.Db.Sequences.GetAsync(id))!;
var sidecar = File.ReadAllText(fx.Paths.Sequence(source.Folder).ManifestPath);
var job = await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id));
var made = Assert.NotNull(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, id)));
var copy = (await fx.Db.Sequences.GetAsync(made))!;
Assert.Equal(("Roof (TIFF)", 4, "From the roof"), (copy.Title, copy.Rating, copy.Notes));
Assert.SequenceEqual(["milan", "sunset"], copy.Tags.Order());
var frames = await FramesAsync(fx, made);
Assert.SequenceEqual(originals.Select(f => f.CapturedAt), frames.Select(f => f.CapturedAt));
Assert.True(frames.All(f => f.Format == ImageFileFormat.Tiff));
// Filed as the ingest files every photo: the archive's template in the new sequence, {name} the card's, {ext} the target's.
Assert.SequenceEqual(originals.Select(f => Path.ChangeExtension(f.FileName, ".tif")), frames.Select(f => f.FileName));
Assert.SequenceEqual(originals.Select(f => f.OriginalFileName), frames.Select(f => f.OriginalFileName));
var settings = ProjectSettings.FromJson(copy.SettingsJson!);
Assert.SequenceEqual([1], settings.ExcludedFrames);
Assert.Equal((2, frames[2].Id), (settings.HeroFrame, copy.HeroFrameId));
// The source: not a byte of its sidecar, nor a row, changed; nothing went to the trash.
Assert.Equal(sidecar, File.ReadAllText(fx.Paths.Sequence(source.Folder).ManifestPath));
var now = (await fx.Db.Sequences.GetAsync(id))!;
Assert.Equal((source.FramesVersion, source.SettingsVersion, source.Title, source.Folder), (now.FramesVersion, now.SettingsVersion, now.Title, now.Folder));
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256)), (await FramesAsync(fx, id)).Select(f => (f.Id, f.Sha256)));
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
Assert.False(StagedAnything(fx));
}
[Test]
public async Task APhotoThatFailsLeavesItsWholeSequenceAsItWasAndTheOthersConvert()
{
await using var fx = await FixtureAsync();
var bad = await UploadedAsync(fx, 3);
var good = await UploadedAsync(fx, 2, "upload-2", seed: 300);
var engine = await fx.StartEngineAsync();
var damaged = (await FramesAsync(fx, bad))[1];
var originals = await FramesAsync(fx, bad);
// Rot after the plan: the photo no longer matches the hash it was archived with.
var path = fx.Paths.FromRelative(damaged.RelativePath);
var bytes = File.ReadAllBytes(path);
bytes[^20] ^= 0x10;
File.WriteAllBytes(path, bytes);
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, bad, good);
Assert.Equal(5, plan.Frames);
var job = await RunAsync(fx, engine, plan, JobState.Failed);
Assert.Equal(Loc.Plural("Archive.Error.ConvertKept", 1), job.Error);
Assert.Contains(damaged.FileName, job.LogTail);
// All or nothing: not one frame of that sequence changed, and nothing of it is in the trash.
var rows = await fx.Db.Frames.ListBySequenceAsync(bad);
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256, f.RelativePath)), rows.Select(f => (f.Id, f.Sha256, f.RelativePath)));
Assert.True((await FramesAsync(fx, good)).All(f => f.Format == ImageFileFormat.Tiff));
Assert.SequenceEqual([good], (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.SequenceId));
Assert.False(StagedAnything(fx));
// The journal stays for a retry, which tries the failed sequence again and leaves the converted one.
var journal = Assert.NotNull(ConvertJournal.Read(fx.Paths, job.Id));
Assert.Equal(ConvertOutcome.Converted, journal.Of(good)!.Outcome);
Assert.Equal(ConvertOutcome.Failed, journal.Of(bad)!.Outcome);
}
[Test]
public async Task AResumedJobKeepsTheStagedFilesThatStillVerify()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 3);
var engine = await fx.StartEngineAsync();
// To JPEG needs other originals than the JPEGs uploaded: they become TIFF first.
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id));
var plan = await PlanAsync(fx, ConvertTarget.Jpeg, ConvertMode.Replace, id);
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
var folder = fx.Paths.Sequence(sequence.Folder);
var manifest = Sidecar(fx, sequence);
var jobId = Guid.CreateVersion7();
var staging = Path.Combine(Staging(fx), $"convert-{jobId:N}", $"{id:N}");
var log = StagedLog.Open(ConvertJournal.StagedPathFor(fx.Paths, jobId));
// An earlier attempt staged the first photo (at another quality, so its use shows) and the second, damaged since.
StagedConversion Stage(SidecarFrame frame, int quality)
{
var target = Path.Combine(staging, frame.Id.ToString("N") + ".jpg");
var file = FrameConverter.Convert(new ConvertSource(folder.File(frame.File), frame, manifest.Camera, manifest.Lens), new ConvertOptions(ConvertTarget.Jpeg, quality),
DecoderFactory.CreateServer(), target, cancellationToken: fx.Ct);
var conversion = new StagedConversion(frame.Id, frame.Sha256!, Path.GetFileName(target), file.Size, file.Sha256, file.Width, file.Height);
log.Append(id, manifest.Versions.Frames, conversion);
return conversion;
}
var kept = Stage(manifest.Frames[0], 40);
var torn = Stage(manifest.Frames[1], 95);
File.WriteAllBytes(Path.Combine(staging, torn.File), [1, 2, 3]);
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Convert(fx.Archive, plan), "convert", jobId);
await fx.WaitForAsync(job.Id, JobState.Done);
var frames = await FramesAsync(fx, id);
Assert.Equal(kept.Sha256, frames[0].Sha256);
Assert.Equal(torn.Sha256, frames[1].Sha256);
Assert.True(frames.All(f => f.Format == ImageFileFormat.Jpeg));
}
[Test]
public async Task AJobThatStoppedAfterItsReplacementReportsTheSequenceConverted()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
var engine = await fx.StartEngineAsync();
var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id);
var request = ArchiveJobs.Convert(fx.Archive, plan);
await RunAsync(fx, engine, plan);
// As a crash between the replacement and the journal's note leaves it: the staged log names the conversions done.
var sidecar = Sidecar(fx, (await fx.Db.Sequences.GetAsync(id))!);
var jobId = Guid.CreateVersion7();
var log = StagedLog.Open(ConvertJournal.StagedPathFor(fx.Paths, jobId));
foreach (var frame in sidecar.Frames)
log.Append(id, plan.Sequences.Single().FramesVersion, new StagedConversion(frame.Id, frame.Source!.Sha256!, "gone.dng", frame.Size, frame.Sha256!, frame.W, frame.H));
var job = await fx.EnqueueWithCheckpointAsync(engine, request, "convert", jobId);
var done = await fx.WaitForAsync(job.Id, JobState.Done);
Assert.Contains(Loc.L("Archive.Log.Converted", (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle, 2), done.LogTail);
Assert.Count(1, await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
}
[Test]
public async Task ASequenceThatChangedAfterThePlanIsLeftAsItWas()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
var engine = await fx.StartEngineAsync();
var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id);
// Another photo arrives before the job runs.
var (path, size, sha) = fx.Stage(Start.AddSeconds(60), 999);
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, null, path, "IMG_0099.JPG", size, sha, null, null), fx.Ct);
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
var job = await RunAsync(fx, engine, plan, JobState.Failed);
Assert.Equal(Loc.Plural("Archive.Error.ConvertKept", 1), job.Error);
Assert.Contains(Loc.L("Archive.Log.ConvertChanged", (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle), job.LogTail);
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
}
[Test]
public async Task AVolumeWithoutTheSpaceConvertsNothing()
{
await using var fx = await FixtureAsync(o => o with { FreeSpace = _ => 1_000 });
var id = await UploadedAsync(fx, 2);
var engine = await fx.StartEngineAsync();
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
Assert.False(plan.Space.Enough);
Assert.Equal(1_000, plan.Space.Free);
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
var job = await RunAsync(fx, engine, plan, JobState.Failed);
Assert.True(job.Error!.StartsWith("The conversion needs", StringComparison.Ordinal), job.Error);
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
Assert.False(StagedAnything(fx));
}
[Test]
public async Task ACardKeepsTheCopyOfAPhotoThatWasConverted()
{
await using var fx = await FixtureAsync();
var card = context.CreateTempDirectory("card");
Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, 3);
var media = await fx.Service.Imports.RegisterCardAsync(new CardVolume("3F2A9C11-EOS_DIGITAL-127865454592", 0x3F2A9C11, "EOS_DIGITAL", 127_865_454_592), fx.Ct);
var import = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
await fx.Service.Imports.ExecuteAsync(new ImportJobParameters(fx.Archive.Id, import, media.Id, true, null), Guid.NewGuid(), 0,
_ => ValueTask.CompletedTask, null, fx.Ct);
var id = (await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, import.Sessions.Single().SessionKey))!.Value;
Assert.Equal(3, (await fx.Service.PlanCardEraseAsync(fx.Archive.Id, media.Id, card, null, fx.Ct)).ToDelete.Count());
var engine = await fx.StartEngineAsync();
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id));
// The frames now hold converted files: nothing in the archive matches the card's photos byte for byte, so they stay.
var erase = await fx.Service.PlanCardEraseAsync(fx.Archive.Id, media.Id, card, null, fx.Ct);
Assert.Empty(erase.ToDelete);
Assert.True(erase.Items.All(i => i.Verdict == EraseVerdict.CopyMismatch));
Assert.Equal(0, CardEraser.Execute(erase).Deleted);
Assert.Count(3, Directory.EnumerateFiles(card, "*.JPG", SearchOption.AllDirectories));
}
[Test]
public async Task AReplacedEntryIsErasedOnlyWhileTheConvertedFilesStillHashToTheirRecord()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
var engine = await fx.StartEngineAsync();
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id));
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
var converted = (await FramesAsync(fx, id))[0];
var file = fx.Paths.FromRelative(converted.RelativePath);
var bytes = File.ReadAllBytes(file);
var damaged = (byte[])bytes.Clone();
damaged[^10] ^= 0x01;
File.WriteAllBytes(file, damaged);
var plan = await PlanEmptyAsync(fx);
var kept = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, plan), cancellationToken: fx.Ct);
await fx.WaitForAsync(kept.Id, JobState.Failed);
Assert.Contains(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name));
File.WriteAllBytes(file, bytes);
var erased = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, await PlanEmptyAsync(fx)), cancellationToken: fx.Ct);
await fx.WaitForAsync(erased.Id, JobState.Done);
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
static Task<TrashEmptyPlan> PlanEmptyAsync(ArchiveFixture fx) => fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
}
[Test]
public async Task AReplacementACrashInterruptedBeforeItsSidecarIsUndoneByTheNextWriter()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
var folder = fx.Paths.Sequence(sequence.Folder);
var manifest = Sidecar(fx, sequence);
var frame = manifest.Frames[0];
// Half a replacement: the original in its entry, the converted file in its place, the record pending, the sidecar as it was.
var staged = Path.Combine(Staging(fx), "convert-test", frame.Id.ToString("N") + ".tif");
FrameConverter.Convert(new ConvertSource(folder.File(frame.File), frame), new ConvertOptions(ConvertTarget.Tiff16), DecoderFactory.CreateServer(), staged,
cancellationToken: fx.Ct);
var entryFolder = Path.Combine(FrameTrash.Directory(fx.Paths), $"{id:N}-20261027120000");
var record = entryFolder + ".json";
var trashed = Path.Combine(entryFolder, "originals", Path.GetFileName(frame.File));
var placed = Path.ChangeExtension(frame.File, ".tif");
var journal = new FrameReplaceJournal(manifest.Versions.Frames, fx.Paths.ToRelative(record), fx.Paths.ToRelative(entryFolder),
[new SwapMove(frame.File, fx.Paths.ToRelative(trashed))], [new SwapMove(placed, fx.Paths.ToRelative(staged))], false, null);
journal.Write(folder);
Directory.CreateDirectory(FrameTrash.Directory(fx.Paths));
File.WriteAllText(FrameRemovalJournal.PendingRecord(record), "{}");
Directory.CreateDirectory(Path.GetDirectoryName(trashed)!);
File.Move(folder.File(frame.File), trashed);
File.Move(staged, folder.File(placed));
await fx.Service.EditSequenceInfoAsync(id, info => info with { Rating = 2 }, fx.Ct);
Assert.FileExists(folder.File(frame.File));
Assert.FileMissing(folder.File(placed));
Assert.FileExists(staged);
Assert.False(Directory.Exists(entryFolder));
Assert.FileMissing(FrameRemovalJournal.PendingRecord(record));
Assert.FileMissing(folder.ReplaceJournalPath);
Assert.Equal(frame.Sha256, Sha(folder.File(frame.File)));
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
}
[Test]
public async Task AReplacementACrashInterruptedAfterItsSidecarIsCompletedByTheNextWriter()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
var engine = await fx.StartEngineAsync();
var version = (await fx.Db.Sequences.GetAsync(id))!.FramesVersion;
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id));
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
var (entryFolder, record) = FrameTrash.Location(fx.Paths, entry.Name);
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
var folder = fx.Paths.Sequence(sequence.Folder);
// As a crash right after the sidecar was written leaves it: the record still pending, the journal there.
File.Move(record, FrameRemovalJournal.PendingRecord(record));
new FrameReplaceJournal(version, fx.Paths.ToRelative(record), fx.Paths.ToRelative(entryFolder), [], [], true, null).Write(folder);
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
await fx.Service.EditSequenceInfoAsync(id, info => info with { Rating = 2 }, fx.Ct);
Assert.FileMissing(folder.ReplaceJournalPath);
Assert.Equal(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name);
Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Tiff));
}
[Test]
public async Task APhotoTheServerCannotReadAtFullResolutionIsSkipped()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
// The sidecar says the second photo is a HEIC (a desktop with WIC filed it): the managed decoders have only its preview.
var manifest = Sidecar(fx, sequence);
ManifestStore.WriteSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath,
manifest with { Frames = [manifest.Frames[0], manifest.Frames[1] with { Format = ImageFileFormat.Heif }] });
var step = (await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.NewSequence, id)).Sequences.Single();
Assert.Equal(1, step.Frames.Count);
Assert.Equal((ConvertSkip.NoNativeDecoder, 1), (step.Skipped.Single().Reason, step.Skipped.Single().Frames));
}
[Test]
public async Task TheJournalsSurviveARoundTrip()
{
await using var fx = await FixtureAsync();
var journal = new ConvertJournal { JobId = Guid.NewGuid() };
var first = Guid.NewGuid();
journal.Note(new ConvertSequenceResult(first, "Roof", ConvertOutcome.Created, 3, Guid.NewGuid()));
journal.Note(new ConvertSequenceResult(Guid.NewGuid(), "Harbour", ConvertOutcome.Failed, 0, Error: "no"));
journal.Write(fx.Paths);
var read = Assert.NotNull(ConvertJournal.Read(fx.Paths, journal.JobId));
Assert.SequenceEqual(journal.Sequences.Select(s => (s.Id, s.Outcome, s.Frames, s.NewSequenceId, s.Final)),
read.Sequences.Select(s => (s.Id, s.Outcome, s.Frames, s.NewSequenceId, s.Final)));
Assert.True(read.Of(first)!.Final);
// A line torn by a crash is skipped; a later line for a frame replaces an earlier one.
var path = ConvertJournal.StagedPathFor(fx.Paths, journal.JobId);
var log = StagedLog.Open(path);
var frame = Guid.NewGuid();
log.Append(first, 4, new StagedConversion(frame, "a", "x.dng", 1, "b", 2, 3));
log.Append(first, 4, new StagedConversion(frame, "a", "x.dng", 1, "c", 2, 3));
File.AppendAllText(path, "{\"sequence\":");
var reread = StagedLog.Open(path);
Assert.Equal("c", reread.Of(first, 4).Single().Sha256);
Assert.Empty(reread.Of(first, 5));
ConvertJournal.Delete(fx.Paths, journal.JobId);
Assert.FileMissing(path);
Assert.Null(ConvertJournal.Read(fx.Paths, journal.JobId));
_ = JsonSerializer.Serialize(new ConvertJobParameters(fx.Archive.Id, ConvertTarget.Jpeg, 80, ConvertMode.NewSequence, [new ConvertSequence(first, 2)], "f"),
ArchiveJobJson.Default.ConvertJobParameters);
}
[Test]
public async Task ConvertedFilesAreNamedAsFilingNamesThemAndNothingIsOverwritten()
{
await using var fx = await FixtureAsync();
await fx.Service.UpdateSettingsAsync(fx.Archive.Id, s => s with { Templates = s.Templates with { File = "{name}_{n:00}" } }, fx.Ct);
var roof = await UploadedAsync(fx, 2);
var lake = await UploadedAsync(fx, 2, "upload-2", seed: 200);
Assert.SequenceEqual(["IMG_0001_01.JPG", "IMG_0002_02.JPG"], (await FramesAsync(fx, roof)).Select(f => f.FileName));
await fx.Service.SetSequenceInfoAsync(roof, "Roof", [], 0, null, fx.Ct);
await fx.Service.SetSequenceInfoAsync(lake, "Lake", [], 0, null, fx.Ct);
// Filing names a photo with the template of the moment: a conversion names its files as filing would now.
await fx.Service.UpdateSettingsAsync(fx.Archive.Id, s => s with { Templates = s.Templates with { File = "{title}_{n:00}" } }, fx.Ct);
// A file that is no frame holds the first name: it is never written over, the converted file takes the next.
var stray = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(roof))!.Folder).File("originals/Roof_01.dng");
File.WriteAllBytes(stray, [1, 2, 3]);
var engine = await fx.StartEngineAsync();
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, roof));
var replaced = await FramesAsync(fx, roof);
Assert.SequenceEqual(["Roof_01_2.dng", "Roof_02.dng"], replaced.Select(f => f.FileName));
Assert.SequenceEqual(["IMG_0001.JPG", "IMG_0002.JPG"], replaced.Select(f => f.OriginalFileName));
Assert.SequenceEqual(new byte[] { 1, 2, 3 }, File.ReadAllBytes(stray));
var job = await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, lake));
var made = Assert.NotNull(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, lake)));
var copies = await FramesAsync(fx, made);
Assert.SequenceEqual(["Lake (TIFF)_01.tif", "Lake (TIFF)_02.tif"], copies.Select(f => f.FileName));
Assert.SequenceEqual(["IMG_0001.JPG", "IMG_0002.JPG"], copies.Select(f => f.OriginalFileName));
}
[Test]
public async Task AFolderMovedWhileTheJobWaitsForItsLeaseIsWhereTheConvertedFilesGo()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
var engine = await fx.StartEngineAsync();
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
var row = (await fx.Db.Sequences.GetAsync(id))!;
var from = fx.Paths.Sequence(row.Folder).Folder;
Job job;
var planned = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
void OnProgress(JobProgressInfo info)
{
if (info.Kind == JobKind.Convert && info.Total == 2) planned.TrySetResult();
}
engine.Progress += OnProgress;
using (await LeaseFile.AcquireAsync(fx.Paths.LocksDirectory, $"seq-{id:N}", "editor", TimeSpan.FromSeconds(5), cancellationToken: fx.Ct))
{
job = await engine.EnqueueAsync(ArchiveJobs.Convert(fx.Archive, plan), cancellationToken: fx.Ct);
// Planned again where the index had the sequence then: the job now waits for its lease to convert it.
await planned.Task.WaitAsync(TimeSpan.FromSeconds(TestTiming.Budget(30)), fx.Ct);
// What a Reorganize step does under that lease meanwhile: the folder moves, and its index row follows.
var to = fx.Paths.Sequence("moved/Roof").Folder;
Directory.CreateDirectory(Path.GetDirectoryName(to)!);
Directory.Move(from, to);
Assert.True(await fx.Db.Sequences.RelocateAsync(id, "moved/Roof", null, fx.Ct));
}
engine.Progress -= OnProgress;
await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60));
var frames = await FramesAsync(fx, id);
Assert.True(frames.All(f => f.Format == ImageFileFormat.Tiff && f.RelativePath.StartsWith("moved/Roof/", StringComparison.Ordinal)));
Assert.False(Directory.Exists(from), "the folder the sequence left is never made again");
Assert.Equal("moved/Roof", (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Folder);
}
[Test]
public async Task AReorganizeTheArchiveWaitsForKeepsTheConversionOffItsSequences()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
var originals = await FramesAsync(fx, id);
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
// A Reorganize paused by its user holds the archive: its plan waits for it.
await fx.Db.Jobs.EnqueueAsync(new Job
{
Id = Guid.CreateVersion7(),
Kind = JobKind.Move,
TargetIds = [fx.Archive.Id],
State = JobState.Paused,
Created = DateTimeOffset.UtcNow,
});
Assert.SequenceEqual([ConvertBlock.Busy], (await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id)).Sequences.Single().Blocked);
var engine = await fx.StartEngineAsync();
var job = await RunAsync(fx, engine, plan, JobState.Failed);
Assert.Equal(Loc.Plural("Archive.Error.ConvertKept", 1), job.Error);
Assert.Contains(Loc.L("Archive.Log.ConvertBlocked", (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle), job.LogTail);
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256, f.RelativePath)), (await FramesAsync(fx, id)).Select(f => (f.Id, f.Sha256, f.RelativePath)));
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
}
}