Files
Chronos/tests/Chronos.Data.Tests/Archiving/ConvertTests.cs
T
Alby96 c669d0504d fix(convert): the job converts exactly the photos its preview listed, once
A conversion job could still do something its confirmed dry run did not show,
or refuse what it should have converted:

- An upload landing between the confirmation and the sequence's turn made the
  sequence Changed at every retry. Each ConvertSequence now names the photos
  the plan converts (Photos, their frame ids); the job takes those from its
  re-plan, checks their content against the confirmed digest and leaves photos
  that came in since as they are. A photo the plan did not list is never
  converted, whatever changed on disk or in the index since the dry run.
- A file another program held during the dry run was planned as Missing and
  then converted or not depending on the re-plan. It is now its own reason
  (inaccessible, "held open or cannot be opened"); a truncated file reads as
  undecodable. At job time a confirmed photo that cannot be opened leaves the
  sequence Blocked for the retry; a held photo the job does not convert does
  not hide a real change.
- The same sequences could be converted twice (two dialogs or tabs, a lost
  answer, no dry run at all). MassConverter.QueueAsync checks the queued,
  running and paused conversions and queues under one lock, in the caller's
  context (the desktop's enqueue shows a toast on the UI thread); a job's own
  re-plan treats sequences of conversions queued before it as busy.
- The per-photo space check reserved a full batch of photos even when fewer
  were left, so a job whose preview had the space could stop at every retry.
  It now reserves the photo and those after it that may convert at once.
- New sequence now says in the dry run, the panel's totals and confirmation,
  and a new desktop confirmation, how many photos each new sequence lacks;
  only sequences that are copied use those words.
- On the server, a confirmation takes only the plan kept for the index as it
  is now (503 plan_dropped otherwise, never a plan made inside the request),
  and busy sequences and free space are as they are at every request. The
  panel's Apply asks the dry run again first, with Close still available,
  and confirms only the same fingerprint.
- Maintenance keeps another host's conversion staging as long as its journal
  and purges both together; the unstaged-copy failure and blocked sequences
  are logged in catalog sentences.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit fefd9c34ebfeed1f9e84161610ef286026d38355)
2026-11-02 10:33:34 +01:00

1359 lines
82 KiB
C#

using System.Runtime.Versioning;
using System.Security.Cryptography;
using System.Text.Json;
using Chronos.Core.Archiving;
using Chronos.Core.Conversion;
using Chronos.Core.Import;
using Chronos.Core.Imaging;
using Chronos.Core.Imaging.Decoders;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Data.Archiving;
using Chronos.Data.Archiving.Jobs;
using Chronos.Data.Housekeeping;
using Chronos.Data.Jobs;
using Chronos.Data.Repositories;
using Chronos.Testing;
using static Chronos.Data.Tests.Archiving.ArchiveFixture;
namespace Chronos.Data.Tests.Archiving;
/// <summary>
/// Mass conversion after import (owner review 2, R7): the dry run, the <c>convert</c> job in both modes, its undo through the
/// trash, all or nothing per sequence, resumes in stage A and B, plans gone stale, the space check, and the gates that must
/// keep holding (the card erase, the emptying of a replaced entry).
/// </summary>
[Category("archive")]
public sealed class ConvertTests(TestContext context)
{
private Task<ArchiveFixture> FixtureAsync(Func<ArchiveServiceOptions, ArchiveServiceOptions>? more = null) =>
CreateAsync(context, configure: o =>
{
var options = o with { Decoder = DecoderFactory.CreateServer(), Parallelism = () => 2 };
return more is null ? options : more(options);
});
/// <summary>A sequence of <paramref name="count"/> uploaded photos (hashed, under <c>originals/</c>), ten seconds apart, by <paramref name="user"/>.</summary>
private static async Task<Guid> UploadedAsync(ArchiveFixture fx, int count, string session = "upload-1", int seed = 100, Guid? user = null)
{
Guid? id = null;
for (var i = 0; i < count; i++)
{
var (path, size, sha) = fx.Stage(Start.AddSeconds(10 * i).AddTicks(1_234_567), seed + i);
var result = await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, id is null ? session : null, path, $"IMG_{i + 1:0000}.JPG",
size, sha, null, user), fx.Ct);
id = result.SequenceId;
}
return id!.Value;
}
private static Task<ConvertPlan> PlanAsync(ArchiveFixture fx, ConvertTarget target, ConvertMode mode, params Guid[] ids) =>
fx.Service.PlanConvertAsync(fx.Archive.Id, ids, new ConvertOptions(target), mode, fx.Ct);
/// <summary>The parameters the job of <paramref name="plan"/> runs with.</summary>
private static ConvertJobParameters Parameters(ArchiveFixture fx, ConvertPlan plan) =>
JsonSerializer.Deserialize(ArchiveJobs.Convert(fx.Archive, plan).ParametersJson!, ArchiveJobJson.Default.ConvertJobParameters)!;
/// <summary>Callbacks that report nothing, but for <paramref name="progress"/> and <paramref name="done"/> when given.</summary>
private static ConvertCallbacks Callbacks(Action<long, long>? progress = null, Action<ConvertSequenceResult>? done = null,
Action<IReadOnlyList<ArchiveJobRequest>>? queue = null) => new(
(n, total) =>
{
progress?.Invoke(n, total);
return ValueTask.CompletedTask;
},
(_, _) => ValueTask.CompletedTask,
_ => { },
s =>
{
done?.Invoke(s);
return ValueTask.CompletedTask;
},
requests =>
{
queue?.Invoke(requests);
return ValueTask.CompletedTask;
});
/// <summary>The conversion job <paramref name="jobId"/> of <paramref name="plan"/>, run here as its handler runs it.</summary>
private static Task<ConvertResult> RunHereAsync(ArchiveFixture fx, ConvertPlan plan, Guid jobId, ConvertCallbacks? callbacks = null) =>
fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), jobId, new ConvertRequester(null, null), callbacks ?? Callbacks(), fx.Ct);
private static string StagingOf(ArchiveFixture fx, Guid job, Guid sequence) => Path.Combine(Staging(fx), $"convert-{job:N}", $"{sequence:N}");
private static async Task<Job> RunAsync(ArchiveFixture fx, JobEngine engine, ConvertPlan plan, JobState state = JobState.Done)
{
var job = await engine.EnqueueAsync(ArchiveJobs.Convert(fx.Archive, plan), cancellationToken: fx.Ct);
return await fx.WaitForAsync(job.Id, state, (int)TestTiming.Budget(60));
}
private static SequenceManifest Sidecar(ArchiveFixture fx, Sequence sequence) => ManifestStore.ReadSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath);
private static string Sha(string path) => Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(path)));
/// <summary>The index rows of a sequence, after checking that each file holds its row's content and that the sidecar agrees.</summary>
private static async Task<List<Frame>> FramesAsync(ArchiveFixture fx, Guid id)
{
var sequence = Assert.NotNull(await fx.Db.Sequences.GetAsync(id));
var rows = (await fx.Db.Frames.ListBySequenceAsync(id)).ToList();
var sidecar = Sidecar(fx, sequence);
Assert.Equal(sidecar.Versions.Frames, sequence.FramesVersion);
Assert.SequenceEqual(sidecar.Frames.Select(f => (f.Id, f.I, f.Sha256)), rows.Select(r => (r.Id, r.Index, r.Sha256)));
foreach (var row in rows) Assert.Equal(row.Sha256, Sha(fx.Paths.FromRelative(row.RelativePath)));
Assert.FileMissing(fx.Paths.Sequence(sequence.Folder).ReplaceJournalPath);
return rows;
}
private static string Staging(ArchiveFixture fx) => fx.Paths.StagingDirectory;
private static bool StagedAnything(ArchiveFixture fx) =>
Directory.Exists(Staging(fx)) && Directory.EnumerateDirectories(Staging(fx), "convert-*").Any();
/// <summary>
/// Makes <paramref name="path"/> unreadable until disposed, as another program holding it does: without read permission
/// where that stops the test process (UnauthorizedAccessException), otherwise (Windows, root) held open without sharing
/// (IOException).
/// </summary>
private static IDisposable Unreadable(string path)
{
if (!OperatingSystem.IsWindows())
{
var mode = File.GetUnixFileMode(path);
File.SetUnixFileMode(path, UnixFileMode.None);
try
{
File.OpenRead(path).Dispose();
File.SetUnixFileMode(path, mode);
}
catch (UnauthorizedAccessException)
{
return new UnixPermissions(path, mode);
}
}
return new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.None);
}
/// <summary>Gives a file its permissions back.</summary>
[UnsupportedOSPlatform("windows")]
private sealed class UnixPermissions(string path, UnixFileMode mode) : IDisposable
{
public void Dispose() => File.SetUnixFileMode(path, mode);
}
/// <summary>A queued conversion job of <paramref name="plan"/>, as the engine would hold it, created at <paramref name="created"/>.</summary>
private static async Task<Job> QueuedAsync(ArchiveFixture fx, ConvertPlan plan, DateTimeOffset created)
{
var request = ArchiveJobs.Convert(fx.Archive, plan);
var job = new Job
{
Id = Guid.CreateVersion7(),
Kind = request.Kind,
TargetIds = request.Targets,
ParametersJson = request.ParametersJson,
State = JobState.Queued,
Created = created,
};
await fx.Db.Jobs.EnqueueAsync(job, fx.Ct);
return job;
}
[Test]
public async Task TheDryRunCountsWhatConvertsAndChangesNothing()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 3);
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id);
var step = plan.Sequences.Single();
Assert.Equal((3, 0, true), (step.Frames.Count, step.SkippedFrames, step.Converts));
Assert.Equal((await fx.Db.Sequences.GetAsync(id))!.FramesVersion, step.FramesVersion);
Assert.Equal(step.Frames.Sum(f => f.Size), plan.BytesBefore);
Assert.Equal(3 * ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.LinearDng), 48, 32), plan.BytesAfter);
Assert.Equal(ConvertPlanner.Required(plan.BytesAfter), plan.Space.Required);
Assert.True(plan.Space.Enough);
Assert.Equal(plan.Fingerprint(), (await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id)).Fingerprint());
Assert.NotEqual(plan.Fingerprint(), (await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.NewSequence, id)).Fingerprint());
// JPEG to JPEG converts nothing.
var same = (await PlanAsync(fx, ConvertTarget.Jpeg, ConvertMode.Replace, id)).Sequences.Single();
Assert.Equal((0, ConvertSkip.SameFormat, 3), (same.Frames.Count, same.Skipped.Single().Reason, same.Skipped.Single().Frames));
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
Assert.False(StagedAnything(fx));
await Assert.ThrowsAsync<ArgumentException>(() => PlanAsync(fx, ConvertTarget.Jpeg, ConvertMode.Replace, Guid.NewGuid()));
}
[Test]
public async Task ASequenceBusyOrUnreadableIsBlocked()
{
await using var fx = await FixtureAsync();
var busy = await UploadedAsync(fx, 2);
var unreadable = await UploadedAsync(fx, 2, "upload-2", seed: 200);
await fx.Db.Jobs.EnqueueAsync(new Job
{
Id = Guid.CreateVersion7(),
Kind = JobKind.Render,
TargetIds = [busy],
State = JobState.Running,
NodeId = "test#0",
Created = DateTimeOffset.UtcNow,
Started = DateTimeOffset.UtcNow,
});
File.WriteAllText(fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(unreadable))!.Folder).ManifestPath, "{ not json");
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, busy, unreadable);
Assert.SequenceEqual([ConvertBlock.Busy], plan.Sequences.Single(s => s.SequenceId == busy).Blocked);
Assert.SequenceEqual([ConvertBlock.Unreadable], plan.Sequences.Single(s => s.SequenceId == unreadable).Blocked);
Assert.Equal(0, plan.Frames);
Assert.Throws<ArgumentException>(() => ArchiveJobs.Convert(fx.Archive, plan));
}
[Test]
public async Task AReplaceKeepsTheFramesAndTrashesTheOriginalsWithTheirReplacements()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 3);
var engine = await fx.StartEngineAsync();
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
var originals = await FramesAsync(fx, id);
var project = new ProjectSettings { ExcludedFrames = [1], HeroFrame = 2 }.ToJson();
await fx.Db.Sequences.EditAsync(id, new SequenceEdit { SettingsJson = project, HeroFrameId = originals[2].Id }, null, fx.Ct);
var folder = fx.Paths.Sequence(sequence.Folder);
Directory.CreateDirectory(folder.MetaDirectory);
File.WriteAllText(folder.AnalysisPath, "{\"frames\":3}");
var before = Sidecar(fx, sequence);
var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id);
var job = await RunAsync(fx, engine, plan);
Assert.Contains(Loc.L("Archive.Log.Converted", sequence.DisplayTitle, 3), job.LogTail);
var row = (await fx.Db.Sequences.GetAsync(id))!;
var frames = await FramesAsync(fx, id);
Assert.Equal(before.Versions.Frames + 1, row.FramesVersion);
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Index)), frames.Select(f => (f.Id, f.Index)));
var after = Sidecar(fx, row);
foreach (var (was, now) in before.Frames.Zip(after.Frames))
{
// Named as filing named the original, with the target's extension: the same template, frame and sequence.
Assert.Equal((Path.ChangeExtension(was.File, ".dng"), ImageFileFormat.Dng, DecodeTier.Native, 1), (now.File, now.Format, now.Tier, now.Orientation));
Assert.Equal((was.CapturedAt, was.Original, was.Flags), (now.CapturedAt, now.Original, now.Flags));
var source = Assert.NotNull(now.Source);
Assert.Equal((was.Sha256, was.Format, was.File, was.Size), (source.Sha256, source.Format, source.File, source.Size));
Assert.FileMissing(folder.File(was.File));
}
// The project's positions stay on their frames.
var settings = ProjectSettings.FromJson(row.SettingsJson!);
Assert.SequenceEqual([1], settings.ExcludedFrames);
Assert.Equal((2, originals[2].Id), (settings.HeroFrame, row.HeroFrameId));
// One replaced entry with the originals, each naming its replacement, and the analysis as it was.
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
Assert.Equal((TrashKind.Replaced, id, 3), (entry.Kind, entry.SequenceId, entry.Frames));
var record = Assert.NotNull(FrameTrash.ReadEntry(fx.Paths, entry.Name));
var entryFolder = FrameTrash.Location(fx.Paths, entry.Name).Folder;
foreach (var photo in record.Photos)
{
var now = after.Frames.Single(f => f.Id == photo.Frame.Id);
Assert.Equal(now.Sha256, photo.Replacement);
Assert.Equal(photo.Frame.Sha256, Sha(Path.Combine(entryFolder, photo.Files[0].Path)));
}
Assert.Equal("{\"frames\":3}", File.ReadAllText(Path.Combine(entryFolder, FrameTrash.AnalysisCopy)));
// Tombstones alias the old content to the new, in the sidecar and the index.
var tombstones = Assert.NotNull(after.Removed);
Assert.SequenceEqual(before.Frames.Select(f => (f.Sha256, after.Frames.Single(n => n.Id == f.Id).Sha256, RemovalReason.Replaced)).Order(),
tombstones.Select(t => ((string?)t.Sha256, t.Replacement, t.Reason)).Order());
Assert.SequenceEqual(tombstones.Select(t => (t.Sha256, t.Replacement)).Order(),
(await fx.Db.FrameRemovals.ListBySequenceAsync(id)).Select(r => (r.Sha256, r.ReplacementSha256)).Order());
Assert.Equal((SequenceStatus)0, row.Status & (SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt));
Assert.False(StagedAnything(fx));
Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, job.Id));
Assert.FileMissing(ConvertJournal.StagedPathFor(fx.Paths, job.Id));
// Converted again, nothing is left to convert.
Assert.Equal(0, (await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id)).Frames);
}
[Test]
public async Task RestoringTheReplacedEntrySwapsTheOriginalsBack()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 3);
var engine = await fx.StartEngineAsync();
var originals = await FramesAsync(fx, id);
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
var before = Sidecar(fx, sequence);
var analysis = fx.Paths.Sequence(sequence.Folder).AnalysisPath;
Directory.CreateDirectory(Path.GetDirectoryName(analysis)!);
File.WriteAllText(analysis, "of the originals");
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id));
var converted = Sidecar(fx, (await fx.Db.Sequences.GetAsync(id))!);
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
// The light analysis of the converted frames, as it would be written again.
File.WriteAllText(analysis, "of the conversions");
var result = await fx.Service.RestoreEntryAsync(fx.Archive.Id, entry.Name, null, fx.Ct);
Assert.Equal((TrashKind.Replaced, 3, false, id), (result.Kind, result.Photos, result.NewSequence, result.Sequence.Id));
var frames = await FramesAsync(fx, id);
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Index, f.Sha256, f.RelativePath, f.Format)), frames.Select(f => (f.Id, f.Index, f.Sha256, f.RelativePath, f.Format)));
var back = Sidecar(fx, (await fx.Db.Sequences.GetAsync(id))!);
Assert.True(back.Frames.All(f => f.Source is null));
// The converted files are in the trash in turn, naming the originals as their replacements; the old entry is gone.
var turned = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
Assert.NotEqual(entry.Name, turned.Name);
Assert.Equal(TrashKind.Replaced, turned.Kind);
var record = FrameTrash.ReadEntry(fx.Paths, turned.Name)!;
Assert.SequenceEqual(converted.Frames.Select(f => (f.Sha256, before.Frames.Single(o => o.Id == f.Id).Sha256)).Order(),
record.Photos.Select(p => (p.Frame.Sha256, p.Replacement)).Order());
// Each analysis goes with its frames: the originals' back in the sequence, the conversions' into the new entry.
Assert.Equal("of the originals", File.ReadAllText(analysis));
Assert.Equal("of the conversions", File.ReadAllText(Path.Combine(FrameTrash.Location(fx.Paths, turned.Name).Folder, FrameTrash.AnalysisCopy)));
Assert.False(Directory.Exists(FrameTrash.Location(fx.Paths, entry.Name).Folder));
Assert.SequenceEqual(converted.Frames.Select(f => (f.Sha256, before.Frames.Single(o => o.Id == f.Id).Sha256)).Order(),
back.Removed!.Select(t => ((string?)t.Sha256, t.Replacement)).Order());
// That entry erases: its replacements (the originals) are where it says, re-hashed now.
var empty = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
var job = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, empty), cancellationToken: fx.Ct);
await fx.WaitForAsync(job.Id, JobState.Done);
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
await FramesAsync(fx, id);
}
[Test]
public async Task AnUndoAfterTheFramesChangedBringsTheOriginalsBackAsANewSequence()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 3);
var engine = await fx.StartEngineAsync();
var originals = await FramesAsync(fx, id);
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id));
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
var converted = await FramesAsync(fx, id);
// A converted frame deleted since: the frames are no longer the ones the conversion left.
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
var delete = await fx.Service.FrameTrash.PlanDeleteAsync(id, [converted[1].Id], sequence.FramesVersion, fx.Ct);
await fx.Service.FrameTrash.DeleteAsync(delete, "alberto", null, fx.Ct);
var result = await fx.Service.RestoreEntryAsync(fx.Archive.Id, entry.Name, null, fx.Ct);
Assert.Equal((TrashKind.Replaced, 3, true), (result.Kind, result.Photos, result.NewSequence));
Assert.NotEqual(id, result.Sequence.Id);
Assert.SequenceEqual(originals.Select(f => f.Sha256), (await FramesAsync(fx, result.Sequence.Id)).Select(f => f.Sha256));
Assert.Count(2, await FramesAsync(fx, id));
// The replaced entry is gone with its last photo (and its copy of the analysis); the deleted frame's entry stays.
Assert.DoesNotContain(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name));
}
[Test]
public async Task ANewSequenceTakesTheCopiesAndLeavesTheSourceAsItIs()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 3);
var engine = await fx.StartEngineAsync();
await fx.Service.SetSequenceInfoAsync(id, "Roof", ["sunset", "milan"], 4, "From the roof", fx.Ct);
var originals = await FramesAsync(fx, id);
await fx.Db.Sequences.EditAsync(id, new SequenceEdit { SettingsJson = new ProjectSettings { ExcludedFrames = [1], HeroFrame = 2 }.ToJson() }, null, fx.Ct);
var source = (await fx.Db.Sequences.GetAsync(id))!;
var sidecar = File.ReadAllText(fx.Paths.Sequence(source.Folder).ManifestPath);
var job = await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id));
var made = Assert.NotNull(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, id)));
var copy = (await fx.Db.Sequences.GetAsync(made))!;
Assert.Equal(("Roof (TIFF)", 4, "From the roof"), (copy.Title, copy.Rating, copy.Notes));
Assert.SequenceEqual(["milan", "sunset"], copy.Tags.Order());
var frames = await FramesAsync(fx, made);
Assert.SequenceEqual(originals.Select(f => f.CapturedAt), frames.Select(f => f.CapturedAt));
Assert.True(frames.All(f => f.Format == ImageFileFormat.Tiff));
// Filed as the ingest files every photo: the archive's template in the new sequence, {name} the card's, {ext} the target's.
Assert.SequenceEqual(originals.Select(f => Path.ChangeExtension(f.FileName, ".tif")), frames.Select(f => f.FileName));
Assert.SequenceEqual(originals.Select(f => f.OriginalFileName), frames.Select(f => f.OriginalFileName));
var settings = ProjectSettings.FromJson(copy.SettingsJson!);
Assert.SequenceEqual([1], settings.ExcludedFrames);
Assert.Equal((2, frames[2].Id), (settings.HeroFrame, copy.HeroFrameId));
// The source: not a byte of its sidecar, nor a row, changed; nothing went to the trash.
Assert.Equal(sidecar, File.ReadAllText(fx.Paths.Sequence(source.Folder).ManifestPath));
var now = (await fx.Db.Sequences.GetAsync(id))!;
Assert.Equal((source.FramesVersion, source.SettingsVersion, source.Title, source.Folder), (now.FramesVersion, now.SettingsVersion, now.Title, now.Folder));
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256)), (await FramesAsync(fx, id)).Select(f => (f.Id, f.Sha256)));
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
Assert.False(StagedAnything(fx));
}
[Test]
public async Task APhotoThatFailsLeavesItsWholeSequenceAsItWasAndTheOthersConvert()
{
await using var fx = await FixtureAsync();
var bad = await UploadedAsync(fx, 3);
var good = await UploadedAsync(fx, 2, "upload-2", seed: 300);
var engine = await fx.StartEngineAsync();
var damaged = (await FramesAsync(fx, bad))[1];
var originals = await FramesAsync(fx, bad);
// Rot after the plan: the photo no longer matches the hash it was archived with.
var path = fx.Paths.FromRelative(damaged.RelativePath);
var bytes = File.ReadAllBytes(path);
bytes[^20] ^= 0x10;
File.WriteAllBytes(path, bytes);
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, bad, good);
Assert.Equal(5, plan.Frames);
var job = await RunAsync(fx, engine, plan, JobState.Failed);
Assert.Equal(Loc.Plural("Archive.Error.ConvertKept", 1), job.Error);
Assert.Contains(damaged.FileName, job.LogTail);
// The log says why in one catalog sentence; the exception's English text stays in the diagnostics log.
Assert.Contains(Loc.L("Archive.Convert.Failed.Changed"), job.LogTail);
Assert.False(job.LogTail!.Contains("SHA-256", StringComparison.Ordinal), job.LogTail);
// All or nothing: not one frame of that sequence changed, and nothing of it is in the trash.
var rows = await fx.Db.Frames.ListBySequenceAsync(bad);
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256, f.RelativePath)), rows.Select(f => (f.Id, f.Sha256, f.RelativePath)));
Assert.True((await FramesAsync(fx, good)).All(f => f.Format == ImageFileFormat.Tiff));
Assert.SequenceEqual([good], (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.SequenceId));
// The journal stays for a retry, which tries the failed sequence again and leaves the converted one; so does what the
// failed sequence staged before its photo failed, and only that.
var journal = Assert.NotNull(ConvertJournal.Read(fx.Paths, job.Id));
Assert.Equal(ConvertOutcome.Converted, journal.Of(good)!.Outcome);
Assert.Equal(ConvertOutcome.Failed, journal.Of(bad)!.Outcome);
Assert.False(Directory.Exists(StagingOf(fx, job.Id, good)));
var noted = StagedLog.Open(ConvertJournal.StagedPathFor(fx.Paths, job.Id)).Of(bad, (await fx.Db.Sequences.GetAsync(bad))!.FramesVersion);
Assert.SequenceEqual(noted.Select(c => c.File).Order(), Directory.EnumerateFiles(StagingOf(fx, job.Id, bad)).Select(f => Path.GetFileName(f)).Order());
Assert.DoesNotContain(damaged.Id, noted.Select(c => c.FrameId));
var first = Path.Combine(StagingOf(fx, job.Id, bad), noted.Single(c => c.FrameId == originals[0].Id).File);
File.SetLastWriteTimeUtc(first, new DateTime(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc));
// The photo is put right: the retry takes the staged conversion that still verifies and converts the rest.
bytes[^20] ^= 0x10;
File.WriteAllBytes(path, bytes);
await engine.ResumeAsync(job.Id, fx.Ct);
await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60));
var converted = await FramesAsync(fx, bad);
Assert.True(converted.All(f => f.Format == ImageFileFormat.Tiff));
Assert.Equal(2020, File.GetLastWriteTimeUtc(fx.Paths.FromRelative(converted[0].RelativePath)).Year);
Assert.False(StagedAnything(fx));
Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, job.Id));
}
[Test]
public async Task AResumedJobKeepsTheStagedFilesThatStillVerify()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 3);
var engine = await fx.StartEngineAsync();
// To JPEG needs other originals than the JPEGs uploaded: they become TIFF first.
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id));
var plan = await PlanAsync(fx, ConvertTarget.Jpeg, ConvertMode.Replace, id);
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
var folder = fx.Paths.Sequence(sequence.Folder);
var manifest = Sidecar(fx, sequence);
var jobId = Guid.CreateVersion7();
var staging = Path.Combine(Staging(fx), $"convert-{jobId:N}", $"{id:N}");
var log = StagedLog.Open(ConvertJournal.StagedPathFor(fx.Paths, jobId));
// An earlier attempt staged the first photo (at another quality, so its use shows) and the second, damaged since.
StagedConversion Stage(SidecarFrame frame, int quality)
{
var target = Path.Combine(staging, frame.Id.ToString("N") + ".jpg");
var file = FrameConverter.Convert(new ConvertSource(folder.File(frame.File), frame, manifest.Camera, manifest.Lens), new ConvertOptions(ConvertTarget.Jpeg, quality),
DecoderFactory.CreateServer(), target, cancellationToken: fx.Ct);
var conversion = new StagedConversion(frame.Id, frame.Sha256!, Path.GetFileName(target), file.Size, file.Sha256, file.Width, file.Height);
log.Append(id, manifest.Versions.Frames, conversion);
return conversion;
}
var kept = Stage(manifest.Frames[0], 40);
var torn = Stage(manifest.Frames[1], 95);
File.WriteAllBytes(Path.Combine(staging, torn.File), [1, 2, 3]);
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Convert(fx.Archive, plan), "convert", jobId);
await fx.WaitForAsync(job.Id, JobState.Done);
var frames = await FramesAsync(fx, id);
Assert.Equal(kept.Sha256, frames[0].Sha256);
Assert.Equal(torn.Sha256, frames[1].Sha256);
Assert.True(frames.All(f => f.Format == ImageFileFormat.Jpeg));
}
[Test]
public async Task AJobThatStoppedAfterItsReplacementReportsTheSequenceConverted()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
var engine = await fx.StartEngineAsync();
var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id);
var request = ArchiveJobs.Convert(fx.Archive, plan);
await RunAsync(fx, engine, plan);
// As a crash between the replacement and the journal's note leaves it: the staged log names the conversions done.
var sidecar = Sidecar(fx, (await fx.Db.Sequences.GetAsync(id))!);
var jobId = Guid.CreateVersion7();
var log = StagedLog.Open(ConvertJournal.StagedPathFor(fx.Paths, jobId));
foreach (var frame in sidecar.Frames)
log.Append(id, plan.Sequences.Single().FramesVersion, new StagedConversion(frame.Id, frame.Source!.Sha256!, "gone.dng", frame.Size, frame.Sha256!, frame.W, frame.H));
var job = await fx.EnqueueWithCheckpointAsync(engine, request, "convert", jobId);
var done = await fx.WaitForAsync(job.Id, JobState.Done);
Assert.Contains(Loc.L("Archive.Log.Converted", (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle, 2), done.LogTail);
Assert.Count(1, await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
}
[Test]
public async Task ASequenceThatChangedAfterThePlanIsLeftAsItWas()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 3);
var engine = await fx.StartEngineAsync();
var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id);
// A photo of the plan is deleted before the job runs.
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
var delete = await fx.Service.FrameTrash.PlanDeleteAsync(id, [(await FramesAsync(fx, id))[1].Id], sequence.FramesVersion, fx.Ct);
await fx.Service.FrameTrash.DeleteAsync(delete, "alberto", null, fx.Ct);
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
var job = await RunAsync(fx, engine, plan, JobState.Failed);
Assert.Equal(Loc.Plural("Archive.Error.ConvertKept", 1), job.Error);
var changed = Loc.L("Archive.Log.ConvertChanged", sequence.DisplayTitle);
Assert.Contains(changed, job.LogTail);
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
Assert.DoesNotContain(TrashKind.Replaced, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Kind));
// The refusal holds on a retry too, and says so: the user plans again.
await engine.ResumeAsync(job.Id, fx.Ct);
Assert.Contains(changed, (await fx.WaitForAsync(job.Id, JobState.Failed)).LogTail);
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
}
[Test]
public async Task PhotosThatComeInWhileTheJobWaitsItsTurnStayAsTheyAreAndThoseConfirmedConvert()
{
foreach (var mode in new[] { ConvertMode.Replace, ConvertMode.NewSequence })
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
var engine = await fx.StartEngineAsync();
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, mode, id);
var title = (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle;
// An upload lands after the confirmation, before the job comes to the sequence (it waited behind a render, or
// behind other sequences).
var (path, size, sha) = fx.Stage(Start.AddSeconds(60), 999);
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, null, path, "IMG_0099.JPG", size, sha, null, null), fx.Ct);
var job = await RunAsync(fx, engine, plan);
var frames = await FramesAsync(fx, id);
Assert.Equal((sha, ImageFileFormat.Jpeg), frames.Where(f => f.Sha256 == sha).Select(f => (f.Sha256, f.Format)).Single(), mode.ToString());
if (mode == ConvertMode.Replace)
{
Assert.Contains(Loc.L("Archive.Log.Converted", title, 2), job.LogTail);
Assert.Equal(2, frames.Count(f => f.Format == ImageFileFormat.Tiff));
Assert.Equal(2, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Frames);
}
else
{
Assert.Contains(Loc.L("Archive.Log.ConvertedNew", title, 2), job.LogTail);
Assert.Contains(Loc.L("Archive.Log.ConvertLeftOut", title, 1), job.LogTail);
var made = Assert.NotNull(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, id)));
Assert.True((await FramesAsync(fx, made)) is { Count: 2 } copies && copies.All(f => f.Format == ImageFileFormat.Tiff));
Assert.True(frames.All(f => f.Format == ImageFileFormat.Jpeg));
}
}
}
[Test]
public async Task AVolumeWithoutTheSpaceConvertsNothing()
{
await using var fx = await FixtureAsync(o => o with { FreeSpace = _ => 1_000 });
var id = await UploadedAsync(fx, 2);
var engine = await fx.StartEngineAsync();
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
Assert.False(plan.Space.Enough);
Assert.Equal(1_000, plan.Space.Free);
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
var job = await RunAsync(fx, engine, plan, JobState.Failed);
Assert.True(job.Error!.StartsWith("The conversion needs", StringComparison.Ordinal), job.Error);
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
Assert.False(StagedAnything(fx));
}
[Test]
public async Task ACardKeepsTheCopyOfAPhotoThatWasConverted()
{
await using var fx = await FixtureAsync();
var card = context.CreateTempDirectory("card");
Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, 3);
var media = await fx.Service.Imports.RegisterCardAsync(new CardVolume("3F2A9C11-EOS_DIGITAL-127865454592", 0x3F2A9C11, "EOS_DIGITAL", 127_865_454_592), fx.Ct);
var import = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
await fx.Service.Imports.ExecuteAsync(new ImportJobParameters(fx.Archive.Id, import, media.Id, true, null), Guid.NewGuid(), 0,
_ => ValueTask.CompletedTask, null, fx.Ct);
var id = (await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, import.Sessions.Single().SessionKey))!.Value;
Assert.Equal(3, (await fx.Service.PlanCardEraseAsync(fx.Archive.Id, media.Id, card, null, fx.Ct)).ToDelete.Count());
var engine = await fx.StartEngineAsync();
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id));
// The frames now hold converted files: nothing in the archive matches the card's photos byte for byte, so they stay.
var erase = await fx.Service.PlanCardEraseAsync(fx.Archive.Id, media.Id, card, null, fx.Ct);
Assert.Empty(erase.ToDelete);
Assert.True(erase.Items.All(i => i.Verdict == EraseVerdict.CopyMismatch));
Assert.Equal(0, CardEraser.Execute(erase).Deleted);
Assert.Count(3, Directory.EnumerateFiles(card, "*.JPG", SearchOption.AllDirectories));
}
[Test]
public async Task AReplacedEntryIsErasedOnlyWhileTheConvertedFilesStillHashToTheirRecord()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
var engine = await fx.StartEngineAsync();
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id));
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
var converted = (await FramesAsync(fx, id))[0];
var file = fx.Paths.FromRelative(converted.RelativePath);
var bytes = File.ReadAllBytes(file);
var damaged = (byte[])bytes.Clone();
damaged[^10] ^= 0x01;
File.WriteAllBytes(file, damaged);
var plan = await PlanEmptyAsync(fx);
var kept = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, plan), cancellationToken: fx.Ct);
await fx.WaitForAsync(kept.Id, JobState.Failed);
Assert.Contains(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name));
File.WriteAllBytes(file, bytes);
var erased = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, await PlanEmptyAsync(fx)), cancellationToken: fx.Ct);
await fx.WaitForAsync(erased.Id, JobState.Done);
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
static Task<TrashEmptyPlan> PlanEmptyAsync(ArchiveFixture fx) => fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
}
[Test]
public async Task AReplacementACrashInterruptedBeforeItsSidecarIsUndoneByTheNextWriter()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
var folder = fx.Paths.Sequence(sequence.Folder);
var manifest = Sidecar(fx, sequence);
var frame = manifest.Frames[0];
// Half a replacement: the original in its entry, the converted file in its place, the record pending, the sidecar as it was.
var staged = Path.Combine(Staging(fx), "convert-test", frame.Id.ToString("N") + ".tif");
FrameConverter.Convert(new ConvertSource(folder.File(frame.File), frame), new ConvertOptions(ConvertTarget.Tiff16), DecoderFactory.CreateServer(), staged,
cancellationToken: fx.Ct);
var entryFolder = Path.Combine(FrameTrash.Directory(fx.Paths), $"{id:N}-20261027120000");
var record = entryFolder + ".json";
var trashed = Path.Combine(entryFolder, "originals", Path.GetFileName(frame.File));
var placed = Path.ChangeExtension(frame.File, ".tif");
var journal = new FrameReplaceJournal(manifest.Versions.Frames, fx.Paths.ToRelative(record), fx.Paths.ToRelative(entryFolder),
[new SwapMove(frame.File, fx.Paths.ToRelative(trashed))], [new SwapMove(placed, fx.Paths.ToRelative(staged))], false, null);
journal.Write(folder);
Directory.CreateDirectory(FrameTrash.Directory(fx.Paths));
File.WriteAllText(FrameRemovalJournal.PendingRecord(record), "{}");
Directory.CreateDirectory(Path.GetDirectoryName(trashed)!);
File.Move(folder.File(frame.File), trashed);
File.Move(staged, folder.File(placed));
await fx.Service.EditSequenceInfoAsync(id, info => info with { Rating = 2 }, fx.Ct);
Assert.FileExists(folder.File(frame.File));
Assert.FileMissing(folder.File(placed));
Assert.FileExists(staged);
Assert.False(Directory.Exists(entryFolder));
Assert.FileMissing(FrameRemovalJournal.PendingRecord(record));
Assert.FileMissing(folder.ReplaceJournalPath);
Assert.Equal(frame.Sha256, Sha(folder.File(frame.File)));
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
}
[Test]
public async Task AReplacementACrashInterruptedAfterItsSidecarIsCompletedByTheNextWriter()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
var engine = await fx.StartEngineAsync();
var version = (await fx.Db.Sequences.GetAsync(id))!.FramesVersion;
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id));
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
var (entryFolder, record) = FrameTrash.Location(fx.Paths, entry.Name);
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
var folder = fx.Paths.Sequence(sequence.Folder);
// As a crash right after the sidecar was written leaves it: the record still pending, the journal there.
File.Move(record, FrameRemovalJournal.PendingRecord(record));
new FrameReplaceJournal(version, fx.Paths.ToRelative(record), fx.Paths.ToRelative(entryFolder), [], [], true, null).Write(folder);
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
await fx.Service.EditSequenceInfoAsync(id, info => info with { Rating = 2 }, fx.Ct);
Assert.FileMissing(folder.ReplaceJournalPath);
Assert.Equal(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name);
Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Tiff));
}
[Test]
public async Task APhotoTheServerCannotReadAtFullResolutionIsSkipped()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
// The sidecar says the second photo is a HEIC (a desktop with WIC filed it): the managed decoders have only its preview.
var manifest = Sidecar(fx, sequence);
ManifestStore.WriteSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath,
manifest with { Frames = [manifest.Frames[0], manifest.Frames[1] with { Format = ImageFileFormat.Heif }] });
var step = (await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.NewSequence, id)).Sequences.Single();
Assert.Equal(1, step.Frames.Count);
Assert.Equal((ConvertSkip.NoNativeDecoder, 1), (step.Skipped.Single().Reason, step.Skipped.Single().Frames));
}
[Test]
public async Task TheJournalsSurviveARoundTrip()
{
await using var fx = await FixtureAsync();
var journal = new ConvertJournal { JobId = Guid.NewGuid() };
var first = Guid.NewGuid();
var harbour = Guid.NewGuid();
journal.Begin(harbour, "Harbour", SequenceStatus.AnalyzedLight | SequenceStatus.ProxiesBuilt);
journal.Note(new ConvertSequenceResult(first, "Roof", ConvertOutcome.Created, 3, Guid.NewGuid()), SequenceStatus.AnalyzedDeep);
journal.Of(first)!.Queued = true;
journal.Note(new ConvertSequenceResult(harbour, "Harbour", ConvertOutcome.Failed, 0, Error: "no"));
journal.Write(fx.Paths);
var read = Assert.NotNull(ConvertJournal.Read(fx.Paths, journal.JobId));
Assert.SequenceEqual(journal.Sequences.Select(s => (s.Id, s.Outcome, s.Frames, s.NewSequenceId, s.Final, s.Had, s.Queued)),
read.Sequences.Select(s => (s.Id, s.Outcome, s.Frames, s.NewSequenceId, s.Final, s.Had, s.Queued)));
Assert.True(read.Of(first)!.Final);
// A failure keeps the statuses the sequence had when the job came to it.
Assert.Equal(SequenceStatus.AnalyzedLight | SequenceStatus.ProxiesBuilt, read.Of(harbour)!.Had);
// The photos pinned: a later line for a sequence replaces an earlier one, a torn one is skipped.
var photo = Guid.NewGuid();
var pins = read.Pins(fx.Paths);
Assert.Null(pins.Of(first));
ConvertSequencePlan Pinned(int version, string sha) =>
new(first, "Roof", "roof", version, [new ConvertFramePlan(photo, 0, "originals/a.jpg", ImageFileFormat.Jpeg, 1, 2, sha)], [], []);
pins.Append(first, Pinned(4, "aa"));
pins.Append(first, Pinned(5, "bb"));
File.AppendAllText(ConvertJournal.PinsPathFor(fx.Paths, journal.JobId), "{\"sequence\":");
var pinned = Assert.NotNull(read.Pins(fx.Paths).Of(first));
Assert.Equal((5, photo, "bb"), (pinned.FramesVersion, pinned.Frames.Single().Id, pinned.Frames.Single().Sha256));
// Its files and its staging folder are known by their names.
Assert.SequenceEqual([journal.JobId, journal.JobId, journal.JobId],
ConvertJournal.Files(fx.Paths, journal.JobId).Select(p => ConvertJournal.JobOf(Path.GetFileName(p))));
Assert.Equal(journal.JobId, ConvertJournal.JobOf(MassConverter.StagingFolder(journal.JobId)));
Assert.Null(ConvertJournal.JobOf("convert-test"));
Assert.Null(ConvertJournal.JobOf("rename-" + journal.JobId.ToString("N") + ".json"));
// A line torn by a crash is skipped; a later line for a frame replaces an earlier one.
var path = ConvertJournal.StagedPathFor(fx.Paths, journal.JobId);
var log = StagedLog.Open(path);
var frame = Guid.NewGuid();
log.Append(first, 4, new StagedConversion(frame, "a", "x.dng", 1, "b", 2, 3));
log.Append(first, 4, new StagedConversion(frame, "a", "x.dng", 1, "c", 2, 3));
File.AppendAllText(path, "{\"sequence\":");
var reread = StagedLog.Open(path);
Assert.Equal("c", reread.Of(first, 4).Single().Sha256);
Assert.Empty(reread.Of(first, 5));
ConvertJournal.Delete(fx.Paths, journal.JobId);
Assert.FileMissing(path);
Assert.FileMissing(ConvertJournal.PinsPathFor(fx.Paths, journal.JobId));
Assert.Null(ConvertJournal.Read(fx.Paths, journal.JobId));
// A job an earlier build queued names no digest: it reads with none.
var parameters = JsonSerializer.Serialize(new ConvertJobParameters(fx.Archive.Id, ConvertTarget.Jpeg, 80, ConvertMode.NewSequence,
[new ConvertSequence(first, 2, "d")], "f"), ArchiveJobJson.Default.ConvertJobParameters);
Assert.Equal("d", JsonSerializer.Deserialize(parameters, ArchiveJobJson.Default.ConvertJobParameters)!.Sequences.Single().Frames);
var older = $$"""{"archiveId":"{{fx.Archive.Id}}","target":"Jpeg","quality":80,"mode":"Replace","sequences":[{"id":"{{first}}","framesVersion":2}],"fingerprint":"f"}""";
Assert.Null(JsonSerializer.Deserialize(older, ArchiveJobJson.Default.ConvertJobParameters)!.Sequences.Single().Frames);
}
[Test]
public async Task ConvertedFilesAreNamedAsFilingNamesThemAndNothingIsOverwritten()
{
await using var fx = await FixtureAsync();
await fx.Service.UpdateSettingsAsync(fx.Archive.Id, s => s with { Templates = s.Templates with { File = "{name}_{n:00}" } }, fx.Ct);
var roof = await UploadedAsync(fx, 2);
var lake = await UploadedAsync(fx, 2, "upload-2", seed: 200);
Assert.SequenceEqual(["IMG_0001_01.JPG", "IMG_0002_02.JPG"], (await FramesAsync(fx, roof)).Select(f => f.FileName));
await fx.Service.SetSequenceInfoAsync(roof, "Roof", [], 0, null, fx.Ct);
await fx.Service.SetSequenceInfoAsync(lake, "Lake", [], 0, null, fx.Ct);
// Filing names a photo with the template of the moment: a conversion names its files as filing would now.
await fx.Service.UpdateSettingsAsync(fx.Archive.Id, s => s with { Templates = s.Templates with { File = "{title}_{n:00}" } }, fx.Ct);
// A file that is no frame holds the first name: it is never written over, the converted file takes the next.
var stray = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(roof))!.Folder).File("originals/Roof_01.dng");
File.WriteAllBytes(stray, [1, 2, 3]);
var engine = await fx.StartEngineAsync();
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, roof));
var replaced = await FramesAsync(fx, roof);
Assert.SequenceEqual(["Roof_01_2.dng", "Roof_02.dng"], replaced.Select(f => f.FileName));
Assert.SequenceEqual(["IMG_0001.JPG", "IMG_0002.JPG"], replaced.Select(f => f.OriginalFileName));
Assert.SequenceEqual(new byte[] { 1, 2, 3 }, File.ReadAllBytes(stray));
var job = await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, lake));
var made = Assert.NotNull(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, lake)));
var copies = await FramesAsync(fx, made);
Assert.SequenceEqual(["Lake (TIFF)_01.tif", "Lake (TIFF)_02.tif"], copies.Select(f => f.FileName));
Assert.SequenceEqual(["IMG_0001.JPG", "IMG_0002.JPG"], copies.Select(f => f.OriginalFileName));
}
[Test]
public async Task AFolderMovedWhileTheJobWaitsForItsLeaseIsWhereTheConvertedFilesGo()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
var engine = await fx.StartEngineAsync();
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
var row = (await fx.Db.Sequences.GetAsync(id))!;
var from = fx.Paths.Sequence(row.Folder).Folder;
Job job;
var planned = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
void OnProgress(JobProgressInfo info)
{
if (info.Kind == JobKind.Convert && info.Total == 2) planned.TrySetResult();
}
engine.Progress += OnProgress;
using (await LeaseFile.AcquireAsync(fx.Paths.LocksDirectory, $"seq-{id:N}", "editor", TimeSpan.FromSeconds(5), cancellationToken: fx.Ct))
{
job = await engine.EnqueueAsync(ArchiveJobs.Convert(fx.Archive, plan), cancellationToken: fx.Ct);
// Planned again where the index had the sequence then: the job now waits for its lease to convert it.
await planned.Task.WaitAsync(TimeSpan.FromSeconds(TestTiming.Budget(30)), fx.Ct);
// What a Reorganize step does under that lease meanwhile: the folder moves, and its index row follows.
var to = fx.Paths.Sequence("moved/Roof").Folder;
Directory.CreateDirectory(Path.GetDirectoryName(to)!);
Directory.Move(from, to);
Assert.True(await fx.Db.Sequences.RelocateAsync(id, "moved/Roof", null, fx.Ct));
}
engine.Progress -= OnProgress;
await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60));
var frames = await FramesAsync(fx, id);
Assert.True(frames.All(f => f.Format == ImageFileFormat.Tiff && f.RelativePath.StartsWith("moved/Roof/", StringComparison.Ordinal)));
Assert.False(Directory.Exists(from), "the folder the sequence left is never made again");
Assert.Equal("moved/Roof", (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Folder);
}
[Test]
public async Task ASequenceAnotherIndexMovedIsLeftAsItIsUntilThisIndexFindsIt()
{
// The folder the sequence left holds another sequence now, or nothing.
foreach (var taken in new[] { true, false })
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
Guid? stranger = taken ? await UploadedAsync(fx, 2, "upload-2", seed: 200) : null;
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
var row = (await fx.Db.Sequences.GetAsync(id))!;
var folder = fx.Paths.Sequence(row.Folder).Folder;
// Another index moves the folder away, then the other sequence into the folder it left; this index has scanned neither.
var (other, archive) = await fx.OtherWriterAsync();
var moves = stranger is not { } s ? [(id, "moved/{date}")] : new[] { (id, "moved/{date}"), (s, "parked/{date}"), (s, row.Folder) };
foreach (var (sequence, template) in moves)
await other.ApplyRenameAsync(await other.PlanRenameAsync(archive.Id, [sequence], template, null, fx.Ct), null, fx.Ct);
var before = taken ? ArchiveFixture.FilesOf(folder) : null;
var job = Guid.NewGuid();
var left = (await RunHereAsync(fx, plan, job)).Sequences.Single();
Assert.Equal((ConvertOutcome.Blocked, Loc.L("Errors.Archive.FolderMoved")), (left.Outcome, left.Error), taken.ToString());
if (taken) Assert.SequenceEqual(before!, ArchiveFixture.FilesOf(folder), "the other sequence's files are untouched");
else Assert.False(Directory.Exists(folder), "the folder it left is not created again");
// Once this index's scan found it there, the retry converts it where it is now.
await fx.ScanAsync();
Assert.Equal(ConvertOutcome.Converted, (await RunHereAsync(fx, plan, job)).Sequences.Single().Outcome, taken.ToString());
var moved = (await fx.Db.Sequences.GetAsync(id))!.Folder;
Assert.True(moved.StartsWith("moved/", StringComparison.Ordinal), moved);
Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Tiff && f.RelativePath.StartsWith(moved + "/", StringComparison.Ordinal)));
if (taken) Assert.SequenceEqual(before!, ArchiveFixture.FilesOf(folder), "the other sequence's files are still untouched");
else Assert.False(Directory.Exists(folder));
}
}
[Test]
public async Task AReorganizeTheArchiveWaitsForKeepsTheConversionOffItsSequences()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
var originals = await FramesAsync(fx, id);
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
// A Reorganize paused by its user holds the archive: its plan waits for it.
await fx.Db.Jobs.EnqueueAsync(new Job
{
Id = Guid.CreateVersion7(),
Kind = JobKind.Move,
TargetIds = [fx.Archive.Id],
State = JobState.Paused,
Created = DateTimeOffset.UtcNow,
});
Assert.SequenceEqual([ConvertBlock.Busy], (await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id)).Sequences.Single().Blocked);
var engine = await fx.StartEngineAsync();
var job = await RunAsync(fx, engine, plan, JobState.Failed);
Assert.Equal(Loc.Plural("Archive.Error.ConvertKept", 1), job.Error);
Assert.Contains(Loc.L("Archive.Log.ConvertBlocked", (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle), job.LogTail);
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256, f.RelativePath)), (await FramesAsync(fx, id)).Select(f => (f.Id, f.Sha256, f.RelativePath)));
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
}
[Test]
public async Task PhotosVerifiedAfterThePlanAreNotReplacedWithoutAConfirmation()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 3);
var engine = await fx.StartEngineAsync();
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
var path = fx.Paths.Sequence(sequence.Folder).ManifestPath;
var verified = ManifestStore.ReadSequence(path);
// Never verified when the plan is made (adopted): the dry run says that photo stays as it is.
ManifestStore.WriteSequence(path, verified with { Frames = [verified.Frames[0], verified.Frames[1] with { Sha256 = null }, verified.Frames[2]] });
var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id);
Assert.Equal((2, ConvertSkip.Unhashed), (plan.Frames, plan.Sequences.Single().Skipped.Single().Reason));
// A verify then hashes it before the job runs, as ArchiveVerifier does: no new frames version.
ManifestStore.WriteSequence(path, verified);
// The job converts the two photos the plan listed, and leaves the third as it is, as the dry run said.
var job = await RunAsync(fx, engine, plan);
Assert.Contains(Loc.L("Archive.Log.Converted", sequence.DisplayTitle, 2), job.LogTail);
var frames = await FramesAsync(fx, id);
Assert.SequenceEqual(verified.Frames.Select(f => f.Id), frames.Select(f => f.Id));
Assert.Equal((verified.Frames[1].Sha256, ImageFileFormat.Jpeg), (frames[1].Sha256, frames[1].Format));
Assert.True(frames[0].Format == ImageFileFormat.Dng && frames[2].Format == ImageFileFormat.Dng);
Assert.Equal(2, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Frames);
Assert.False(StagedAnything(fx));
}
[Test]
public async Task PhotosThatComeInOnceTheSequenceIsPinnedStayAsTheyAreAndARetryConvertsTheRest()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
var engine = await fx.StartEngineAsync();
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
var title = (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle;
// An earlier attempt pinned the sequence's photos and stopped (stage B blocked, a pause); an upload came in since.
var jobId = Guid.CreateVersion7();
PinLog.Open(ConvertJournal.PinsPathFor(fx.Paths, jobId)).Append(id, plan.Sequences.Single());
var (path, size, sha) = fx.Stage(Start.AddSeconds(60), 999);
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, null, path, "IMG_0099.JPG", size, sha, null, null), fx.Ct);
var added = (await FramesAsync(fx, id)).Single(f => f.Sha256 == sha);
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Convert(fx.Archive, plan), "convert", jobId);
var done = await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60));
Assert.Contains(Loc.L("Archive.Log.Converted", title, 2), done.LogTail);
var frames = await FramesAsync(fx, id);
Assert.Equal(2, frames.Count(f => f.Format == ImageFileFormat.Tiff));
Assert.Equal((sha, ImageFileFormat.Jpeg), frames.Where(f => f.Id == added.Id).Select(f => (f.Sha256, f.Format)).Single());
Assert.Equal(2, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Frames);
}
[Test]
public async Task AJobResumedAfterACrashInItsReplacementSettlesItAndConvertsTheWholeSequence()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
var engine = await fx.StartEngineAsync();
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
var folder = fx.Paths.Sequence(sequence.Folder);
var manifest = Sidecar(fx, sequence);
// The attempt that crashed: the photos pinned and staged, then its replacement cut short once every original had left
// for the trash, none of the converted files in yet.
var jobId = Guid.CreateVersion7();
PinLog.Open(ConvertJournal.PinsPathFor(fx.Paths, jobId)).Append(id, plan.Sequences.Single());
var log = StagedLog.Open(ConvertJournal.StagedPathFor(fx.Paths, jobId));
var entryFolder = Path.Combine(FrameTrash.Directory(fx.Paths), $"{id:N}-20261027120000");
var record = entryFolder + ".json";
List<SwapMove> outgoing = [];
List<SwapMove> incoming = [];
foreach (var frame in manifest.Frames)
{
var staged = Path.Combine(StagingOf(fx, jobId, id), frame.Id.ToString("N") + ".tif");
var file = FrameConverter.Convert(new ConvertSource(folder.File(frame.File), frame), new ConvertOptions(ConvertTarget.Tiff16), DecoderFactory.CreateServer(),
staged, cancellationToken: fx.Ct);
log.Append(id, manifest.Versions.Frames, new StagedConversion(frame.Id, frame.Sha256!, Path.GetFileName(staged), file.Size, file.Sha256, file.Width, file.Height));
outgoing.Add(new SwapMove(frame.File, fx.Paths.ToRelative(Path.Combine(entryFolder, "originals", Path.GetFileName(frame.File)))));
incoming.Add(new SwapMove(Path.ChangeExtension(frame.File, ".tif"), fx.Paths.ToRelative(staged)));
}
new FrameReplaceJournal(manifest.Versions.Frames, fx.Paths.ToRelative(record), fx.Paths.ToRelative(entryFolder), outgoing, incoming, false, null).Write(folder);
Directory.CreateDirectory(FrameTrash.Directory(fx.Paths));
File.WriteAllText(FrameRemovalJournal.PendingRecord(record), "{}");
foreach (var move in outgoing)
{
var trashed = fx.Paths.FromRelative(move.Other);
Directory.CreateDirectory(Path.GetDirectoryName(trashed)!);
File.Move(folder.File(move.Place), trashed);
}
// Not a sequence with nothing to convert, nor half of it converted: the originals come back, then all of it converts.
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Convert(fx.Archive, plan), "convert", jobId);
var done = await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60));
Assert.Contains(Loc.L("Archive.Log.Converted", sequence.DisplayTitle, 2), done.LogTail);
Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Tiff));
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
Assert.Equal((TrashKind.Replaced, 2), (entry.Kind, entry.Frames));
Assert.False(Directory.Exists(entryFolder));
}
[Test]
public async Task TheFollowUpsOfEachSequenceAreQueuedAsItEndsAndOnResumeWhenAPauseCameFirst()
{
await using var fx = await FixtureAsync();
var first = await UploadedAsync(fx, 2);
var second = await UploadedAsync(fx, 2, "upload-2", seed: 200);
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, first, second);
var events = new List<string>();
var callbacks = Callbacks(done: s => events.Add($"done {s.SequenceId}"), queue: r => events.AddRange(r.Select(q => $"{q.Kind} {q.Targets.Single()}")));
await RunHereAsync(fx, plan, Guid.CreateVersion7(), callbacks);
// Queued as each sequence is journaled, before the next starts: a pause, a yield or a crash later loses none of them.
Assert.Less(events.IndexOf($"done {first}"), events.IndexOf($"{JobKind.LightAnalyse} {first}"));
Assert.Less(events.IndexOf($"{JobKind.LightAnalyse} {first}"), events.IndexOf($"done {second}"));
Assert.Contains($"{JobKind.LightAnalyse} {second}", events);
// As a pause right after the first sequence's outcome was journaled leaves the job: its passes not queued yet.
var jobId = Guid.CreateVersion7();
var journal = new ConvertJournal { JobId = jobId };
journal.Note(new ConvertSequenceResult(first, "Roof", ConvertOutcome.Converted, 2), SequenceStatus.AnalyzedDeep | SequenceStatus.ThumbsBuilt);
journal.Note(new ConvertSequenceResult(second, "Lake", ConvertOutcome.Converted, 2), SequenceStatus.AnalyzedLight);
journal.Of(second)!.Queued = true;
journal.Write(fx.Paths);
List<ArchiveJobRequest> queued = [];
var resumed = await RunHereAsync(fx, plan, jobId, Callbacks(queue: queued.AddRange));
Assert.Equal(4, resumed.Frames);
Assert.SequenceEqual([(JobKind.LightAnalyse, first), (JobKind.DeepAnalyse, first), (JobKind.Thumbs, first)],
queued.Select(r => (r.Kind, r.Targets.Single())));
Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, jobId));
}
[Test]
public async Task AConversionWaitingInTheQueueKeepsItsSequencesFromASecondConfirmation()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id);
var request = ArchiveJobs.Convert(fx.Archive, plan);
var job = new Job
{
Id = Guid.CreateVersion7(),
Kind = request.Kind,
TargetIds = request.Targets,
ParametersJson = request.ParametersJson,
State = JobState.Queued,
Created = DateTimeOffset.UtcNow,
};
await fx.Db.Jobs.EnqueueAsync(job, fx.Ct);
// Another tab, or Apply pressed again after an answer was lost: the same copies are not queued twice.
var again = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id);
Assert.SequenceEqual([ConvertBlock.Busy], again.Sequences.Single().Blocked);
Assert.NotEqual(plan.Fingerprint(), again.Fingerprint());
Assert.Throws<ArgumentException>(() => ArchiveJobs.Convert(fx.Archive, again));
// The job itself plans again with its sequences free, and converts them.
var engine = await fx.StartEngineAsync();
engine.Wake();
await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60));
Assert.NotNull(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, id)));
Assert.Empty((await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id)).Sequences.Single().Blocked);
}
[Test]
public async Task ANewSequenceIsOwnedAsItsSourceIsOrByWhoAskedForIt()
{
await using var fx = await FixtureAsync();
var owner = Guid.NewGuid();
var owned = await UploadedAsync(fx, 2, user: owner);
var unowned = await UploadedAsync(fx, 2, "upload-2", seed: 200);
Assert.Equal(owner, (await fx.Db.Sequences.GetAsync(owned))!.OwnerUserId);
var engine = await fx.StartEngineAsync();
var asker = Guid.NewGuid();
var job = await engine.EnqueueAsync(ArchiveJobs.Convert(fx.Archive, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, owned, unowned)), asker, fx.Ct);
await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60));
foreach (var (source, expected) in new[] { (owned, owner), (unowned, asker) })
{
var made = (await fx.Db.Sequences.GetAsync((await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, source)))!.Value))!;
Assert.Equal(expected, made.OwnerUserId);
Assert.Equal(expected.ToString(), Sidecar(fx, made).CreatedBy);
}
}
[Test]
public async Task AStagedCopyThatVanishesFailsItsNewSequenceInsteadOfLeavingThePhotoOut()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id);
var first = Sidecar(fx, (await fx.Db.Sequences.GetAsync(id))!).Frames[0];
var jobId = Guid.CreateVersion7();
// A clean-up takes the first copy once both are staged, before they are filed.
var callbacks = Callbacks(progress: (n, total) =>
{
if (n == 2 && total == 2) File.Delete(Path.Combine(StagingOf(fx, jobId, id), first.Id.ToString("N") + ".tif"));
});
var failed = (await RunHereAsync(fx, plan, jobId, callbacks)).Sequences.Single();
Assert.Equal(ConvertOutcome.Failed, failed.Outcome);
// Why, in the catalog's words: the job log has no English exception text.
Assert.Equal(Loc.L("Archive.Convert.NotPlaced", Loc.L("Archive.Convert.Unstaged", first.File)), failed.Error);
Assert.Null(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(jobId, id)));
// The retry converts that photo again and makes the whole copy.
var made = (await RunHereAsync(fx, plan, jobId)).Sequences.Single();
Assert.Equal((ConvertOutcome.Created, 2, 0), (made.Outcome, made.Frames, made.LeftOut));
Assert.Count(2, await FramesAsync(fx, made.NewSequenceId!.Value));
}
[Test]
public async Task ANewSequenceSaysWhichPhotosItLeftInTheSourceAlone()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 3);
var engine = await fx.StartEngineAsync();
// Three TIFFs, then a JPEG uploaded: to JPEG, the TIFFs convert and the JPEG stays in the source alone.
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id));
var (path, size, sha) = fx.Stage(Start.AddSeconds(60), 999);
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, null, path, "IMG_0099.JPG", size, sha, null, null), fx.Ct);
var job = await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Jpeg, ConvertMode.NewSequence, id));
Assert.Contains(Loc.L("Archive.Log.ConvertedNew", sequence.DisplayTitle, 3), job.LogTail);
Assert.Contains(Loc.L("Archive.Log.ConvertLeftOut", sequence.DisplayTitle, 1), job.LogTail);
var made = Assert.NotNull(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, id)));
Assert.Count(3, await FramesAsync(fx, made));
}
[Test]
public async Task AVolumeThatFillsDuringTheJobStopsItBeforeTheNextSequenceAndTheNextPhoto()
{
long free = 1L << 50;
await using var fx = await FixtureAsync(o => o with { FreeSpace = _ => free, Parallelism = () => 1 });
var first = await UploadedAsync(fx, 2);
var second = await UploadedAsync(fx, 2, "upload-2", seed: 200);
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, first, second);
var jobId = Guid.CreateVersion7();
var originals = await FramesAsync(fx, second);
// Another writer fills the volume while the first sequence converts: the second does not start.
var full = await Assert.ThrowsAsync<IOException>(() => RunHereAsync(fx, plan, jobId, Callbacks(done: _ => free = 1_000)));
Assert.True(full.Message.StartsWith("The conversion needs", StringComparison.Ordinal), full.Message);
Assert.True((await FramesAsync(fx, first)).All(f => f.Format == ImageFileFormat.Tiff));
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256)), (await FramesAsync(fx, second)).Select(f => (f.Id, f.Sha256)));
Assert.False(Directory.Exists(StagingOf(fx, jobId, second)));
// Space again, until the first photo of the second sequence is staged: the next one does not start, what was staged stays.
free = 1L << 50;
await Assert.ThrowsAsync<IOException>(() => RunHereAsync(fx, plan, jobId, Callbacks(progress: (n, _) =>
{
if (n == 1) free = 1_000;
})));
Assert.Count(1, Directory.EnumerateFiles(StagingOf(fx, jobId, second)));
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256)), (await FramesAsync(fx, second)).Select(f => (f.Id, f.Sha256)));
free = 1L << 50;
var done = await RunHereAsync(fx, plan, jobId);
Assert.True(done.Sequences.All(s => s.Outcome == ConvertOutcome.Converted));
Assert.True((await FramesAsync(fx, second)).All(f => f.Format == ImageFileFormat.Tiff));
Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, jobId));
}
[Test]
public async Task MaintenanceKeepsWhatAWaitingConversionStagedAndClearsWhatAFinishedOneLeft()
{
await using var fx = await FixtureAsync();
var paused = Guid.CreateVersion7();
var finished = Guid.CreateVersion7();
var forgotten = Guid.CreateVersion7();
var stray = Guid.CreateVersion7();
foreach (var (id, state) in new[] { (paused, JobState.Paused), (finished, JobState.Done) })
await fx.Db.Jobs.EnqueueAsync(new Job { Id = id, Kind = JobKind.Convert, State = state, Created = DateTimeOffset.UtcNow }, fx.Ct);
string Staged(Guid job)
{
var file = Path.Combine(StagingOf(fx, job, Guid.NewGuid()), "frame.tif");
Directory.CreateDirectory(Path.GetDirectoryName(file)!);
File.WriteAllText(file, "x");
File.SetLastWriteTimeUtc(file, DateTime.UtcNow.AddDays(-3));
return file;
}
string Journal(Guid job)
{
new ConvertJournal { JobId = job }.Write(fx.Paths);
return ConvertJournal.PathFor(fx.Paths, job);
}
var (pausedFile, pausedJournal) = (Staged(paused), Journal(paused));
var (finishedFile, finishedJournal) = (Staged(finished), Journal(finished));
var (forgottenFile, forgottenJournal) = (Staged(forgotten), Journal(forgotten));
var strayFile = Staged(stray);
var dry = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: true, 0, _ => ValueTask.CompletedTask, fx.Ct);
Assert.SequenceEqual(new[] { finishedFile, finishedJournal, strayFile }.Select(fx.Paths.ToRelative).Order(), dry.TemporaryFiles.Order());
await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
// A paused job's staged conversions stay however old; a finished job's go with its journal. A job this index does not
// know (another host's on a shared archive, or forgotten by the history) may still resume where it is known: its
// staged conversions stay as long as its journal; without a journal no resume can use them, and they go once idle.
Assert.FileExists(pausedFile);
Assert.FileExists(pausedJournal);
Assert.False(Directory.Exists(Path.Combine(Staging(fx), MassConverter.StagingFolder(finished))));
Assert.FileMissing(finishedJournal);
Assert.FileExists(forgottenFile);
Assert.FileExists(forgottenJournal);
Assert.False(Directory.Exists(Path.Combine(Staging(fx), MassConverter.StagingFolder(stray))));
// Its journal untouched for as long as the history keeps jobs, but a staged file written since: both stay.
var longAgo = DateTime.UtcNow.AddDays(-CleanupSteps.DefaultHistoryDays - 1);
File.SetLastWriteTimeUtc(forgottenJournal, longAgo);
await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
Assert.FileExists(forgottenJournal);
Assert.FileExists(forgottenFile);
// Neither written for that long: they go together.
File.SetLastWriteTimeUtc(forgottenFile, longAgo);
await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
Assert.FileMissing(forgottenJournal);
Assert.False(Directory.Exists(Path.Combine(Staging(fx), MassConverter.StagingFolder(forgotten))));
Assert.FileExists(pausedJournal);
}
[Test]
public async Task APhotoAnotherProgramHoldsIsSaidSoAndNeverConvertedWithoutAPlanThatListsIt()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 3);
var title = (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle;
var originals = await FramesAsync(fx, id);
var held = fx.Paths.FromRelative(originals[1].RelativePath);
// A sync client holds it while the dry run is made: not missing, and not planned, with why.
ConvertPlan plan;
using (Unreadable(held)) plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
var step = plan.Sequences.Single();
Assert.Equal((2, ConvertSkip.Inaccessible, 1), (step.Frames.Count, step.Skipped.Single().Reason, step.Skipped.Single().Frames));
Assert.DoesNotContain(originals[1].Id, step.Frames.Select(f => f.FrameId));
// Free again when the job runs: it converts the two photos the plan listed, not that one.
var done = (await RunHereAsync(fx, plan, Guid.CreateVersion7())).Sequences.Single();
Assert.Equal((ConvertOutcome.Converted, 2), (done.Outcome, done.Frames));
var frames = await FramesAsync(fx, id);
Assert.Equal((originals[1].Sha256, ImageFileFormat.Jpeg), (frames[1].Sha256, frames[1].Format));
Assert.True(frames[0].Format == ImageFileFormat.Tiff && frames[2].Format == ImageFileFormat.Tiff);
// Held again while the job that converts it comes to it: left as it is, saying why, and the retry converts it once free.
var rest = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
Assert.Equal(1, rest.Frames);
var jobId = Guid.CreateVersion7();
var lines = new List<string>();
ConvertSequenceResult left;
using (Unreadable(held))
left = (await RunHereAsync(fx, rest, jobId, Callbacks() with { Log = lines.Add })).Sequences.Single();
Assert.Equal((ConvertOutcome.Blocked, Loc.L("Archive.Convert.PhotoHeld")), (left.Outcome, left.Error));
Assert.SequenceEqual([Loc.L("Archive.Log.ConvertFailed", title, Loc.L("Archive.Convert.PhotoHeld"))], lines);
Assert.Equal(ImageFileFormat.Jpeg, (await FramesAsync(fx, id))[1].Format);
Assert.Equal(ConvertOutcome.Converted, (await RunHereAsync(fx, rest, jobId)).Sequences.Single().Outcome);
Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Tiff));
}
/// <summary>
/// Only a photo the job converts makes a sequence wait for its retry while another program holds it: one the plan left
/// out being held does not hide that a photo confirmed is gone, which no retry can mend.
/// </summary>
[Test]
public async Task APhotoHeldThatTheJobDoesNotConvertDoesNotHideAChange()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 3);
var originals = await FramesAsync(fx, id);
var held = fx.Paths.FromRelative(originals[0].RelativePath);
ConvertPlan plan;
using (Unreadable(held)) plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
Assert.Equal(2, plan.Frames);
// A photo of the plan is deleted before the job comes to the sequence, and the one left out is held again then.
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
var delete = await fx.Service.FrameTrash.PlanDeleteAsync(id, [originals[1].Id], sequence.FramesVersion, fx.Ct);
await fx.Service.FrameTrash.DeleteAsync(delete, "alberto", null, fx.Ct);
ConvertSequenceResult left;
using (Unreadable(held)) left = (await RunHereAsync(fx, plan, Guid.CreateVersion7())).Sequences.Single();
Assert.Equal((ConvertOutcome.Changed, (string?)null), (left.Outcome, left.Error));
Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Jpeg));
}
/// <summary>
/// Before each photo of stage A, the space is checked for it and the photos after it that may convert at once with it:
/// fewer near the end, so a job whose dry run had the space never stops short for photos that are not there.
/// </summary>
[Test]
public async Task TheLastPhotosToStartNeedOnlyTheirOwnSpace()
{
ConvertFramePlan[] todo = [.. new long[] { 10, 20, 30, 40 }.Select((bytes, i) => new ConvertFramePlan(Guid.NewGuid(), i, $"originals/{i}.JPG", ImageFileFormat.Jpeg, 1, bytes))];
Assert.SequenceEqual([60L, 90, 70, 40], Enumerable.Range(0, todo.Length).Select(i => MassConverter.Ahead(todo, i, 3)));
Assert.SequenceEqual([10L, 20, 30, 40], Enumerable.Range(0, todo.Length).Select(i => MassConverter.Ahead(todo, i, 1)));
long free = 1L << 50;
await using var fx = await FixtureAsync(o => o with { FreeSpace = _ => free, Parallelism = () => 4 });
var id = await UploadedAsync(fx, 1);
// Exactly what the dry run asks for: four photos may convert at once, but there is one.
free = ConvertPlanner.Required(ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.Tiff16), 48, 32));
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
Assert.True(plan.Space.Enough);
Assert.Equal(free, plan.Space.Required);
var done = (await RunHereAsync(fx, plan, Guid.CreateVersion7())).Sequences.Single();
Assert.Equal((ConvertOutcome.Converted, 1), (done.Outcome, done.Frames));
}
[Test]
public async Task ASecondConversionOfTheSameSequencesIsRefusedWithOrWithoutADryRun()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id);
async Task Enqueue(ArchiveJobRequest request)
{
// Long enough for the other confirmation to check meanwhile, were it not held back.
await Task.Delay(50, fx.Ct);
await fx.Db.Jobs.EnqueueAsync(new Job
{
Id = Guid.CreateVersion7(),
Kind = request.Kind,
TargetIds = request.Targets,
ParametersJson = request.ParametersJson,
State = JobState.Queued,
Created = DateTimeOffset.UtcNow,
}, fx.Ct);
}
async Task<bool> ConfirmAsync()
{
try
{
await fx.Service.Converter.QueueAsync(fx.Archive, plan, Enqueue, fx.Ct);
return true;
}
catch (ConversionQueuedException ex)
{
Assert.SequenceEqual([id], ex.SequenceIds);
Assert.Equal(Loc.L("Archive.Convert.AlreadyQueued"), ex.Message);
return false;
}
}
// The Library's dialog and the Archive manager's confirm the same plan at once: one job.
var confirmed = await Task.WhenAll(ConfirmAsync(), ConfirmAsync());
Assert.SequenceEqual([false, true], confirmed.Order());
// And again later from that plan, made before the job was queued and never planned again.
Assert.False(await ConfirmAsync());
Assert.Count(1, (await fx.Db.Jobs.ListAsync(JobState.Queued, JobKind.Convert)).Items);
}
[Test]
public async Task AConversionQueuedFirstHoldsItsSequencesFromADuplicateQueuedAfterIt()
{
await using var fx = await FixtureAsync();
var id = await UploadedAsync(fx, 2);
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id);
// Queued twice anyway (an earlier build, two processes on one index): the copies are made once.
var first = await QueuedAsync(fx, plan, DateTimeOffset.UtcNow.AddMinutes(-2));
var second = await QueuedAsync(fx, plan, DateTimeOffset.UtcNow.AddMinutes(-1));
var held = (await fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), second.Id, new ConvertRequester(null, null), Callbacks(), fx.Ct))
.Sequences.Single();
Assert.Equal(ConvertOutcome.Blocked, held.Outcome);
Assert.Null(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(second.Id, id)));
// The one queued first is not held back by the one after it.
var made = (await fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), first.Id, new ConvertRequester(null, null), Callbacks(), fx.Ct))
.Sequences.Single();
Assert.Equal((ConvertOutcome.Created, 2), (made.Outcome, made.Frames));
}
[Test]
public void AJobsParametersNameThePhotosItConverts()
{
Guid[] ids = [Guid.CreateVersion7(), Guid.CreateVersion7(), Guid.NewGuid()];
var step = new ConvertSequence(Guid.NewGuid(), 3, "digest", ConvertSequence.EncodePhotos(ids));
Assert.SequenceEqual(ids, step.PhotoIds()!);
var json = JsonSerializer.Serialize(step, ArchiveJobJson.Default.ConvertSequence);
Assert.SequenceEqual(ids, JsonSerializer.Deserialize(json, ArchiveJobJson.Default.ConvertSequence)!.PhotoIds()!);
// A job an earlier build queued names none.
Assert.Null(new ConvertSequence(Guid.NewGuid(), 3, "digest").PhotoIds());
Assert.Empty(new ConvertSequence(Guid.NewGuid(), 3, "digest", ConvertSequence.EncodePhotos([])).PhotoIds()!);
}
}