using System.Runtime.Versioning; using System.Security.Cryptography; using System.Text.Json; using Chronos.Core.Archiving; using Chronos.Core.Conversion; using Chronos.Core.Import; using Chronos.Core.Imaging; using Chronos.Core.Imaging.Decoders; using Chronos.Core.Localization; using Chronos.Core.Model; using Chronos.Core.Project; using Chronos.Data.Archiving; using Chronos.Data.Archiving.Jobs; using Chronos.Data.Housekeeping; using Chronos.Data.Jobs; using Chronos.Data.Repositories; using Chronos.Testing; using static Chronos.Data.Tests.Archiving.ArchiveFixture; namespace Chronos.Data.Tests.Archiving; /// /// Mass conversion after import (owner review 2, R7): the dry run, the convert job in both modes, its undo through the /// trash, all or nothing per sequence, resumes in stage A and B, plans gone stale, the space check, and the gates that must /// keep holding (the card erase, the emptying of a replaced entry). /// [Category("archive")] public sealed class ConvertTests(TestContext context) { private Task FixtureAsync(Func? more = null) => CreateAsync(context, configure: o => { var options = o with { Decoder = DecoderFactory.CreateServer(), Parallelism = () => 2 }; return more is null ? options : more(options); }); /// A sequence of uploaded photos (hashed, under originals/), ten seconds apart, by . private static async Task UploadedAsync(ArchiveFixture fx, int count, string session = "upload-1", int seed = 100, Guid? user = null) { Guid? id = null; for (var i = 0; i < count; i++) { var (path, size, sha) = fx.Stage(Start.AddSeconds(10 * i).AddTicks(1_234_567), seed + i); var result = await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, id is null ? session : null, path, $"IMG_{i + 1:0000}.JPG", size, sha, null, user), fx.Ct); id = result.SequenceId; } return id!.Value; } private static Task PlanAsync(ArchiveFixture fx, ConvertTarget target, ConvertMode mode, params Guid[] ids) => fx.Service.PlanConvertAsync(fx.Archive.Id, ids, new ConvertOptions(target), mode, fx.Ct); /// The parameters the job of runs with. private static ConvertJobParameters Parameters(ArchiveFixture fx, ConvertPlan plan) => JsonSerializer.Deserialize(ArchiveJobs.Convert(fx.Archive, plan).ParametersJson!, ArchiveJobJson.Default.ConvertJobParameters)!; /// Callbacks that report nothing, but for and when given. private static ConvertCallbacks Callbacks(Action? progress = null, Action? done = null, Action>? queue = null) => new( (n, total) => { progress?.Invoke(n, total); return ValueTask.CompletedTask; }, (_, _) => ValueTask.CompletedTask, _ => { }, s => { done?.Invoke(s); return ValueTask.CompletedTask; }, requests => { queue?.Invoke(requests); return ValueTask.CompletedTask; }); /// /// The conversion job of , run here as its handler runs it, with the photos /// confirmed written beside its journal as writes them (once: a run again of the /// same job finds them), and as the time it was queued. /// private static Task RunHereAsync(ArchiveFixture fx, ConvertPlan plan, Guid jobId, ConvertCallbacks? callbacks = null, DateTimeOffset? confirmed = null) { if (!File.Exists(ConfirmedPhotos.PathFor(fx.Paths, jobId))) Assert.True(ConfirmedPhotos.TryWrite(fx.Paths, jobId, plan)); return fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), jobId, new ConvertRequester(null, null, confirmed), callbacks ?? Callbacks(), fx.Ct); } private static string StagingOf(ArchiveFixture fx, Guid job, Guid sequence) => Path.Combine(Staging(fx), $"convert-{job:N}", $"{sequence:N}"); /// The job of , queued as the Library, the Archive manager and the panel queue it (). private static async Task QueueAsync(ArchiveFixture fx, JobEngine engine, ConvertPlan plan, Guid? createdBy = null) { Job? job = null; await fx.Service.Converter.QueueAsync(fx.Archive, plan, async r => job = await engine.EnqueueAsync(r, createdBy, fx.Ct), fx.Ct); return Assert.NotNull(job); } private static async Task RunAsync(ArchiveFixture fx, JobEngine engine, ConvertPlan plan, JobState state = JobState.Done) => await fx.WaitForAsync((await QueueAsync(fx, engine, plan)).Id, state, (int)TestTiming.Budget(60)); private static SequenceManifest Sidecar(ArchiveFixture fx, Sequence sequence) => ManifestStore.ReadSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath); private static string Sha(string path) => Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(path))); /// The index rows of a sequence, after checking that each file holds its row's content and that the sidecar agrees. private static async Task> FramesAsync(ArchiveFixture fx, Guid id) { var sequence = Assert.NotNull(await fx.Db.Sequences.GetAsync(id)); var rows = (await fx.Db.Frames.ListBySequenceAsync(id)).ToList(); var sidecar = Sidecar(fx, sequence); Assert.Equal(sidecar.Versions.Frames, sequence.FramesVersion); Assert.SequenceEqual(sidecar.Frames.Select(f => (f.Id, f.I, f.Sha256)), rows.Select(r => (r.Id, r.Index, r.Sha256))); foreach (var row in rows) Assert.Equal(row.Sha256, Sha(fx.Paths.FromRelative(row.RelativePath))); Assert.FileMissing(fx.Paths.Sequence(sequence.Folder).ReplaceJournalPath); return rows; } private static string Staging(ArchiveFixture fx) => fx.Paths.StagingDirectory; private static bool StagedAnything(ArchiveFixture fx) => Directory.Exists(Staging(fx)) && Directory.EnumerateDirectories(Staging(fx), "convert-*").Any(); /// /// Makes unreadable until disposed, as another program holding it does: without read permission /// where that stops the test process (UnauthorizedAccessException), otherwise (Windows, root) held open without sharing /// (IOException). /// private static IDisposable Unreadable(string path) { if (!OperatingSystem.IsWindows()) { var mode = File.GetUnixFileMode(path); File.SetUnixFileMode(path, UnixFileMode.None); try { File.OpenRead(path).Dispose(); File.SetUnixFileMode(path, mode); } catch (UnauthorizedAccessException) { return new UnixPermissions(path, mode); } } return new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.None); } /// Gives a file its permissions back. [UnsupportedOSPlatform("windows")] private sealed class UnixPermissions(string path, UnixFileMode mode) : IDisposable { public void Dispose() => File.SetUnixFileMode(path, mode); } /// A queued conversion job of , as the engine would hold it, created at . private static async Task QueuedAsync(ArchiveFixture fx, ConvertPlan plan, DateTimeOffset created) { var request = ArchiveJobs.Convert(fx.Archive, plan); var job = new Job { Id = Guid.CreateVersion7(), Kind = request.Kind, TargetIds = request.Targets, ParametersJson = request.ParametersJson, State = JobState.Queued, Created = created, }; await fx.Db.Jobs.EnqueueAsync(job, fx.Ct); return job; } [Test] public async Task TheDryRunCountsWhatConvertsAndChangesNothing() { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 3); var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id); var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id); var step = plan.Sequences.Single(); Assert.Equal((3, 0, true), (step.Frames.Count, step.SkippedFrames, step.Converts)); Assert.Equal((await fx.Db.Sequences.GetAsync(id))!.FramesVersion, step.FramesVersion); Assert.Equal(step.Frames.Sum(f => f.Size), plan.BytesBefore); Assert.Equal(3 * ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.LinearDng), 48, 32), plan.BytesAfter); Assert.Equal(ConvertPlanner.Required(plan.BytesAfter), plan.Space.Required); Assert.True(plan.Space.Enough); Assert.Equal(plan.Fingerprint(), (await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id)).Fingerprint()); Assert.NotEqual(plan.Fingerprint(), (await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.NewSequence, id)).Fingerprint()); // JPEG to JPEG converts nothing. var same = (await PlanAsync(fx, ConvertTarget.Jpeg, ConvertMode.Replace, id)).Sequences.Single(); Assert.Equal((0, ConvertSkip.SameFormat, 3), (same.Frames.Count, same.Skipped.Single().Reason, same.Skipped.Single().Frames)); Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id)); Assert.False(StagedAnything(fx)); await Assert.ThrowsAsync(() => PlanAsync(fx, ConvertTarget.Jpeg, ConvertMode.Replace, Guid.NewGuid())); } [Test] public async Task ASequenceBusyOrUnreadableIsBlocked() { await using var fx = await FixtureAsync(); var busy = await UploadedAsync(fx, 2); var unreadable = await UploadedAsync(fx, 2, "upload-2", seed: 200); await fx.Db.Jobs.EnqueueAsync(new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Render, TargetIds = [busy], State = JobState.Running, NodeId = "test#0", Created = DateTimeOffset.UtcNow, Started = DateTimeOffset.UtcNow, }); File.WriteAllText(fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(unreadable))!.Folder).ManifestPath, "{ not json"); var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, busy, unreadable); Assert.SequenceEqual([ConvertBlock.Busy], plan.Sequences.Single(s => s.SequenceId == busy).Blocked); Assert.SequenceEqual([ConvertBlock.Unreadable], plan.Sequences.Single(s => s.SequenceId == unreadable).Blocked); Assert.Equal(0, plan.Frames); Assert.Throws(() => ArchiveJobs.Convert(fx.Archive, plan)); } [Test] public async Task AReplaceKeepsTheFramesAndTrashesTheOriginalsWithTheirReplacements() { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 3); var engine = await fx.StartEngineAsync(); var sequence = (await fx.Db.Sequences.GetAsync(id))!; var originals = await FramesAsync(fx, id); var project = new ProjectSettings { ExcludedFrames = [1], HeroFrame = 2 }.ToJson(); await fx.Db.Sequences.EditAsync(id, new SequenceEdit { SettingsJson = project, HeroFrameId = originals[2].Id }, null, fx.Ct); var folder = fx.Paths.Sequence(sequence.Folder); Directory.CreateDirectory(folder.MetaDirectory); File.WriteAllText(folder.AnalysisPath, "{\"frames\":3}"); var before = Sidecar(fx, sequence); var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id); var job = await RunAsync(fx, engine, plan); Assert.Contains(Loc.L("Archive.Log.Converted", sequence.DisplayTitle, 3), job.LogTail); var row = (await fx.Db.Sequences.GetAsync(id))!; var frames = await FramesAsync(fx, id); Assert.Equal(before.Versions.Frames + 1, row.FramesVersion); Assert.SequenceEqual(originals.Select(f => (f.Id, f.Index)), frames.Select(f => (f.Id, f.Index))); var after = Sidecar(fx, row); foreach (var (was, now) in before.Frames.Zip(after.Frames)) { // Named as filing named the original, with the target's extension: the same template, frame and sequence. Assert.Equal((Path.ChangeExtension(was.File, ".dng"), ImageFileFormat.Dng, DecodeTier.Native, 1), (now.File, now.Format, now.Tier, now.Orientation)); Assert.Equal((was.CapturedAt, was.Original, was.Flags), (now.CapturedAt, now.Original, now.Flags)); var source = Assert.NotNull(now.Source); Assert.Equal((was.Sha256, was.Format, was.File, was.Size), (source.Sha256, source.Format, source.File, source.Size)); Assert.FileMissing(folder.File(was.File)); } // The project's positions stay on their frames. var settings = ProjectSettings.FromJson(row.SettingsJson!); Assert.SequenceEqual([1], settings.ExcludedFrames); Assert.Equal((2, originals[2].Id), (settings.HeroFrame, row.HeroFrameId)); // One replaced entry with the originals, each naming its replacement, and the analysis as it was. var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single(); Assert.Equal((TrashKind.Replaced, id, 3), (entry.Kind, entry.SequenceId, entry.Frames)); var record = Assert.NotNull(FrameTrash.ReadEntry(fx.Paths, entry.Name)); var entryFolder = FrameTrash.Location(fx.Paths, entry.Name).Folder; foreach (var photo in record.Photos) { var now = after.Frames.Single(f => f.Id == photo.Frame.Id); Assert.Equal(now.Sha256, photo.Replacement); Assert.Equal(photo.Frame.Sha256, Sha(Path.Combine(entryFolder, photo.Files[0].Path))); } Assert.Equal("{\"frames\":3}", File.ReadAllText(Path.Combine(entryFolder, FrameTrash.AnalysisCopy))); // Tombstones alias the old content to the new, in the sidecar and the index. var tombstones = Assert.NotNull(after.Removed); Assert.SequenceEqual(before.Frames.Select(f => (f.Sha256, after.Frames.Single(n => n.Id == f.Id).Sha256, RemovalReason.Replaced)).Order(), tombstones.Select(t => ((string?)t.Sha256, t.Replacement, t.Reason)).Order()); Assert.SequenceEqual(tombstones.Select(t => (t.Sha256, t.Replacement)).Order(), (await fx.Db.FrameRemovals.ListBySequenceAsync(id)).Select(r => (r.Sha256, r.ReplacementSha256)).Order()); Assert.Equal((SequenceStatus)0, row.Status & (SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt)); Assert.False(StagedAnything(fx)); Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, job.Id)); Assert.FileMissing(ConvertJournal.StagedPathFor(fx.Paths, job.Id)); // Converted again, nothing is left to convert. Assert.Equal(0, (await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id)).Frames); } [Test] public async Task RestoringTheReplacedEntrySwapsTheOriginalsBack() { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 3); var engine = await fx.StartEngineAsync(); var originals = await FramesAsync(fx, id); var sequence = (await fx.Db.Sequences.GetAsync(id))!; var before = Sidecar(fx, sequence); var analysis = fx.Paths.Sequence(sequence.Folder).AnalysisPath; Directory.CreateDirectory(Path.GetDirectoryName(analysis)!); File.WriteAllText(analysis, "of the originals"); await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id)); var converted = Sidecar(fx, (await fx.Db.Sequences.GetAsync(id))!); var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single(); // The light analysis of the converted frames, as it would be written again. File.WriteAllText(analysis, "of the conversions"); var result = await fx.Service.RestoreEntryAsync(fx.Archive.Id, entry.Name, null, fx.Ct); Assert.Equal((TrashKind.Replaced, 3, false, id), (result.Kind, result.Photos, result.NewSequence, result.Sequence.Id)); var frames = await FramesAsync(fx, id); Assert.SequenceEqual(originals.Select(f => (f.Id, f.Index, f.Sha256, f.RelativePath, f.Format)), frames.Select(f => (f.Id, f.Index, f.Sha256, f.RelativePath, f.Format))); var back = Sidecar(fx, (await fx.Db.Sequences.GetAsync(id))!); Assert.True(back.Frames.All(f => f.Source is null)); // The converted files are in the trash in turn, naming the originals as their replacements; the old entry is gone. var turned = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single(); Assert.NotEqual(entry.Name, turned.Name); Assert.Equal(TrashKind.Replaced, turned.Kind); var record = FrameTrash.ReadEntry(fx.Paths, turned.Name)!; Assert.SequenceEqual(converted.Frames.Select(f => (f.Sha256, before.Frames.Single(o => o.Id == f.Id).Sha256)).Order(), record.Photos.Select(p => (p.Frame.Sha256, p.Replacement)).Order()); // Each analysis goes with its frames: the originals' back in the sequence, the conversions' into the new entry. Assert.Equal("of the originals", File.ReadAllText(analysis)); Assert.Equal("of the conversions", File.ReadAllText(Path.Combine(FrameTrash.Location(fx.Paths, turned.Name).Folder, FrameTrash.AnalysisCopy))); Assert.False(Directory.Exists(FrameTrash.Location(fx.Paths, entry.Name).Folder)); Assert.SequenceEqual(converted.Frames.Select(f => (f.Sha256, before.Frames.Single(o => o.Id == f.Id).Sha256)).Order(), back.Removed!.Select(t => ((string?)t.Sha256, t.Replacement)).Order()); // That entry erases: its replacements (the originals) are where it says, re-hashed now. var empty = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); var job = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, empty), cancellationToken: fx.Ct); await fx.WaitForAsync(job.Id, JobState.Done); Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); await FramesAsync(fx, id); } [Test] public async Task AnUndoAfterTheFramesChangedBringsTheOriginalsBackAsANewSequence() { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 3); var engine = await fx.StartEngineAsync(); var originals = await FramesAsync(fx, id); await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id)); var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single(); var converted = await FramesAsync(fx, id); // A converted frame deleted since: the frames are no longer the ones the conversion left. var sequence = (await fx.Db.Sequences.GetAsync(id))!; var delete = await fx.Service.FrameTrash.PlanDeleteAsync(id, [converted[1].Id], sequence.FramesVersion, fx.Ct); await fx.Service.FrameTrash.DeleteAsync(delete, "alberto", null, fx.Ct); var result = await fx.Service.RestoreEntryAsync(fx.Archive.Id, entry.Name, null, fx.Ct); Assert.Equal((TrashKind.Replaced, 3, true), (result.Kind, result.Photos, result.NewSequence)); Assert.NotEqual(id, result.Sequence.Id); Assert.SequenceEqual(originals.Select(f => f.Sha256), (await FramesAsync(fx, result.Sequence.Id)).Select(f => f.Sha256)); Assert.Count(2, await FramesAsync(fx, id)); // The replaced entry is gone with its last photo (and its copy of the analysis); the deleted frame's entry stays. Assert.DoesNotContain(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name)); } [Test] public async Task ANewSequenceTakesTheCopiesAndLeavesTheSourceAsItIs() { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 3); var engine = await fx.StartEngineAsync(); await fx.Service.SetSequenceInfoAsync(id, "Roof", ["sunset", "milan"], 4, "From the roof", fx.Ct); var originals = await FramesAsync(fx, id); await fx.Db.Sequences.EditAsync(id, new SequenceEdit { SettingsJson = new ProjectSettings { ExcludedFrames = [1], HeroFrame = 2 }.ToJson() }, null, fx.Ct); var source = (await fx.Db.Sequences.GetAsync(id))!; var sidecar = File.ReadAllText(fx.Paths.Sequence(source.Folder).ManifestPath); var job = await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id)); var made = Assert.NotNull(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, id))); var copy = (await fx.Db.Sequences.GetAsync(made))!; Assert.Equal(("Roof (TIFF)", 4, "From the roof"), (copy.Title, copy.Rating, copy.Notes)); Assert.SequenceEqual(["milan", "sunset"], copy.Tags.Order()); var frames = await FramesAsync(fx, made); Assert.SequenceEqual(originals.Select(f => f.CapturedAt), frames.Select(f => f.CapturedAt)); Assert.True(frames.All(f => f.Format == ImageFileFormat.Tiff)); // Filed as the ingest files every photo: the archive's template in the new sequence, {name} the card's, {ext} the target's. Assert.SequenceEqual(originals.Select(f => Path.ChangeExtension(f.FileName, ".tif")), frames.Select(f => f.FileName)); Assert.SequenceEqual(originals.Select(f => f.OriginalFileName), frames.Select(f => f.OriginalFileName)); var settings = ProjectSettings.FromJson(copy.SettingsJson!); Assert.SequenceEqual([1], settings.ExcludedFrames); Assert.Equal((2, frames[2].Id), (settings.HeroFrame, copy.HeroFrameId)); // The source: not a byte of its sidecar, nor a row, changed; nothing went to the trash. Assert.Equal(sidecar, File.ReadAllText(fx.Paths.Sequence(source.Folder).ManifestPath)); var now = (await fx.Db.Sequences.GetAsync(id))!; Assert.Equal((source.FramesVersion, source.SettingsVersion, source.Title, source.Folder), (now.FramesVersion, now.SettingsVersion, now.Title, now.Folder)); Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256)), (await FramesAsync(fx, id)).Select(f => (f.Id, f.Sha256))); Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); Assert.False(StagedAnything(fx)); } [Test] public async Task APhotoThatFailsLeavesItsWholeSequenceAsItWasAndTheOthersConvert() { await using var fx = await FixtureAsync(); var bad = await UploadedAsync(fx, 3); var good = await UploadedAsync(fx, 2, "upload-2", seed: 300); var engine = await fx.StartEngineAsync(); var damaged = (await FramesAsync(fx, bad))[1]; var originals = await FramesAsync(fx, bad); // Rot after the plan: the photo no longer matches the hash it was archived with. var path = fx.Paths.FromRelative(damaged.RelativePath); var bytes = File.ReadAllBytes(path); bytes[^20] ^= 0x10; File.WriteAllBytes(path, bytes); var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, bad, good); Assert.Equal(5, plan.Frames); var job = await RunAsync(fx, engine, plan, JobState.Failed); Assert.Equal(Loc.Plural("Archive.Error.ConvertKept", 1), job.Error); Assert.Contains(damaged.FileName, job.LogTail); // The log says why in one catalog sentence; the exception's English text stays in the diagnostics log. Assert.Contains(Loc.L("Archive.Convert.Failed.Changed"), job.LogTail); Assert.False(job.LogTail!.Contains("SHA-256", StringComparison.Ordinal), job.LogTail); // All or nothing: not one frame of that sequence changed, and nothing of it is in the trash. var rows = await fx.Db.Frames.ListBySequenceAsync(bad); Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256, f.RelativePath)), rows.Select(f => (f.Id, f.Sha256, f.RelativePath))); Assert.True((await FramesAsync(fx, good)).All(f => f.Format == ImageFileFormat.Tiff)); Assert.SequenceEqual([good], (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.SequenceId)); // The journal stays for a retry, which tries the failed sequence again and leaves the converted one; so does what the // failed sequence staged before its photo failed, and only that. var journal = Assert.NotNull(ConvertJournal.Read(fx.Paths, job.Id)); Assert.Equal(ConvertOutcome.Converted, journal.Of(good)!.Outcome); Assert.Equal(ConvertOutcome.Failed, journal.Of(bad)!.Outcome); Assert.False(Directory.Exists(StagingOf(fx, job.Id, good))); var noted = StagedLog.Open(ConvertJournal.StagedPathFor(fx.Paths, job.Id)).Of(bad, (await fx.Db.Sequences.GetAsync(bad))!.FramesVersion); Assert.SequenceEqual(noted.Select(c => c.File).Order(), Directory.EnumerateFiles(StagingOf(fx, job.Id, bad)).Select(f => Path.GetFileName(f)).Order()); Assert.DoesNotContain(damaged.Id, noted.Select(c => c.FrameId)); var first = Path.Combine(StagingOf(fx, job.Id, bad), noted.Single(c => c.FrameId == originals[0].Id).File); File.SetLastWriteTimeUtc(first, new DateTime(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc)); // The photo is put right: the retry takes the staged conversion that still verifies and converts the rest. bytes[^20] ^= 0x10; File.WriteAllBytes(path, bytes); await engine.ResumeAsync(job.Id, fx.Ct); await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60)); var converted = await FramesAsync(fx, bad); Assert.True(converted.All(f => f.Format == ImageFileFormat.Tiff)); Assert.Equal(2020, File.GetLastWriteTimeUtc(fx.Paths.FromRelative(converted[0].RelativePath)).Year); Assert.False(StagedAnything(fx)); Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, job.Id)); } [Test] public async Task AResumedJobKeepsTheStagedFilesThatStillVerify() { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 3); var engine = await fx.StartEngineAsync(); // To JPEG needs other originals than the JPEGs uploaded: they become TIFF first. await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id)); var plan = await PlanAsync(fx, ConvertTarget.Jpeg, ConvertMode.Replace, id); var sequence = (await fx.Db.Sequences.GetAsync(id))!; var folder = fx.Paths.Sequence(sequence.Folder); var manifest = Sidecar(fx, sequence); var jobId = Guid.CreateVersion7(); var staging = Path.Combine(Staging(fx), $"convert-{jobId:N}", $"{id:N}"); var log = StagedLog.Open(ConvertJournal.StagedPathFor(fx.Paths, jobId)); // An earlier attempt staged the first photo (at another quality, so its use shows) and the second, damaged since. StagedConversion Stage(SidecarFrame frame, int quality) { var target = Path.Combine(staging, frame.Id.ToString("N") + ".jpg"); var file = FrameConverter.Convert(new ConvertSource(folder.File(frame.File), frame, manifest.Camera, manifest.Lens), new ConvertOptions(ConvertTarget.Jpeg, quality), DecoderFactory.CreateServer(), target, cancellationToken: fx.Ct); var conversion = new StagedConversion(frame.Id, frame.Sha256!, Path.GetFileName(target), file.Size, file.Sha256, file.Width, file.Height); log.Append(id, manifest.Versions.Frames, conversion); return conversion; } var kept = Stage(manifest.Frames[0], 40); var torn = Stage(manifest.Frames[1], 95); File.WriteAllBytes(Path.Combine(staging, torn.File), [1, 2, 3]); var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Convert(fx.Archive, plan), "convert", jobId); await fx.WaitForAsync(job.Id, JobState.Done); var frames = await FramesAsync(fx, id); Assert.Equal(kept.Sha256, frames[0].Sha256); Assert.Equal(torn.Sha256, frames[1].Sha256); Assert.True(frames.All(f => f.Format == ImageFileFormat.Jpeg)); } [Test] public async Task AJobThatStoppedAfterItsReplacementReportsTheSequenceConverted() { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 2); var engine = await fx.StartEngineAsync(); var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id); var request = ArchiveJobs.Convert(fx.Archive, plan); await RunAsync(fx, engine, plan); // As a crash between the replacement and the journal's note leaves it: the staged log names the conversions done. var sidecar = Sidecar(fx, (await fx.Db.Sequences.GetAsync(id))!); var jobId = Guid.CreateVersion7(); var log = StagedLog.Open(ConvertJournal.StagedPathFor(fx.Paths, jobId)); foreach (var frame in sidecar.Frames) log.Append(id, plan.Sequences.Single().FramesVersion, new StagedConversion(frame.Id, frame.Source!.Sha256!, "gone.dng", frame.Size, frame.Sha256!, frame.W, frame.H)); var job = await fx.EnqueueWithCheckpointAsync(engine, request, "convert", jobId); var done = await fx.WaitForAsync(job.Id, JobState.Done); Assert.Contains(Loc.L("Archive.Log.Converted", (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle, 2), done.LogTail); Assert.Count(1, await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); } [Test] public async Task ASequenceThatChangedAfterThePlanIsLeftAsItWas() { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 3); var engine = await fx.StartEngineAsync(); var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id); // A photo of the plan is deleted before the job runs. var sequence = (await fx.Db.Sequences.GetAsync(id))!; var delete = await fx.Service.FrameTrash.PlanDeleteAsync(id, [(await FramesAsync(fx, id))[1].Id], sequence.FramesVersion, fx.Ct); await fx.Service.FrameTrash.DeleteAsync(delete, "alberto", null, fx.Ct); var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id); var job = await RunAsync(fx, engine, plan, JobState.Failed); Assert.Equal(Loc.Plural("Archive.Error.ConvertKept", 1), job.Error); var changed = Loc.L("Archive.Log.ConvertChanged", sequence.DisplayTitle); Assert.Contains(changed, job.LogTail); Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id)); Assert.DoesNotContain(TrashKind.Replaced, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Kind)); // The refusal holds on a retry too, and says so: the user plans again. await engine.ResumeAsync(job.Id, fx.Ct); Assert.Contains(changed, (await fx.WaitForAsync(job.Id, JobState.Failed)).LogTail); Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id)); } [Test] public async Task PhotosThatComeInWhileTheJobWaitsItsTurnStayAsTheyAreAndThoseConfirmedConvert() { foreach (var mode in new[] { ConvertMode.Replace, ConvertMode.NewSequence }) { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 2); var engine = await fx.StartEngineAsync(); var plan = await PlanAsync(fx, ConvertTarget.Tiff16, mode, id); var title = (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle; // An upload lands after the confirmation, before the job comes to the sequence (it waited behind a render, or // behind other sequences). var (path, size, sha) = fx.Stage(Start.AddSeconds(60), 999); await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, null, path, "IMG_0099.JPG", size, sha, null, null), fx.Ct); var job = await RunAsync(fx, engine, plan); var frames = await FramesAsync(fx, id); Assert.Equal((sha, ImageFileFormat.Jpeg), frames.Where(f => f.Sha256 == sha).Select(f => (f.Sha256, f.Format)).Single(), mode.ToString()); if (mode == ConvertMode.Replace) { Assert.Contains(Loc.L("Archive.Log.Converted", title, 2), job.LogTail); Assert.Equal(2, frames.Count(f => f.Format == ImageFileFormat.Tiff)); Assert.Equal(2, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Frames); } else { Assert.Contains(Loc.L("Archive.Log.ConvertedNew", title, 2), job.LogTail); Assert.Contains(Loc.L("Archive.Log.ConvertLeftOut", title, 1), job.LogTail); var made = Assert.NotNull(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, id))); Assert.True((await FramesAsync(fx, made)) is { Count: 2 } copies && copies.All(f => f.Format == ImageFileFormat.Tiff)); Assert.True(frames.All(f => f.Format == ImageFileFormat.Jpeg)); } } } [Test] public async Task AVolumeWithoutTheSpaceConvertsNothing() { await using var fx = await FixtureAsync(o => o with { FreeSpace = _ => 1_000 }); var id = await UploadedAsync(fx, 2); var engine = await fx.StartEngineAsync(); var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id); Assert.False(plan.Space.Enough); Assert.Equal(1_000, plan.Space.Free); var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id); var job = await RunAsync(fx, engine, plan, JobState.Failed); Assert.True(job.Error!.StartsWith("The conversion needs", StringComparison.Ordinal), job.Error); Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id)); Assert.False(StagedAnything(fx)); } [Test] public async Task ACardKeepsTheCopyOfAPhotoThatWasConverted() { await using var fx = await FixtureAsync(); var card = context.CreateTempDirectory("card"); Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, 3); var media = await fx.Service.Imports.RegisterCardAsync(new CardVolume("3F2A9C11-EOS_DIGITAL-127865454592", 0x3F2A9C11, "EOS_DIGITAL", 127_865_454_592), fx.Ct); var import = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct); await fx.Service.Imports.ExecuteAsync(new ImportJobParameters(fx.Archive.Id, import, media.Id, true, null), Guid.NewGuid(), 0, _ => ValueTask.CompletedTask, null, fx.Ct); var id = (await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, import.Sessions.Single().SessionKey))!.Value; Assert.Equal(3, (await fx.Service.PlanCardEraseAsync(fx.Archive.Id, media.Id, card, null, fx.Ct)).ToDelete.Count()); var engine = await fx.StartEngineAsync(); await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id)); // The frames now hold converted files: nothing in the archive matches the card's photos byte for byte, so they stay. var erase = await fx.Service.PlanCardEraseAsync(fx.Archive.Id, media.Id, card, null, fx.Ct); Assert.Empty(erase.ToDelete); Assert.True(erase.Items.All(i => i.Verdict == EraseVerdict.CopyMismatch)); Assert.Equal(0, CardEraser.Execute(erase).Deleted); Assert.Count(3, Directory.EnumerateFiles(card, "*.JPG", SearchOption.AllDirectories)); } [Test] public async Task AReplacedEntryIsErasedOnlyWhileTheConvertedFilesStillHashToTheirRecord() { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 2); var engine = await fx.StartEngineAsync(); await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id)); var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single(); var converted = (await FramesAsync(fx, id))[0]; var file = fx.Paths.FromRelative(converted.RelativePath); var bytes = File.ReadAllBytes(file); var damaged = (byte[])bytes.Clone(); damaged[^10] ^= 0x01; File.WriteAllBytes(file, damaged); var plan = await PlanEmptyAsync(fx); var kept = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, plan), cancellationToken: fx.Ct); await fx.WaitForAsync(kept.Id, JobState.Failed); Assert.Contains(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name)); File.WriteAllBytes(file, bytes); var erased = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, await PlanEmptyAsync(fx)), cancellationToken: fx.Ct); await fx.WaitForAsync(erased.Id, JobState.Done); Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); static Task PlanEmptyAsync(ArchiveFixture fx) => fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct); } [Test] public async Task AReplacementACrashInterruptedBeforeItsSidecarIsUndoneByTheNextWriter() { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 2); var sequence = (await fx.Db.Sequences.GetAsync(id))!; var folder = fx.Paths.Sequence(sequence.Folder); var manifest = Sidecar(fx, sequence); var frame = manifest.Frames[0]; // Half a replacement: the original in its entry, the converted file in its place, the record pending, the sidecar as it was. var staged = Path.Combine(Staging(fx), "convert-test", frame.Id.ToString("N") + ".tif"); FrameConverter.Convert(new ConvertSource(folder.File(frame.File), frame), new ConvertOptions(ConvertTarget.Tiff16), DecoderFactory.CreateServer(), staged, cancellationToken: fx.Ct); var entryFolder = Path.Combine(FrameTrash.Directory(fx.Paths), $"{id:N}-20261027120000"); var record = entryFolder + ".json"; var trashed = Path.Combine(entryFolder, "originals", Path.GetFileName(frame.File)); var placed = Path.ChangeExtension(frame.File, ".tif"); var journal = new FrameReplaceJournal(manifest.Versions.Frames, fx.Paths.ToRelative(record), fx.Paths.ToRelative(entryFolder), [new SwapMove(frame.File, fx.Paths.ToRelative(trashed))], [new SwapMove(placed, fx.Paths.ToRelative(staged))], false, null); journal.Write(folder); Directory.CreateDirectory(FrameTrash.Directory(fx.Paths)); File.WriteAllText(FrameRemovalJournal.PendingRecord(record), "{}"); Directory.CreateDirectory(Path.GetDirectoryName(trashed)!); File.Move(folder.File(frame.File), trashed); File.Move(staged, folder.File(placed)); await fx.Service.EditSequenceInfoAsync(id, info => info with { Rating = 2 }, fx.Ct); Assert.FileExists(folder.File(frame.File)); Assert.FileMissing(folder.File(placed)); Assert.FileExists(staged); Assert.False(Directory.Exists(entryFolder)); Assert.FileMissing(FrameRemovalJournal.PendingRecord(record)); Assert.FileMissing(folder.ReplaceJournalPath); Assert.Equal(frame.Sha256, Sha(folder.File(frame.File))); Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); } [Test] public async Task AReplacementACrashInterruptedAfterItsSidecarIsCompletedByTheNextWriter() { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 2); var engine = await fx.StartEngineAsync(); var version = (await fx.Db.Sequences.GetAsync(id))!.FramesVersion; await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id)); var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single(); var (entryFolder, record) = FrameTrash.Location(fx.Paths, entry.Name); var sequence = (await fx.Db.Sequences.GetAsync(id))!; var folder = fx.Paths.Sequence(sequence.Folder); // As a crash right after the sidecar was written leaves it: the record still pending, the journal there. File.Move(record, FrameRemovalJournal.PendingRecord(record)); new FrameReplaceJournal(version, fx.Paths.ToRelative(record), fx.Paths.ToRelative(entryFolder), [], [], true, null).Write(folder); Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); await fx.Service.EditSequenceInfoAsync(id, info => info with { Rating = 2 }, fx.Ct); Assert.FileMissing(folder.ReplaceJournalPath); Assert.Equal(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name); Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Tiff)); } [Test] public async Task APhotoTheServerCannotReadAtFullResolutionIsSkipped() { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 2); var sequence = (await fx.Db.Sequences.GetAsync(id))!; // The sidecar says the second photo is a HEIC (a desktop with WIC filed it): the managed decoders have only its preview. var manifest = Sidecar(fx, sequence); ManifestStore.WriteSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath, manifest with { Frames = [manifest.Frames[0], manifest.Frames[1] with { Format = ImageFileFormat.Heif }] }); var step = (await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.NewSequence, id)).Sequences.Single(); Assert.Equal(1, step.Frames.Count); Assert.Equal((ConvertSkip.NoNativeDecoder, 1), (step.Skipped.Single().Reason, step.Skipped.Single().Frames)); } [Test] public async Task TheJournalsSurviveARoundTrip() { await using var fx = await FixtureAsync(); var journal = new ConvertJournal { JobId = Guid.NewGuid() }; var first = Guid.NewGuid(); var harbour = Guid.NewGuid(); journal.Begin(harbour, "Harbour", SequenceStatus.AnalyzedLight | SequenceStatus.ProxiesBuilt); journal.Note(new ConvertSequenceResult(first, "Roof", ConvertOutcome.Created, 3, Guid.NewGuid()), SequenceStatus.AnalyzedDeep); journal.Of(first)!.Queued = true; journal.Note(new ConvertSequenceResult(harbour, "Harbour", ConvertOutcome.Failed, 0, Error: "no")); journal.Write(fx.Paths); var read = Assert.NotNull(ConvertJournal.Read(fx.Paths, journal.JobId)); Assert.SequenceEqual(journal.Sequences.Select(s => (s.Id, s.Outcome, s.Frames, s.NewSequenceId, s.Final, s.Had, s.Queued)), read.Sequences.Select(s => (s.Id, s.Outcome, s.Frames, s.NewSequenceId, s.Final, s.Had, s.Queued))); Assert.True(read.Of(first)!.Final); // A failure keeps the statuses the sequence had when the job came to it. Assert.Equal(SequenceStatus.AnalyzedLight | SequenceStatus.ProxiesBuilt, read.Of(harbour)!.Had); // The photos pinned: a later line for a sequence replaces an earlier one, a torn one is skipped. var photo = Guid.NewGuid(); var pins = read.Pins(fx.Paths); Assert.Null(pins.Of(first)); ConvertSequencePlan Pinned(int version, string sha) => new(first, "Roof", "roof", version, [new ConvertFramePlan(photo, 0, "originals/a.jpg", ImageFileFormat.Jpeg, 1, 2, sha)], [], []); pins.Append(first, Pinned(4, "aa")); pins.Append(first, Pinned(5, "bb")); File.AppendAllText(ConvertJournal.PinsPathFor(fx.Paths, journal.JobId), "{\"sequence\":"); var pinned = Assert.NotNull(read.Pins(fx.Paths).Of(first)); Assert.Equal((5, photo, "bb"), (pinned.FramesVersion, pinned.Frames.Single().Id, pinned.Frames.Single().Sha256)); // Its files (the photos confirmed among them) and its staging folder are known by their names. Assert.SequenceEqual([journal.JobId, journal.JobId, journal.JobId, journal.JobId], ConvertJournal.Files(fx.Paths, journal.JobId).Select(p => ConvertJournal.JobOf(Path.GetFileName(p)))); Assert.Contains(ConfirmedPhotos.PathFor(fx.Paths, journal.JobId), ConvertJournal.Files(fx.Paths, journal.JobId)); // The copy a New sequence conversion made stays named whatever the sequence comes to after it. var copy = Guid.NewGuid(); journal.Made(harbour, copy); journal.Note(new ConvertSequenceResult(harbour, "Harbour", ConvertOutcome.Failed, 0, Error: "no")); journal.Write(fx.Paths); Assert.Equal(copy, ConvertJournal.Read(fx.Paths, journal.JobId)!.Of(harbour)!.NewSequenceId); Assert.Equal(journal.JobId, ConvertJournal.JobOf(MassConverter.StagingFolder(journal.JobId))); Assert.Null(ConvertJournal.JobOf("convert-test")); Assert.Null(ConvertJournal.JobOf("rename-" + journal.JobId.ToString("N") + ".json")); // A line torn by a crash is skipped; a later line for a frame replaces an earlier one. var path = ConvertJournal.StagedPathFor(fx.Paths, journal.JobId); var log = StagedLog.Open(path); var frame = Guid.NewGuid(); log.Append(first, 4, new StagedConversion(frame, "a", "x.dng", 1, "b", 2, 3)); log.Append(first, 4, new StagedConversion(frame, "a", "x.dng", 1, "c", 2, 3)); File.AppendAllText(path, "{\"sequence\":"); var reread = StagedLog.Open(path); Assert.Equal("c", reread.Of(first, 4).Single().Sha256); Assert.Empty(reread.Of(first, 5)); ConvertJournal.Delete(fx.Paths, journal.JobId); Assert.FileMissing(path); Assert.FileMissing(ConvertJournal.PinsPathFor(fx.Paths, journal.JobId)); Assert.Null(ConvertJournal.Read(fx.Paths, journal.JobId)); // A job an earlier build queued names no digest: it reads with none. var parameters = JsonSerializer.Serialize(new ConvertJobParameters(fx.Archive.Id, ConvertTarget.Jpeg, 80, ConvertMode.NewSequence, [new ConvertSequence(first, 2, "d")], "f"), ArchiveJobJson.Default.ConvertJobParameters); Assert.Equal("d", JsonSerializer.Deserialize(parameters, ArchiveJobJson.Default.ConvertJobParameters)!.Sequences.Single().Frames); var older = $$"""{"archiveId":"{{fx.Archive.Id}}","target":"Jpeg","quality":80,"mode":"Replace","sequences":[{"id":"{{first}}","framesVersion":2}],"fingerprint":"f"}"""; Assert.Null(JsonSerializer.Deserialize(older, ArchiveJobJson.Default.ConvertJobParameters)!.Sequences.Single().Frames); } [Test] public async Task ConvertedFilesAreNamedAsFilingNamesThemAndNothingIsOverwritten() { await using var fx = await FixtureAsync(); await fx.Service.UpdateSettingsAsync(fx.Archive.Id, s => s with { Templates = s.Templates with { File = "{name}_{n:00}" } }, fx.Ct); var roof = await UploadedAsync(fx, 2); var lake = await UploadedAsync(fx, 2, "upload-2", seed: 200); Assert.SequenceEqual(["IMG_0001_01.JPG", "IMG_0002_02.JPG"], (await FramesAsync(fx, roof)).Select(f => f.FileName)); await fx.Service.SetSequenceInfoAsync(roof, "Roof", [], 0, null, fx.Ct); await fx.Service.SetSequenceInfoAsync(lake, "Lake", [], 0, null, fx.Ct); // Filing names a photo with the template of the moment: a conversion names its files as filing would now. await fx.Service.UpdateSettingsAsync(fx.Archive.Id, s => s with { Templates = s.Templates with { File = "{title}_{n:00}" } }, fx.Ct); // A file that is no frame holds the first name: it is never written over, the converted file takes the next. var stray = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(roof))!.Folder).File("originals/Roof_01.dng"); File.WriteAllBytes(stray, [1, 2, 3]); var engine = await fx.StartEngineAsync(); await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, roof)); var replaced = await FramesAsync(fx, roof); Assert.SequenceEqual(["Roof_01_2.dng", "Roof_02.dng"], replaced.Select(f => f.FileName)); Assert.SequenceEqual(["IMG_0001.JPG", "IMG_0002.JPG"], replaced.Select(f => f.OriginalFileName)); Assert.SequenceEqual(new byte[] { 1, 2, 3 }, File.ReadAllBytes(stray)); var job = await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, lake)); var made = Assert.NotNull(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, lake))); var copies = await FramesAsync(fx, made); Assert.SequenceEqual(["Lake (TIFF)_01.tif", "Lake (TIFF)_02.tif"], copies.Select(f => f.FileName)); Assert.SequenceEqual(["IMG_0001.JPG", "IMG_0002.JPG"], copies.Select(f => f.OriginalFileName)); } [Test] public async Task AFolderMovedWhileTheJobWaitsForItsLeaseIsWhereTheConvertedFilesGo() { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 2); var engine = await fx.StartEngineAsync(); var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id); var row = (await fx.Db.Sequences.GetAsync(id))!; var from = fx.Paths.Sequence(row.Folder).Folder; Job job; var planned = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); void OnProgress(JobProgressInfo info) { if (info.Kind == JobKind.Convert && info.Total == 2) planned.TrySetResult(); } engine.Progress += OnProgress; using (await LeaseFile.AcquireAsync(fx.Paths.LocksDirectory, $"seq-{id:N}", "editor", TimeSpan.FromSeconds(5), cancellationToken: fx.Ct)) { job = await engine.EnqueueAsync(ArchiveJobs.Convert(fx.Archive, plan), cancellationToken: fx.Ct); // Planned again where the index had the sequence then: the job now waits for its lease to convert it. await planned.Task.WaitAsync(TimeSpan.FromSeconds(TestTiming.Budget(30)), fx.Ct); // What a Reorganize step does under that lease meanwhile: the folder moves, and its index row follows. var to = fx.Paths.Sequence("moved/Roof").Folder; Directory.CreateDirectory(Path.GetDirectoryName(to)!); Directory.Move(from, to); Assert.True(await fx.Db.Sequences.RelocateAsync(id, "moved/Roof", null, fx.Ct)); } engine.Progress -= OnProgress; await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60)); var frames = await FramesAsync(fx, id); Assert.True(frames.All(f => f.Format == ImageFileFormat.Tiff && f.RelativePath.StartsWith("moved/Roof/", StringComparison.Ordinal))); Assert.False(Directory.Exists(from), "the folder the sequence left is never made again"); Assert.Equal("moved/Roof", (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Folder); } [Test] public async Task ASequenceAnotherIndexMovedIsLeftAsItIsUntilThisIndexFindsIt() { // The folder the sequence left holds another sequence now, or nothing. foreach (var taken in new[] { true, false }) { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 2); Guid? stranger = taken ? await UploadedAsync(fx, 2, "upload-2", seed: 200) : null; var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id); var row = (await fx.Db.Sequences.GetAsync(id))!; var folder = fx.Paths.Sequence(row.Folder).Folder; // Another index moves the folder away, then the other sequence into the folder it left; this index has scanned neither. var (other, archive) = await fx.OtherWriterAsync(); var moves = stranger is not { } s ? [(id, "moved/{date}")] : new[] { (id, "moved/{date}"), (s, "parked/{date}"), (s, row.Folder) }; foreach (var (sequence, template) in moves) await other.ApplyRenameAsync(await other.PlanRenameAsync(archive.Id, [sequence], template, null, fx.Ct), null, fx.Ct); var before = taken ? ArchiveFixture.FilesOf(folder) : null; var job = Guid.NewGuid(); var left = (await RunHereAsync(fx, plan, job)).Sequences.Single(); Assert.Equal((ConvertOutcome.Blocked, Loc.L("Errors.Archive.FolderMoved")), (left.Outcome, left.Error), taken.ToString()); if (taken) Assert.SequenceEqual(before!, ArchiveFixture.FilesOf(folder), "the other sequence's files are untouched"); else Assert.False(Directory.Exists(folder), "the folder it left is not created again"); // Once this index's scan found it there, the retry converts it where it is now. await fx.ScanAsync(); Assert.Equal(ConvertOutcome.Converted, (await RunHereAsync(fx, plan, job)).Sequences.Single().Outcome, taken.ToString()); var moved = (await fx.Db.Sequences.GetAsync(id))!.Folder; Assert.True(moved.StartsWith("moved/", StringComparison.Ordinal), moved); Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Tiff && f.RelativePath.StartsWith(moved + "/", StringComparison.Ordinal))); if (taken) Assert.SequenceEqual(before!, ArchiveFixture.FilesOf(folder), "the other sequence's files are still untouched"); else Assert.False(Directory.Exists(folder)); } } [Test] public async Task AReorganizeTheArchiveWaitsForKeepsTheConversionOffItsSequences() { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 2); var originals = await FramesAsync(fx, id); var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id); // A Reorganize paused by its user holds the archive: its plan waits for it. await fx.Db.Jobs.EnqueueAsync(new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Move, TargetIds = [fx.Archive.Id], State = JobState.Paused, Created = DateTimeOffset.UtcNow, }); Assert.SequenceEqual([ConvertBlock.Busy], (await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id)).Sequences.Single().Blocked); var engine = await fx.StartEngineAsync(); var job = await RunAsync(fx, engine, plan, JobState.Failed); Assert.Equal(Loc.Plural("Archive.Error.ConvertKept", 1), job.Error); Assert.Contains(Loc.L("Archive.Log.ConvertBlocked", (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle), job.LogTail); Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256, f.RelativePath)), (await FramesAsync(fx, id)).Select(f => (f.Id, f.Sha256, f.RelativePath))); Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); } [Test] public async Task PhotosVerifiedAfterThePlanAreNotReplacedWithoutAConfirmation() { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 3); var engine = await fx.StartEngineAsync(); var sequence = (await fx.Db.Sequences.GetAsync(id))!; var path = fx.Paths.Sequence(sequence.Folder).ManifestPath; var verified = ManifestStore.ReadSequence(path); // Never verified when the plan is made (adopted): the dry run says that photo stays as it is. ManifestStore.WriteSequence(path, verified with { Frames = [verified.Frames[0], verified.Frames[1] with { Sha256 = null }, verified.Frames[2]] }); var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id); Assert.Equal((2, ConvertSkip.Unhashed), (plan.Frames, plan.Sequences.Single().Skipped.Single().Reason)); // A verify then hashes it before the job runs, as ArchiveVerifier does: no new frames version. ManifestStore.WriteSequence(path, verified); // The job converts the two photos the plan listed, and leaves the third as it is, as the dry run said. var job = await RunAsync(fx, engine, plan); Assert.Contains(Loc.L("Archive.Log.Converted", sequence.DisplayTitle, 2), job.LogTail); var frames = await FramesAsync(fx, id); Assert.SequenceEqual(verified.Frames.Select(f => f.Id), frames.Select(f => f.Id)); Assert.Equal((verified.Frames[1].Sha256, ImageFileFormat.Jpeg), (frames[1].Sha256, frames[1].Format)); Assert.True(frames[0].Format == ImageFileFormat.Dng && frames[2].Format == ImageFileFormat.Dng); Assert.Equal(2, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Frames); Assert.False(StagedAnything(fx)); } [Test] public async Task PhotosThatComeInOnceTheSequenceIsPinnedStayAsTheyAreAndARetryConvertsTheRest() { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 2); var engine = await fx.StartEngineAsync(); var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id); var title = (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle; // An earlier attempt pinned the sequence's photos and stopped (stage B blocked, a pause); an upload came in since. var jobId = Guid.CreateVersion7(); PinLog.Open(ConvertJournal.PinsPathFor(fx.Paths, jobId)).Append(id, plan.Sequences.Single()); var (path, size, sha) = fx.Stage(Start.AddSeconds(60), 999); await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, null, path, "IMG_0099.JPG", size, sha, null, null), fx.Ct); var added = (await FramesAsync(fx, id)).Single(f => f.Sha256 == sha); var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Convert(fx.Archive, plan), "convert", jobId); var done = await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60)); Assert.Contains(Loc.L("Archive.Log.Converted", title, 2), done.LogTail); var frames = await FramesAsync(fx, id); Assert.Equal(2, frames.Count(f => f.Format == ImageFileFormat.Tiff)); Assert.Equal((sha, ImageFileFormat.Jpeg), frames.Where(f => f.Id == added.Id).Select(f => (f.Sha256, f.Format)).Single()); Assert.Equal(2, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Frames); } [Test] public async Task AJobResumedAfterACrashInItsReplacementSettlesItAndConvertsTheWholeSequence() { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 2); var engine = await fx.StartEngineAsync(); var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id); var sequence = (await fx.Db.Sequences.GetAsync(id))!; var folder = fx.Paths.Sequence(sequence.Folder); var manifest = Sidecar(fx, sequence); // The attempt that crashed: the photos pinned and staged, then its replacement cut short once every original had left // for the trash, none of the converted files in yet. var jobId = Guid.CreateVersion7(); PinLog.Open(ConvertJournal.PinsPathFor(fx.Paths, jobId)).Append(id, plan.Sequences.Single()); var log = StagedLog.Open(ConvertJournal.StagedPathFor(fx.Paths, jobId)); var entryFolder = Path.Combine(FrameTrash.Directory(fx.Paths), $"{id:N}-20261027120000"); var record = entryFolder + ".json"; List outgoing = []; List incoming = []; foreach (var frame in manifest.Frames) { var staged = Path.Combine(StagingOf(fx, jobId, id), frame.Id.ToString("N") + ".tif"); var file = FrameConverter.Convert(new ConvertSource(folder.File(frame.File), frame), new ConvertOptions(ConvertTarget.Tiff16), DecoderFactory.CreateServer(), staged, cancellationToken: fx.Ct); log.Append(id, manifest.Versions.Frames, new StagedConversion(frame.Id, frame.Sha256!, Path.GetFileName(staged), file.Size, file.Sha256, file.Width, file.Height)); outgoing.Add(new SwapMove(frame.File, fx.Paths.ToRelative(Path.Combine(entryFolder, "originals", Path.GetFileName(frame.File))))); incoming.Add(new SwapMove(Path.ChangeExtension(frame.File, ".tif"), fx.Paths.ToRelative(staged))); } new FrameReplaceJournal(manifest.Versions.Frames, fx.Paths.ToRelative(record), fx.Paths.ToRelative(entryFolder), outgoing, incoming, false, null).Write(folder); Directory.CreateDirectory(FrameTrash.Directory(fx.Paths)); File.WriteAllText(FrameRemovalJournal.PendingRecord(record), "{}"); foreach (var move in outgoing) { var trashed = fx.Paths.FromRelative(move.Other); Directory.CreateDirectory(Path.GetDirectoryName(trashed)!); File.Move(folder.File(move.Place), trashed); } // Not a sequence with nothing to convert, nor half of it converted: the originals come back, then all of it converts. var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Convert(fx.Archive, plan), "convert", jobId); var done = await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60)); Assert.Contains(Loc.L("Archive.Log.Converted", sequence.DisplayTitle, 2), done.LogTail); Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Tiff)); var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single(); Assert.Equal((TrashKind.Replaced, 2), (entry.Kind, entry.Frames)); Assert.False(Directory.Exists(entryFolder)); } [Test] public async Task TheFollowUpsOfEachSequenceAreQueuedAsItEndsAndOnResumeWhenAPauseCameFirst() { await using var fx = await FixtureAsync(); var first = await UploadedAsync(fx, 2); var second = await UploadedAsync(fx, 2, "upload-2", seed: 200); var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, first, second); var events = new List(); var callbacks = Callbacks(done: s => events.Add($"done {s.SequenceId}"), queue: r => events.AddRange(r.Select(q => $"{q.Kind} {q.Targets.Single()}"))); await RunHereAsync(fx, plan, Guid.CreateVersion7(), callbacks); // Queued as each sequence is journaled, before the next starts: a pause, a yield or a crash later loses none of them. Assert.Less(events.IndexOf($"done {first}"), events.IndexOf($"{JobKind.LightAnalyse} {first}")); Assert.Less(events.IndexOf($"{JobKind.LightAnalyse} {first}"), events.IndexOf($"done {second}")); Assert.Contains($"{JobKind.LightAnalyse} {second}", events); // As a pause right after the first sequence's outcome was journaled leaves the job: its passes not queued yet. var jobId = Guid.CreateVersion7(); var journal = new ConvertJournal { JobId = jobId }; journal.Note(new ConvertSequenceResult(first, "Roof", ConvertOutcome.Converted, 2), SequenceStatus.AnalyzedDeep | SequenceStatus.ThumbsBuilt); journal.Note(new ConvertSequenceResult(second, "Lake", ConvertOutcome.Converted, 2), SequenceStatus.AnalyzedLight); journal.Of(second)!.Queued = true; journal.Write(fx.Paths); List queued = []; var resumed = await RunHereAsync(fx, plan, jobId, Callbacks(queue: queued.AddRange)); Assert.Equal(4, resumed.Frames); Assert.SequenceEqual([(JobKind.LightAnalyse, first), (JobKind.DeepAnalyse, first), (JobKind.Thumbs, first)], queued.Select(r => (r.Kind, r.Targets.Single()))); Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, jobId)); } [Test] public async Task AConversionWaitingInTheQueueKeepsItsSequencesFromASecondConfirmation() { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 2); var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id); var request = ArchiveJobs.Convert(fx.Archive, plan); var job = new Job { Id = Guid.CreateVersion7(), Kind = request.Kind, TargetIds = request.Targets, ParametersJson = request.ParametersJson, State = JobState.Queued, Created = DateTimeOffset.UtcNow, }; await fx.Db.Jobs.EnqueueAsync(job, fx.Ct); // Another tab, or Apply pressed again after an answer was lost: the same copies are not queued twice. var again = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id); Assert.SequenceEqual([ConvertBlock.Busy], again.Sequences.Single().Blocked); Assert.NotEqual(plan.Fingerprint(), again.Fingerprint()); Assert.Throws(() => ArchiveJobs.Convert(fx.Archive, again)); // The job itself plans again with its sequences free, and converts them. var engine = await fx.StartEngineAsync(); engine.Wake(); await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60)); Assert.NotNull(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, id))); Assert.Empty((await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id)).Sequences.Single().Blocked); } [Test] public async Task ANewSequenceIsOwnedAsItsSourceIsOrByWhoAskedForIt() { await using var fx = await FixtureAsync(); var owner = Guid.NewGuid(); var owned = await UploadedAsync(fx, 2, user: owner); var unowned = await UploadedAsync(fx, 2, "upload-2", seed: 200); Assert.Equal(owner, (await fx.Db.Sequences.GetAsync(owned))!.OwnerUserId); var engine = await fx.StartEngineAsync(); var asker = Guid.NewGuid(); var job = await engine.EnqueueAsync(ArchiveJobs.Convert(fx.Archive, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, owned, unowned)), asker, fx.Ct); await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60)); foreach (var (source, expected) in new[] { (owned, owner), (unowned, asker) }) { var made = (await fx.Db.Sequences.GetAsync((await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, source)))!.Value))!; Assert.Equal(expected, made.OwnerUserId); Assert.Equal(expected.ToString(), Sidecar(fx, made).CreatedBy); } } [Test] public async Task AStagedCopyThatVanishesFailsItsNewSequenceInsteadOfLeavingThePhotoOut() { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 2); var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id); var first = Sidecar(fx, (await fx.Db.Sequences.GetAsync(id))!).Frames[0]; var jobId = Guid.CreateVersion7(); // A clean-up takes the first copy once both are staged, before they are filed. var callbacks = Callbacks(progress: (n, total) => { if (n == 2 && total == 2) File.Delete(Path.Combine(StagingOf(fx, jobId, id), first.Id.ToString("N") + ".tif")); }); var failed = (await RunHereAsync(fx, plan, jobId, callbacks)).Sequences.Single(); Assert.Equal(ConvertOutcome.Failed, failed.Outcome); // Why, in the catalog's words: the job log has no English exception text. Assert.Equal(Loc.L("Archive.Convert.NotPlaced", Loc.L("Archive.Convert.Unstaged", first.File)), failed.Error); Assert.Null(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(jobId, id))); // The retry converts that photo again and makes the whole copy. var made = (await RunHereAsync(fx, plan, jobId)).Sequences.Single(); Assert.Equal((ConvertOutcome.Created, 2, 0), (made.Outcome, made.Frames, made.LeftOut)); Assert.Count(2, await FramesAsync(fx, made.NewSequenceId!.Value)); } [Test] public async Task ANewSequenceSaysWhichPhotosItLeftInTheSourceAlone() { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 3); var engine = await fx.StartEngineAsync(); // Three TIFFs, then a JPEG uploaded: to JPEG, the TIFFs convert and the JPEG stays in the source alone. var sequence = (await fx.Db.Sequences.GetAsync(id))!; await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id)); var (path, size, sha) = fx.Stage(Start.AddSeconds(60), 999); await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, null, path, "IMG_0099.JPG", size, sha, null, null), fx.Ct); var job = await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Jpeg, ConvertMode.NewSequence, id)); Assert.Contains(Loc.L("Archive.Log.ConvertedNew", sequence.DisplayTitle, 3), job.LogTail); Assert.Contains(Loc.L("Archive.Log.ConvertLeftOut", sequence.DisplayTitle, 1), job.LogTail); var made = Assert.NotNull(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, id))); Assert.Count(3, await FramesAsync(fx, made)); } [Test] public async Task AVolumeThatFillsDuringTheJobStopsItBeforeTheNextSequenceAndTheNextPhoto() { long free = 1L << 50; await using var fx = await FixtureAsync(o => o with { FreeSpace = _ => free, Parallelism = () => 1 }); var first = await UploadedAsync(fx, 2); var second = await UploadedAsync(fx, 2, "upload-2", seed: 200); var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, first, second); var jobId = Guid.CreateVersion7(); var originals = await FramesAsync(fx, second); // Another writer fills the volume while the first sequence converts: the second does not start. var full = await Assert.ThrowsAsync(() => RunHereAsync(fx, plan, jobId, Callbacks(done: _ => free = 1_000))); Assert.True(full.Message.StartsWith("The conversion needs", StringComparison.Ordinal), full.Message); Assert.True((await FramesAsync(fx, first)).All(f => f.Format == ImageFileFormat.Tiff)); Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256)), (await FramesAsync(fx, second)).Select(f => (f.Id, f.Sha256))); Assert.False(Directory.Exists(StagingOf(fx, jobId, second))); // Space again, until the first photo of the second sequence is staged: the next one does not start, what was staged stays. free = 1L << 50; await Assert.ThrowsAsync(() => RunHereAsync(fx, plan, jobId, Callbacks(progress: (n, _) => { if (n == 1) free = 1_000; }))); Assert.Count(1, Directory.EnumerateFiles(StagingOf(fx, jobId, second))); Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256)), (await FramesAsync(fx, second)).Select(f => (f.Id, f.Sha256))); free = 1L << 50; var done = await RunHereAsync(fx, plan, jobId); Assert.True(done.Sequences.All(s => s.Outcome == ConvertOutcome.Converted)); Assert.True((await FramesAsync(fx, second)).All(f => f.Format == ImageFileFormat.Tiff)); Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, jobId)); } [Test] public async Task MaintenanceKeepsWhatAWaitingConversionStagedAndClearsWhatAFinishedOneLeft() { await using var fx = await FixtureAsync(); var paused = Guid.CreateVersion7(); var finished = Guid.CreateVersion7(); var forgotten = Guid.CreateVersion7(); var stray = Guid.CreateVersion7(); foreach (var (id, state) in new[] { (paused, JobState.Paused), (finished, JobState.Done) }) await fx.Db.Jobs.EnqueueAsync(new Job { Id = id, Kind = JobKind.Convert, State = state, Created = DateTimeOffset.UtcNow }, fx.Ct); string Staged(Guid job) { var file = Path.Combine(StagingOf(fx, job, Guid.NewGuid()), "frame.tif"); Directory.CreateDirectory(Path.GetDirectoryName(file)!); File.WriteAllText(file, "x"); File.SetLastWriteTimeUtc(file, DateTime.UtcNow.AddDays(-3)); return file; } string Journal(Guid job) { new ConvertJournal { JobId = job }.Write(fx.Paths); return ConvertJournal.PathFor(fx.Paths, job); } var (pausedFile, pausedJournal) = (Staged(paused), Journal(paused)); var (finishedFile, finishedJournal) = (Staged(finished), Journal(finished)); var (forgottenFile, forgottenJournal) = (Staged(forgotten), Journal(forgotten)); var strayFile = Staged(stray); var dry = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: true, 0, _ => ValueTask.CompletedTask, fx.Ct); Assert.SequenceEqual(new[] { finishedFile, finishedJournal, strayFile }.Select(fx.Paths.ToRelative).Order(), dry.TemporaryFiles.Order()); await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct); // A paused job's staged conversions stay however old; a finished job's go with its journal. A job this index does not // know (another host's on a shared archive, or forgotten by the history) may still resume where it is known: its // staged conversions stay as long as its journal; without a journal no resume can use them, and they go once idle. Assert.FileExists(pausedFile); Assert.FileExists(pausedJournal); Assert.False(Directory.Exists(Path.Combine(Staging(fx), MassConverter.StagingFolder(finished)))); Assert.FileMissing(finishedJournal); Assert.FileExists(forgottenFile); Assert.FileExists(forgottenJournal); Assert.False(Directory.Exists(Path.Combine(Staging(fx), MassConverter.StagingFolder(stray)))); // Its journal untouched for as long as the history keeps jobs, but a staged file written since: both stay. var longAgo = DateTime.UtcNow.AddDays(-CleanupSteps.DefaultHistoryDays - 1); File.SetLastWriteTimeUtc(forgottenJournal, longAgo); await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct); Assert.FileExists(forgottenJournal); Assert.FileExists(forgottenFile); // Neither written for that long: they go together. File.SetLastWriteTimeUtc(forgottenFile, longAgo); await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct); Assert.FileMissing(forgottenJournal); Assert.False(Directory.Exists(Path.Combine(Staging(fx), MassConverter.StagingFolder(forgotten)))); Assert.FileExists(pausedJournal); } [Test] public async Task APhotoAnotherProgramHoldsIsSaidSoAndNeverConvertedWithoutAPlanThatListsIt() { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 3); var title = (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle; var originals = await FramesAsync(fx, id); var held = fx.Paths.FromRelative(originals[1].RelativePath); // A sync client holds it while the dry run is made: not missing, and not planned, with why. ConvertPlan plan; using (Unreadable(held)) plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id); var step = plan.Sequences.Single(); Assert.Equal((2, ConvertSkip.Inaccessible, 1), (step.Frames.Count, step.Skipped.Single().Reason, step.Skipped.Single().Frames)); Assert.DoesNotContain(originals[1].Id, step.Frames.Select(f => f.FrameId)); // Free again when the job runs: it converts the two photos the plan listed, not that one. var done = (await RunHereAsync(fx, plan, Guid.CreateVersion7())).Sequences.Single(); Assert.Equal((ConvertOutcome.Converted, 2), (done.Outcome, done.Frames)); var frames = await FramesAsync(fx, id); Assert.Equal((originals[1].Sha256, ImageFileFormat.Jpeg), (frames[1].Sha256, frames[1].Format)); Assert.True(frames[0].Format == ImageFileFormat.Tiff && frames[2].Format == ImageFileFormat.Tiff); // Held again while the job that converts it comes to it: left as it is, saying why, and the retry converts it once free. var rest = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id); Assert.Equal(1, rest.Frames); var jobId = Guid.CreateVersion7(); var lines = new List(); ConvertSequenceResult left; using (Unreadable(held)) left = (await RunHereAsync(fx, rest, jobId, Callbacks() with { Log = lines.Add })).Sequences.Single(); Assert.Equal((ConvertOutcome.Blocked, Loc.L("Archive.Convert.PhotoHeld")), (left.Outcome, left.Error)); Assert.SequenceEqual([Loc.L("Archive.Log.ConvertFailed", title, Loc.L("Archive.Convert.PhotoHeld"))], lines); Assert.Equal(ImageFileFormat.Jpeg, (await FramesAsync(fx, id))[1].Format); Assert.Equal(ConvertOutcome.Converted, (await RunHereAsync(fx, rest, jobId)).Sequences.Single().Outcome); Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Tiff)); } /// /// Only a photo the job converts makes a sequence wait for its retry while another program holds it: one the plan left /// out being held does not hide that a photo confirmed is gone, which no retry can mend. /// [Test] public async Task APhotoHeldThatTheJobDoesNotConvertDoesNotHideAChange() { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 3); var originals = await FramesAsync(fx, id); var held = fx.Paths.FromRelative(originals[0].RelativePath); ConvertPlan plan; using (Unreadable(held)) plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id); Assert.Equal(2, plan.Frames); // A photo of the plan is deleted before the job comes to the sequence, and the one left out is held again then. var sequence = (await fx.Db.Sequences.GetAsync(id))!; var delete = await fx.Service.FrameTrash.PlanDeleteAsync(id, [originals[1].Id], sequence.FramesVersion, fx.Ct); await fx.Service.FrameTrash.DeleteAsync(delete, "alberto", null, fx.Ct); ConvertSequenceResult left; using (Unreadable(held)) left = (await RunHereAsync(fx, plan, Guid.CreateVersion7())).Sequences.Single(); Assert.Equal((ConvertOutcome.Changed, (string?)null), (left.Outcome, left.Error)); Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Jpeg)); } /// /// Before each photo of stage A, the space is checked for it and for every photo handed to the work before it whose file /// has not landed yet: a converted file lands whole, by a rename, before its photo's work verifies it, so the free space /// measured holds the files landed and none of what the photos still converting will write. Near the end of a sequence /// those are as many as anywhere else, never fewer; photos that are not there to start are never counted. /// [Test] public void APhotoStartsOnlyWithTheSpaceForItAndThePhotosStillWriting() { const long photo = 100_000_000; var staging = context.CreateTempDirectory("staging"); string Target(int i) => Path.Combine(staging, $"{i:00}.tif"); // The volume: what it had, less the files landed so far (those still converting are not on it yet). var landed = 0L; var volume = ConvertPlanner.Required(3 * photo); var space = new MassConverter.SpaceWatch(() => volume - landed); void Land(int i) { File.WriteAllBytes(Target(i), [1]); landed += photo; } // The last photos of a sequence, four converting at once: none of the three started has landed, so the fourth, the // last one, needs the space of all four, not its own alone. for (var i = 0; i < 3; i++) Assert.Null(space.Start(photo, Target(i))); Assert.Equal(3 * photo, space.Writing); var refused = Assert.NotNull(space.Start(photo, Target(3))); Assert.True(refused.StartsWith("The conversion needs", StringComparison.Ordinal), refused); Assert.Equal(3 * photo, space.Writing); // One lands, and its work verifies it: the volume holds it, so the watch counts it no more (never twice). The last // photo still waits: the two others and it need more than what is left. Land(0); Assert.Equal(2 * photo, space.Writing); Assert.NotNull(space.Start(photo, Target(3))); space.Ended(Target(0)); Land(1); Land(2); Assert.Equal(0, space.Writing); space.Ended(Target(1)); space.Ended(Target(2)); // All three landed: what is left, with the margin and the reserve, cannot take a fourth; the reserve is whole. Assert.NotNull(space.Start(photo, Target(3))); Assert.True(volume - landed >= ConvertPlanner.SpaceReserve); // Room for one more: it starts, and a photo whose work ends without a file (it failed) counts no more either. volume = landed + ConvertPlanner.Required(photo); Assert.Null(space.Start(photo, Target(3))); Assert.NotNull(space.Start(photo, Target(4))); space.Ended(Target(3)); Assert.Null(space.Start(photo, Target(4))); space.Ended(Target(4)); // A file a crash left where a photo converts is not its file: the photo counts until the new one replaces it. File.WriteAllBytes(Target(5), [1, 2, 3]); File.SetLastWriteTimeUtc(Target(5), new DateTime(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc)); Assert.Null(space.Start(photo, Target(5))); Assert.Equal(photo, space.Writing); File.WriteAllBytes(Target(5), new byte[10]); Assert.Equal(0, space.Writing); // Files staged over their estimate scale what is still writing too. var scaled = new MassConverter.SpaceWatch(() => ConvertPlanner.Required(3 * photo)); scaled.Staged(2 * photo, photo); Assert.Equal(2, scaled.Ratio); Assert.Null(scaled.Start(photo, Target(6))); Assert.NotNull(scaled.Start(photo, Target(7))); scaled.Ended(Target(6)); Assert.Null(scaled.Start(photo, Target(7))); // A volume that cannot be measured stops nothing. Assert.Null(new MassConverter.SpaceWatch(() => null).Start(long.MaxValue / 4, Target(8))); } /// /// A job whose dry run had the space never stops short at the end of a sequence: the last photo to start counts only /// the photos that started and have not written yet, none that are not there. /// [Test] public async Task TheLastPhotosToStartNeedOnlyTheSpaceOfThePhotosStillWriting() { long free = 1L << 50; await using var fx = await FixtureAsync(o => o with { FreeSpace = _ => free, Parallelism = () => 4 }); var id = await UploadedAsync(fx, 1); // Exactly what the dry run asks for: four photos may convert at once, but there is one. free = ConvertPlanner.Required(ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.Tiff16), 48, 32)); var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id); Assert.True(plan.Space.Enough); Assert.Equal(free, plan.Space.Required); var done = (await RunHereAsync(fx, plan, Guid.CreateVersion7())).Sequences.Single(); Assert.Equal((ConvertOutcome.Converted, 1), (done.Outcome, done.Frames)); } /// /// A sequence that fits converts to its last photo, four at once, while its converted files take the volume as they land: /// at the end the photos still writing are counted, those landed (verified meanwhile) are in the free space measured, and /// none is counted twice. /// [Test] public async Task ASequenceThatFitsConvertsToItsLastPhotoWhileItsFilesFillTheVolume() { long volume = 1L << 50; string? staging = null; // The volume as the converted files land on it; an atomic write's temporary file is left out, being there only while // its photo still counts. long Landed() => staging is not null && Directory.Exists(staging) ? Directory.EnumerateFiles(staging).Where(f => !Path.GetFileName(f).StartsWith('.')).Sum(f => new FileInfo(f).Length) : 0; await using var fx = await FixtureAsync(o => o with { FreeSpace = _ => volume - Landed(), Parallelism = () => 4 }); var id = await UploadedAsync(fx, 12); var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id); // What the twelve converted files take, each converted beforehand as the job converts it. var sequence = (await fx.Db.Sequences.GetAsync(id))!; var folder = fx.Paths.Sequence(sequence.Folder); var manifest = Sidecar(fx, sequence); var sizes = context.CreateTempDirectory("sizes"); var converted = manifest.Frames.Sum(f => FrameConverter.Convert(new ConvertSource(folder.File(f.File), f, manifest.Camera, manifest.Lens), new ConvertOptions(ConvertTarget.Tiff16), DecoderFactory.CreateServer(), Path.Combine(sizes, f.Id.ToString("N") + ".tif"), cancellationToken: fx.Ct).Size); // Just the space they need with the margin and the reserve, and never less than the dry run asks. volume = ConvertPlanner.Required(Math.Max(converted, plan.BytesAfter)); var jobId = Guid.CreateVersion7(); staging = StagingOf(fx, jobId, id); var done = (await RunHereAsync(fx, plan, jobId)).Sequences.Single(); Assert.Equal((ConvertOutcome.Converted, 12), (done.Outcome, done.Frames)); Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Tiff)); } [Test] public async Task ASecondConversionOfTheSameSequencesIsRefusedWithOrWithoutADryRun() { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 2); var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id); async Task Enqueue(ArchiveJobRequest request) { // Long enough for the other confirmation to check meanwhile, were it not held back. await Task.Delay(50, fx.Ct); await fx.Db.Jobs.EnqueueAsync(new Job { Id = Guid.CreateVersion7(), Kind = request.Kind, TargetIds = request.Targets, ParametersJson = request.ParametersJson, State = JobState.Queued, Created = DateTimeOffset.UtcNow, }, fx.Ct); } async Task ConfirmAsync() { try { await fx.Service.Converter.QueueAsync(fx.Archive, plan, Enqueue, fx.Ct); return true; } catch (ConversionQueuedException ex) { Assert.SequenceEqual([id], ex.SequenceIds); Assert.Equal(Loc.L("Archive.Convert.AlreadyQueued"), ex.Message); return false; } } // The Library's dialog and the Archive manager's confirm the same plan at once: one job. var confirmed = await Task.WhenAll(ConfirmAsync(), ConfirmAsync()); Assert.SequenceEqual([false, true], confirmed.Order()); // And again later from that plan, made before the job was queued and never planned again. Assert.False(await ConfirmAsync()); Assert.Count(1, (await fx.Db.Jobs.ListAsync(JobState.Queued, JobKind.Convert)).Items); } [Test] public async Task AConversionQueuedFirstHoldsItsSequencesFromADuplicateQueuedAfterIt() { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 2); var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id); // Queued twice anyway (an earlier build, two processes on one index): the copies are made once. var first = await QueuedAsync(fx, plan, DateTimeOffset.UtcNow.AddMinutes(-2)); var second = await QueuedAsync(fx, plan, DateTimeOffset.UtcNow.AddMinutes(-1)); var held = (await fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), second.Id, new ConvertRequester(null, null, second.Created), Callbacks(), fx.Ct)) .Sequences.Single(); Assert.Equal(ConvertOutcome.Blocked, held.Outcome); Assert.Null(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(second.Id, id))); // The one queued first is not held back by the one after it. var made = (await fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), first.Id, new ConvertRequester(null, null, first.Created), Callbacks(), fx.Ct)) .Sequences.Single(); Assert.Equal((ConvertOutcome.Created, 2), (made.Outcome, made.Frames)); await fx.Db.Jobs.SetStateAsync(first.Id, JobState.Done, null, fx.Ct); // The duplicate, tried again once the first is done, finds the copy made since it was queued: it makes no second one. var again = (await fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), second.Id, new ConvertRequester(null, null, second.Created), Callbacks(), fx.Ct)) .Sequences.Single(); Assert.Equal(ConvertOutcome.Superseded, again.Outcome); Assert.Null(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(second.Id, id))); } /// /// A job's row names no photo, so it stays as small as the number of sequences however many photos convert: the photos /// confirmed are written whole beside the journal, under the job's id, before it is queued, and go with the journal. /// [Test] public async Task TheJobRowNamesNoPhotoAndThePhotosConfirmedWaitBesideItsJournal() { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 3); var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id); var frames = plan.Sequences.Single().Frames.Select(f => f.FrameId).ToList(); // Queued and not run yet: an engine that is not started. await using var idle = new JobEngine(fx.Db, []); Job? job = null; await fx.Service.Converter.QueueAsync(fx.Archive, plan, async r => { Assert.FileExists(ConfirmedPhotos.PathFor(fx.Paths, r.Id!.Value)); job = await idle.EnqueueAsync(r, cancellationToken: fx.Ct); }, fx.Ct); var queued = Assert.NotNull(await fx.Db.Jobs.GetAsync(job!.Id, fx.Ct)); Assert.SequenceEqual(frames, ConfirmedPhotos.Read(fx.Paths, queued.Id).Of(id)!); Assert.Null(ConfirmedPhotos.Read(fx.Paths, queued.Id).Of(Guid.NewGuid())); foreach (var frame in frames) { Assert.False(queued.ParametersJson!.Contains(frame.ToString(), StringComparison.OrdinalIgnoreCase), queued.ParametersJson); Assert.False(queued.ParametersJson!.Contains(ConfirmedPhotos.Encode([frame])[..20], StringComparison.Ordinal), queued.ParametersJson); } Assert.Equal(plan.Sequences.Single().Digest(), Parameters(fx, plan).Sequences.Single().Frames); // Lists of jobs read none of their parameters. Assert.Null((await fx.Db.Jobs.ListBriefAsync(JobState.Queued, JobKind.Convert, 10, null, fx.Ct)).Items.Single().ParametersJson); // A confirmation the queue refuses leaves nothing behind. var other = await UploadedAsync(fx, 1, "upload-2", seed: 300); var refused = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, other); Guid? named = null; await Assert.ThrowsAsync(() => fx.Service.Converter.QueueAsync(fx.Archive, refused, r => { named = r.Id; throw new InvalidOperationException("the index is away"); }, fx.Ct)); Assert.FileMissing(ConfirmedPhotos.PathFor(fx.Paths, named!.Value)); // The job runs, and once done with every sequence, its photos go with its journal. var engine = await fx.StartEngineAsync(); engine.Wake(); await fx.WaitForAsync(queued.Id, JobState.Done, (int)TestTiming.Budget(60)); Assert.FileMissing(ConfirmedPhotos.PathFor(fx.Paths, queued.Id)); Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Tiff)); } [Test] public void ThePhotosConfirmedReadBackAsWrittenAndALineThatCannotBeReadNamesNothing() { Guid[] ids = [Guid.CreateVersion7(), Guid.CreateVersion7(), Guid.NewGuid()]; Assert.SequenceEqual(ids, ConfirmedPhotos.Decode(ConfirmedPhotos.Encode(ids))); Assert.Empty(ConfirmedPhotos.Decode(ConfirmedPhotos.Encode([]))); Assert.Throws(() => ConfirmedPhotos.Decode(Convert.ToBase64String(new byte[15]))); var paths = new ArchivePaths(context.CreateTempDirectory("archive")); var job = Guid.CreateVersion7(); var (good, bad) = (Guid.NewGuid(), Guid.NewGuid()); Directory.CreateDirectory(paths.JournalDirectory); File.WriteAllLines(ConfirmedPhotos.PathFor(paths, job), [ JsonSerializer.Serialize(new ConfirmedLine(good, ConfirmedPhotos.Encode(ids)), ArchiveJobJson.Default.ConfirmedLine), JsonSerializer.Serialize(new ConfirmedLine(bad, "not ids"), ArchiveJobJson.Default.ConfirmedLine), "{\"sequence\":", ]); var read = ConfirmedPhotos.Read(paths, job); Assert.SequenceEqual(ids, read.Of(good)!); Assert.Null(read.Of(bad)); // None at all where nothing was written: every sequence is then held to its whole plan. Assert.Null(ConfirmedPhotos.Read(paths, Guid.CreateVersion7()).Of(good)); } /// /// Without the photos confirmed (they could not be written when the job was queued, or an earlier build queued it), a /// sequence converts only while its whole plan made again is the one confirmed: an upload since leaves it as it is. /// [Test] public async Task WithoutThePhotosConfirmedASequenceIsHeldToItsWholePlan() { await using var fx = await FixtureAsync(); var still = await UploadedAsync(fx, 2); var grown = await UploadedAsync(fx, 2, "upload-2", seed: 300); var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, still, grown); var (path, size, sha) = fx.Stage(Start.AddSeconds(60), 999); await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, grown, null, null, path, "IMG_0099.JPG", size, sha, null, null), fx.Ct); var before = await FramesAsync(fx, grown); var result = await fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), Guid.CreateVersion7(), new ConvertRequester(null, null), Callbacks(), fx.Ct); Assert.SequenceEqual([ConvertOutcome.Converted, ConvertOutcome.Changed], result.Sequences.Select(s => s.Outcome)); Assert.SequenceEqual(before.Select(f => (f.Id, f.Sha256)), (await FramesAsync(fx, grown)).Select(f => (f.Id, f.Sha256))); } /// /// A New sequence job that stopped once its copy held the photos, before its journal said the sequence was done (a crash, /// a stop): retried after the source changed, it finishes that copy and makes no other; the copy is reported as made. /// [Test] public async Task ARetryAfterACrashOnceTheCopyWasFiledFinishesThatCopyAndMakesNoOther() { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 3); await fx.Service.SetSequenceInfoAsync(id, "Roof", ["sunset"], 4, "From the roof", fx.Ct); var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id); var jobId = Guid.CreateVersion7(); // What the attempt that stopped there left: the photos confirmed, pinned and staged, and the copy filed, the journal // not saying more (it is written once the copy is finished). var left = new Dictionary(); var callbacks = Callbacks(done: _ => { foreach (var file in new[] { ConfirmedPhotos.PathFor(fx.Paths, jobId), ConvertJournal.PinsPathFor(fx.Paths, jobId), ConvertJournal.StagedPathFor(fx.Paths, jobId) }) left[file] = File.ReadAllBytes(file); }); var first = (await RunHereAsync(fx, plan, jobId, callbacks)).Sequences.Single(); Assert.Equal(ConvertOutcome.Created, first.Outcome); var copy = first.NewSequenceId!.Value; await fx.Db.Sequences.EditAsync(copy, new SequenceEdit { Tags = ["other"], Rating = 1, Notes = "Scrambled" }, null, fx.Ct); foreach (var (file, bytes) in left) File.WriteAllBytes(file, bytes); // The source changed since: a photo of it deleted. var sequence = (await fx.Db.Sequences.GetAsync(id))!; var delete = await fx.Service.FrameTrash.PlanDeleteAsync(id, [(await FramesAsync(fx, id))[1].Id], sequence.FramesVersion, fx.Ct); await fx.Service.FrameTrash.DeleteAsync(delete, "alberto", null, fx.Ct); var sequences = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count; var retried = (await RunHereAsync(fx, plan, jobId)).Sequences.Single(); Assert.Equal((ConvertOutcome.Created, 3, (Guid?)copy, 0), (retried.Outcome, retried.Frames, retried.NewSequenceId, retried.LeftOut)); Assert.Equal(sequences, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count); Assert.Count(3, await FramesAsync(fx, copy)); // Finished as stage B finishes a copy: the source's tags, rating and notes. var finished = (await fx.Db.Sequences.GetAsync(copy))!; Assert.Equal((4, "From the roof"), (finished.Rating, finished.Notes)); Assert.SequenceEqual(["sunset"], finished.Tags); Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, jobId)); // Its journal names the copy as soon as the copy holds its photos: a copy deleted since is not made again. var again = Guid.CreateVersion7(); var journal = new ConvertJournal { JobId = again }; journal.Made(id, Guid.CreateVersion7()); journal.Write(fx.Paths); var gone = (await RunHereAsync(fx, plan, again)).Sequences.Single(); Assert.Equal(ConvertOutcome.Changed, gone.Outcome); Assert.Null(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(again, id))); Assert.Equal(sequences, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count); } /// The photos a New sequence job filed into its copy before it stopped take no space again: its retry finishes on a full volume. [Test] public async Task ARetryOnceTheCopyWasFiledNeedsNoSpaceForItsPhotosAgain() { long free = 1L << 50; await using var fx = await FixtureAsync(o => o with { FreeSpace = _ => free }); var id = await UploadedAsync(fx, 2); var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id); var jobId = Guid.CreateVersion7(); var left = new Dictionary(); var callbacks = Callbacks(done: _ => { foreach (var file in new[] { ConfirmedPhotos.PathFor(fx.Paths, jobId), ConvertJournal.PinsPathFor(fx.Paths, jobId), ConvertJournal.StagedPathFor(fx.Paths, jobId) }) left[file] = File.ReadAllBytes(file); }); var copy = (await RunHereAsync(fx, plan, jobId, callbacks)).Sequences.Single().NewSequenceId; foreach (var (file, bytes) in left) File.WriteAllBytes(file, bytes); free = 1_000; var retried = (await RunHereAsync(fx, plan, jobId)).Sequences.Single(); Assert.Equal((ConvertOutcome.Created, copy), (retried.Outcome, retried.NewSequenceId)); Assert.Count(2, await FramesAsync(fx, copy!.Value)); } /// /// A conversion into new sequences retried after another conversion, confirmed later, copied the same source in the same /// format: it makes no second copy, and is done with that sequence. /// [Test] public async Task ARetriedConversionMakesNoSecondCopyOfASourceAnotherConversionCopiedSince() { await using var fx = await FixtureAsync(); var id = await UploadedAsync(fx, 2); var title = (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle; var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id); // Job A finds the sequence busy (a render works on it), and fails. var a = await QueuedAsync(fx, plan, DateTimeOffset.UtcNow.AddMinutes(-10)); var render = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Render, TargetIds = [id], State = JobState.Running, Created = DateTimeOffset.UtcNow }; await fx.Db.Jobs.EnqueueAsync(render, fx.Ct); Assert.Equal(ConvertOutcome.Blocked, (await RunHereAsync(fx, plan, a.Id, confirmed: a.Created)).Sequences.Single().Outcome); await fx.Db.Jobs.SetStateAsync(render.Id, JobState.Done, null, fx.Ct); await fx.Db.Jobs.SetStateAsync(a.Id, JobState.Failed, "kept", fx.Ct); // The user plans again: job B makes the copy. A JPEG copy made meanwhile is another format, which does not count. var again = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id); var b = await QueuedAsync(fx, again, DateTimeOffset.UtcNow.AddMinutes(-5)); var made = (await RunHereAsync(fx, again, b.Id, confirmed: b.Created)).Sequences.Single(); Assert.Equal(ConvertOutcome.Created, made.Outcome); var sequences = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count; // While job B has not finished (it failed after filing its copy), Retry on A waits for it: no second copy, not done with. await fx.Db.Jobs.SetStateAsync(b.Id, JobState.Failed, "kept", fx.Ct); var waiting = (await RunHereAsync(fx, plan, a.Id, confirmed: a.Created)).Sequences.Single(); Assert.Equal((ConvertOutcome.Blocked, Loc.L("Archive.Convert.CopyUnfinished")), (waiting.Outcome, waiting.Error)); Assert.Null(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(a.Id, id))); Assert.Equal(sequences, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count); await fx.Db.Jobs.SetStateAsync(b.Id, JobState.Done, null, fx.Ct); // Retry on A: done with the sequence, no second copy. var lines = new List(); var retried = (await RunHereAsync(fx, plan, a.Id, Callbacks() with { Log = lines.Add }, a.Created)).Sequences.Single(); Assert.Equal((ConvertOutcome.Superseded, true), (retried.Outcome, retried.Final)); Assert.SequenceEqual([Loc.L("Archive.Log.ConvertSuperseded", title)], lines); Assert.Null(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(a.Id, id))); Assert.Equal(sequences, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count); Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, a.Id)); // A conversion confirmed after that copy was made is the user asking again: it makes its copy. var latest = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id); var later = await QueuedAsync(fx, latest, DateTimeOffset.UtcNow); Assert.Equal(ConvertOutcome.Created, (await RunHereAsync(fx, latest, later.Id, confirmed: later.Created)).Sequences.Single().Outcome); // A copy in another format is no copy of this one: a TIFF conversion confirmed before a DNG copy was made makes its own. var other = await UploadedAsync(fx, 2, "upload-2", seed: 300); var tiff = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, other); var confirmedFirst = DateTimeOffset.UtcNow.AddMinutes(-1); var dng = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.NewSequence, other); Assert.Equal(ConvertOutcome.Created, (await RunHereAsync(fx, dng, Guid.CreateVersion7(), confirmed: DateTimeOffset.UtcNow)).Sequences.Single().Outcome); Assert.Equal(ConvertOutcome.Created, (await RunHereAsync(fx, tiff, Guid.CreateVersion7(), confirmed: confirmedFirst)).Sequences.Single().Outcome); } }