Phase 4b and its first fix round were written on phase 4a before its rounds 4-6. Moved onto the released 4a, the conversion now follows 4a's rule for writers into a sequence's folder (SequenceFolders.Holds), and the parallel fixes meet: - The job leaves a sequence another index moved away (its folder missing, or holding another sequence's sidecar) as Blocked with Errors.Archive.FolderMoved, nothing settled, staged or replaced there; the retry after this index's scan converts it where it is. It read the sidecar there and reported the reader's technical text. - New sequence reads the source's project under the source's lease, where its sidecar was just read as its own, and writes the new sequence's project under that sequence's lease only into a folder that holds it. It read the project from the folder the row named afterwards, without a lease, and wrote through the store without the check. - Download originals (resolved in the cherry-pick of the trash and sync fix, tested here) swaps a copy fetched in another encoding into its missing frame after the transfers, as round 6 restores originals, and only once the sequence is found in the folder the index has; otherwise the copies stay staged and the job fails with FolderMoved after its other work. The swap ran inside the transfer loop and failed the job at the first refusal. - The New sequence wording of the three skip reasons the codecs fix added (Archive.Convert. NotCopied.EightBit, ColorProfile, TooLarge, en and it): the server and the desktop look them up by reason. - The server's free space test sets the staging folder's figures too: the job fix measures the tighter of the archive root's and the staging folder's volumes. Tests: a conversion of a sequence another index moved (its folder taken or empty), and a download whose missing frame takes the server's conversion where the sequence is now. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1095 lines
68 KiB
C#
1095 lines
68 KiB
C#
using System.Security.Cryptography;
|
|
using System.Text.Json;
|
|
using Chronos.Core.Archiving;
|
|
using Chronos.Core.Conversion;
|
|
using Chronos.Core.Import;
|
|
using Chronos.Core.Imaging;
|
|
using Chronos.Core.Imaging.Decoders;
|
|
using Chronos.Core.Localization;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Core.Project;
|
|
using Chronos.Data.Archiving;
|
|
using Chronos.Data.Archiving.Jobs;
|
|
using Chronos.Data.Housekeeping;
|
|
using Chronos.Data.Jobs;
|
|
using Chronos.Data.Repositories;
|
|
using Chronos.Testing;
|
|
using static Chronos.Data.Tests.Archiving.ArchiveFixture;
|
|
|
|
namespace Chronos.Data.Tests.Archiving;
|
|
|
|
/// <summary>
|
|
/// Mass conversion after import (owner review 2, R7): the dry run, the <c>convert</c> job in both modes, its undo through the
|
|
/// trash, all or nothing per sequence, resumes in stage A and B, plans gone stale, the space check, and the gates that must
|
|
/// keep holding (the card erase, the emptying of a replaced entry).
|
|
/// </summary>
|
|
[Category("archive")]
|
|
public sealed class ConvertTests(TestContext context)
|
|
{
|
|
private Task<ArchiveFixture> FixtureAsync(Func<ArchiveServiceOptions, ArchiveServiceOptions>? more = null) =>
|
|
CreateAsync(context, configure: o =>
|
|
{
|
|
var options = o with { Decoder = DecoderFactory.CreateServer(), Parallelism = () => 2 };
|
|
return more is null ? options : more(options);
|
|
});
|
|
|
|
/// <summary>A sequence of <paramref name="count"/> uploaded photos (hashed, under <c>originals/</c>), ten seconds apart, by <paramref name="user"/>.</summary>
|
|
private static async Task<Guid> UploadedAsync(ArchiveFixture fx, int count, string session = "upload-1", int seed = 100, Guid? user = null)
|
|
{
|
|
Guid? id = null;
|
|
for (var i = 0; i < count; i++)
|
|
{
|
|
var (path, size, sha) = fx.Stage(Start.AddSeconds(10 * i).AddTicks(1_234_567), seed + i);
|
|
var result = await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, id is null ? session : null, path, $"IMG_{i + 1:0000}.JPG",
|
|
size, sha, null, user), fx.Ct);
|
|
id = result.SequenceId;
|
|
}
|
|
return id!.Value;
|
|
}
|
|
|
|
private static Task<ConvertPlan> PlanAsync(ArchiveFixture fx, ConvertTarget target, ConvertMode mode, params Guid[] ids) =>
|
|
fx.Service.PlanConvertAsync(fx.Archive.Id, ids, new ConvertOptions(target), mode, fx.Ct);
|
|
|
|
/// <summary>The parameters the job of <paramref name="plan"/> runs with.</summary>
|
|
private static ConvertJobParameters Parameters(ArchiveFixture fx, ConvertPlan plan) =>
|
|
JsonSerializer.Deserialize(ArchiveJobs.Convert(fx.Archive, plan).ParametersJson!, ArchiveJobJson.Default.ConvertJobParameters)!;
|
|
|
|
/// <summary>Callbacks that report nothing, but for <paramref name="progress"/> and <paramref name="done"/> when given.</summary>
|
|
private static ConvertCallbacks Callbacks(Action<long, long>? progress = null, Action<ConvertSequenceResult>? done = null,
|
|
Action<IReadOnlyList<ArchiveJobRequest>>? queue = null) => new(
|
|
(n, total) =>
|
|
{
|
|
progress?.Invoke(n, total);
|
|
return ValueTask.CompletedTask;
|
|
},
|
|
(_, _) => ValueTask.CompletedTask,
|
|
_ => { },
|
|
s =>
|
|
{
|
|
done?.Invoke(s);
|
|
return ValueTask.CompletedTask;
|
|
},
|
|
requests =>
|
|
{
|
|
queue?.Invoke(requests);
|
|
return ValueTask.CompletedTask;
|
|
});
|
|
|
|
/// <summary>The conversion job <paramref name="jobId"/> of <paramref name="plan"/>, run here as its handler runs it.</summary>
|
|
private static Task<ConvertResult> RunHereAsync(ArchiveFixture fx, ConvertPlan plan, Guid jobId, ConvertCallbacks? callbacks = null) =>
|
|
fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), jobId, new ConvertRequester(null, null), callbacks ?? Callbacks(), fx.Ct);
|
|
|
|
private static string StagingOf(ArchiveFixture fx, Guid job, Guid sequence) => Path.Combine(Staging(fx), $"convert-{job:N}", $"{sequence:N}");
|
|
|
|
private static async Task<Job> RunAsync(ArchiveFixture fx, JobEngine engine, ConvertPlan plan, JobState state = JobState.Done)
|
|
{
|
|
var job = await engine.EnqueueAsync(ArchiveJobs.Convert(fx.Archive, plan), cancellationToken: fx.Ct);
|
|
return await fx.WaitForAsync(job.Id, state, (int)TestTiming.Budget(60));
|
|
}
|
|
|
|
private static SequenceManifest Sidecar(ArchiveFixture fx, Sequence sequence) => ManifestStore.ReadSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath);
|
|
|
|
private static string Sha(string path) => Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(path)));
|
|
|
|
/// <summary>The index rows of a sequence, after checking that each file holds its row's content and that the sidecar agrees.</summary>
|
|
private static async Task<List<Frame>> FramesAsync(ArchiveFixture fx, Guid id)
|
|
{
|
|
var sequence = Assert.NotNull(await fx.Db.Sequences.GetAsync(id));
|
|
var rows = (await fx.Db.Frames.ListBySequenceAsync(id)).ToList();
|
|
var sidecar = Sidecar(fx, sequence);
|
|
Assert.Equal(sidecar.Versions.Frames, sequence.FramesVersion);
|
|
Assert.SequenceEqual(sidecar.Frames.Select(f => (f.Id, f.I, f.Sha256)), rows.Select(r => (r.Id, r.Index, r.Sha256)));
|
|
foreach (var row in rows) Assert.Equal(row.Sha256, Sha(fx.Paths.FromRelative(row.RelativePath)));
|
|
Assert.FileMissing(fx.Paths.Sequence(sequence.Folder).ReplaceJournalPath);
|
|
return rows;
|
|
}
|
|
|
|
private static string Staging(ArchiveFixture fx) => fx.Paths.StagingDirectory;
|
|
|
|
private static bool StagedAnything(ArchiveFixture fx) =>
|
|
Directory.Exists(Staging(fx)) && Directory.EnumerateDirectories(Staging(fx), "convert-*").Any();
|
|
|
|
[Test]
|
|
public async Task TheDryRunCountsWhatConvertsAndChangesNothing()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
|
|
|
|
var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id);
|
|
var step = plan.Sequences.Single();
|
|
Assert.Equal((3, 0, true), (step.Frames.Count, step.SkippedFrames, step.Converts));
|
|
Assert.Equal((await fx.Db.Sequences.GetAsync(id))!.FramesVersion, step.FramesVersion);
|
|
Assert.Equal(step.Frames.Sum(f => f.Size), plan.BytesBefore);
|
|
Assert.Equal(3 * ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.LinearDng), 48, 32), plan.BytesAfter);
|
|
Assert.Equal(ConvertPlanner.Required(plan.BytesAfter), plan.Space.Required);
|
|
Assert.True(plan.Space.Enough);
|
|
Assert.Equal(plan.Fingerprint(), (await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id)).Fingerprint());
|
|
Assert.NotEqual(plan.Fingerprint(), (await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.NewSequence, id)).Fingerprint());
|
|
|
|
// JPEG to JPEG converts nothing.
|
|
var same = (await PlanAsync(fx, ConvertTarget.Jpeg, ConvertMode.Replace, id)).Sequences.Single();
|
|
Assert.Equal((0, ConvertSkip.SameFormat, 3), (same.Frames.Count, same.Skipped.Single().Reason, same.Skipped.Single().Frames));
|
|
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
|
|
Assert.False(StagedAnything(fx));
|
|
await Assert.ThrowsAsync<ArgumentException>(() => PlanAsync(fx, ConvertTarget.Jpeg, ConvertMode.Replace, Guid.NewGuid()));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ASequenceBusyOrUnreadableIsBlocked()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var busy = await UploadedAsync(fx, 2);
|
|
var unreadable = await UploadedAsync(fx, 2, "upload-2", seed: 200);
|
|
await fx.Db.Jobs.EnqueueAsync(new Job
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
Kind = JobKind.Render,
|
|
TargetIds = [busy],
|
|
State = JobState.Running,
|
|
NodeId = "test#0",
|
|
Created = DateTimeOffset.UtcNow,
|
|
Started = DateTimeOffset.UtcNow,
|
|
});
|
|
File.WriteAllText(fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(unreadable))!.Folder).ManifestPath, "{ not json");
|
|
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, busy, unreadable);
|
|
Assert.SequenceEqual([ConvertBlock.Busy], plan.Sequences.Single(s => s.SequenceId == busy).Blocked);
|
|
Assert.SequenceEqual([ConvertBlock.Unreadable], plan.Sequences.Single(s => s.SequenceId == unreadable).Blocked);
|
|
Assert.Equal(0, plan.Frames);
|
|
Assert.Throws<ArgumentException>(() => ArchiveJobs.Convert(fx.Archive, plan));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AReplaceKeepsTheFramesAndTrashesTheOriginalsWithTheirReplacements()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
var engine = await fx.StartEngineAsync();
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var originals = await FramesAsync(fx, id);
|
|
var project = new ProjectSettings { ExcludedFrames = [1], HeroFrame = 2 }.ToJson();
|
|
await fx.Db.Sequences.EditAsync(id, new SequenceEdit { SettingsJson = project, HeroFrameId = originals[2].Id }, null, fx.Ct);
|
|
var folder = fx.Paths.Sequence(sequence.Folder);
|
|
Directory.CreateDirectory(folder.MetaDirectory);
|
|
File.WriteAllText(folder.AnalysisPath, "{\"frames\":3}");
|
|
var before = Sidecar(fx, sequence);
|
|
|
|
var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id);
|
|
var job = await RunAsync(fx, engine, plan);
|
|
Assert.Contains(Loc.L("Archive.Log.Converted", sequence.DisplayTitle, 3), job.LogTail);
|
|
|
|
var row = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var frames = await FramesAsync(fx, id);
|
|
Assert.Equal(before.Versions.Frames + 1, row.FramesVersion);
|
|
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Index)), frames.Select(f => (f.Id, f.Index)));
|
|
var after = Sidecar(fx, row);
|
|
foreach (var (was, now) in before.Frames.Zip(after.Frames))
|
|
{
|
|
// Named as filing named the original, with the target's extension: the same template, frame and sequence.
|
|
Assert.Equal((Path.ChangeExtension(was.File, ".dng"), ImageFileFormat.Dng, DecodeTier.Native, 1), (now.File, now.Format, now.Tier, now.Orientation));
|
|
Assert.Equal((was.CapturedAt, was.Original, was.Flags), (now.CapturedAt, now.Original, now.Flags));
|
|
var source = Assert.NotNull(now.Source);
|
|
Assert.Equal((was.Sha256, was.Format, was.File, was.Size), (source.Sha256, source.Format, source.File, source.Size));
|
|
Assert.FileMissing(folder.File(was.File));
|
|
}
|
|
// The project's positions stay on their frames.
|
|
var settings = ProjectSettings.FromJson(row.SettingsJson!);
|
|
Assert.SequenceEqual([1], settings.ExcludedFrames);
|
|
Assert.Equal((2, originals[2].Id), (settings.HeroFrame, row.HeroFrameId));
|
|
|
|
// One replaced entry with the originals, each naming its replacement, and the analysis as it was.
|
|
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
|
|
Assert.Equal((TrashKind.Replaced, id, 3), (entry.Kind, entry.SequenceId, entry.Frames));
|
|
var record = Assert.NotNull(FrameTrash.ReadEntry(fx.Paths, entry.Name));
|
|
var entryFolder = FrameTrash.Location(fx.Paths, entry.Name).Folder;
|
|
foreach (var photo in record.Photos)
|
|
{
|
|
var now = after.Frames.Single(f => f.Id == photo.Frame.Id);
|
|
Assert.Equal(now.Sha256, photo.Replacement);
|
|
Assert.Equal(photo.Frame.Sha256, Sha(Path.Combine(entryFolder, photo.Files[0].Path)));
|
|
}
|
|
Assert.Equal("{\"frames\":3}", File.ReadAllText(Path.Combine(entryFolder, FrameTrash.AnalysisCopy)));
|
|
// Tombstones alias the old content to the new, in the sidecar and the index.
|
|
var tombstones = Assert.NotNull(after.Removed);
|
|
Assert.SequenceEqual(before.Frames.Select(f => (f.Sha256, after.Frames.Single(n => n.Id == f.Id).Sha256, RemovalReason.Replaced)).Order(),
|
|
tombstones.Select(t => ((string?)t.Sha256, t.Replacement, t.Reason)).Order());
|
|
Assert.SequenceEqual(tombstones.Select(t => (t.Sha256, t.Replacement)).Order(),
|
|
(await fx.Db.FrameRemovals.ListBySequenceAsync(id)).Select(r => (r.Sha256, r.ReplacementSha256)).Order());
|
|
Assert.Equal((SequenceStatus)0, row.Status & (SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt));
|
|
Assert.False(StagedAnything(fx));
|
|
Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, job.Id));
|
|
Assert.FileMissing(ConvertJournal.StagedPathFor(fx.Paths, job.Id));
|
|
// Converted again, nothing is left to convert.
|
|
Assert.Equal(0, (await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id)).Frames);
|
|
}
|
|
|
|
[Test]
|
|
public async Task RestoringTheReplacedEntrySwapsTheOriginalsBack()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
var engine = await fx.StartEngineAsync();
|
|
var originals = await FramesAsync(fx, id);
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var before = Sidecar(fx, sequence);
|
|
var analysis = fx.Paths.Sequence(sequence.Folder).AnalysisPath;
|
|
Directory.CreateDirectory(Path.GetDirectoryName(analysis)!);
|
|
File.WriteAllText(analysis, "of the originals");
|
|
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id));
|
|
var converted = Sidecar(fx, (await fx.Db.Sequences.GetAsync(id))!);
|
|
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
|
|
// The light analysis of the converted frames, as it would be written again.
|
|
File.WriteAllText(analysis, "of the conversions");
|
|
|
|
var result = await fx.Service.RestoreEntryAsync(fx.Archive.Id, entry.Name, null, fx.Ct);
|
|
Assert.Equal((TrashKind.Replaced, 3, false, id), (result.Kind, result.Photos, result.NewSequence, result.Sequence.Id));
|
|
var frames = await FramesAsync(fx, id);
|
|
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Index, f.Sha256, f.RelativePath, f.Format)), frames.Select(f => (f.Id, f.Index, f.Sha256, f.RelativePath, f.Format)));
|
|
var back = Sidecar(fx, (await fx.Db.Sequences.GetAsync(id))!);
|
|
Assert.True(back.Frames.All(f => f.Source is null));
|
|
// The converted files are in the trash in turn, naming the originals as their replacements; the old entry is gone.
|
|
var turned = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
|
|
Assert.NotEqual(entry.Name, turned.Name);
|
|
Assert.Equal(TrashKind.Replaced, turned.Kind);
|
|
var record = FrameTrash.ReadEntry(fx.Paths, turned.Name)!;
|
|
Assert.SequenceEqual(converted.Frames.Select(f => (f.Sha256, before.Frames.Single(o => o.Id == f.Id).Sha256)).Order(),
|
|
record.Photos.Select(p => (p.Frame.Sha256, p.Replacement)).Order());
|
|
// Each analysis goes with its frames: the originals' back in the sequence, the conversions' into the new entry.
|
|
Assert.Equal("of the originals", File.ReadAllText(analysis));
|
|
Assert.Equal("of the conversions", File.ReadAllText(Path.Combine(FrameTrash.Location(fx.Paths, turned.Name).Folder, FrameTrash.AnalysisCopy)));
|
|
Assert.False(Directory.Exists(FrameTrash.Location(fx.Paths, entry.Name).Folder));
|
|
Assert.SequenceEqual(converted.Frames.Select(f => (f.Sha256, before.Frames.Single(o => o.Id == f.Id).Sha256)).Order(),
|
|
back.Removed!.Select(t => ((string?)t.Sha256, t.Replacement)).Order());
|
|
|
|
// That entry erases: its replacements (the originals) are where it says, re-hashed now.
|
|
var empty = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
|
|
var job = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, empty), cancellationToken: fx.Ct);
|
|
await fx.WaitForAsync(job.Id, JobState.Done);
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
await FramesAsync(fx, id);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnUndoAfterTheFramesChangedBringsTheOriginalsBackAsANewSequence()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
var engine = await fx.StartEngineAsync();
|
|
var originals = await FramesAsync(fx, id);
|
|
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id));
|
|
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
|
|
var converted = await FramesAsync(fx, id);
|
|
// A converted frame deleted since: the frames are no longer the ones the conversion left.
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var delete = await fx.Service.FrameTrash.PlanDeleteAsync(id, [converted[1].Id], sequence.FramesVersion, fx.Ct);
|
|
await fx.Service.FrameTrash.DeleteAsync(delete, "alberto", null, fx.Ct);
|
|
|
|
var result = await fx.Service.RestoreEntryAsync(fx.Archive.Id, entry.Name, null, fx.Ct);
|
|
Assert.Equal((TrashKind.Replaced, 3, true), (result.Kind, result.Photos, result.NewSequence));
|
|
Assert.NotEqual(id, result.Sequence.Id);
|
|
Assert.SequenceEqual(originals.Select(f => f.Sha256), (await FramesAsync(fx, result.Sequence.Id)).Select(f => f.Sha256));
|
|
Assert.Count(2, await FramesAsync(fx, id));
|
|
// The replaced entry is gone with its last photo (and its copy of the analysis); the deleted frame's entry stays.
|
|
Assert.DoesNotContain(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ANewSequenceTakesTheCopiesAndLeavesTheSourceAsItIs()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
var engine = await fx.StartEngineAsync();
|
|
await fx.Service.SetSequenceInfoAsync(id, "Roof", ["sunset", "milan"], 4, "From the roof", fx.Ct);
|
|
var originals = await FramesAsync(fx, id);
|
|
await fx.Db.Sequences.EditAsync(id, new SequenceEdit { SettingsJson = new ProjectSettings { ExcludedFrames = [1], HeroFrame = 2 }.ToJson() }, null, fx.Ct);
|
|
var source = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var sidecar = File.ReadAllText(fx.Paths.Sequence(source.Folder).ManifestPath);
|
|
|
|
var job = await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id));
|
|
var made = Assert.NotNull(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, id)));
|
|
var copy = (await fx.Db.Sequences.GetAsync(made))!;
|
|
Assert.Equal(("Roof (TIFF)", 4, "From the roof"), (copy.Title, copy.Rating, copy.Notes));
|
|
Assert.SequenceEqual(["milan", "sunset"], copy.Tags.Order());
|
|
var frames = await FramesAsync(fx, made);
|
|
Assert.SequenceEqual(originals.Select(f => f.CapturedAt), frames.Select(f => f.CapturedAt));
|
|
Assert.True(frames.All(f => f.Format == ImageFileFormat.Tiff));
|
|
// Filed as the ingest files every photo: the archive's template in the new sequence, {name} the card's, {ext} the target's.
|
|
Assert.SequenceEqual(originals.Select(f => Path.ChangeExtension(f.FileName, ".tif")), frames.Select(f => f.FileName));
|
|
Assert.SequenceEqual(originals.Select(f => f.OriginalFileName), frames.Select(f => f.OriginalFileName));
|
|
var settings = ProjectSettings.FromJson(copy.SettingsJson!);
|
|
Assert.SequenceEqual([1], settings.ExcludedFrames);
|
|
Assert.Equal((2, frames[2].Id), (settings.HeroFrame, copy.HeroFrameId));
|
|
// The source: not a byte of its sidecar, nor a row, changed; nothing went to the trash.
|
|
Assert.Equal(sidecar, File.ReadAllText(fx.Paths.Sequence(source.Folder).ManifestPath));
|
|
var now = (await fx.Db.Sequences.GetAsync(id))!;
|
|
Assert.Equal((source.FramesVersion, source.SettingsVersion, source.Title, source.Folder), (now.FramesVersion, now.SettingsVersion, now.Title, now.Folder));
|
|
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256)), (await FramesAsync(fx, id)).Select(f => (f.Id, f.Sha256)));
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
Assert.False(StagedAnything(fx));
|
|
}
|
|
|
|
[Test]
|
|
public async Task APhotoThatFailsLeavesItsWholeSequenceAsItWasAndTheOthersConvert()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var bad = await UploadedAsync(fx, 3);
|
|
var good = await UploadedAsync(fx, 2, "upload-2", seed: 300);
|
|
var engine = await fx.StartEngineAsync();
|
|
var damaged = (await FramesAsync(fx, bad))[1];
|
|
var originals = await FramesAsync(fx, bad);
|
|
// Rot after the plan: the photo no longer matches the hash it was archived with.
|
|
var path = fx.Paths.FromRelative(damaged.RelativePath);
|
|
var bytes = File.ReadAllBytes(path);
|
|
bytes[^20] ^= 0x10;
|
|
File.WriteAllBytes(path, bytes);
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, bad, good);
|
|
Assert.Equal(5, plan.Frames);
|
|
|
|
var job = await RunAsync(fx, engine, plan, JobState.Failed);
|
|
Assert.Equal(Loc.Plural("Archive.Error.ConvertKept", 1), job.Error);
|
|
Assert.Contains(damaged.FileName, job.LogTail);
|
|
// The log says why in one catalog sentence; the exception's English text stays in the diagnostics log.
|
|
Assert.Contains(Loc.L("Archive.Convert.Failed.Changed"), job.LogTail);
|
|
Assert.False(job.LogTail!.Contains("SHA-256", StringComparison.Ordinal), job.LogTail);
|
|
// All or nothing: not one frame of that sequence changed, and nothing of it is in the trash.
|
|
var rows = await fx.Db.Frames.ListBySequenceAsync(bad);
|
|
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256, f.RelativePath)), rows.Select(f => (f.Id, f.Sha256, f.RelativePath)));
|
|
Assert.True((await FramesAsync(fx, good)).All(f => f.Format == ImageFileFormat.Tiff));
|
|
Assert.SequenceEqual([good], (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.SequenceId));
|
|
// The journal stays for a retry, which tries the failed sequence again and leaves the converted one; so does what the
|
|
// failed sequence staged before its photo failed, and only that.
|
|
var journal = Assert.NotNull(ConvertJournal.Read(fx.Paths, job.Id));
|
|
Assert.Equal(ConvertOutcome.Converted, journal.Of(good)!.Outcome);
|
|
Assert.Equal(ConvertOutcome.Failed, journal.Of(bad)!.Outcome);
|
|
Assert.False(Directory.Exists(StagingOf(fx, job.Id, good)));
|
|
var noted = StagedLog.Open(ConvertJournal.StagedPathFor(fx.Paths, job.Id)).Of(bad, (await fx.Db.Sequences.GetAsync(bad))!.FramesVersion);
|
|
Assert.SequenceEqual(noted.Select(c => c.File).Order(), Directory.EnumerateFiles(StagingOf(fx, job.Id, bad)).Select(f => Path.GetFileName(f)).Order());
|
|
Assert.DoesNotContain(damaged.Id, noted.Select(c => c.FrameId));
|
|
var first = Path.Combine(StagingOf(fx, job.Id, bad), noted.Single(c => c.FrameId == originals[0].Id).File);
|
|
File.SetLastWriteTimeUtc(first, new DateTime(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc));
|
|
|
|
// The photo is put right: the retry takes the staged conversion that still verifies and converts the rest.
|
|
bytes[^20] ^= 0x10;
|
|
File.WriteAllBytes(path, bytes);
|
|
await engine.ResumeAsync(job.Id, fx.Ct);
|
|
await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60));
|
|
var converted = await FramesAsync(fx, bad);
|
|
Assert.True(converted.All(f => f.Format == ImageFileFormat.Tiff));
|
|
Assert.Equal(2020, File.GetLastWriteTimeUtc(fx.Paths.FromRelative(converted[0].RelativePath)).Year);
|
|
Assert.False(StagedAnything(fx));
|
|
Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, job.Id));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AResumedJobKeepsTheStagedFilesThatStillVerify()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
var engine = await fx.StartEngineAsync();
|
|
// To JPEG needs other originals than the JPEGs uploaded: they become TIFF first.
|
|
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id));
|
|
var plan = await PlanAsync(fx, ConvertTarget.Jpeg, ConvertMode.Replace, id);
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var folder = fx.Paths.Sequence(sequence.Folder);
|
|
var manifest = Sidecar(fx, sequence);
|
|
var jobId = Guid.CreateVersion7();
|
|
var staging = Path.Combine(Staging(fx), $"convert-{jobId:N}", $"{id:N}");
|
|
var log = StagedLog.Open(ConvertJournal.StagedPathFor(fx.Paths, jobId));
|
|
// An earlier attempt staged the first photo (at another quality, so its use shows) and the second, damaged since.
|
|
StagedConversion Stage(SidecarFrame frame, int quality)
|
|
{
|
|
var target = Path.Combine(staging, frame.Id.ToString("N") + ".jpg");
|
|
var file = FrameConverter.Convert(new ConvertSource(folder.File(frame.File), frame, manifest.Camera, manifest.Lens), new ConvertOptions(ConvertTarget.Jpeg, quality),
|
|
DecoderFactory.CreateServer(), target, cancellationToken: fx.Ct);
|
|
var conversion = new StagedConversion(frame.Id, frame.Sha256!, Path.GetFileName(target), file.Size, file.Sha256, file.Width, file.Height);
|
|
log.Append(id, manifest.Versions.Frames, conversion);
|
|
return conversion;
|
|
}
|
|
var kept = Stage(manifest.Frames[0], 40);
|
|
var torn = Stage(manifest.Frames[1], 95);
|
|
File.WriteAllBytes(Path.Combine(staging, torn.File), [1, 2, 3]);
|
|
|
|
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Convert(fx.Archive, plan), "convert", jobId);
|
|
await fx.WaitForAsync(job.Id, JobState.Done);
|
|
var frames = await FramesAsync(fx, id);
|
|
Assert.Equal(kept.Sha256, frames[0].Sha256);
|
|
Assert.Equal(torn.Sha256, frames[1].Sha256);
|
|
Assert.True(frames.All(f => f.Format == ImageFileFormat.Jpeg));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AJobThatStoppedAfterItsReplacementReportsTheSequenceConverted()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id);
|
|
var request = ArchiveJobs.Convert(fx.Archive, plan);
|
|
await RunAsync(fx, engine, plan);
|
|
// As a crash between the replacement and the journal's note leaves it: the staged log names the conversions done.
|
|
var sidecar = Sidecar(fx, (await fx.Db.Sequences.GetAsync(id))!);
|
|
var jobId = Guid.CreateVersion7();
|
|
var log = StagedLog.Open(ConvertJournal.StagedPathFor(fx.Paths, jobId));
|
|
foreach (var frame in sidecar.Frames)
|
|
log.Append(id, plan.Sequences.Single().FramesVersion, new StagedConversion(frame.Id, frame.Source!.Sha256!, "gone.dng", frame.Size, frame.Sha256!, frame.W, frame.H));
|
|
|
|
var job = await fx.EnqueueWithCheckpointAsync(engine, request, "convert", jobId);
|
|
var done = await fx.WaitForAsync(job.Id, JobState.Done);
|
|
Assert.Contains(Loc.L("Archive.Log.Converted", (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle, 2), done.LogTail);
|
|
Assert.Count(1, await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ASequenceThatChangedAfterThePlanIsLeftAsItWas()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id);
|
|
// Another photo arrives before the job runs.
|
|
var (path, size, sha) = fx.Stage(Start.AddSeconds(60), 999);
|
|
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, null, path, "IMG_0099.JPG", size, sha, null, null), fx.Ct);
|
|
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
|
|
|
|
var job = await RunAsync(fx, engine, plan, JobState.Failed);
|
|
Assert.Equal(Loc.Plural("Archive.Error.ConvertKept", 1), job.Error);
|
|
Assert.Contains(Loc.L("Archive.Log.ConvertChanged", (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle), job.LogTail);
|
|
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AVolumeWithoutTheSpaceConvertsNothing()
|
|
{
|
|
await using var fx = await FixtureAsync(o => o with { FreeSpace = _ => 1_000 });
|
|
var id = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
|
|
Assert.False(plan.Space.Enough);
|
|
Assert.Equal(1_000, plan.Space.Free);
|
|
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
|
|
|
|
var job = await RunAsync(fx, engine, plan, JobState.Failed);
|
|
Assert.True(job.Error!.StartsWith("The conversion needs", StringComparison.Ordinal), job.Error);
|
|
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
|
|
Assert.False(StagedAnything(fx));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ACardKeepsTheCopyOfAPhotoThatWasConverted()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var card = context.CreateTempDirectory("card");
|
|
Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, 3);
|
|
var media = await fx.Service.Imports.RegisterCardAsync(new CardVolume("3F2A9C11-EOS_DIGITAL-127865454592", 0x3F2A9C11, "EOS_DIGITAL", 127_865_454_592), fx.Ct);
|
|
var import = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
|
|
await fx.Service.Imports.ExecuteAsync(new ImportJobParameters(fx.Archive.Id, import, media.Id, true, null), Guid.NewGuid(), 0,
|
|
_ => ValueTask.CompletedTask, null, fx.Ct);
|
|
var id = (await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, import.Sessions.Single().SessionKey))!.Value;
|
|
Assert.Equal(3, (await fx.Service.PlanCardEraseAsync(fx.Archive.Id, media.Id, card, null, fx.Ct)).ToDelete.Count());
|
|
var engine = await fx.StartEngineAsync();
|
|
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id));
|
|
|
|
// The frames now hold converted files: nothing in the archive matches the card's photos byte for byte, so they stay.
|
|
var erase = await fx.Service.PlanCardEraseAsync(fx.Archive.Id, media.Id, card, null, fx.Ct);
|
|
Assert.Empty(erase.ToDelete);
|
|
Assert.True(erase.Items.All(i => i.Verdict == EraseVerdict.CopyMismatch));
|
|
Assert.Equal(0, CardEraser.Execute(erase).Deleted);
|
|
Assert.Count(3, Directory.EnumerateFiles(card, "*.JPG", SearchOption.AllDirectories));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AReplacedEntryIsErasedOnlyWhileTheConvertedFilesStillHashToTheirRecord()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id));
|
|
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
|
|
var converted = (await FramesAsync(fx, id))[0];
|
|
var file = fx.Paths.FromRelative(converted.RelativePath);
|
|
var bytes = File.ReadAllBytes(file);
|
|
var damaged = (byte[])bytes.Clone();
|
|
damaged[^10] ^= 0x01;
|
|
File.WriteAllBytes(file, damaged);
|
|
|
|
var plan = await PlanEmptyAsync(fx);
|
|
var kept = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, plan), cancellationToken: fx.Ct);
|
|
await fx.WaitForAsync(kept.Id, JobState.Failed);
|
|
Assert.Contains(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name));
|
|
|
|
File.WriteAllBytes(file, bytes);
|
|
var erased = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, await PlanEmptyAsync(fx)), cancellationToken: fx.Ct);
|
|
await fx.WaitForAsync(erased.Id, JobState.Done);
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
|
|
static Task<TrashEmptyPlan> PlanEmptyAsync(ArchiveFixture fx) => fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AReplacementACrashInterruptedBeforeItsSidecarIsUndoneByTheNextWriter()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var folder = fx.Paths.Sequence(sequence.Folder);
|
|
var manifest = Sidecar(fx, sequence);
|
|
var frame = manifest.Frames[0];
|
|
// Half a replacement: the original in its entry, the converted file in its place, the record pending, the sidecar as it was.
|
|
var staged = Path.Combine(Staging(fx), "convert-test", frame.Id.ToString("N") + ".tif");
|
|
FrameConverter.Convert(new ConvertSource(folder.File(frame.File), frame), new ConvertOptions(ConvertTarget.Tiff16), DecoderFactory.CreateServer(), staged,
|
|
cancellationToken: fx.Ct);
|
|
var entryFolder = Path.Combine(FrameTrash.Directory(fx.Paths), $"{id:N}-20261027120000");
|
|
var record = entryFolder + ".json";
|
|
var trashed = Path.Combine(entryFolder, "originals", Path.GetFileName(frame.File));
|
|
var placed = Path.ChangeExtension(frame.File, ".tif");
|
|
var journal = new FrameReplaceJournal(manifest.Versions.Frames, fx.Paths.ToRelative(record), fx.Paths.ToRelative(entryFolder),
|
|
[new SwapMove(frame.File, fx.Paths.ToRelative(trashed))], [new SwapMove(placed, fx.Paths.ToRelative(staged))], false, null);
|
|
journal.Write(folder);
|
|
Directory.CreateDirectory(FrameTrash.Directory(fx.Paths));
|
|
File.WriteAllText(FrameRemovalJournal.PendingRecord(record), "{}");
|
|
Directory.CreateDirectory(Path.GetDirectoryName(trashed)!);
|
|
File.Move(folder.File(frame.File), trashed);
|
|
File.Move(staged, folder.File(placed));
|
|
|
|
await fx.Service.EditSequenceInfoAsync(id, info => info with { Rating = 2 }, fx.Ct);
|
|
Assert.FileExists(folder.File(frame.File));
|
|
Assert.FileMissing(folder.File(placed));
|
|
Assert.FileExists(staged);
|
|
Assert.False(Directory.Exists(entryFolder));
|
|
Assert.FileMissing(FrameRemovalJournal.PendingRecord(record));
|
|
Assert.FileMissing(folder.ReplaceJournalPath);
|
|
Assert.Equal(frame.Sha256, Sha(folder.File(frame.File)));
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AReplacementACrashInterruptedAfterItsSidecarIsCompletedByTheNextWriter()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
var version = (await fx.Db.Sequences.GetAsync(id))!.FramesVersion;
|
|
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id));
|
|
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
|
|
var (entryFolder, record) = FrameTrash.Location(fx.Paths, entry.Name);
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var folder = fx.Paths.Sequence(sequence.Folder);
|
|
// As a crash right after the sidecar was written leaves it: the record still pending, the journal there.
|
|
File.Move(record, FrameRemovalJournal.PendingRecord(record));
|
|
new FrameReplaceJournal(version, fx.Paths.ToRelative(record), fx.Paths.ToRelative(entryFolder), [], [], true, null).Write(folder);
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
|
|
await fx.Service.EditSequenceInfoAsync(id, info => info with { Rating = 2 }, fx.Ct);
|
|
Assert.FileMissing(folder.ReplaceJournalPath);
|
|
Assert.Equal(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name);
|
|
Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Tiff));
|
|
}
|
|
|
|
[Test]
|
|
public async Task APhotoTheServerCannotReadAtFullResolutionIsSkipped()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
// The sidecar says the second photo is a HEIC (a desktop with WIC filed it): the managed decoders have only its preview.
|
|
var manifest = Sidecar(fx, sequence);
|
|
ManifestStore.WriteSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath,
|
|
manifest with { Frames = [manifest.Frames[0], manifest.Frames[1] with { Format = ImageFileFormat.Heif }] });
|
|
var step = (await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.NewSequence, id)).Sequences.Single();
|
|
Assert.Equal(1, step.Frames.Count);
|
|
Assert.Equal((ConvertSkip.NoNativeDecoder, 1), (step.Skipped.Single().Reason, step.Skipped.Single().Frames));
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheJournalsSurviveARoundTrip()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var journal = new ConvertJournal { JobId = Guid.NewGuid() };
|
|
var first = Guid.NewGuid();
|
|
var harbour = Guid.NewGuid();
|
|
journal.Begin(harbour, "Harbour", SequenceStatus.AnalyzedLight | SequenceStatus.ProxiesBuilt);
|
|
journal.Note(new ConvertSequenceResult(first, "Roof", ConvertOutcome.Created, 3, Guid.NewGuid()), SequenceStatus.AnalyzedDeep);
|
|
journal.Of(first)!.Queued = true;
|
|
journal.Note(new ConvertSequenceResult(harbour, "Harbour", ConvertOutcome.Failed, 0, Error: "no"));
|
|
journal.Write(fx.Paths);
|
|
var read = Assert.NotNull(ConvertJournal.Read(fx.Paths, journal.JobId));
|
|
Assert.SequenceEqual(journal.Sequences.Select(s => (s.Id, s.Outcome, s.Frames, s.NewSequenceId, s.Final, s.Had, s.Queued)),
|
|
read.Sequences.Select(s => (s.Id, s.Outcome, s.Frames, s.NewSequenceId, s.Final, s.Had, s.Queued)));
|
|
Assert.True(read.Of(first)!.Final);
|
|
// A failure keeps the statuses the sequence had when the job came to it.
|
|
Assert.Equal(SequenceStatus.AnalyzedLight | SequenceStatus.ProxiesBuilt, read.Of(harbour)!.Had);
|
|
// The photos pinned: a later line for a sequence replaces an earlier one, a torn one is skipped.
|
|
var photo = Guid.NewGuid();
|
|
var pins = read.Pins(fx.Paths);
|
|
Assert.Null(pins.Of(first));
|
|
ConvertSequencePlan Pinned(int version, string sha) =>
|
|
new(first, "Roof", "roof", version, [new ConvertFramePlan(photo, 0, "originals/a.jpg", ImageFileFormat.Jpeg, 1, 2, sha)], [], []);
|
|
pins.Append(first, Pinned(4, "aa"));
|
|
pins.Append(first, Pinned(5, "bb"));
|
|
File.AppendAllText(ConvertJournal.PinsPathFor(fx.Paths, journal.JobId), "{\"sequence\":");
|
|
var pinned = Assert.NotNull(read.Pins(fx.Paths).Of(first));
|
|
Assert.Equal((5, photo, "bb"), (pinned.FramesVersion, pinned.Frames.Single().Id, pinned.Frames.Single().Sha256));
|
|
// Its files and its staging folder are known by their names.
|
|
Assert.SequenceEqual([journal.JobId, journal.JobId, journal.JobId],
|
|
ConvertJournal.Files(fx.Paths, journal.JobId).Select(p => ConvertJournal.JobOf(Path.GetFileName(p))));
|
|
Assert.Equal(journal.JobId, ConvertJournal.JobOf(MassConverter.StagingFolder(journal.JobId)));
|
|
Assert.Null(ConvertJournal.JobOf("convert-test"));
|
|
Assert.Null(ConvertJournal.JobOf("rename-" + journal.JobId.ToString("N") + ".json"));
|
|
// A line torn by a crash is skipped; a later line for a frame replaces an earlier one.
|
|
var path = ConvertJournal.StagedPathFor(fx.Paths, journal.JobId);
|
|
var log = StagedLog.Open(path);
|
|
var frame = Guid.NewGuid();
|
|
log.Append(first, 4, new StagedConversion(frame, "a", "x.dng", 1, "b", 2, 3));
|
|
log.Append(first, 4, new StagedConversion(frame, "a", "x.dng", 1, "c", 2, 3));
|
|
File.AppendAllText(path, "{\"sequence\":");
|
|
var reread = StagedLog.Open(path);
|
|
Assert.Equal("c", reread.Of(first, 4).Single().Sha256);
|
|
Assert.Empty(reread.Of(first, 5));
|
|
ConvertJournal.Delete(fx.Paths, journal.JobId);
|
|
Assert.FileMissing(path);
|
|
Assert.FileMissing(ConvertJournal.PinsPathFor(fx.Paths, journal.JobId));
|
|
Assert.Null(ConvertJournal.Read(fx.Paths, journal.JobId));
|
|
// A job an earlier build queued names no digest: it reads with none.
|
|
var parameters = JsonSerializer.Serialize(new ConvertJobParameters(fx.Archive.Id, ConvertTarget.Jpeg, 80, ConvertMode.NewSequence,
|
|
[new ConvertSequence(first, 2, "d")], "f"), ArchiveJobJson.Default.ConvertJobParameters);
|
|
Assert.Equal("d", JsonSerializer.Deserialize(parameters, ArchiveJobJson.Default.ConvertJobParameters)!.Sequences.Single().Frames);
|
|
var older = $$"""{"archiveId":"{{fx.Archive.Id}}","target":"Jpeg","quality":80,"mode":"Replace","sequences":[{"id":"{{first}}","framesVersion":2}],"fingerprint":"f"}""";
|
|
Assert.Null(JsonSerializer.Deserialize(older, ArchiveJobJson.Default.ConvertJobParameters)!.Sequences.Single().Frames);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ConvertedFilesAreNamedAsFilingNamesThemAndNothingIsOverwritten()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
await fx.Service.UpdateSettingsAsync(fx.Archive.Id, s => s with { Templates = s.Templates with { File = "{name}_{n:00}" } }, fx.Ct);
|
|
var roof = await UploadedAsync(fx, 2);
|
|
var lake = await UploadedAsync(fx, 2, "upload-2", seed: 200);
|
|
Assert.SequenceEqual(["IMG_0001_01.JPG", "IMG_0002_02.JPG"], (await FramesAsync(fx, roof)).Select(f => f.FileName));
|
|
await fx.Service.SetSequenceInfoAsync(roof, "Roof", [], 0, null, fx.Ct);
|
|
await fx.Service.SetSequenceInfoAsync(lake, "Lake", [], 0, null, fx.Ct);
|
|
// Filing names a photo with the template of the moment: a conversion names its files as filing would now.
|
|
await fx.Service.UpdateSettingsAsync(fx.Archive.Id, s => s with { Templates = s.Templates with { File = "{title}_{n:00}" } }, fx.Ct);
|
|
// A file that is no frame holds the first name: it is never written over, the converted file takes the next.
|
|
var stray = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(roof))!.Folder).File("originals/Roof_01.dng");
|
|
File.WriteAllBytes(stray, [1, 2, 3]);
|
|
var engine = await fx.StartEngineAsync();
|
|
|
|
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, roof));
|
|
var replaced = await FramesAsync(fx, roof);
|
|
Assert.SequenceEqual(["Roof_01_2.dng", "Roof_02.dng"], replaced.Select(f => f.FileName));
|
|
Assert.SequenceEqual(["IMG_0001.JPG", "IMG_0002.JPG"], replaced.Select(f => f.OriginalFileName));
|
|
Assert.SequenceEqual(new byte[] { 1, 2, 3 }, File.ReadAllBytes(stray));
|
|
|
|
var job = await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, lake));
|
|
var made = Assert.NotNull(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, lake)));
|
|
var copies = await FramesAsync(fx, made);
|
|
Assert.SequenceEqual(["Lake (TIFF)_01.tif", "Lake (TIFF)_02.tif"], copies.Select(f => f.FileName));
|
|
Assert.SequenceEqual(["IMG_0001.JPG", "IMG_0002.JPG"], copies.Select(f => f.OriginalFileName));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AFolderMovedWhileTheJobWaitsForItsLeaseIsWhereTheConvertedFilesGo()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
|
|
var row = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var from = fx.Paths.Sequence(row.Folder).Folder;
|
|
Job job;
|
|
var planned = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
void OnProgress(JobProgressInfo info)
|
|
{
|
|
if (info.Kind == JobKind.Convert && info.Total == 2) planned.TrySetResult();
|
|
}
|
|
engine.Progress += OnProgress;
|
|
using (await LeaseFile.AcquireAsync(fx.Paths.LocksDirectory, $"seq-{id:N}", "editor", TimeSpan.FromSeconds(5), cancellationToken: fx.Ct))
|
|
{
|
|
job = await engine.EnqueueAsync(ArchiveJobs.Convert(fx.Archive, plan), cancellationToken: fx.Ct);
|
|
// Planned again where the index had the sequence then: the job now waits for its lease to convert it.
|
|
await planned.Task.WaitAsync(TimeSpan.FromSeconds(TestTiming.Budget(30)), fx.Ct);
|
|
// What a Reorganize step does under that lease meanwhile: the folder moves, and its index row follows.
|
|
var to = fx.Paths.Sequence("moved/Roof").Folder;
|
|
Directory.CreateDirectory(Path.GetDirectoryName(to)!);
|
|
Directory.Move(from, to);
|
|
Assert.True(await fx.Db.Sequences.RelocateAsync(id, "moved/Roof", null, fx.Ct));
|
|
}
|
|
engine.Progress -= OnProgress;
|
|
|
|
await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60));
|
|
var frames = await FramesAsync(fx, id);
|
|
Assert.True(frames.All(f => f.Format == ImageFileFormat.Tiff && f.RelativePath.StartsWith("moved/Roof/", StringComparison.Ordinal)));
|
|
Assert.False(Directory.Exists(from), "the folder the sequence left is never made again");
|
|
Assert.Equal("moved/Roof", (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Folder);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ASequenceAnotherIndexMovedIsLeftAsItIsUntilThisIndexFindsIt()
|
|
{
|
|
// The folder the sequence left holds another sequence now, or nothing.
|
|
foreach (var taken in new[] { true, false })
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
Guid? stranger = taken ? await UploadedAsync(fx, 2, "upload-2", seed: 200) : null;
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
|
|
var row = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var folder = fx.Paths.Sequence(row.Folder).Folder;
|
|
|
|
// Another index moves the folder away, then the other sequence into the folder it left; this index has scanned neither.
|
|
var (other, archive) = await fx.OtherWriterAsync();
|
|
var moves = stranger is not { } s ? [(id, "moved/{date}")] : new[] { (id, "moved/{date}"), (s, "parked/{date}"), (s, row.Folder) };
|
|
foreach (var (sequence, template) in moves)
|
|
await other.ApplyRenameAsync(await other.PlanRenameAsync(archive.Id, [sequence], template, null, fx.Ct), null, fx.Ct);
|
|
var before = taken ? ArchiveFixture.FilesOf(folder) : null;
|
|
|
|
var job = Guid.NewGuid();
|
|
var left = (await RunHereAsync(fx, plan, job)).Sequences.Single();
|
|
Assert.Equal((ConvertOutcome.Blocked, Loc.L("Errors.Archive.FolderMoved")), (left.Outcome, left.Error), taken.ToString());
|
|
if (taken) Assert.SequenceEqual(before!, ArchiveFixture.FilesOf(folder), "the other sequence's files are untouched");
|
|
else Assert.False(Directory.Exists(folder), "the folder it left is not created again");
|
|
|
|
// Once this index's scan found it there, the retry converts it where it is now.
|
|
await fx.ScanAsync();
|
|
Assert.Equal(ConvertOutcome.Converted, (await RunHereAsync(fx, plan, job)).Sequences.Single().Outcome, taken.ToString());
|
|
var moved = (await fx.Db.Sequences.GetAsync(id))!.Folder;
|
|
Assert.True(moved.StartsWith("moved/", StringComparison.Ordinal), moved);
|
|
Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Tiff && f.RelativePath.StartsWith(moved + "/", StringComparison.Ordinal)));
|
|
if (taken) Assert.SequenceEqual(before!, ArchiveFixture.FilesOf(folder), "the other sequence's files are still untouched");
|
|
else Assert.False(Directory.Exists(folder));
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public async Task AReorganizeTheArchiveWaitsForKeepsTheConversionOffItsSequences()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var originals = await FramesAsync(fx, id);
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
|
|
// A Reorganize paused by its user holds the archive: its plan waits for it.
|
|
await fx.Db.Jobs.EnqueueAsync(new Job
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
Kind = JobKind.Move,
|
|
TargetIds = [fx.Archive.Id],
|
|
State = JobState.Paused,
|
|
Created = DateTimeOffset.UtcNow,
|
|
});
|
|
Assert.SequenceEqual([ConvertBlock.Busy], (await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id)).Sequences.Single().Blocked);
|
|
|
|
var engine = await fx.StartEngineAsync();
|
|
var job = await RunAsync(fx, engine, plan, JobState.Failed);
|
|
Assert.Equal(Loc.Plural("Archive.Error.ConvertKept", 1), job.Error);
|
|
Assert.Contains(Loc.L("Archive.Log.ConvertBlocked", (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle), job.LogTail);
|
|
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256, f.RelativePath)), (await FramesAsync(fx, id)).Select(f => (f.Id, f.Sha256, f.RelativePath)));
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task PhotosVerifiedAfterThePlanAreNotReplacedWithoutAConfirmation()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
var engine = await fx.StartEngineAsync();
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var path = fx.Paths.Sequence(sequence.Folder).ManifestPath;
|
|
var verified = ManifestStore.ReadSequence(path);
|
|
// Never verified when the plan is made (adopted): the dry run says that photo stays as it is.
|
|
ManifestStore.WriteSequence(path, verified with { Frames = [verified.Frames[0], verified.Frames[1] with { Sha256 = null }, verified.Frames[2]] });
|
|
var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id);
|
|
Assert.Equal((2, ConvertSkip.Unhashed), (plan.Frames, plan.Sequences.Single().Skipped.Single().Reason));
|
|
// A verify then hashes it before the job runs, as ArchiveVerifier does: no new frames version.
|
|
ManifestStore.WriteSequence(path, verified);
|
|
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
|
|
|
|
var job = await RunAsync(fx, engine, plan, JobState.Failed);
|
|
Assert.Equal(Loc.Plural("Archive.Error.ConvertKept", 1), job.Error);
|
|
Assert.Contains(Loc.L("Archive.Log.ConvertChanged", sequence.DisplayTitle), job.LogTail);
|
|
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
|
|
Assert.SequenceEqual(verified.Frames.Select(f => (f.Id, f.Sha256, f.File)), ManifestStore.ReadSequence(path).Frames.Select(f => (f.Id, f.Sha256, f.File)));
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
Assert.False(StagedAnything(fx));
|
|
// A retry does not convert it either: the user plans again.
|
|
await engine.ResumeAsync(job.Id, fx.Ct);
|
|
await fx.WaitForAsync(job.Id, JobState.Failed);
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task PhotosThatComeInOnceTheSequenceIsPinnedStayAsTheyAreAndARetryConvertsTheRest()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
|
|
var title = (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle;
|
|
// An earlier attempt pinned the sequence's photos and stopped (stage B blocked, a pause); an upload came in since.
|
|
var jobId = Guid.CreateVersion7();
|
|
PinLog.Open(ConvertJournal.PinsPathFor(fx.Paths, jobId)).Append(id, plan.Sequences.Single());
|
|
var (path, size, sha) = fx.Stage(Start.AddSeconds(60), 999);
|
|
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, null, path, "IMG_0099.JPG", size, sha, null, null), fx.Ct);
|
|
var added = (await FramesAsync(fx, id)).Single(f => f.Sha256 == sha);
|
|
|
|
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Convert(fx.Archive, plan), "convert", jobId);
|
|
var done = await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60));
|
|
Assert.Contains(Loc.L("Archive.Log.Converted", title, 2), done.LogTail);
|
|
var frames = await FramesAsync(fx, id);
|
|
Assert.Equal(2, frames.Count(f => f.Format == ImageFileFormat.Tiff));
|
|
Assert.Equal((sha, ImageFileFormat.Jpeg), frames.Where(f => f.Id == added.Id).Select(f => (f.Sha256, f.Format)).Single());
|
|
Assert.Equal(2, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Frames);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AJobResumedAfterACrashInItsReplacementSettlesItAndConvertsTheWholeSequence()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var folder = fx.Paths.Sequence(sequence.Folder);
|
|
var manifest = Sidecar(fx, sequence);
|
|
// The attempt that crashed: the photos pinned and staged, then its replacement cut short once every original had left
|
|
// for the trash, none of the converted files in yet.
|
|
var jobId = Guid.CreateVersion7();
|
|
PinLog.Open(ConvertJournal.PinsPathFor(fx.Paths, jobId)).Append(id, plan.Sequences.Single());
|
|
var log = StagedLog.Open(ConvertJournal.StagedPathFor(fx.Paths, jobId));
|
|
var entryFolder = Path.Combine(FrameTrash.Directory(fx.Paths), $"{id:N}-20261027120000");
|
|
var record = entryFolder + ".json";
|
|
List<SwapMove> outgoing = [];
|
|
List<SwapMove> incoming = [];
|
|
foreach (var frame in manifest.Frames)
|
|
{
|
|
var staged = Path.Combine(StagingOf(fx, jobId, id), frame.Id.ToString("N") + ".tif");
|
|
var file = FrameConverter.Convert(new ConvertSource(folder.File(frame.File), frame), new ConvertOptions(ConvertTarget.Tiff16), DecoderFactory.CreateServer(),
|
|
staged, cancellationToken: fx.Ct);
|
|
log.Append(id, manifest.Versions.Frames, new StagedConversion(frame.Id, frame.Sha256!, Path.GetFileName(staged), file.Size, file.Sha256, file.Width, file.Height));
|
|
outgoing.Add(new SwapMove(frame.File, fx.Paths.ToRelative(Path.Combine(entryFolder, "originals", Path.GetFileName(frame.File)))));
|
|
incoming.Add(new SwapMove(Path.ChangeExtension(frame.File, ".tif"), fx.Paths.ToRelative(staged)));
|
|
}
|
|
new FrameReplaceJournal(manifest.Versions.Frames, fx.Paths.ToRelative(record), fx.Paths.ToRelative(entryFolder), outgoing, incoming, false, null).Write(folder);
|
|
Directory.CreateDirectory(FrameTrash.Directory(fx.Paths));
|
|
File.WriteAllText(FrameRemovalJournal.PendingRecord(record), "{}");
|
|
foreach (var move in outgoing)
|
|
{
|
|
var trashed = fx.Paths.FromRelative(move.Other);
|
|
Directory.CreateDirectory(Path.GetDirectoryName(trashed)!);
|
|
File.Move(folder.File(move.Place), trashed);
|
|
}
|
|
|
|
// Not a sequence with nothing to convert, nor half of it converted: the originals come back, then all of it converts.
|
|
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Convert(fx.Archive, plan), "convert", jobId);
|
|
var done = await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60));
|
|
Assert.Contains(Loc.L("Archive.Log.Converted", sequence.DisplayTitle, 2), done.LogTail);
|
|
Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Tiff));
|
|
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
|
|
Assert.Equal((TrashKind.Replaced, 2), (entry.Kind, entry.Frames));
|
|
Assert.False(Directory.Exists(entryFolder));
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheFollowUpsOfEachSequenceAreQueuedAsItEndsAndOnResumeWhenAPauseCameFirst()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var first = await UploadedAsync(fx, 2);
|
|
var second = await UploadedAsync(fx, 2, "upload-2", seed: 200);
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, first, second);
|
|
var events = new List<string>();
|
|
var callbacks = Callbacks(done: s => events.Add($"done {s.SequenceId}"), queue: r => events.AddRange(r.Select(q => $"{q.Kind} {q.Targets.Single()}")));
|
|
await RunHereAsync(fx, plan, Guid.CreateVersion7(), callbacks);
|
|
// Queued as each sequence is journaled, before the next starts: a pause, a yield or a crash later loses none of them.
|
|
Assert.Less(events.IndexOf($"done {first}"), events.IndexOf($"{JobKind.LightAnalyse} {first}"));
|
|
Assert.Less(events.IndexOf($"{JobKind.LightAnalyse} {first}"), events.IndexOf($"done {second}"));
|
|
Assert.Contains($"{JobKind.LightAnalyse} {second}", events);
|
|
|
|
// As a pause right after the first sequence's outcome was journaled leaves the job: its passes not queued yet.
|
|
var jobId = Guid.CreateVersion7();
|
|
var journal = new ConvertJournal { JobId = jobId };
|
|
journal.Note(new ConvertSequenceResult(first, "Roof", ConvertOutcome.Converted, 2), SequenceStatus.AnalyzedDeep | SequenceStatus.ThumbsBuilt);
|
|
journal.Note(new ConvertSequenceResult(second, "Lake", ConvertOutcome.Converted, 2), SequenceStatus.AnalyzedLight);
|
|
journal.Of(second)!.Queued = true;
|
|
journal.Write(fx.Paths);
|
|
List<ArchiveJobRequest> queued = [];
|
|
var resumed = await RunHereAsync(fx, plan, jobId, Callbacks(queue: queued.AddRange));
|
|
Assert.Equal(4, resumed.Frames);
|
|
Assert.SequenceEqual([(JobKind.LightAnalyse, first), (JobKind.DeepAnalyse, first), (JobKind.Thumbs, first)],
|
|
queued.Select(r => (r.Kind, r.Targets.Single())));
|
|
Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, jobId));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AConversionWaitingInTheQueueKeepsItsSequencesFromASecondConfirmation()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id);
|
|
var request = ArchiveJobs.Convert(fx.Archive, plan);
|
|
var job = new Job
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
Kind = request.Kind,
|
|
TargetIds = request.Targets,
|
|
ParametersJson = request.ParametersJson,
|
|
State = JobState.Queued,
|
|
Created = DateTimeOffset.UtcNow,
|
|
};
|
|
await fx.Db.Jobs.EnqueueAsync(job, fx.Ct);
|
|
|
|
// Another tab, or Apply pressed again after an answer was lost: the same copies are not queued twice.
|
|
var again = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id);
|
|
Assert.SequenceEqual([ConvertBlock.Busy], again.Sequences.Single().Blocked);
|
|
Assert.NotEqual(plan.Fingerprint(), again.Fingerprint());
|
|
Assert.Throws<ArgumentException>(() => ArchiveJobs.Convert(fx.Archive, again));
|
|
// The job itself plans again with its sequences free, and converts them.
|
|
var engine = await fx.StartEngineAsync();
|
|
engine.Wake();
|
|
await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60));
|
|
Assert.NotNull(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, id)));
|
|
Assert.Empty((await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id)).Sequences.Single().Blocked);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ANewSequenceIsOwnedAsItsSourceIsOrByWhoAskedForIt()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var owner = Guid.NewGuid();
|
|
var owned = await UploadedAsync(fx, 2, user: owner);
|
|
var unowned = await UploadedAsync(fx, 2, "upload-2", seed: 200);
|
|
Assert.Equal(owner, (await fx.Db.Sequences.GetAsync(owned))!.OwnerUserId);
|
|
var engine = await fx.StartEngineAsync();
|
|
var asker = Guid.NewGuid();
|
|
|
|
var job = await engine.EnqueueAsync(ArchiveJobs.Convert(fx.Archive, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, owned, unowned)), asker, fx.Ct);
|
|
await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60));
|
|
foreach (var (source, expected) in new[] { (owned, owner), (unowned, asker) })
|
|
{
|
|
var made = (await fx.Db.Sequences.GetAsync((await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, source)))!.Value))!;
|
|
Assert.Equal(expected, made.OwnerUserId);
|
|
Assert.Equal(expected.ToString(), Sidecar(fx, made).CreatedBy);
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public async Task AStagedCopyThatVanishesFailsItsNewSequenceInsteadOfLeavingThePhotoOut()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id);
|
|
var first = Sidecar(fx, (await fx.Db.Sequences.GetAsync(id))!).Frames[0];
|
|
var jobId = Guid.CreateVersion7();
|
|
// A clean-up takes the first copy once both are staged, before they are filed.
|
|
var callbacks = Callbacks(progress: (n, total) =>
|
|
{
|
|
if (n == 2 && total == 2) File.Delete(Path.Combine(StagingOf(fx, jobId, id), first.Id.ToString("N") + ".tif"));
|
|
});
|
|
|
|
var failed = (await RunHereAsync(fx, plan, jobId, callbacks)).Sequences.Single();
|
|
Assert.Equal(ConvertOutcome.Failed, failed.Outcome);
|
|
Assert.Null(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(jobId, id)));
|
|
// The retry converts that photo again and makes the whole copy.
|
|
var made = (await RunHereAsync(fx, plan, jobId)).Sequences.Single();
|
|
Assert.Equal((ConvertOutcome.Created, 2, 0), (made.Outcome, made.Frames, made.LeftOut));
|
|
Assert.Count(2, await FramesAsync(fx, made.NewSequenceId!.Value));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ANewSequenceSaysWhichPhotosItLeftInTheSourceAlone()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
var engine = await fx.StartEngineAsync();
|
|
// Three TIFFs, then a JPEG uploaded: to JPEG, the TIFFs convert and the JPEG stays in the source alone.
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id));
|
|
var (path, size, sha) = fx.Stage(Start.AddSeconds(60), 999);
|
|
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, null, path, "IMG_0099.JPG", size, sha, null, null), fx.Ct);
|
|
|
|
var job = await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Jpeg, ConvertMode.NewSequence, id));
|
|
Assert.Contains(Loc.L("Archive.Log.ConvertedNew", sequence.DisplayTitle, 3), job.LogTail);
|
|
Assert.Contains(Loc.L("Archive.Log.ConvertLeftOut", sequence.DisplayTitle, 1), job.LogTail);
|
|
var made = Assert.NotNull(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, id)));
|
|
Assert.Count(3, await FramesAsync(fx, made));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AVolumeThatFillsDuringTheJobStopsItBeforeTheNextSequenceAndTheNextPhoto()
|
|
{
|
|
long free = 1L << 50;
|
|
await using var fx = await FixtureAsync(o => o with { FreeSpace = _ => free, Parallelism = () => 1 });
|
|
var first = await UploadedAsync(fx, 2);
|
|
var second = await UploadedAsync(fx, 2, "upload-2", seed: 200);
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, first, second);
|
|
var jobId = Guid.CreateVersion7();
|
|
var originals = await FramesAsync(fx, second);
|
|
|
|
// Another writer fills the volume while the first sequence converts: the second does not start.
|
|
var full = await Assert.ThrowsAsync<IOException>(() => RunHereAsync(fx, plan, jobId, Callbacks(done: _ => free = 1_000)));
|
|
Assert.True(full.Message.StartsWith("The conversion needs", StringComparison.Ordinal), full.Message);
|
|
Assert.True((await FramesAsync(fx, first)).All(f => f.Format == ImageFileFormat.Tiff));
|
|
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256)), (await FramesAsync(fx, second)).Select(f => (f.Id, f.Sha256)));
|
|
Assert.False(Directory.Exists(StagingOf(fx, jobId, second)));
|
|
|
|
// Space again, until the first photo of the second sequence is staged: the next one does not start, what was staged stays.
|
|
free = 1L << 50;
|
|
await Assert.ThrowsAsync<IOException>(() => RunHereAsync(fx, plan, jobId, Callbacks(progress: (n, _) =>
|
|
{
|
|
if (n == 1) free = 1_000;
|
|
})));
|
|
Assert.Count(1, Directory.EnumerateFiles(StagingOf(fx, jobId, second)));
|
|
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256)), (await FramesAsync(fx, second)).Select(f => (f.Id, f.Sha256)));
|
|
|
|
free = 1L << 50;
|
|
var done = await RunHereAsync(fx, plan, jobId);
|
|
Assert.True(done.Sequences.All(s => s.Outcome == ConvertOutcome.Converted));
|
|
Assert.True((await FramesAsync(fx, second)).All(f => f.Format == ImageFileFormat.Tiff));
|
|
Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, jobId));
|
|
}
|
|
|
|
[Test]
|
|
public async Task MaintenanceKeepsWhatAWaitingConversionStagedAndClearsWhatAFinishedOneLeft()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var paused = Guid.CreateVersion7();
|
|
var finished = Guid.CreateVersion7();
|
|
var forgotten = Guid.CreateVersion7();
|
|
foreach (var (id, state) in new[] { (paused, JobState.Paused), (finished, JobState.Done) })
|
|
await fx.Db.Jobs.EnqueueAsync(new Job { Id = id, Kind = JobKind.Convert, State = state, Created = DateTimeOffset.UtcNow }, fx.Ct);
|
|
string Staged(Guid job)
|
|
{
|
|
var file = Path.Combine(StagingOf(fx, job, Guid.NewGuid()), "frame.tif");
|
|
Directory.CreateDirectory(Path.GetDirectoryName(file)!);
|
|
File.WriteAllText(file, "x");
|
|
File.SetLastWriteTimeUtc(file, DateTime.UtcNow.AddDays(-3));
|
|
return file;
|
|
}
|
|
string Journal(Guid job)
|
|
{
|
|
new ConvertJournal { JobId = job }.Write(fx.Paths);
|
|
return ConvertJournal.PathFor(fx.Paths, job);
|
|
}
|
|
var (pausedFile, pausedJournal) = (Staged(paused), Journal(paused));
|
|
var (finishedFile, finishedJournal) = (Staged(finished), Journal(finished));
|
|
var (forgottenFile, forgottenJournal) = (Staged(forgotten), Journal(forgotten));
|
|
|
|
var dry = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: true, 0, _ => ValueTask.CompletedTask, fx.Ct);
|
|
Assert.SequenceEqual(new[] { finishedFile, finishedJournal, forgottenFile }.Select(fx.Paths.ToRelative).Order(), dry.TemporaryFiles.Order());
|
|
await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
|
|
// A paused job's staged conversions stay however old; a finished job's go with its journal; a job this index does not
|
|
// know (another host's, or forgotten by the history) loses staged files idle for a day, and keeps its journal a while.
|
|
Assert.FileExists(pausedFile);
|
|
Assert.FileExists(pausedJournal);
|
|
Assert.False(Directory.Exists(Path.Combine(Staging(fx), MassConverter.StagingFolder(finished))));
|
|
Assert.FileMissing(finishedJournal);
|
|
Assert.False(Directory.Exists(Path.Combine(Staging(fx), MassConverter.StagingFolder(forgotten))));
|
|
Assert.FileMissing(forgottenFile);
|
|
Assert.FileExists(forgottenJournal);
|
|
File.SetLastWriteTimeUtc(forgottenJournal, DateTime.UtcNow.AddDays(-CleanupSteps.DefaultHistoryDays - 1));
|
|
await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
|
|
Assert.FileMissing(forgottenJournal);
|
|
Assert.FileExists(pausedJournal);
|
|
}
|
|
}
|