The convert job planned every sequence again when it ran and checked only the frames version, so a verify, a repaired upload or a tier change between the dry run and the job let it replace photos the confirmed plan had listed as staying unchanged. - Each sequence of a convert job now carries the digest of the photos it converts, each with its SHA-256 (ConvertSequencePlan.Digest, also in the plan's fingerprint). Under the sequence's lease, its sidecar settled, the job plans it again and goes on only while the digest matches; it then pins those photos (convert-<job>.pins), so photos that come in or are verified later stay as they are, a retry included. Stage B checks under the lease that every pinned photo still holds the content converted instead of holding the frames version, so a sequence still receiving uploads converts what was confirmed. - A resumed job settles the journals a crash left in a sequence (a half-done replacement among them) before it plans, and reports a sequence its replacement completed as Converted, never Nothing or a partial swap. - The passes a converted or new sequence needs are queued as soon as its outcome is journaled; the journal records the statuses they follow from and whether they were queued, so a pause, yield, stop or crash no longer loses them. - Staged conversions of a blocked or failed sequence are kept for the retry, which re-verifies and takes them; maintenance keeps a convert job's staging while the job is queued, running or paused, and removes its journal and staging once the job is done or gone from the history. - A dry run counts the sequences of a queued conversion as busy, so the same conversion cannot be confirmed twice. - New sequence copies are owned by the source's owner, else by the requesting user's id (never the user name); a staged copy that vanished fails the sequence instead of leaving its photo out; the job log states how many photos stayed in the source alone. - Free space is measured where the conversion writes: the server takes the tighter of the space path's volume and the archive's own (Unraid cache), the desktop asks Windows (GetDiskFreeSpaceEx), which also measures UNC archives; the job checks again before each sequence and each photo, scaling estimates by the files staged so far, and stops before the volume fills with what it staged kept. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> (cherry picked from commit 2d7d461c33b9f257bf57793cc7cf331d0425c14e)
1058 lines
65 KiB
C#
1058 lines
65 KiB
C#
using System.Security.Cryptography;
|
|
using System.Text.Json;
|
|
using Chronos.Core.Archiving;
|
|
using Chronos.Core.Conversion;
|
|
using Chronos.Core.Import;
|
|
using Chronos.Core.Imaging;
|
|
using Chronos.Core.Imaging.Decoders;
|
|
using Chronos.Core.Localization;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Core.Project;
|
|
using Chronos.Data.Archiving;
|
|
using Chronos.Data.Archiving.Jobs;
|
|
using Chronos.Data.Housekeeping;
|
|
using Chronos.Data.Jobs;
|
|
using Chronos.Data.Repositories;
|
|
using Chronos.Testing;
|
|
using static Chronos.Data.Tests.Archiving.ArchiveFixture;
|
|
|
|
namespace Chronos.Data.Tests.Archiving;
|
|
|
|
/// <summary>
|
|
/// Mass conversion after import (owner review 2, R7): the dry run, the <c>convert</c> job in both modes, its undo through the
|
|
/// trash, all or nothing per sequence, resumes in stage A and B, plans gone stale, the space check, and the gates that must
|
|
/// keep holding (the card erase, the emptying of a replaced entry).
|
|
/// </summary>
|
|
[Category("archive")]
|
|
public sealed class ConvertTests(TestContext context)
|
|
{
|
|
private Task<ArchiveFixture> FixtureAsync(Func<ArchiveServiceOptions, ArchiveServiceOptions>? more = null) =>
|
|
CreateAsync(context, configure: o =>
|
|
{
|
|
var options = o with { Decoder = DecoderFactory.CreateServer(), Parallelism = () => 2 };
|
|
return more is null ? options : more(options);
|
|
});
|
|
|
|
/// <summary>A sequence of <paramref name="count"/> uploaded photos (hashed, under <c>originals/</c>), ten seconds apart, by <paramref name="user"/>.</summary>
|
|
private static async Task<Guid> UploadedAsync(ArchiveFixture fx, int count, string session = "upload-1", int seed = 100, Guid? user = null)
|
|
{
|
|
Guid? id = null;
|
|
for (var i = 0; i < count; i++)
|
|
{
|
|
var (path, size, sha) = fx.Stage(Start.AddSeconds(10 * i).AddTicks(1_234_567), seed + i);
|
|
var result = await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, id is null ? session : null, path, $"IMG_{i + 1:0000}.JPG",
|
|
size, sha, null, user), fx.Ct);
|
|
id = result.SequenceId;
|
|
}
|
|
return id!.Value;
|
|
}
|
|
|
|
private static Task<ConvertPlan> PlanAsync(ArchiveFixture fx, ConvertTarget target, ConvertMode mode, params Guid[] ids) =>
|
|
fx.Service.PlanConvertAsync(fx.Archive.Id, ids, new ConvertOptions(target), mode, fx.Ct);
|
|
|
|
/// <summary>The parameters the job of <paramref name="plan"/> runs with.</summary>
|
|
private static ConvertJobParameters Parameters(ArchiveFixture fx, ConvertPlan plan) =>
|
|
JsonSerializer.Deserialize(ArchiveJobs.Convert(fx.Archive, plan).ParametersJson!, ArchiveJobJson.Default.ConvertJobParameters)!;
|
|
|
|
/// <summary>Callbacks that report nothing, but for <paramref name="progress"/> and <paramref name="done"/> when given.</summary>
|
|
private static ConvertCallbacks Callbacks(Action<long, long>? progress = null, Action<ConvertSequenceResult>? done = null,
|
|
Action<IReadOnlyList<ArchiveJobRequest>>? queue = null) => new(
|
|
(n, total) =>
|
|
{
|
|
progress?.Invoke(n, total);
|
|
return ValueTask.CompletedTask;
|
|
},
|
|
(_, _) => ValueTask.CompletedTask,
|
|
_ => { },
|
|
s =>
|
|
{
|
|
done?.Invoke(s);
|
|
return ValueTask.CompletedTask;
|
|
},
|
|
requests =>
|
|
{
|
|
queue?.Invoke(requests);
|
|
return ValueTask.CompletedTask;
|
|
});
|
|
|
|
/// <summary>The conversion job <paramref name="jobId"/> of <paramref name="plan"/>, run here as its handler runs it.</summary>
|
|
private static Task<ConvertResult> RunHereAsync(ArchiveFixture fx, ConvertPlan plan, Guid jobId, ConvertCallbacks? callbacks = null) =>
|
|
fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), jobId, new ConvertRequester(null, null), callbacks ?? Callbacks(), fx.Ct);
|
|
|
|
private static string StagingOf(ArchiveFixture fx, Guid job, Guid sequence) => Path.Combine(Staging(fx), $"convert-{job:N}", $"{sequence:N}");
|
|
|
|
private static async Task<Job> RunAsync(ArchiveFixture fx, JobEngine engine, ConvertPlan plan, JobState state = JobState.Done)
|
|
{
|
|
var job = await engine.EnqueueAsync(ArchiveJobs.Convert(fx.Archive, plan), cancellationToken: fx.Ct);
|
|
return await fx.WaitForAsync(job.Id, state, (int)TestTiming.Budget(60));
|
|
}
|
|
|
|
private static SequenceManifest Sidecar(ArchiveFixture fx, Sequence sequence) => ManifestStore.ReadSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath);
|
|
|
|
private static string Sha(string path) => Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(path)));
|
|
|
|
/// <summary>The index rows of a sequence, after checking that each file holds its row's content and that the sidecar agrees.</summary>
|
|
private static async Task<List<Frame>> FramesAsync(ArchiveFixture fx, Guid id)
|
|
{
|
|
var sequence = Assert.NotNull(await fx.Db.Sequences.GetAsync(id));
|
|
var rows = (await fx.Db.Frames.ListBySequenceAsync(id)).ToList();
|
|
var sidecar = Sidecar(fx, sequence);
|
|
Assert.Equal(sidecar.Versions.Frames, sequence.FramesVersion);
|
|
Assert.SequenceEqual(sidecar.Frames.Select(f => (f.Id, f.I, f.Sha256)), rows.Select(r => (r.Id, r.Index, r.Sha256)));
|
|
foreach (var row in rows) Assert.Equal(row.Sha256, Sha(fx.Paths.FromRelative(row.RelativePath)));
|
|
Assert.FileMissing(fx.Paths.Sequence(sequence.Folder).ReplaceJournalPath);
|
|
return rows;
|
|
}
|
|
|
|
private static string Staging(ArchiveFixture fx) => fx.Paths.StagingDirectory;
|
|
|
|
private static bool StagedAnything(ArchiveFixture fx) =>
|
|
Directory.Exists(Staging(fx)) && Directory.EnumerateDirectories(Staging(fx), "convert-*").Any();
|
|
|
|
[Test]
|
|
public async Task TheDryRunCountsWhatConvertsAndChangesNothing()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
|
|
|
|
var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id);
|
|
var step = plan.Sequences.Single();
|
|
Assert.Equal((3, 0, true), (step.Frames.Count, step.SkippedFrames, step.Converts));
|
|
Assert.Equal((await fx.Db.Sequences.GetAsync(id))!.FramesVersion, step.FramesVersion);
|
|
Assert.Equal(step.Frames.Sum(f => f.Size), plan.BytesBefore);
|
|
Assert.Equal(3 * ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.LinearDng), 48, 32), plan.BytesAfter);
|
|
Assert.Equal(ConvertPlanner.Required(plan.BytesAfter), plan.Space.Required);
|
|
Assert.True(plan.Space.Enough);
|
|
Assert.Equal(plan.Fingerprint(), (await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id)).Fingerprint());
|
|
Assert.NotEqual(plan.Fingerprint(), (await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.NewSequence, id)).Fingerprint());
|
|
|
|
// JPEG to JPEG converts nothing.
|
|
var same = (await PlanAsync(fx, ConvertTarget.Jpeg, ConvertMode.Replace, id)).Sequences.Single();
|
|
Assert.Equal((0, ConvertSkip.SameFormat, 3), (same.Frames.Count, same.Skipped.Single().Reason, same.Skipped.Single().Frames));
|
|
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
|
|
Assert.False(StagedAnything(fx));
|
|
await Assert.ThrowsAsync<ArgumentException>(() => PlanAsync(fx, ConvertTarget.Jpeg, ConvertMode.Replace, Guid.NewGuid()));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ASequenceBusyOrUnreadableIsBlocked()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var busy = await UploadedAsync(fx, 2);
|
|
var unreadable = await UploadedAsync(fx, 2, "upload-2", seed: 200);
|
|
await fx.Db.Jobs.EnqueueAsync(new Job
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
Kind = JobKind.Render,
|
|
TargetIds = [busy],
|
|
State = JobState.Running,
|
|
NodeId = "test#0",
|
|
Created = DateTimeOffset.UtcNow,
|
|
Started = DateTimeOffset.UtcNow,
|
|
});
|
|
File.WriteAllText(fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(unreadable))!.Folder).ManifestPath, "{ not json");
|
|
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, busy, unreadable);
|
|
Assert.SequenceEqual([ConvertBlock.Busy], plan.Sequences.Single(s => s.SequenceId == busy).Blocked);
|
|
Assert.SequenceEqual([ConvertBlock.Unreadable], plan.Sequences.Single(s => s.SequenceId == unreadable).Blocked);
|
|
Assert.Equal(0, plan.Frames);
|
|
Assert.Throws<ArgumentException>(() => ArchiveJobs.Convert(fx.Archive, plan));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AReplaceKeepsTheFramesAndTrashesTheOriginalsWithTheirReplacements()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
var engine = await fx.StartEngineAsync();
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var originals = await FramesAsync(fx, id);
|
|
var project = new ProjectSettings { ExcludedFrames = [1], HeroFrame = 2 }.ToJson();
|
|
await fx.Db.Sequences.EditAsync(id, new SequenceEdit { SettingsJson = project, HeroFrameId = originals[2].Id }, null, fx.Ct);
|
|
var folder = fx.Paths.Sequence(sequence.Folder);
|
|
Directory.CreateDirectory(folder.MetaDirectory);
|
|
File.WriteAllText(folder.AnalysisPath, "{\"frames\":3}");
|
|
var before = Sidecar(fx, sequence);
|
|
|
|
var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id);
|
|
var job = await RunAsync(fx, engine, plan);
|
|
Assert.Contains(Loc.L("Archive.Log.Converted", sequence.DisplayTitle, 3), job.LogTail);
|
|
|
|
var row = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var frames = await FramesAsync(fx, id);
|
|
Assert.Equal(before.Versions.Frames + 1, row.FramesVersion);
|
|
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Index)), frames.Select(f => (f.Id, f.Index)));
|
|
var after = Sidecar(fx, row);
|
|
foreach (var (was, now) in before.Frames.Zip(after.Frames))
|
|
{
|
|
// Named as filing named the original, with the target's extension: the same template, frame and sequence.
|
|
Assert.Equal((Path.ChangeExtension(was.File, ".dng"), ImageFileFormat.Dng, DecodeTier.Native, 1), (now.File, now.Format, now.Tier, now.Orientation));
|
|
Assert.Equal((was.CapturedAt, was.Original, was.Flags), (now.CapturedAt, now.Original, now.Flags));
|
|
var source = Assert.NotNull(now.Source);
|
|
Assert.Equal((was.Sha256, was.Format, was.File, was.Size), (source.Sha256, source.Format, source.File, source.Size));
|
|
Assert.FileMissing(folder.File(was.File));
|
|
}
|
|
// The project's positions stay on their frames.
|
|
var settings = ProjectSettings.FromJson(row.SettingsJson!);
|
|
Assert.SequenceEqual([1], settings.ExcludedFrames);
|
|
Assert.Equal((2, originals[2].Id), (settings.HeroFrame, row.HeroFrameId));
|
|
|
|
// One replaced entry with the originals, each naming its replacement, and the analysis as it was.
|
|
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
|
|
Assert.Equal((TrashKind.Replaced, id, 3), (entry.Kind, entry.SequenceId, entry.Frames));
|
|
var record = Assert.NotNull(FrameTrash.ReadEntry(fx.Paths, entry.Name));
|
|
var entryFolder = FrameTrash.Location(fx.Paths, entry.Name).Folder;
|
|
foreach (var photo in record.Photos)
|
|
{
|
|
var now = after.Frames.Single(f => f.Id == photo.Frame.Id);
|
|
Assert.Equal(now.Sha256, photo.Replacement);
|
|
Assert.Equal(photo.Frame.Sha256, Sha(Path.Combine(entryFolder, photo.Files[0].Path)));
|
|
}
|
|
Assert.Equal("{\"frames\":3}", File.ReadAllText(Path.Combine(entryFolder, FrameTrash.AnalysisCopy)));
|
|
// Tombstones alias the old content to the new, in the sidecar and the index.
|
|
var tombstones = Assert.NotNull(after.Removed);
|
|
Assert.SequenceEqual(before.Frames.Select(f => (f.Sha256, after.Frames.Single(n => n.Id == f.Id).Sha256, RemovalReason.Replaced)).Order(),
|
|
tombstones.Select(t => ((string?)t.Sha256, t.Replacement, t.Reason)).Order());
|
|
Assert.SequenceEqual(tombstones.Select(t => (t.Sha256, t.Replacement)).Order(),
|
|
(await fx.Db.FrameRemovals.ListBySequenceAsync(id)).Select(r => (r.Sha256, r.ReplacementSha256)).Order());
|
|
Assert.Equal((SequenceStatus)0, row.Status & (SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt));
|
|
Assert.False(StagedAnything(fx));
|
|
Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, job.Id));
|
|
Assert.FileMissing(ConvertJournal.StagedPathFor(fx.Paths, job.Id));
|
|
// Converted again, nothing is left to convert.
|
|
Assert.Equal(0, (await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id)).Frames);
|
|
}
|
|
|
|
[Test]
|
|
public async Task RestoringTheReplacedEntrySwapsTheOriginalsBack()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
var engine = await fx.StartEngineAsync();
|
|
var originals = await FramesAsync(fx, id);
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var before = Sidecar(fx, sequence);
|
|
var analysis = fx.Paths.Sequence(sequence.Folder).AnalysisPath;
|
|
Directory.CreateDirectory(Path.GetDirectoryName(analysis)!);
|
|
File.WriteAllText(analysis, "of the originals");
|
|
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id));
|
|
var converted = Sidecar(fx, (await fx.Db.Sequences.GetAsync(id))!);
|
|
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
|
|
// The light analysis of the converted frames, as it would be written again.
|
|
File.WriteAllText(analysis, "of the conversions");
|
|
|
|
var result = await fx.Service.RestoreEntryAsync(fx.Archive.Id, entry.Name, null, fx.Ct);
|
|
Assert.Equal((TrashKind.Replaced, 3, false, id), (result.Kind, result.Photos, result.NewSequence, result.Sequence.Id));
|
|
var frames = await FramesAsync(fx, id);
|
|
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Index, f.Sha256, f.RelativePath, f.Format)), frames.Select(f => (f.Id, f.Index, f.Sha256, f.RelativePath, f.Format)));
|
|
var back = Sidecar(fx, (await fx.Db.Sequences.GetAsync(id))!);
|
|
Assert.True(back.Frames.All(f => f.Source is null));
|
|
// The converted files are in the trash in turn, naming the originals as their replacements; the old entry is gone.
|
|
var turned = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
|
|
Assert.NotEqual(entry.Name, turned.Name);
|
|
Assert.Equal(TrashKind.Replaced, turned.Kind);
|
|
var record = FrameTrash.ReadEntry(fx.Paths, turned.Name)!;
|
|
Assert.SequenceEqual(converted.Frames.Select(f => (f.Sha256, before.Frames.Single(o => o.Id == f.Id).Sha256)).Order(),
|
|
record.Photos.Select(p => (p.Frame.Sha256, p.Replacement)).Order());
|
|
// Each analysis goes with its frames: the originals' back in the sequence, the conversions' into the new entry.
|
|
Assert.Equal("of the originals", File.ReadAllText(analysis));
|
|
Assert.Equal("of the conversions", File.ReadAllText(Path.Combine(FrameTrash.Location(fx.Paths, turned.Name).Folder, FrameTrash.AnalysisCopy)));
|
|
Assert.False(Directory.Exists(FrameTrash.Location(fx.Paths, entry.Name).Folder));
|
|
Assert.SequenceEqual(converted.Frames.Select(f => (f.Sha256, before.Frames.Single(o => o.Id == f.Id).Sha256)).Order(),
|
|
back.Removed!.Select(t => ((string?)t.Sha256, t.Replacement)).Order());
|
|
|
|
// That entry erases: its replacements (the originals) are where it says, re-hashed now.
|
|
var empty = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
|
|
var job = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, empty), cancellationToken: fx.Ct);
|
|
await fx.WaitForAsync(job.Id, JobState.Done);
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
await FramesAsync(fx, id);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnUndoAfterTheFramesChangedBringsTheOriginalsBackAsANewSequence()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
var engine = await fx.StartEngineAsync();
|
|
var originals = await FramesAsync(fx, id);
|
|
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id));
|
|
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
|
|
var converted = await FramesAsync(fx, id);
|
|
// A converted frame deleted since: the frames are no longer the ones the conversion left.
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var delete = await fx.Service.FrameTrash.PlanDeleteAsync(id, [converted[1].Id], sequence.FramesVersion, fx.Ct);
|
|
await fx.Service.FrameTrash.DeleteAsync(delete, "alberto", null, fx.Ct);
|
|
|
|
var result = await fx.Service.RestoreEntryAsync(fx.Archive.Id, entry.Name, null, fx.Ct);
|
|
Assert.Equal((TrashKind.Replaced, 3, true), (result.Kind, result.Photos, result.NewSequence));
|
|
Assert.NotEqual(id, result.Sequence.Id);
|
|
Assert.SequenceEqual(originals.Select(f => f.Sha256), (await FramesAsync(fx, result.Sequence.Id)).Select(f => f.Sha256));
|
|
Assert.Count(2, await FramesAsync(fx, id));
|
|
// The replaced entry is gone with its last photo (and its copy of the analysis); the deleted frame's entry stays.
|
|
Assert.DoesNotContain(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ANewSequenceTakesTheCopiesAndLeavesTheSourceAsItIs()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
var engine = await fx.StartEngineAsync();
|
|
await fx.Service.SetSequenceInfoAsync(id, "Roof", ["sunset", "milan"], 4, "From the roof", fx.Ct);
|
|
var originals = await FramesAsync(fx, id);
|
|
await fx.Db.Sequences.EditAsync(id, new SequenceEdit { SettingsJson = new ProjectSettings { ExcludedFrames = [1], HeroFrame = 2 }.ToJson() }, null, fx.Ct);
|
|
var source = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var sidecar = File.ReadAllText(fx.Paths.Sequence(source.Folder).ManifestPath);
|
|
|
|
var job = await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id));
|
|
var made = Assert.NotNull(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, id)));
|
|
var copy = (await fx.Db.Sequences.GetAsync(made))!;
|
|
Assert.Equal(("Roof (TIFF)", 4, "From the roof"), (copy.Title, copy.Rating, copy.Notes));
|
|
Assert.SequenceEqual(["milan", "sunset"], copy.Tags.Order());
|
|
var frames = await FramesAsync(fx, made);
|
|
Assert.SequenceEqual(originals.Select(f => f.CapturedAt), frames.Select(f => f.CapturedAt));
|
|
Assert.True(frames.All(f => f.Format == ImageFileFormat.Tiff));
|
|
// Filed as the ingest files every photo: the archive's template in the new sequence, {name} the card's, {ext} the target's.
|
|
Assert.SequenceEqual(originals.Select(f => Path.ChangeExtension(f.FileName, ".tif")), frames.Select(f => f.FileName));
|
|
Assert.SequenceEqual(originals.Select(f => f.OriginalFileName), frames.Select(f => f.OriginalFileName));
|
|
var settings = ProjectSettings.FromJson(copy.SettingsJson!);
|
|
Assert.SequenceEqual([1], settings.ExcludedFrames);
|
|
Assert.Equal((2, frames[2].Id), (settings.HeroFrame, copy.HeroFrameId));
|
|
// The source: not a byte of its sidecar, nor a row, changed; nothing went to the trash.
|
|
Assert.Equal(sidecar, File.ReadAllText(fx.Paths.Sequence(source.Folder).ManifestPath));
|
|
var now = (await fx.Db.Sequences.GetAsync(id))!;
|
|
Assert.Equal((source.FramesVersion, source.SettingsVersion, source.Title, source.Folder), (now.FramesVersion, now.SettingsVersion, now.Title, now.Folder));
|
|
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256)), (await FramesAsync(fx, id)).Select(f => (f.Id, f.Sha256)));
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
Assert.False(StagedAnything(fx));
|
|
}
|
|
|
|
[Test]
|
|
public async Task APhotoThatFailsLeavesItsWholeSequenceAsItWasAndTheOthersConvert()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var bad = await UploadedAsync(fx, 3);
|
|
var good = await UploadedAsync(fx, 2, "upload-2", seed: 300);
|
|
var engine = await fx.StartEngineAsync();
|
|
var damaged = (await FramesAsync(fx, bad))[1];
|
|
var originals = await FramesAsync(fx, bad);
|
|
// Rot after the plan: the photo no longer matches the hash it was archived with.
|
|
var path = fx.Paths.FromRelative(damaged.RelativePath);
|
|
var bytes = File.ReadAllBytes(path);
|
|
bytes[^20] ^= 0x10;
|
|
File.WriteAllBytes(path, bytes);
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, bad, good);
|
|
Assert.Equal(5, plan.Frames);
|
|
|
|
var job = await RunAsync(fx, engine, plan, JobState.Failed);
|
|
Assert.Equal(Loc.Plural("Archive.Error.ConvertKept", 1), job.Error);
|
|
Assert.Contains(damaged.FileName, job.LogTail);
|
|
// The log says why in one catalog sentence; the exception's English text stays in the diagnostics log.
|
|
Assert.Contains(Loc.L("Archive.Convert.Failed.Changed"), job.LogTail);
|
|
Assert.False(job.LogTail!.Contains("SHA-256", StringComparison.Ordinal), job.LogTail);
|
|
// All or nothing: not one frame of that sequence changed, and nothing of it is in the trash.
|
|
var rows = await fx.Db.Frames.ListBySequenceAsync(bad);
|
|
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256, f.RelativePath)), rows.Select(f => (f.Id, f.Sha256, f.RelativePath)));
|
|
Assert.True((await FramesAsync(fx, good)).All(f => f.Format == ImageFileFormat.Tiff));
|
|
Assert.SequenceEqual([good], (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.SequenceId));
|
|
// The journal stays for a retry, which tries the failed sequence again and leaves the converted one; so does what the
|
|
// failed sequence staged before its photo failed, and only that.
|
|
var journal = Assert.NotNull(ConvertJournal.Read(fx.Paths, job.Id));
|
|
Assert.Equal(ConvertOutcome.Converted, journal.Of(good)!.Outcome);
|
|
Assert.Equal(ConvertOutcome.Failed, journal.Of(bad)!.Outcome);
|
|
Assert.False(Directory.Exists(StagingOf(fx, job.Id, good)));
|
|
var noted = StagedLog.Open(ConvertJournal.StagedPathFor(fx.Paths, job.Id)).Of(bad, (await fx.Db.Sequences.GetAsync(bad))!.FramesVersion);
|
|
Assert.SequenceEqual(noted.Select(c => c.File).Order(), Directory.EnumerateFiles(StagingOf(fx, job.Id, bad)).Select(f => Path.GetFileName(f)).Order());
|
|
Assert.DoesNotContain(damaged.Id, noted.Select(c => c.FrameId));
|
|
var first = Path.Combine(StagingOf(fx, job.Id, bad), noted.Single(c => c.FrameId == originals[0].Id).File);
|
|
File.SetLastWriteTimeUtc(first, new DateTime(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc));
|
|
|
|
// The photo is put right: the retry takes the staged conversion that still verifies and converts the rest.
|
|
bytes[^20] ^= 0x10;
|
|
File.WriteAllBytes(path, bytes);
|
|
await engine.ResumeAsync(job.Id, fx.Ct);
|
|
await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60));
|
|
var converted = await FramesAsync(fx, bad);
|
|
Assert.True(converted.All(f => f.Format == ImageFileFormat.Tiff));
|
|
Assert.Equal(2020, File.GetLastWriteTimeUtc(fx.Paths.FromRelative(converted[0].RelativePath)).Year);
|
|
Assert.False(StagedAnything(fx));
|
|
Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, job.Id));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AResumedJobKeepsTheStagedFilesThatStillVerify()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
var engine = await fx.StartEngineAsync();
|
|
// To JPEG needs other originals than the JPEGs uploaded: they become TIFF first.
|
|
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id));
|
|
var plan = await PlanAsync(fx, ConvertTarget.Jpeg, ConvertMode.Replace, id);
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var folder = fx.Paths.Sequence(sequence.Folder);
|
|
var manifest = Sidecar(fx, sequence);
|
|
var jobId = Guid.CreateVersion7();
|
|
var staging = Path.Combine(Staging(fx), $"convert-{jobId:N}", $"{id:N}");
|
|
var log = StagedLog.Open(ConvertJournal.StagedPathFor(fx.Paths, jobId));
|
|
// An earlier attempt staged the first photo (at another quality, so its use shows) and the second, damaged since.
|
|
StagedConversion Stage(SidecarFrame frame, int quality)
|
|
{
|
|
var target = Path.Combine(staging, frame.Id.ToString("N") + ".jpg");
|
|
var file = FrameConverter.Convert(new ConvertSource(folder.File(frame.File), frame, manifest.Camera, manifest.Lens), new ConvertOptions(ConvertTarget.Jpeg, quality),
|
|
DecoderFactory.CreateServer(), target, cancellationToken: fx.Ct);
|
|
var conversion = new StagedConversion(frame.Id, frame.Sha256!, Path.GetFileName(target), file.Size, file.Sha256, file.Width, file.Height);
|
|
log.Append(id, manifest.Versions.Frames, conversion);
|
|
return conversion;
|
|
}
|
|
var kept = Stage(manifest.Frames[0], 40);
|
|
var torn = Stage(manifest.Frames[1], 95);
|
|
File.WriteAllBytes(Path.Combine(staging, torn.File), [1, 2, 3]);
|
|
|
|
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Convert(fx.Archive, plan), "convert", jobId);
|
|
await fx.WaitForAsync(job.Id, JobState.Done);
|
|
var frames = await FramesAsync(fx, id);
|
|
Assert.Equal(kept.Sha256, frames[0].Sha256);
|
|
Assert.Equal(torn.Sha256, frames[1].Sha256);
|
|
Assert.True(frames.All(f => f.Format == ImageFileFormat.Jpeg));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AJobThatStoppedAfterItsReplacementReportsTheSequenceConverted()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id);
|
|
var request = ArchiveJobs.Convert(fx.Archive, plan);
|
|
await RunAsync(fx, engine, plan);
|
|
// As a crash between the replacement and the journal's note leaves it: the staged log names the conversions done.
|
|
var sidecar = Sidecar(fx, (await fx.Db.Sequences.GetAsync(id))!);
|
|
var jobId = Guid.CreateVersion7();
|
|
var log = StagedLog.Open(ConvertJournal.StagedPathFor(fx.Paths, jobId));
|
|
foreach (var frame in sidecar.Frames)
|
|
log.Append(id, plan.Sequences.Single().FramesVersion, new StagedConversion(frame.Id, frame.Source!.Sha256!, "gone.dng", frame.Size, frame.Sha256!, frame.W, frame.H));
|
|
|
|
var job = await fx.EnqueueWithCheckpointAsync(engine, request, "convert", jobId);
|
|
var done = await fx.WaitForAsync(job.Id, JobState.Done);
|
|
Assert.Contains(Loc.L("Archive.Log.Converted", (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle, 2), done.LogTail);
|
|
Assert.Count(1, await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ASequenceThatChangedAfterThePlanIsLeftAsItWas()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id);
|
|
// Another photo arrives before the job runs.
|
|
var (path, size, sha) = fx.Stage(Start.AddSeconds(60), 999);
|
|
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, null, path, "IMG_0099.JPG", size, sha, null, null), fx.Ct);
|
|
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
|
|
|
|
var job = await RunAsync(fx, engine, plan, JobState.Failed);
|
|
Assert.Equal(Loc.Plural("Archive.Error.ConvertKept", 1), job.Error);
|
|
Assert.Contains(Loc.L("Archive.Log.ConvertChanged", (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle), job.LogTail);
|
|
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AVolumeWithoutTheSpaceConvertsNothing()
|
|
{
|
|
await using var fx = await FixtureAsync(o => o with { FreeSpace = _ => 1_000 });
|
|
var id = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
|
|
Assert.False(plan.Space.Enough);
|
|
Assert.Equal(1_000, plan.Space.Free);
|
|
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
|
|
|
|
var job = await RunAsync(fx, engine, plan, JobState.Failed);
|
|
Assert.True(job.Error!.StartsWith("The conversion needs", StringComparison.Ordinal), job.Error);
|
|
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
|
|
Assert.False(StagedAnything(fx));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ACardKeepsTheCopyOfAPhotoThatWasConverted()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var card = context.CreateTempDirectory("card");
|
|
Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, 3);
|
|
var media = await fx.Service.Imports.RegisterCardAsync(new CardVolume("3F2A9C11-EOS_DIGITAL-127865454592", 0x3F2A9C11, "EOS_DIGITAL", 127_865_454_592), fx.Ct);
|
|
var import = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
|
|
await fx.Service.Imports.ExecuteAsync(new ImportJobParameters(fx.Archive.Id, import, media.Id, true, null), Guid.NewGuid(), 0,
|
|
_ => ValueTask.CompletedTask, null, fx.Ct);
|
|
var id = (await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, import.Sessions.Single().SessionKey))!.Value;
|
|
Assert.Equal(3, (await fx.Service.PlanCardEraseAsync(fx.Archive.Id, media.Id, card, null, fx.Ct)).ToDelete.Count());
|
|
var engine = await fx.StartEngineAsync();
|
|
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id));
|
|
|
|
// The frames now hold converted files: nothing in the archive matches the card's photos byte for byte, so they stay.
|
|
var erase = await fx.Service.PlanCardEraseAsync(fx.Archive.Id, media.Id, card, null, fx.Ct);
|
|
Assert.Empty(erase.ToDelete);
|
|
Assert.True(erase.Items.All(i => i.Verdict == EraseVerdict.CopyMismatch));
|
|
Assert.Equal(0, CardEraser.Execute(erase).Deleted);
|
|
Assert.Count(3, Directory.EnumerateFiles(card, "*.JPG", SearchOption.AllDirectories));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AReplacedEntryIsErasedOnlyWhileTheConvertedFilesStillHashToTheirRecord()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id));
|
|
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
|
|
var converted = (await FramesAsync(fx, id))[0];
|
|
var file = fx.Paths.FromRelative(converted.RelativePath);
|
|
var bytes = File.ReadAllBytes(file);
|
|
var damaged = (byte[])bytes.Clone();
|
|
damaged[^10] ^= 0x01;
|
|
File.WriteAllBytes(file, damaged);
|
|
|
|
var plan = await PlanEmptyAsync(fx);
|
|
var kept = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, plan), cancellationToken: fx.Ct);
|
|
await fx.WaitForAsync(kept.Id, JobState.Failed);
|
|
Assert.Contains(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name));
|
|
|
|
File.WriteAllBytes(file, bytes);
|
|
var erased = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, await PlanEmptyAsync(fx)), cancellationToken: fx.Ct);
|
|
await fx.WaitForAsync(erased.Id, JobState.Done);
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
|
|
static Task<TrashEmptyPlan> PlanEmptyAsync(ArchiveFixture fx) => fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AReplacementACrashInterruptedBeforeItsSidecarIsUndoneByTheNextWriter()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var folder = fx.Paths.Sequence(sequence.Folder);
|
|
var manifest = Sidecar(fx, sequence);
|
|
var frame = manifest.Frames[0];
|
|
// Half a replacement: the original in its entry, the converted file in its place, the record pending, the sidecar as it was.
|
|
var staged = Path.Combine(Staging(fx), "convert-test", frame.Id.ToString("N") + ".tif");
|
|
FrameConverter.Convert(new ConvertSource(folder.File(frame.File), frame), new ConvertOptions(ConvertTarget.Tiff16), DecoderFactory.CreateServer(), staged,
|
|
cancellationToken: fx.Ct);
|
|
var entryFolder = Path.Combine(FrameTrash.Directory(fx.Paths), $"{id:N}-20261027120000");
|
|
var record = entryFolder + ".json";
|
|
var trashed = Path.Combine(entryFolder, "originals", Path.GetFileName(frame.File));
|
|
var placed = Path.ChangeExtension(frame.File, ".tif");
|
|
var journal = new FrameReplaceJournal(manifest.Versions.Frames, fx.Paths.ToRelative(record), fx.Paths.ToRelative(entryFolder),
|
|
[new SwapMove(frame.File, fx.Paths.ToRelative(trashed))], [new SwapMove(placed, fx.Paths.ToRelative(staged))], false, null);
|
|
journal.Write(folder);
|
|
Directory.CreateDirectory(FrameTrash.Directory(fx.Paths));
|
|
File.WriteAllText(FrameRemovalJournal.PendingRecord(record), "{}");
|
|
Directory.CreateDirectory(Path.GetDirectoryName(trashed)!);
|
|
File.Move(folder.File(frame.File), trashed);
|
|
File.Move(staged, folder.File(placed));
|
|
|
|
await fx.Service.EditSequenceInfoAsync(id, info => info with { Rating = 2 }, fx.Ct);
|
|
Assert.FileExists(folder.File(frame.File));
|
|
Assert.FileMissing(folder.File(placed));
|
|
Assert.FileExists(staged);
|
|
Assert.False(Directory.Exists(entryFolder));
|
|
Assert.FileMissing(FrameRemovalJournal.PendingRecord(record));
|
|
Assert.FileMissing(folder.ReplaceJournalPath);
|
|
Assert.Equal(frame.Sha256, Sha(folder.File(frame.File)));
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AReplacementACrashInterruptedAfterItsSidecarIsCompletedByTheNextWriter()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
var version = (await fx.Db.Sequences.GetAsync(id))!.FramesVersion;
|
|
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id));
|
|
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
|
|
var (entryFolder, record) = FrameTrash.Location(fx.Paths, entry.Name);
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var folder = fx.Paths.Sequence(sequence.Folder);
|
|
// As a crash right after the sidecar was written leaves it: the record still pending, the journal there.
|
|
File.Move(record, FrameRemovalJournal.PendingRecord(record));
|
|
new FrameReplaceJournal(version, fx.Paths.ToRelative(record), fx.Paths.ToRelative(entryFolder), [], [], true, null).Write(folder);
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
|
|
await fx.Service.EditSequenceInfoAsync(id, info => info with { Rating = 2 }, fx.Ct);
|
|
Assert.FileMissing(folder.ReplaceJournalPath);
|
|
Assert.Equal(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name);
|
|
Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Tiff));
|
|
}
|
|
|
|
[Test]
|
|
public async Task APhotoTheServerCannotReadAtFullResolutionIsSkipped()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
// The sidecar says the second photo is a HEIC (a desktop with WIC filed it): the managed decoders have only its preview.
|
|
var manifest = Sidecar(fx, sequence);
|
|
ManifestStore.WriteSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath,
|
|
manifest with { Frames = [manifest.Frames[0], manifest.Frames[1] with { Format = ImageFileFormat.Heif }] });
|
|
var step = (await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.NewSequence, id)).Sequences.Single();
|
|
Assert.Equal(1, step.Frames.Count);
|
|
Assert.Equal((ConvertSkip.NoNativeDecoder, 1), (step.Skipped.Single().Reason, step.Skipped.Single().Frames));
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheJournalsSurviveARoundTrip()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var journal = new ConvertJournal { JobId = Guid.NewGuid() };
|
|
var first = Guid.NewGuid();
|
|
var harbour = Guid.NewGuid();
|
|
journal.Begin(harbour, "Harbour", SequenceStatus.AnalyzedLight | SequenceStatus.ProxiesBuilt);
|
|
journal.Note(new ConvertSequenceResult(first, "Roof", ConvertOutcome.Created, 3, Guid.NewGuid()), SequenceStatus.AnalyzedDeep);
|
|
journal.Of(first)!.Queued = true;
|
|
journal.Note(new ConvertSequenceResult(harbour, "Harbour", ConvertOutcome.Failed, 0, Error: "no"));
|
|
journal.Write(fx.Paths);
|
|
var read = Assert.NotNull(ConvertJournal.Read(fx.Paths, journal.JobId));
|
|
Assert.SequenceEqual(journal.Sequences.Select(s => (s.Id, s.Outcome, s.Frames, s.NewSequenceId, s.Final, s.Had, s.Queued)),
|
|
read.Sequences.Select(s => (s.Id, s.Outcome, s.Frames, s.NewSequenceId, s.Final, s.Had, s.Queued)));
|
|
Assert.True(read.Of(first)!.Final);
|
|
// A failure keeps the statuses the sequence had when the job came to it.
|
|
Assert.Equal(SequenceStatus.AnalyzedLight | SequenceStatus.ProxiesBuilt, read.Of(harbour)!.Had);
|
|
// The photos pinned: a later line for a sequence replaces an earlier one, a torn one is skipped.
|
|
var photo = Guid.NewGuid();
|
|
var pins = read.Pins(fx.Paths);
|
|
Assert.Null(pins.Of(first));
|
|
ConvertSequencePlan Pinned(int version, string sha) =>
|
|
new(first, "Roof", "roof", version, [new ConvertFramePlan(photo, 0, "originals/a.jpg", ImageFileFormat.Jpeg, 1, 2, sha)], [], []);
|
|
pins.Append(first, Pinned(4, "aa"));
|
|
pins.Append(first, Pinned(5, "bb"));
|
|
File.AppendAllText(ConvertJournal.PinsPathFor(fx.Paths, journal.JobId), "{\"sequence\":");
|
|
var pinned = Assert.NotNull(read.Pins(fx.Paths).Of(first));
|
|
Assert.Equal((5, photo, "bb"), (pinned.FramesVersion, pinned.Frames.Single().Id, pinned.Frames.Single().Sha256));
|
|
// Its files and its staging folder are known by their names.
|
|
Assert.SequenceEqual([journal.JobId, journal.JobId, journal.JobId],
|
|
ConvertJournal.Files(fx.Paths, journal.JobId).Select(p => ConvertJournal.JobOf(Path.GetFileName(p))));
|
|
Assert.Equal(journal.JobId, ConvertJournal.JobOf(MassConverter.StagingFolder(journal.JobId)));
|
|
Assert.Null(ConvertJournal.JobOf("convert-test"));
|
|
Assert.Null(ConvertJournal.JobOf("rename-" + journal.JobId.ToString("N") + ".json"));
|
|
// A line torn by a crash is skipped; a later line for a frame replaces an earlier one.
|
|
var path = ConvertJournal.StagedPathFor(fx.Paths, journal.JobId);
|
|
var log = StagedLog.Open(path);
|
|
var frame = Guid.NewGuid();
|
|
log.Append(first, 4, new StagedConversion(frame, "a", "x.dng", 1, "b", 2, 3));
|
|
log.Append(first, 4, new StagedConversion(frame, "a", "x.dng", 1, "c", 2, 3));
|
|
File.AppendAllText(path, "{\"sequence\":");
|
|
var reread = StagedLog.Open(path);
|
|
Assert.Equal("c", reread.Of(first, 4).Single().Sha256);
|
|
Assert.Empty(reread.Of(first, 5));
|
|
ConvertJournal.Delete(fx.Paths, journal.JobId);
|
|
Assert.FileMissing(path);
|
|
Assert.FileMissing(ConvertJournal.PinsPathFor(fx.Paths, journal.JobId));
|
|
Assert.Null(ConvertJournal.Read(fx.Paths, journal.JobId));
|
|
// A job an earlier build queued names no digest: it reads with none.
|
|
var parameters = JsonSerializer.Serialize(new ConvertJobParameters(fx.Archive.Id, ConvertTarget.Jpeg, 80, ConvertMode.NewSequence,
|
|
[new ConvertSequence(first, 2, "d")], "f"), ArchiveJobJson.Default.ConvertJobParameters);
|
|
Assert.Equal("d", JsonSerializer.Deserialize(parameters, ArchiveJobJson.Default.ConvertJobParameters)!.Sequences.Single().Frames);
|
|
var older = $$"""{"archiveId":"{{fx.Archive.Id}}","target":"Jpeg","quality":80,"mode":"Replace","sequences":[{"id":"{{first}}","framesVersion":2}],"fingerprint":"f"}""";
|
|
Assert.Null(JsonSerializer.Deserialize(older, ArchiveJobJson.Default.ConvertJobParameters)!.Sequences.Single().Frames);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ConvertedFilesAreNamedAsFilingNamesThemAndNothingIsOverwritten()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
await fx.Service.UpdateSettingsAsync(fx.Archive.Id, s => s with { Templates = s.Templates with { File = "{name}_{n:00}" } }, fx.Ct);
|
|
var roof = await UploadedAsync(fx, 2);
|
|
var lake = await UploadedAsync(fx, 2, "upload-2", seed: 200);
|
|
Assert.SequenceEqual(["IMG_0001_01.JPG", "IMG_0002_02.JPG"], (await FramesAsync(fx, roof)).Select(f => f.FileName));
|
|
await fx.Service.SetSequenceInfoAsync(roof, "Roof", [], 0, null, fx.Ct);
|
|
await fx.Service.SetSequenceInfoAsync(lake, "Lake", [], 0, null, fx.Ct);
|
|
// Filing names a photo with the template of the moment: a conversion names its files as filing would now.
|
|
await fx.Service.UpdateSettingsAsync(fx.Archive.Id, s => s with { Templates = s.Templates with { File = "{title}_{n:00}" } }, fx.Ct);
|
|
// A file that is no frame holds the first name: it is never written over, the converted file takes the next.
|
|
var stray = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(roof))!.Folder).File("originals/Roof_01.dng");
|
|
File.WriteAllBytes(stray, [1, 2, 3]);
|
|
var engine = await fx.StartEngineAsync();
|
|
|
|
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, roof));
|
|
var replaced = await FramesAsync(fx, roof);
|
|
Assert.SequenceEqual(["Roof_01_2.dng", "Roof_02.dng"], replaced.Select(f => f.FileName));
|
|
Assert.SequenceEqual(["IMG_0001.JPG", "IMG_0002.JPG"], replaced.Select(f => f.OriginalFileName));
|
|
Assert.SequenceEqual(new byte[] { 1, 2, 3 }, File.ReadAllBytes(stray));
|
|
|
|
var job = await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, lake));
|
|
var made = Assert.NotNull(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, lake)));
|
|
var copies = await FramesAsync(fx, made);
|
|
Assert.SequenceEqual(["Lake (TIFF)_01.tif", "Lake (TIFF)_02.tif"], copies.Select(f => f.FileName));
|
|
Assert.SequenceEqual(["IMG_0001.JPG", "IMG_0002.JPG"], copies.Select(f => f.OriginalFileName));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AFolderMovedWhileTheJobWaitsForItsLeaseIsWhereTheConvertedFilesGo()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
|
|
var row = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var from = fx.Paths.Sequence(row.Folder).Folder;
|
|
Job job;
|
|
var planned = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
void OnProgress(JobProgressInfo info)
|
|
{
|
|
if (info.Kind == JobKind.Convert && info.Total == 2) planned.TrySetResult();
|
|
}
|
|
engine.Progress += OnProgress;
|
|
using (await LeaseFile.AcquireAsync(fx.Paths.LocksDirectory, $"seq-{id:N}", "editor", TimeSpan.FromSeconds(5), cancellationToken: fx.Ct))
|
|
{
|
|
job = await engine.EnqueueAsync(ArchiveJobs.Convert(fx.Archive, plan), cancellationToken: fx.Ct);
|
|
// Planned again where the index had the sequence then: the job now waits for its lease to convert it.
|
|
await planned.Task.WaitAsync(TimeSpan.FromSeconds(TestTiming.Budget(30)), fx.Ct);
|
|
// What a Reorganize step does under that lease meanwhile: the folder moves, and its index row follows.
|
|
var to = fx.Paths.Sequence("moved/Roof").Folder;
|
|
Directory.CreateDirectory(Path.GetDirectoryName(to)!);
|
|
Directory.Move(from, to);
|
|
Assert.True(await fx.Db.Sequences.RelocateAsync(id, "moved/Roof", null, fx.Ct));
|
|
}
|
|
engine.Progress -= OnProgress;
|
|
|
|
await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60));
|
|
var frames = await FramesAsync(fx, id);
|
|
Assert.True(frames.All(f => f.Format == ImageFileFormat.Tiff && f.RelativePath.StartsWith("moved/Roof/", StringComparison.Ordinal)));
|
|
Assert.False(Directory.Exists(from), "the folder the sequence left is never made again");
|
|
Assert.Equal("moved/Roof", (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Folder);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AReorganizeTheArchiveWaitsForKeepsTheConversionOffItsSequences()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var originals = await FramesAsync(fx, id);
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
|
|
// A Reorganize paused by its user holds the archive: its plan waits for it.
|
|
await fx.Db.Jobs.EnqueueAsync(new Job
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
Kind = JobKind.Move,
|
|
TargetIds = [fx.Archive.Id],
|
|
State = JobState.Paused,
|
|
Created = DateTimeOffset.UtcNow,
|
|
});
|
|
Assert.SequenceEqual([ConvertBlock.Busy], (await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id)).Sequences.Single().Blocked);
|
|
|
|
var engine = await fx.StartEngineAsync();
|
|
var job = await RunAsync(fx, engine, plan, JobState.Failed);
|
|
Assert.Equal(Loc.Plural("Archive.Error.ConvertKept", 1), job.Error);
|
|
Assert.Contains(Loc.L("Archive.Log.ConvertBlocked", (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle), job.LogTail);
|
|
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256, f.RelativePath)), (await FramesAsync(fx, id)).Select(f => (f.Id, f.Sha256, f.RelativePath)));
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task PhotosVerifiedAfterThePlanAreNotReplacedWithoutAConfirmation()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
var engine = await fx.StartEngineAsync();
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var path = fx.Paths.Sequence(sequence.Folder).ManifestPath;
|
|
var verified = ManifestStore.ReadSequence(path);
|
|
// Never verified when the plan is made (adopted): the dry run says that photo stays as it is.
|
|
ManifestStore.WriteSequence(path, verified with { Frames = [verified.Frames[0], verified.Frames[1] with { Sha256 = null }, verified.Frames[2]] });
|
|
var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id);
|
|
Assert.Equal((2, ConvertSkip.Unhashed), (plan.Frames, plan.Sequences.Single().Skipped.Single().Reason));
|
|
// A verify then hashes it before the job runs, as ArchiveVerifier does: no new frames version.
|
|
ManifestStore.WriteSequence(path, verified);
|
|
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
|
|
|
|
var job = await RunAsync(fx, engine, plan, JobState.Failed);
|
|
Assert.Equal(Loc.Plural("Archive.Error.ConvertKept", 1), job.Error);
|
|
Assert.Contains(Loc.L("Archive.Log.ConvertChanged", sequence.DisplayTitle), job.LogTail);
|
|
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
|
|
Assert.SequenceEqual(verified.Frames.Select(f => (f.Id, f.Sha256, f.File)), ManifestStore.ReadSequence(path).Frames.Select(f => (f.Id, f.Sha256, f.File)));
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
Assert.False(StagedAnything(fx));
|
|
// A retry does not convert it either: the user plans again.
|
|
await engine.ResumeAsync(job.Id, fx.Ct);
|
|
await fx.WaitForAsync(job.Id, JobState.Failed);
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task PhotosThatComeInOnceTheSequenceIsPinnedStayAsTheyAreAndARetryConvertsTheRest()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
|
|
var title = (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle;
|
|
// An earlier attempt pinned the sequence's photos and stopped (stage B blocked, a pause); an upload came in since.
|
|
var jobId = Guid.CreateVersion7();
|
|
PinLog.Open(ConvertJournal.PinsPathFor(fx.Paths, jobId)).Append(id, plan.Sequences.Single());
|
|
var (path, size, sha) = fx.Stage(Start.AddSeconds(60), 999);
|
|
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, null, path, "IMG_0099.JPG", size, sha, null, null), fx.Ct);
|
|
var added = (await FramesAsync(fx, id)).Single(f => f.Sha256 == sha);
|
|
|
|
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Convert(fx.Archive, plan), "convert", jobId);
|
|
var done = await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60));
|
|
Assert.Contains(Loc.L("Archive.Log.Converted", title, 2), done.LogTail);
|
|
var frames = await FramesAsync(fx, id);
|
|
Assert.Equal(2, frames.Count(f => f.Format == ImageFileFormat.Tiff));
|
|
Assert.Equal((sha, ImageFileFormat.Jpeg), frames.Where(f => f.Id == added.Id).Select(f => (f.Sha256, f.Format)).Single());
|
|
Assert.Equal(2, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Frames);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AJobResumedAfterACrashInItsReplacementSettlesItAndConvertsTheWholeSequence()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var folder = fx.Paths.Sequence(sequence.Folder);
|
|
var manifest = Sidecar(fx, sequence);
|
|
// The attempt that crashed: the photos pinned and staged, then its replacement cut short once every original had left
|
|
// for the trash, none of the converted files in yet.
|
|
var jobId = Guid.CreateVersion7();
|
|
PinLog.Open(ConvertJournal.PinsPathFor(fx.Paths, jobId)).Append(id, plan.Sequences.Single());
|
|
var log = StagedLog.Open(ConvertJournal.StagedPathFor(fx.Paths, jobId));
|
|
var entryFolder = Path.Combine(FrameTrash.Directory(fx.Paths), $"{id:N}-20261027120000");
|
|
var record = entryFolder + ".json";
|
|
List<SwapMove> outgoing = [];
|
|
List<SwapMove> incoming = [];
|
|
foreach (var frame in manifest.Frames)
|
|
{
|
|
var staged = Path.Combine(StagingOf(fx, jobId, id), frame.Id.ToString("N") + ".tif");
|
|
var file = FrameConverter.Convert(new ConvertSource(folder.File(frame.File), frame), new ConvertOptions(ConvertTarget.Tiff16), DecoderFactory.CreateServer(),
|
|
staged, cancellationToken: fx.Ct);
|
|
log.Append(id, manifest.Versions.Frames, new StagedConversion(frame.Id, frame.Sha256!, Path.GetFileName(staged), file.Size, file.Sha256, file.Width, file.Height));
|
|
outgoing.Add(new SwapMove(frame.File, fx.Paths.ToRelative(Path.Combine(entryFolder, "originals", Path.GetFileName(frame.File)))));
|
|
incoming.Add(new SwapMove(Path.ChangeExtension(frame.File, ".tif"), fx.Paths.ToRelative(staged)));
|
|
}
|
|
new FrameReplaceJournal(manifest.Versions.Frames, fx.Paths.ToRelative(record), fx.Paths.ToRelative(entryFolder), outgoing, incoming, false, null).Write(folder);
|
|
Directory.CreateDirectory(FrameTrash.Directory(fx.Paths));
|
|
File.WriteAllText(FrameRemovalJournal.PendingRecord(record), "{}");
|
|
foreach (var move in outgoing)
|
|
{
|
|
var trashed = fx.Paths.FromRelative(move.Other);
|
|
Directory.CreateDirectory(Path.GetDirectoryName(trashed)!);
|
|
File.Move(folder.File(move.Place), trashed);
|
|
}
|
|
|
|
// Not a sequence with nothing to convert, nor half of it converted: the originals come back, then all of it converts.
|
|
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Convert(fx.Archive, plan), "convert", jobId);
|
|
var done = await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60));
|
|
Assert.Contains(Loc.L("Archive.Log.Converted", sequence.DisplayTitle, 2), done.LogTail);
|
|
Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Tiff));
|
|
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
|
|
Assert.Equal((TrashKind.Replaced, 2), (entry.Kind, entry.Frames));
|
|
Assert.False(Directory.Exists(entryFolder));
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheFollowUpsOfEachSequenceAreQueuedAsItEndsAndOnResumeWhenAPauseCameFirst()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var first = await UploadedAsync(fx, 2);
|
|
var second = await UploadedAsync(fx, 2, "upload-2", seed: 200);
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, first, second);
|
|
var events = new List<string>();
|
|
var callbacks = Callbacks(done: s => events.Add($"done {s.SequenceId}"), queue: r => events.AddRange(r.Select(q => $"{q.Kind} {q.Targets.Single()}")));
|
|
await RunHereAsync(fx, plan, Guid.CreateVersion7(), callbacks);
|
|
// Queued as each sequence is journaled, before the next starts: a pause, a yield or a crash later loses none of them.
|
|
Assert.Less(events.IndexOf($"done {first}"), events.IndexOf($"{JobKind.LightAnalyse} {first}"));
|
|
Assert.Less(events.IndexOf($"{JobKind.LightAnalyse} {first}"), events.IndexOf($"done {second}"));
|
|
Assert.Contains($"{JobKind.LightAnalyse} {second}", events);
|
|
|
|
// As a pause right after the first sequence's outcome was journaled leaves the job: its passes not queued yet.
|
|
var jobId = Guid.CreateVersion7();
|
|
var journal = new ConvertJournal { JobId = jobId };
|
|
journal.Note(new ConvertSequenceResult(first, "Roof", ConvertOutcome.Converted, 2), SequenceStatus.AnalyzedDeep | SequenceStatus.ThumbsBuilt);
|
|
journal.Note(new ConvertSequenceResult(second, "Lake", ConvertOutcome.Converted, 2), SequenceStatus.AnalyzedLight);
|
|
journal.Of(second)!.Queued = true;
|
|
journal.Write(fx.Paths);
|
|
List<ArchiveJobRequest> queued = [];
|
|
var resumed = await RunHereAsync(fx, plan, jobId, Callbacks(queue: queued.AddRange));
|
|
Assert.Equal(4, resumed.Frames);
|
|
Assert.SequenceEqual([(JobKind.LightAnalyse, first), (JobKind.DeepAnalyse, first), (JobKind.Thumbs, first)],
|
|
queued.Select(r => (r.Kind, r.Targets.Single())));
|
|
Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, jobId));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AConversionWaitingInTheQueueKeepsItsSequencesFromASecondConfirmation()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id);
|
|
var request = ArchiveJobs.Convert(fx.Archive, plan);
|
|
var job = new Job
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
Kind = request.Kind,
|
|
TargetIds = request.Targets,
|
|
ParametersJson = request.ParametersJson,
|
|
State = JobState.Queued,
|
|
Created = DateTimeOffset.UtcNow,
|
|
};
|
|
await fx.Db.Jobs.EnqueueAsync(job, fx.Ct);
|
|
|
|
// Another tab, or Apply pressed again after an answer was lost: the same copies are not queued twice.
|
|
var again = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id);
|
|
Assert.SequenceEqual([ConvertBlock.Busy], again.Sequences.Single().Blocked);
|
|
Assert.NotEqual(plan.Fingerprint(), again.Fingerprint());
|
|
Assert.Throws<ArgumentException>(() => ArchiveJobs.Convert(fx.Archive, again));
|
|
// The job itself plans again with its sequences free, and converts them.
|
|
var engine = await fx.StartEngineAsync();
|
|
engine.Wake();
|
|
await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60));
|
|
Assert.NotNull(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, id)));
|
|
Assert.Empty((await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id)).Sequences.Single().Blocked);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ANewSequenceIsOwnedAsItsSourceIsOrByWhoAskedForIt()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var owner = Guid.NewGuid();
|
|
var owned = await UploadedAsync(fx, 2, user: owner);
|
|
var unowned = await UploadedAsync(fx, 2, "upload-2", seed: 200);
|
|
Assert.Equal(owner, (await fx.Db.Sequences.GetAsync(owned))!.OwnerUserId);
|
|
var engine = await fx.StartEngineAsync();
|
|
var asker = Guid.NewGuid();
|
|
|
|
var job = await engine.EnqueueAsync(ArchiveJobs.Convert(fx.Archive, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, owned, unowned)), asker, fx.Ct);
|
|
await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60));
|
|
foreach (var (source, expected) in new[] { (owned, owner), (unowned, asker) })
|
|
{
|
|
var made = (await fx.Db.Sequences.GetAsync((await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, source)))!.Value))!;
|
|
Assert.Equal(expected, made.OwnerUserId);
|
|
Assert.Equal(expected.ToString(), Sidecar(fx, made).CreatedBy);
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public async Task AStagedCopyThatVanishesFailsItsNewSequenceInsteadOfLeavingThePhotoOut()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id);
|
|
var first = Sidecar(fx, (await fx.Db.Sequences.GetAsync(id))!).Frames[0];
|
|
var jobId = Guid.CreateVersion7();
|
|
// A clean-up takes the first copy once both are staged, before they are filed.
|
|
var callbacks = Callbacks(progress: (n, total) =>
|
|
{
|
|
if (n == 2 && total == 2) File.Delete(Path.Combine(StagingOf(fx, jobId, id), first.Id.ToString("N") + ".tif"));
|
|
});
|
|
|
|
var failed = (await RunHereAsync(fx, plan, jobId, callbacks)).Sequences.Single();
|
|
Assert.Equal(ConvertOutcome.Failed, failed.Outcome);
|
|
Assert.Null(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(jobId, id)));
|
|
// The retry converts that photo again and makes the whole copy.
|
|
var made = (await RunHereAsync(fx, plan, jobId)).Sequences.Single();
|
|
Assert.Equal((ConvertOutcome.Created, 2, 0), (made.Outcome, made.Frames, made.LeftOut));
|
|
Assert.Count(2, await FramesAsync(fx, made.NewSequenceId!.Value));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ANewSequenceSaysWhichPhotosItLeftInTheSourceAlone()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
var engine = await fx.StartEngineAsync();
|
|
// Three TIFFs, then a JPEG uploaded: to JPEG, the TIFFs convert and the JPEG stays in the source alone.
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id));
|
|
var (path, size, sha) = fx.Stage(Start.AddSeconds(60), 999);
|
|
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, null, path, "IMG_0099.JPG", size, sha, null, null), fx.Ct);
|
|
|
|
var job = await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Jpeg, ConvertMode.NewSequence, id));
|
|
Assert.Contains(Loc.L("Archive.Log.ConvertedNew", sequence.DisplayTitle, 3), job.LogTail);
|
|
Assert.Contains(Loc.L("Archive.Log.ConvertLeftOut", sequence.DisplayTitle, 1), job.LogTail);
|
|
var made = Assert.NotNull(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, id)));
|
|
Assert.Count(3, await FramesAsync(fx, made));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AVolumeThatFillsDuringTheJobStopsItBeforeTheNextSequenceAndTheNextPhoto()
|
|
{
|
|
long free = 1L << 50;
|
|
await using var fx = await FixtureAsync(o => o with { FreeSpace = _ => free, Parallelism = () => 1 });
|
|
var first = await UploadedAsync(fx, 2);
|
|
var second = await UploadedAsync(fx, 2, "upload-2", seed: 200);
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, first, second);
|
|
var jobId = Guid.CreateVersion7();
|
|
var originals = await FramesAsync(fx, second);
|
|
|
|
// Another writer fills the volume while the first sequence converts: the second does not start.
|
|
var full = await Assert.ThrowsAsync<IOException>(() => RunHereAsync(fx, plan, jobId, Callbacks(done: _ => free = 1_000)));
|
|
Assert.True(full.Message.StartsWith("The conversion needs", StringComparison.Ordinal), full.Message);
|
|
Assert.True((await FramesAsync(fx, first)).All(f => f.Format == ImageFileFormat.Tiff));
|
|
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256)), (await FramesAsync(fx, second)).Select(f => (f.Id, f.Sha256)));
|
|
Assert.False(Directory.Exists(StagingOf(fx, jobId, second)));
|
|
|
|
// Space again, until the first photo of the second sequence is staged: the next one does not start, what was staged stays.
|
|
free = 1L << 50;
|
|
await Assert.ThrowsAsync<IOException>(() => RunHereAsync(fx, plan, jobId, Callbacks(progress: (n, _) =>
|
|
{
|
|
if (n == 1) free = 1_000;
|
|
})));
|
|
Assert.Count(1, Directory.EnumerateFiles(StagingOf(fx, jobId, second)));
|
|
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256)), (await FramesAsync(fx, second)).Select(f => (f.Id, f.Sha256)));
|
|
|
|
free = 1L << 50;
|
|
var done = await RunHereAsync(fx, plan, jobId);
|
|
Assert.True(done.Sequences.All(s => s.Outcome == ConvertOutcome.Converted));
|
|
Assert.True((await FramesAsync(fx, second)).All(f => f.Format == ImageFileFormat.Tiff));
|
|
Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, jobId));
|
|
}
|
|
|
|
[Test]
|
|
public async Task MaintenanceKeepsWhatAWaitingConversionStagedAndClearsWhatAFinishedOneLeft()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var paused = Guid.CreateVersion7();
|
|
var finished = Guid.CreateVersion7();
|
|
var forgotten = Guid.CreateVersion7();
|
|
foreach (var (id, state) in new[] { (paused, JobState.Paused), (finished, JobState.Done) })
|
|
await fx.Db.Jobs.EnqueueAsync(new Job { Id = id, Kind = JobKind.Convert, State = state, Created = DateTimeOffset.UtcNow }, fx.Ct);
|
|
string Staged(Guid job)
|
|
{
|
|
var file = Path.Combine(StagingOf(fx, job, Guid.NewGuid()), "frame.tif");
|
|
Directory.CreateDirectory(Path.GetDirectoryName(file)!);
|
|
File.WriteAllText(file, "x");
|
|
File.SetLastWriteTimeUtc(file, DateTime.UtcNow.AddDays(-3));
|
|
return file;
|
|
}
|
|
string Journal(Guid job)
|
|
{
|
|
new ConvertJournal { JobId = job }.Write(fx.Paths);
|
|
return ConvertJournal.PathFor(fx.Paths, job);
|
|
}
|
|
var (pausedFile, pausedJournal) = (Staged(paused), Journal(paused));
|
|
var (finishedFile, finishedJournal) = (Staged(finished), Journal(finished));
|
|
var (forgottenFile, forgottenJournal) = (Staged(forgotten), Journal(forgotten));
|
|
|
|
var dry = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: true, 0, _ => ValueTask.CompletedTask, fx.Ct);
|
|
Assert.SequenceEqual(new[] { finishedFile, finishedJournal, forgottenFile }.Select(fx.Paths.ToRelative).Order(), dry.TemporaryFiles.Order());
|
|
await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
|
|
// A paused job's staged conversions stay however old; a finished job's go with its journal; a job this index does not
|
|
// know (another host's, or forgotten by the history) loses staged files idle for a day, and keeps its journal a while.
|
|
Assert.FileExists(pausedFile);
|
|
Assert.FileExists(pausedJournal);
|
|
Assert.False(Directory.Exists(Path.Combine(Staging(fx), MassConverter.StagingFolder(finished))));
|
|
Assert.FileMissing(finishedJournal);
|
|
Assert.False(Directory.Exists(Path.Combine(Staging(fx), MassConverter.StagingFolder(forgotten))));
|
|
Assert.FileMissing(forgottenFile);
|
|
Assert.FileExists(forgottenJournal);
|
|
File.SetLastWriteTimeUtc(forgottenJournal, DateTime.UtcNow.AddDays(-CleanupSteps.DefaultHistoryDays - 1));
|
|
await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
|
|
Assert.FileMissing(forgottenJournal);
|
|
Assert.FileExists(pausedJournal);
|
|
}
|
|
}
|