Stage A's space guard reserved, before each photo, only that photo and the ones after it, so at the end of a sequence it counted a single photo while up to twice the parallelism were still converting: a volume filling meanwhile could go into the reserve. It now reserves the photo and every photo started whose converted file has not landed yet (a file lands whole, by a rename, before it is verified), so nothing in flight is missed and nothing landed is counted twice, at the end of a sequence as anywhere. The confirmed photo ids (about 21 MB for a million photos) lived in the job's parameters, read by every dry run's busy check, by each sequence turn of the conversion and by the job lists. They are now written whole, under a job id chosen before the job is queued, into convert-<job>.photos beside the journal, and go with it; the row keeps each sequence's digest, so the job still converts exactly the photos confirmed (without the file, the whole plan made again must match). The busy check reads only ids, kinds and targets (and a verify's parameters), never the asking job's row; a conversion's turn reads only its creation time; the server's job list and dashboard and the engine's queue peeks read jobs without parameters. A retried New sequence conversion could make a second copy. The journal now names the copy as soon as its photos are filed; a retry finishes that copy whatever changed in the source since, never remakes one deleted since, and needs no space again for what it holds; and a source another conversion copied in the same format after this one was confirmed is done with, with no copy made (new outcome Superseded, with its job log line). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1695 lines
103 KiB
C#
1695 lines
103 KiB
C#
using System.Runtime.Versioning;
|
|
using System.Security.Cryptography;
|
|
using System.Text.Json;
|
|
using Chronos.Core.Archiving;
|
|
using Chronos.Core.Conversion;
|
|
using Chronos.Core.Import;
|
|
using Chronos.Core.Imaging;
|
|
using Chronos.Core.Imaging.Decoders;
|
|
using Chronos.Core.Localization;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Core.Project;
|
|
using Chronos.Data.Archiving;
|
|
using Chronos.Data.Archiving.Jobs;
|
|
using Chronos.Data.Housekeeping;
|
|
using Chronos.Data.Jobs;
|
|
using Chronos.Data.Repositories;
|
|
using Chronos.Testing;
|
|
using static Chronos.Data.Tests.Archiving.ArchiveFixture;
|
|
|
|
namespace Chronos.Data.Tests.Archiving;
|
|
|
|
/// <summary>
|
|
/// Mass conversion after import (owner review 2, R7): the dry run, the <c>convert</c> job in both modes, its undo through the
|
|
/// trash, all or nothing per sequence, resumes in stage A and B, plans gone stale, the space check, and the gates that must
|
|
/// keep holding (the card erase, the emptying of a replaced entry).
|
|
/// </summary>
|
|
[Category("archive")]
|
|
public sealed class ConvertTests(TestContext context)
|
|
{
|
|
private Task<ArchiveFixture> FixtureAsync(Func<ArchiveServiceOptions, ArchiveServiceOptions>? more = null) =>
|
|
CreateAsync(context, configure: o =>
|
|
{
|
|
var options = o with { Decoder = DecoderFactory.CreateServer(), Parallelism = () => 2 };
|
|
return more is null ? options : more(options);
|
|
});
|
|
|
|
/// <summary>A sequence of <paramref name="count"/> uploaded photos (hashed, under <c>originals/</c>), ten seconds apart, by <paramref name="user"/>.</summary>
|
|
private static async Task<Guid> UploadedAsync(ArchiveFixture fx, int count, string session = "upload-1", int seed = 100, Guid? user = null)
|
|
{
|
|
Guid? id = null;
|
|
for (var i = 0; i < count; i++)
|
|
{
|
|
var (path, size, sha) = fx.Stage(Start.AddSeconds(10 * i).AddTicks(1_234_567), seed + i);
|
|
var result = await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, id is null ? session : null, path, $"IMG_{i + 1:0000}.JPG",
|
|
size, sha, null, user), fx.Ct);
|
|
id = result.SequenceId;
|
|
}
|
|
return id!.Value;
|
|
}
|
|
|
|
private static Task<ConvertPlan> PlanAsync(ArchiveFixture fx, ConvertTarget target, ConvertMode mode, params Guid[] ids) =>
|
|
fx.Service.PlanConvertAsync(fx.Archive.Id, ids, new ConvertOptions(target), mode, fx.Ct);
|
|
|
|
/// <summary>The parameters the job of <paramref name="plan"/> runs with.</summary>
|
|
private static ConvertJobParameters Parameters(ArchiveFixture fx, ConvertPlan plan) =>
|
|
JsonSerializer.Deserialize(ArchiveJobs.Convert(fx.Archive, plan).ParametersJson!, ArchiveJobJson.Default.ConvertJobParameters)!;
|
|
|
|
/// <summary>Callbacks that report nothing, but for <paramref name="progress"/> and <paramref name="done"/> when given.</summary>
|
|
private static ConvertCallbacks Callbacks(Action<long, long>? progress = null, Action<ConvertSequenceResult>? done = null,
|
|
Action<IReadOnlyList<ArchiveJobRequest>>? queue = null) => new(
|
|
(n, total) =>
|
|
{
|
|
progress?.Invoke(n, total);
|
|
return ValueTask.CompletedTask;
|
|
},
|
|
(_, _) => ValueTask.CompletedTask,
|
|
_ => { },
|
|
s =>
|
|
{
|
|
done?.Invoke(s);
|
|
return ValueTask.CompletedTask;
|
|
},
|
|
requests =>
|
|
{
|
|
queue?.Invoke(requests);
|
|
return ValueTask.CompletedTask;
|
|
});
|
|
|
|
/// <summary>
|
|
/// The conversion job <paramref name="jobId"/> of <paramref name="plan"/>, run here as its handler runs it, with the photos
|
|
/// confirmed written beside its journal as <see cref="MassConverter.QueueAsync"/> writes them (once: a run again of the
|
|
/// same job finds them), and <paramref name="confirmed"/> as the time it was queued.
|
|
/// </summary>
|
|
private static Task<ConvertResult> RunHereAsync(ArchiveFixture fx, ConvertPlan plan, Guid jobId, ConvertCallbacks? callbacks = null, DateTimeOffset? confirmed = null)
|
|
{
|
|
if (!File.Exists(ConfirmedPhotos.PathFor(fx.Paths, jobId))) Assert.True(ConfirmedPhotos.TryWrite(fx.Paths, jobId, plan));
|
|
return fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), jobId, new ConvertRequester(null, null, confirmed), callbacks ?? Callbacks(), fx.Ct);
|
|
}
|
|
|
|
private static string StagingOf(ArchiveFixture fx, Guid job, Guid sequence) => Path.Combine(Staging(fx), $"convert-{job:N}", $"{sequence:N}");
|
|
|
|
/// <summary>The job of <paramref name="plan"/>, queued as the Library, the Archive manager and the panel queue it (<see cref="MassConverter.QueueAsync"/>).</summary>
|
|
private static async Task<Job> QueueAsync(ArchiveFixture fx, JobEngine engine, ConvertPlan plan, Guid? createdBy = null)
|
|
{
|
|
Job? job = null;
|
|
await fx.Service.Converter.QueueAsync(fx.Archive, plan, async r => job = await engine.EnqueueAsync(r, createdBy, fx.Ct), fx.Ct);
|
|
return Assert.NotNull(job);
|
|
}
|
|
|
|
private static async Task<Job> RunAsync(ArchiveFixture fx, JobEngine engine, ConvertPlan plan, JobState state = JobState.Done) =>
|
|
await fx.WaitForAsync((await QueueAsync(fx, engine, plan)).Id, state, (int)TestTiming.Budget(60));
|
|
|
|
private static SequenceManifest Sidecar(ArchiveFixture fx, Sequence sequence) => ManifestStore.ReadSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath);
|
|
|
|
private static string Sha(string path) => Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(path)));
|
|
|
|
/// <summary>The index rows of a sequence, after checking that each file holds its row's content and that the sidecar agrees.</summary>
|
|
private static async Task<List<Frame>> FramesAsync(ArchiveFixture fx, Guid id)
|
|
{
|
|
var sequence = Assert.NotNull(await fx.Db.Sequences.GetAsync(id));
|
|
var rows = (await fx.Db.Frames.ListBySequenceAsync(id)).ToList();
|
|
var sidecar = Sidecar(fx, sequence);
|
|
Assert.Equal(sidecar.Versions.Frames, sequence.FramesVersion);
|
|
Assert.SequenceEqual(sidecar.Frames.Select(f => (f.Id, f.I, f.Sha256)), rows.Select(r => (r.Id, r.Index, r.Sha256)));
|
|
foreach (var row in rows) Assert.Equal(row.Sha256, Sha(fx.Paths.FromRelative(row.RelativePath)));
|
|
Assert.FileMissing(fx.Paths.Sequence(sequence.Folder).ReplaceJournalPath);
|
|
return rows;
|
|
}
|
|
|
|
private static string Staging(ArchiveFixture fx) => fx.Paths.StagingDirectory;
|
|
|
|
private static bool StagedAnything(ArchiveFixture fx) =>
|
|
Directory.Exists(Staging(fx)) && Directory.EnumerateDirectories(Staging(fx), "convert-*").Any();
|
|
|
|
/// <summary>
|
|
/// Makes <paramref name="path"/> unreadable until disposed, as another program holding it does: without read permission
|
|
/// where that stops the test process (UnauthorizedAccessException), otherwise (Windows, root) held open without sharing
|
|
/// (IOException).
|
|
/// </summary>
|
|
private static IDisposable Unreadable(string path)
|
|
{
|
|
if (!OperatingSystem.IsWindows())
|
|
{
|
|
var mode = File.GetUnixFileMode(path);
|
|
File.SetUnixFileMode(path, UnixFileMode.None);
|
|
try
|
|
{
|
|
File.OpenRead(path).Dispose();
|
|
File.SetUnixFileMode(path, mode);
|
|
}
|
|
catch (UnauthorizedAccessException)
|
|
{
|
|
return new UnixPermissions(path, mode);
|
|
}
|
|
}
|
|
return new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.None);
|
|
}
|
|
|
|
/// <summary>Gives a file its permissions back.</summary>
|
|
[UnsupportedOSPlatform("windows")]
|
|
private sealed class UnixPermissions(string path, UnixFileMode mode) : IDisposable
|
|
{
|
|
public void Dispose() => File.SetUnixFileMode(path, mode);
|
|
}
|
|
|
|
/// <summary>A queued conversion job of <paramref name="plan"/>, as the engine would hold it, created at <paramref name="created"/>.</summary>
|
|
private static async Task<Job> QueuedAsync(ArchiveFixture fx, ConvertPlan plan, DateTimeOffset created)
|
|
{
|
|
var request = ArchiveJobs.Convert(fx.Archive, plan);
|
|
var job = new Job
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
Kind = request.Kind,
|
|
TargetIds = request.Targets,
|
|
ParametersJson = request.ParametersJson,
|
|
State = JobState.Queued,
|
|
Created = created,
|
|
};
|
|
await fx.Db.Jobs.EnqueueAsync(job, fx.Ct);
|
|
return job;
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheDryRunCountsWhatConvertsAndChangesNothing()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
|
|
|
|
var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id);
|
|
var step = plan.Sequences.Single();
|
|
Assert.Equal((3, 0, true), (step.Frames.Count, step.SkippedFrames, step.Converts));
|
|
Assert.Equal((await fx.Db.Sequences.GetAsync(id))!.FramesVersion, step.FramesVersion);
|
|
Assert.Equal(step.Frames.Sum(f => f.Size), plan.BytesBefore);
|
|
Assert.Equal(3 * ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.LinearDng), 48, 32), plan.BytesAfter);
|
|
Assert.Equal(ConvertPlanner.Required(plan.BytesAfter), plan.Space.Required);
|
|
Assert.True(plan.Space.Enough);
|
|
Assert.Equal(plan.Fingerprint(), (await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id)).Fingerprint());
|
|
Assert.NotEqual(plan.Fingerprint(), (await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.NewSequence, id)).Fingerprint());
|
|
|
|
// JPEG to JPEG converts nothing.
|
|
var same = (await PlanAsync(fx, ConvertTarget.Jpeg, ConvertMode.Replace, id)).Sequences.Single();
|
|
Assert.Equal((0, ConvertSkip.SameFormat, 3), (same.Frames.Count, same.Skipped.Single().Reason, same.Skipped.Single().Frames));
|
|
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
|
|
Assert.False(StagedAnything(fx));
|
|
await Assert.ThrowsAsync<ArgumentException>(() => PlanAsync(fx, ConvertTarget.Jpeg, ConvertMode.Replace, Guid.NewGuid()));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ASequenceBusyOrUnreadableIsBlocked()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var busy = await UploadedAsync(fx, 2);
|
|
var unreadable = await UploadedAsync(fx, 2, "upload-2", seed: 200);
|
|
await fx.Db.Jobs.EnqueueAsync(new Job
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
Kind = JobKind.Render,
|
|
TargetIds = [busy],
|
|
State = JobState.Running,
|
|
NodeId = "test#0",
|
|
Created = DateTimeOffset.UtcNow,
|
|
Started = DateTimeOffset.UtcNow,
|
|
});
|
|
File.WriteAllText(fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(unreadable))!.Folder).ManifestPath, "{ not json");
|
|
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, busy, unreadable);
|
|
Assert.SequenceEqual([ConvertBlock.Busy], plan.Sequences.Single(s => s.SequenceId == busy).Blocked);
|
|
Assert.SequenceEqual([ConvertBlock.Unreadable], plan.Sequences.Single(s => s.SequenceId == unreadable).Blocked);
|
|
Assert.Equal(0, plan.Frames);
|
|
Assert.Throws<ArgumentException>(() => ArchiveJobs.Convert(fx.Archive, plan));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AReplaceKeepsTheFramesAndTrashesTheOriginalsWithTheirReplacements()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
var engine = await fx.StartEngineAsync();
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var originals = await FramesAsync(fx, id);
|
|
var project = new ProjectSettings { ExcludedFrames = [1], HeroFrame = 2 }.ToJson();
|
|
await fx.Db.Sequences.EditAsync(id, new SequenceEdit { SettingsJson = project, HeroFrameId = originals[2].Id }, null, fx.Ct);
|
|
var folder = fx.Paths.Sequence(sequence.Folder);
|
|
Directory.CreateDirectory(folder.MetaDirectory);
|
|
File.WriteAllText(folder.AnalysisPath, "{\"frames\":3}");
|
|
var before = Sidecar(fx, sequence);
|
|
|
|
var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id);
|
|
var job = await RunAsync(fx, engine, plan);
|
|
Assert.Contains(Loc.L("Archive.Log.Converted", sequence.DisplayTitle, 3), job.LogTail);
|
|
|
|
var row = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var frames = await FramesAsync(fx, id);
|
|
Assert.Equal(before.Versions.Frames + 1, row.FramesVersion);
|
|
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Index)), frames.Select(f => (f.Id, f.Index)));
|
|
var after = Sidecar(fx, row);
|
|
foreach (var (was, now) in before.Frames.Zip(after.Frames))
|
|
{
|
|
// Named as filing named the original, with the target's extension: the same template, frame and sequence.
|
|
Assert.Equal((Path.ChangeExtension(was.File, ".dng"), ImageFileFormat.Dng, DecodeTier.Native, 1), (now.File, now.Format, now.Tier, now.Orientation));
|
|
Assert.Equal((was.CapturedAt, was.Original, was.Flags), (now.CapturedAt, now.Original, now.Flags));
|
|
var source = Assert.NotNull(now.Source);
|
|
Assert.Equal((was.Sha256, was.Format, was.File, was.Size), (source.Sha256, source.Format, source.File, source.Size));
|
|
Assert.FileMissing(folder.File(was.File));
|
|
}
|
|
// The project's positions stay on their frames.
|
|
var settings = ProjectSettings.FromJson(row.SettingsJson!);
|
|
Assert.SequenceEqual([1], settings.ExcludedFrames);
|
|
Assert.Equal((2, originals[2].Id), (settings.HeroFrame, row.HeroFrameId));
|
|
|
|
// One replaced entry with the originals, each naming its replacement, and the analysis as it was.
|
|
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
|
|
Assert.Equal((TrashKind.Replaced, id, 3), (entry.Kind, entry.SequenceId, entry.Frames));
|
|
var record = Assert.NotNull(FrameTrash.ReadEntry(fx.Paths, entry.Name));
|
|
var entryFolder = FrameTrash.Location(fx.Paths, entry.Name).Folder;
|
|
foreach (var photo in record.Photos)
|
|
{
|
|
var now = after.Frames.Single(f => f.Id == photo.Frame.Id);
|
|
Assert.Equal(now.Sha256, photo.Replacement);
|
|
Assert.Equal(photo.Frame.Sha256, Sha(Path.Combine(entryFolder, photo.Files[0].Path)));
|
|
}
|
|
Assert.Equal("{\"frames\":3}", File.ReadAllText(Path.Combine(entryFolder, FrameTrash.AnalysisCopy)));
|
|
// Tombstones alias the old content to the new, in the sidecar and the index.
|
|
var tombstones = Assert.NotNull(after.Removed);
|
|
Assert.SequenceEqual(before.Frames.Select(f => (f.Sha256, after.Frames.Single(n => n.Id == f.Id).Sha256, RemovalReason.Replaced)).Order(),
|
|
tombstones.Select(t => ((string?)t.Sha256, t.Replacement, t.Reason)).Order());
|
|
Assert.SequenceEqual(tombstones.Select(t => (t.Sha256, t.Replacement)).Order(),
|
|
(await fx.Db.FrameRemovals.ListBySequenceAsync(id)).Select(r => (r.Sha256, r.ReplacementSha256)).Order());
|
|
Assert.Equal((SequenceStatus)0, row.Status & (SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt));
|
|
Assert.False(StagedAnything(fx));
|
|
Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, job.Id));
|
|
Assert.FileMissing(ConvertJournal.StagedPathFor(fx.Paths, job.Id));
|
|
// Converted again, nothing is left to convert.
|
|
Assert.Equal(0, (await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id)).Frames);
|
|
}
|
|
|
|
[Test]
|
|
public async Task RestoringTheReplacedEntrySwapsTheOriginalsBack()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
var engine = await fx.StartEngineAsync();
|
|
var originals = await FramesAsync(fx, id);
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var before = Sidecar(fx, sequence);
|
|
var analysis = fx.Paths.Sequence(sequence.Folder).AnalysisPath;
|
|
Directory.CreateDirectory(Path.GetDirectoryName(analysis)!);
|
|
File.WriteAllText(analysis, "of the originals");
|
|
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id));
|
|
var converted = Sidecar(fx, (await fx.Db.Sequences.GetAsync(id))!);
|
|
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
|
|
// The light analysis of the converted frames, as it would be written again.
|
|
File.WriteAllText(analysis, "of the conversions");
|
|
|
|
var result = await fx.Service.RestoreEntryAsync(fx.Archive.Id, entry.Name, null, fx.Ct);
|
|
Assert.Equal((TrashKind.Replaced, 3, false, id), (result.Kind, result.Photos, result.NewSequence, result.Sequence.Id));
|
|
var frames = await FramesAsync(fx, id);
|
|
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Index, f.Sha256, f.RelativePath, f.Format)), frames.Select(f => (f.Id, f.Index, f.Sha256, f.RelativePath, f.Format)));
|
|
var back = Sidecar(fx, (await fx.Db.Sequences.GetAsync(id))!);
|
|
Assert.True(back.Frames.All(f => f.Source is null));
|
|
// The converted files are in the trash in turn, naming the originals as their replacements; the old entry is gone.
|
|
var turned = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
|
|
Assert.NotEqual(entry.Name, turned.Name);
|
|
Assert.Equal(TrashKind.Replaced, turned.Kind);
|
|
var record = FrameTrash.ReadEntry(fx.Paths, turned.Name)!;
|
|
Assert.SequenceEqual(converted.Frames.Select(f => (f.Sha256, before.Frames.Single(o => o.Id == f.Id).Sha256)).Order(),
|
|
record.Photos.Select(p => (p.Frame.Sha256, p.Replacement)).Order());
|
|
// Each analysis goes with its frames: the originals' back in the sequence, the conversions' into the new entry.
|
|
Assert.Equal("of the originals", File.ReadAllText(analysis));
|
|
Assert.Equal("of the conversions", File.ReadAllText(Path.Combine(FrameTrash.Location(fx.Paths, turned.Name).Folder, FrameTrash.AnalysisCopy)));
|
|
Assert.False(Directory.Exists(FrameTrash.Location(fx.Paths, entry.Name).Folder));
|
|
Assert.SequenceEqual(converted.Frames.Select(f => (f.Sha256, before.Frames.Single(o => o.Id == f.Id).Sha256)).Order(),
|
|
back.Removed!.Select(t => ((string?)t.Sha256, t.Replacement)).Order());
|
|
|
|
// That entry erases: its replacements (the originals) are where it says, re-hashed now.
|
|
var empty = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
|
|
var job = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, empty), cancellationToken: fx.Ct);
|
|
await fx.WaitForAsync(job.Id, JobState.Done);
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
await FramesAsync(fx, id);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnUndoAfterTheFramesChangedBringsTheOriginalsBackAsANewSequence()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
var engine = await fx.StartEngineAsync();
|
|
var originals = await FramesAsync(fx, id);
|
|
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id));
|
|
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
|
|
var converted = await FramesAsync(fx, id);
|
|
// A converted frame deleted since: the frames are no longer the ones the conversion left.
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var delete = await fx.Service.FrameTrash.PlanDeleteAsync(id, [converted[1].Id], sequence.FramesVersion, fx.Ct);
|
|
await fx.Service.FrameTrash.DeleteAsync(delete, "alberto", null, fx.Ct);
|
|
|
|
var result = await fx.Service.RestoreEntryAsync(fx.Archive.Id, entry.Name, null, fx.Ct);
|
|
Assert.Equal((TrashKind.Replaced, 3, true), (result.Kind, result.Photos, result.NewSequence));
|
|
Assert.NotEqual(id, result.Sequence.Id);
|
|
Assert.SequenceEqual(originals.Select(f => f.Sha256), (await FramesAsync(fx, result.Sequence.Id)).Select(f => f.Sha256));
|
|
Assert.Count(2, await FramesAsync(fx, id));
|
|
// The replaced entry is gone with its last photo (and its copy of the analysis); the deleted frame's entry stays.
|
|
Assert.DoesNotContain(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ANewSequenceTakesTheCopiesAndLeavesTheSourceAsItIs()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
var engine = await fx.StartEngineAsync();
|
|
await fx.Service.SetSequenceInfoAsync(id, "Roof", ["sunset", "milan"], 4, "From the roof", fx.Ct);
|
|
var originals = await FramesAsync(fx, id);
|
|
await fx.Db.Sequences.EditAsync(id, new SequenceEdit { SettingsJson = new ProjectSettings { ExcludedFrames = [1], HeroFrame = 2 }.ToJson() }, null, fx.Ct);
|
|
var source = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var sidecar = File.ReadAllText(fx.Paths.Sequence(source.Folder).ManifestPath);
|
|
|
|
var job = await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id));
|
|
var made = Assert.NotNull(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, id)));
|
|
var copy = (await fx.Db.Sequences.GetAsync(made))!;
|
|
Assert.Equal(("Roof (TIFF)", 4, "From the roof"), (copy.Title, copy.Rating, copy.Notes));
|
|
Assert.SequenceEqual(["milan", "sunset"], copy.Tags.Order());
|
|
var frames = await FramesAsync(fx, made);
|
|
Assert.SequenceEqual(originals.Select(f => f.CapturedAt), frames.Select(f => f.CapturedAt));
|
|
Assert.True(frames.All(f => f.Format == ImageFileFormat.Tiff));
|
|
// Filed as the ingest files every photo: the archive's template in the new sequence, {name} the card's, {ext} the target's.
|
|
Assert.SequenceEqual(originals.Select(f => Path.ChangeExtension(f.FileName, ".tif")), frames.Select(f => f.FileName));
|
|
Assert.SequenceEqual(originals.Select(f => f.OriginalFileName), frames.Select(f => f.OriginalFileName));
|
|
var settings = ProjectSettings.FromJson(copy.SettingsJson!);
|
|
Assert.SequenceEqual([1], settings.ExcludedFrames);
|
|
Assert.Equal((2, frames[2].Id), (settings.HeroFrame, copy.HeroFrameId));
|
|
// The source: not a byte of its sidecar, nor a row, changed; nothing went to the trash.
|
|
Assert.Equal(sidecar, File.ReadAllText(fx.Paths.Sequence(source.Folder).ManifestPath));
|
|
var now = (await fx.Db.Sequences.GetAsync(id))!;
|
|
Assert.Equal((source.FramesVersion, source.SettingsVersion, source.Title, source.Folder), (now.FramesVersion, now.SettingsVersion, now.Title, now.Folder));
|
|
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256)), (await FramesAsync(fx, id)).Select(f => (f.Id, f.Sha256)));
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
Assert.False(StagedAnything(fx));
|
|
}
|
|
|
|
[Test]
|
|
public async Task APhotoThatFailsLeavesItsWholeSequenceAsItWasAndTheOthersConvert()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var bad = await UploadedAsync(fx, 3);
|
|
var good = await UploadedAsync(fx, 2, "upload-2", seed: 300);
|
|
var engine = await fx.StartEngineAsync();
|
|
var damaged = (await FramesAsync(fx, bad))[1];
|
|
var originals = await FramesAsync(fx, bad);
|
|
// Rot after the plan: the photo no longer matches the hash it was archived with.
|
|
var path = fx.Paths.FromRelative(damaged.RelativePath);
|
|
var bytes = File.ReadAllBytes(path);
|
|
bytes[^20] ^= 0x10;
|
|
File.WriteAllBytes(path, bytes);
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, bad, good);
|
|
Assert.Equal(5, plan.Frames);
|
|
|
|
var job = await RunAsync(fx, engine, plan, JobState.Failed);
|
|
Assert.Equal(Loc.Plural("Archive.Error.ConvertKept", 1), job.Error);
|
|
Assert.Contains(damaged.FileName, job.LogTail);
|
|
// The log says why in one catalog sentence; the exception's English text stays in the diagnostics log.
|
|
Assert.Contains(Loc.L("Archive.Convert.Failed.Changed"), job.LogTail);
|
|
Assert.False(job.LogTail!.Contains("SHA-256", StringComparison.Ordinal), job.LogTail);
|
|
// All or nothing: not one frame of that sequence changed, and nothing of it is in the trash.
|
|
var rows = await fx.Db.Frames.ListBySequenceAsync(bad);
|
|
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256, f.RelativePath)), rows.Select(f => (f.Id, f.Sha256, f.RelativePath)));
|
|
Assert.True((await FramesAsync(fx, good)).All(f => f.Format == ImageFileFormat.Tiff));
|
|
Assert.SequenceEqual([good], (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.SequenceId));
|
|
// The journal stays for a retry, which tries the failed sequence again and leaves the converted one; so does what the
|
|
// failed sequence staged before its photo failed, and only that.
|
|
var journal = Assert.NotNull(ConvertJournal.Read(fx.Paths, job.Id));
|
|
Assert.Equal(ConvertOutcome.Converted, journal.Of(good)!.Outcome);
|
|
Assert.Equal(ConvertOutcome.Failed, journal.Of(bad)!.Outcome);
|
|
Assert.False(Directory.Exists(StagingOf(fx, job.Id, good)));
|
|
var noted = StagedLog.Open(ConvertJournal.StagedPathFor(fx.Paths, job.Id)).Of(bad, (await fx.Db.Sequences.GetAsync(bad))!.FramesVersion);
|
|
Assert.SequenceEqual(noted.Select(c => c.File).Order(), Directory.EnumerateFiles(StagingOf(fx, job.Id, bad)).Select(f => Path.GetFileName(f)).Order());
|
|
Assert.DoesNotContain(damaged.Id, noted.Select(c => c.FrameId));
|
|
var first = Path.Combine(StagingOf(fx, job.Id, bad), noted.Single(c => c.FrameId == originals[0].Id).File);
|
|
File.SetLastWriteTimeUtc(first, new DateTime(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc));
|
|
|
|
// The photo is put right: the retry takes the staged conversion that still verifies and converts the rest.
|
|
bytes[^20] ^= 0x10;
|
|
File.WriteAllBytes(path, bytes);
|
|
await engine.ResumeAsync(job.Id, fx.Ct);
|
|
await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60));
|
|
var converted = await FramesAsync(fx, bad);
|
|
Assert.True(converted.All(f => f.Format == ImageFileFormat.Tiff));
|
|
Assert.Equal(2020, File.GetLastWriteTimeUtc(fx.Paths.FromRelative(converted[0].RelativePath)).Year);
|
|
Assert.False(StagedAnything(fx));
|
|
Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, job.Id));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AResumedJobKeepsTheStagedFilesThatStillVerify()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
var engine = await fx.StartEngineAsync();
|
|
// To JPEG needs other originals than the JPEGs uploaded: they become TIFF first.
|
|
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id));
|
|
var plan = await PlanAsync(fx, ConvertTarget.Jpeg, ConvertMode.Replace, id);
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var folder = fx.Paths.Sequence(sequence.Folder);
|
|
var manifest = Sidecar(fx, sequence);
|
|
var jobId = Guid.CreateVersion7();
|
|
var staging = Path.Combine(Staging(fx), $"convert-{jobId:N}", $"{id:N}");
|
|
var log = StagedLog.Open(ConvertJournal.StagedPathFor(fx.Paths, jobId));
|
|
// An earlier attempt staged the first photo (at another quality, so its use shows) and the second, damaged since.
|
|
StagedConversion Stage(SidecarFrame frame, int quality)
|
|
{
|
|
var target = Path.Combine(staging, frame.Id.ToString("N") + ".jpg");
|
|
var file = FrameConverter.Convert(new ConvertSource(folder.File(frame.File), frame, manifest.Camera, manifest.Lens), new ConvertOptions(ConvertTarget.Jpeg, quality),
|
|
DecoderFactory.CreateServer(), target, cancellationToken: fx.Ct);
|
|
var conversion = new StagedConversion(frame.Id, frame.Sha256!, Path.GetFileName(target), file.Size, file.Sha256, file.Width, file.Height);
|
|
log.Append(id, manifest.Versions.Frames, conversion);
|
|
return conversion;
|
|
}
|
|
var kept = Stage(manifest.Frames[0], 40);
|
|
var torn = Stage(manifest.Frames[1], 95);
|
|
File.WriteAllBytes(Path.Combine(staging, torn.File), [1, 2, 3]);
|
|
|
|
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Convert(fx.Archive, plan), "convert", jobId);
|
|
await fx.WaitForAsync(job.Id, JobState.Done);
|
|
var frames = await FramesAsync(fx, id);
|
|
Assert.Equal(kept.Sha256, frames[0].Sha256);
|
|
Assert.Equal(torn.Sha256, frames[1].Sha256);
|
|
Assert.True(frames.All(f => f.Format == ImageFileFormat.Jpeg));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AJobThatStoppedAfterItsReplacementReportsTheSequenceConverted()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id);
|
|
var request = ArchiveJobs.Convert(fx.Archive, plan);
|
|
await RunAsync(fx, engine, plan);
|
|
// As a crash between the replacement and the journal's note leaves it: the staged log names the conversions done.
|
|
var sidecar = Sidecar(fx, (await fx.Db.Sequences.GetAsync(id))!);
|
|
var jobId = Guid.CreateVersion7();
|
|
var log = StagedLog.Open(ConvertJournal.StagedPathFor(fx.Paths, jobId));
|
|
foreach (var frame in sidecar.Frames)
|
|
log.Append(id, plan.Sequences.Single().FramesVersion, new StagedConversion(frame.Id, frame.Source!.Sha256!, "gone.dng", frame.Size, frame.Sha256!, frame.W, frame.H));
|
|
|
|
var job = await fx.EnqueueWithCheckpointAsync(engine, request, "convert", jobId);
|
|
var done = await fx.WaitForAsync(job.Id, JobState.Done);
|
|
Assert.Contains(Loc.L("Archive.Log.Converted", (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle, 2), done.LogTail);
|
|
Assert.Count(1, await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ASequenceThatChangedAfterThePlanIsLeftAsItWas()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
var engine = await fx.StartEngineAsync();
|
|
var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id);
|
|
// A photo of the plan is deleted before the job runs.
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var delete = await fx.Service.FrameTrash.PlanDeleteAsync(id, [(await FramesAsync(fx, id))[1].Id], sequence.FramesVersion, fx.Ct);
|
|
await fx.Service.FrameTrash.DeleteAsync(delete, "alberto", null, fx.Ct);
|
|
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
|
|
|
|
var job = await RunAsync(fx, engine, plan, JobState.Failed);
|
|
Assert.Equal(Loc.Plural("Archive.Error.ConvertKept", 1), job.Error);
|
|
var changed = Loc.L("Archive.Log.ConvertChanged", sequence.DisplayTitle);
|
|
Assert.Contains(changed, job.LogTail);
|
|
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
|
|
Assert.DoesNotContain(TrashKind.Replaced, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Kind));
|
|
// The refusal holds on a retry too, and says so: the user plans again.
|
|
await engine.ResumeAsync(job.Id, fx.Ct);
|
|
Assert.Contains(changed, (await fx.WaitForAsync(job.Id, JobState.Failed)).LogTail);
|
|
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
|
|
}
|
|
|
|
[Test]
|
|
public async Task PhotosThatComeInWhileTheJobWaitsItsTurnStayAsTheyAreAndThoseConfirmedConvert()
|
|
{
|
|
foreach (var mode in new[] { ConvertMode.Replace, ConvertMode.NewSequence })
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, mode, id);
|
|
var title = (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle;
|
|
// An upload lands after the confirmation, before the job comes to the sequence (it waited behind a render, or
|
|
// behind other sequences).
|
|
var (path, size, sha) = fx.Stage(Start.AddSeconds(60), 999);
|
|
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, null, path, "IMG_0099.JPG", size, sha, null, null), fx.Ct);
|
|
|
|
var job = await RunAsync(fx, engine, plan);
|
|
var frames = await FramesAsync(fx, id);
|
|
Assert.Equal((sha, ImageFileFormat.Jpeg), frames.Where(f => f.Sha256 == sha).Select(f => (f.Sha256, f.Format)).Single(), mode.ToString());
|
|
if (mode == ConvertMode.Replace)
|
|
{
|
|
Assert.Contains(Loc.L("Archive.Log.Converted", title, 2), job.LogTail);
|
|
Assert.Equal(2, frames.Count(f => f.Format == ImageFileFormat.Tiff));
|
|
Assert.Equal(2, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Frames);
|
|
}
|
|
else
|
|
{
|
|
Assert.Contains(Loc.L("Archive.Log.ConvertedNew", title, 2), job.LogTail);
|
|
Assert.Contains(Loc.L("Archive.Log.ConvertLeftOut", title, 1), job.LogTail);
|
|
var made = Assert.NotNull(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, id)));
|
|
Assert.True((await FramesAsync(fx, made)) is { Count: 2 } copies && copies.All(f => f.Format == ImageFileFormat.Tiff));
|
|
Assert.True(frames.All(f => f.Format == ImageFileFormat.Jpeg));
|
|
}
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public async Task AVolumeWithoutTheSpaceConvertsNothing()
|
|
{
|
|
await using var fx = await FixtureAsync(o => o with { FreeSpace = _ => 1_000 });
|
|
var id = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
|
|
Assert.False(plan.Space.Enough);
|
|
Assert.Equal(1_000, plan.Space.Free);
|
|
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
|
|
|
|
var job = await RunAsync(fx, engine, plan, JobState.Failed);
|
|
Assert.True(job.Error!.StartsWith("The conversion needs", StringComparison.Ordinal), job.Error);
|
|
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
|
|
Assert.False(StagedAnything(fx));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ACardKeepsTheCopyOfAPhotoThatWasConverted()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var card = context.CreateTempDirectory("card");
|
|
Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, 3);
|
|
var media = await fx.Service.Imports.RegisterCardAsync(new CardVolume("3F2A9C11-EOS_DIGITAL-127865454592", 0x3F2A9C11, "EOS_DIGITAL", 127_865_454_592), fx.Ct);
|
|
var import = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct);
|
|
await fx.Service.Imports.ExecuteAsync(new ImportJobParameters(fx.Archive.Id, import, media.Id, true, null), Guid.NewGuid(), 0,
|
|
_ => ValueTask.CompletedTask, null, fx.Ct);
|
|
var id = (await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, import.Sessions.Single().SessionKey))!.Value;
|
|
Assert.Equal(3, (await fx.Service.PlanCardEraseAsync(fx.Archive.Id, media.Id, card, null, fx.Ct)).ToDelete.Count());
|
|
var engine = await fx.StartEngineAsync();
|
|
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id));
|
|
|
|
// The frames now hold converted files: nothing in the archive matches the card's photos byte for byte, so they stay.
|
|
var erase = await fx.Service.PlanCardEraseAsync(fx.Archive.Id, media.Id, card, null, fx.Ct);
|
|
Assert.Empty(erase.ToDelete);
|
|
Assert.True(erase.Items.All(i => i.Verdict == EraseVerdict.CopyMismatch));
|
|
Assert.Equal(0, CardEraser.Execute(erase).Deleted);
|
|
Assert.Count(3, Directory.EnumerateFiles(card, "*.JPG", SearchOption.AllDirectories));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AReplacedEntryIsErasedOnlyWhileTheConvertedFilesStillHashToTheirRecord()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id));
|
|
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
|
|
var converted = (await FramesAsync(fx, id))[0];
|
|
var file = fx.Paths.FromRelative(converted.RelativePath);
|
|
var bytes = File.ReadAllBytes(file);
|
|
var damaged = (byte[])bytes.Clone();
|
|
damaged[^10] ^= 0x01;
|
|
File.WriteAllBytes(file, damaged);
|
|
|
|
var plan = await PlanEmptyAsync(fx);
|
|
var kept = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, plan), cancellationToken: fx.Ct);
|
|
await fx.WaitForAsync(kept.Id, JobState.Failed);
|
|
Assert.Contains(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Select(e => e.Name));
|
|
|
|
File.WriteAllBytes(file, bytes);
|
|
var erased = await engine.EnqueueAsync(ArchiveJobs.EmptyTrash(fx.Archive, await PlanEmptyAsync(fx)), cancellationToken: fx.Ct);
|
|
await fx.WaitForAsync(erased.Id, JobState.Done);
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
|
|
static Task<TrashEmptyPlan> PlanEmptyAsync(ArchiveFixture fx) => fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AReplacementACrashInterruptedBeforeItsSidecarIsUndoneByTheNextWriter()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var folder = fx.Paths.Sequence(sequence.Folder);
|
|
var manifest = Sidecar(fx, sequence);
|
|
var frame = manifest.Frames[0];
|
|
// Half a replacement: the original in its entry, the converted file in its place, the record pending, the sidecar as it was.
|
|
var staged = Path.Combine(Staging(fx), "convert-test", frame.Id.ToString("N") + ".tif");
|
|
FrameConverter.Convert(new ConvertSource(folder.File(frame.File), frame), new ConvertOptions(ConvertTarget.Tiff16), DecoderFactory.CreateServer(), staged,
|
|
cancellationToken: fx.Ct);
|
|
var entryFolder = Path.Combine(FrameTrash.Directory(fx.Paths), $"{id:N}-20261027120000");
|
|
var record = entryFolder + ".json";
|
|
var trashed = Path.Combine(entryFolder, "originals", Path.GetFileName(frame.File));
|
|
var placed = Path.ChangeExtension(frame.File, ".tif");
|
|
var journal = new FrameReplaceJournal(manifest.Versions.Frames, fx.Paths.ToRelative(record), fx.Paths.ToRelative(entryFolder),
|
|
[new SwapMove(frame.File, fx.Paths.ToRelative(trashed))], [new SwapMove(placed, fx.Paths.ToRelative(staged))], false, null);
|
|
journal.Write(folder);
|
|
Directory.CreateDirectory(FrameTrash.Directory(fx.Paths));
|
|
File.WriteAllText(FrameRemovalJournal.PendingRecord(record), "{}");
|
|
Directory.CreateDirectory(Path.GetDirectoryName(trashed)!);
|
|
File.Move(folder.File(frame.File), trashed);
|
|
File.Move(staged, folder.File(placed));
|
|
|
|
await fx.Service.EditSequenceInfoAsync(id, info => info with { Rating = 2 }, fx.Ct);
|
|
Assert.FileExists(folder.File(frame.File));
|
|
Assert.FileMissing(folder.File(placed));
|
|
Assert.FileExists(staged);
|
|
Assert.False(Directory.Exists(entryFolder));
|
|
Assert.FileMissing(FrameRemovalJournal.PendingRecord(record));
|
|
Assert.FileMissing(folder.ReplaceJournalPath);
|
|
Assert.Equal(frame.Sha256, Sha(folder.File(frame.File)));
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AReplacementACrashInterruptedAfterItsSidecarIsCompletedByTheNextWriter()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
var version = (await fx.Db.Sequences.GetAsync(id))!.FramesVersion;
|
|
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id));
|
|
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
|
|
var (entryFolder, record) = FrameTrash.Location(fx.Paths, entry.Name);
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var folder = fx.Paths.Sequence(sequence.Folder);
|
|
// As a crash right after the sidecar was written leaves it: the record still pending, the journal there.
|
|
File.Move(record, FrameRemovalJournal.PendingRecord(record));
|
|
new FrameReplaceJournal(version, fx.Paths.ToRelative(record), fx.Paths.ToRelative(entryFolder), [], [], true, null).Write(folder);
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
|
|
await fx.Service.EditSequenceInfoAsync(id, info => info with { Rating = 2 }, fx.Ct);
|
|
Assert.FileMissing(folder.ReplaceJournalPath);
|
|
Assert.Equal(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name);
|
|
Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Tiff));
|
|
}
|
|
|
|
[Test]
|
|
public async Task APhotoTheServerCannotReadAtFullResolutionIsSkipped()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
// The sidecar says the second photo is a HEIC (a desktop with WIC filed it): the managed decoders have only its preview.
|
|
var manifest = Sidecar(fx, sequence);
|
|
ManifestStore.WriteSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath,
|
|
manifest with { Frames = [manifest.Frames[0], manifest.Frames[1] with { Format = ImageFileFormat.Heif }] });
|
|
var step = (await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.NewSequence, id)).Sequences.Single();
|
|
Assert.Equal(1, step.Frames.Count);
|
|
Assert.Equal((ConvertSkip.NoNativeDecoder, 1), (step.Skipped.Single().Reason, step.Skipped.Single().Frames));
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheJournalsSurviveARoundTrip()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var journal = new ConvertJournal { JobId = Guid.NewGuid() };
|
|
var first = Guid.NewGuid();
|
|
var harbour = Guid.NewGuid();
|
|
journal.Begin(harbour, "Harbour", SequenceStatus.AnalyzedLight | SequenceStatus.ProxiesBuilt);
|
|
journal.Note(new ConvertSequenceResult(first, "Roof", ConvertOutcome.Created, 3, Guid.NewGuid()), SequenceStatus.AnalyzedDeep);
|
|
journal.Of(first)!.Queued = true;
|
|
journal.Note(new ConvertSequenceResult(harbour, "Harbour", ConvertOutcome.Failed, 0, Error: "no"));
|
|
journal.Write(fx.Paths);
|
|
var read = Assert.NotNull(ConvertJournal.Read(fx.Paths, journal.JobId));
|
|
Assert.SequenceEqual(journal.Sequences.Select(s => (s.Id, s.Outcome, s.Frames, s.NewSequenceId, s.Final, s.Had, s.Queued)),
|
|
read.Sequences.Select(s => (s.Id, s.Outcome, s.Frames, s.NewSequenceId, s.Final, s.Had, s.Queued)));
|
|
Assert.True(read.Of(first)!.Final);
|
|
// A failure keeps the statuses the sequence had when the job came to it.
|
|
Assert.Equal(SequenceStatus.AnalyzedLight | SequenceStatus.ProxiesBuilt, read.Of(harbour)!.Had);
|
|
// The photos pinned: a later line for a sequence replaces an earlier one, a torn one is skipped.
|
|
var photo = Guid.NewGuid();
|
|
var pins = read.Pins(fx.Paths);
|
|
Assert.Null(pins.Of(first));
|
|
ConvertSequencePlan Pinned(int version, string sha) =>
|
|
new(first, "Roof", "roof", version, [new ConvertFramePlan(photo, 0, "originals/a.jpg", ImageFileFormat.Jpeg, 1, 2, sha)], [], []);
|
|
pins.Append(first, Pinned(4, "aa"));
|
|
pins.Append(first, Pinned(5, "bb"));
|
|
File.AppendAllText(ConvertJournal.PinsPathFor(fx.Paths, journal.JobId), "{\"sequence\":");
|
|
var pinned = Assert.NotNull(read.Pins(fx.Paths).Of(first));
|
|
Assert.Equal((5, photo, "bb"), (pinned.FramesVersion, pinned.Frames.Single().Id, pinned.Frames.Single().Sha256));
|
|
// Its files (the photos confirmed among them) and its staging folder are known by their names.
|
|
Assert.SequenceEqual([journal.JobId, journal.JobId, journal.JobId, journal.JobId],
|
|
ConvertJournal.Files(fx.Paths, journal.JobId).Select(p => ConvertJournal.JobOf(Path.GetFileName(p))));
|
|
Assert.Contains(ConfirmedPhotos.PathFor(fx.Paths, journal.JobId), ConvertJournal.Files(fx.Paths, journal.JobId));
|
|
// The copy a New sequence conversion made stays named whatever the sequence comes to after it.
|
|
var copy = Guid.NewGuid();
|
|
journal.Made(harbour, copy);
|
|
journal.Note(new ConvertSequenceResult(harbour, "Harbour", ConvertOutcome.Failed, 0, Error: "no"));
|
|
journal.Write(fx.Paths);
|
|
Assert.Equal(copy, ConvertJournal.Read(fx.Paths, journal.JobId)!.Of(harbour)!.NewSequenceId);
|
|
Assert.Equal(journal.JobId, ConvertJournal.JobOf(MassConverter.StagingFolder(journal.JobId)));
|
|
Assert.Null(ConvertJournal.JobOf("convert-test"));
|
|
Assert.Null(ConvertJournal.JobOf("rename-" + journal.JobId.ToString("N") + ".json"));
|
|
// A line torn by a crash is skipped; a later line for a frame replaces an earlier one.
|
|
var path = ConvertJournal.StagedPathFor(fx.Paths, journal.JobId);
|
|
var log = StagedLog.Open(path);
|
|
var frame = Guid.NewGuid();
|
|
log.Append(first, 4, new StagedConversion(frame, "a", "x.dng", 1, "b", 2, 3));
|
|
log.Append(first, 4, new StagedConversion(frame, "a", "x.dng", 1, "c", 2, 3));
|
|
File.AppendAllText(path, "{\"sequence\":");
|
|
var reread = StagedLog.Open(path);
|
|
Assert.Equal("c", reread.Of(first, 4).Single().Sha256);
|
|
Assert.Empty(reread.Of(first, 5));
|
|
ConvertJournal.Delete(fx.Paths, journal.JobId);
|
|
Assert.FileMissing(path);
|
|
Assert.FileMissing(ConvertJournal.PinsPathFor(fx.Paths, journal.JobId));
|
|
Assert.Null(ConvertJournal.Read(fx.Paths, journal.JobId));
|
|
// A job an earlier build queued names no digest: it reads with none.
|
|
var parameters = JsonSerializer.Serialize(new ConvertJobParameters(fx.Archive.Id, ConvertTarget.Jpeg, 80, ConvertMode.NewSequence,
|
|
[new ConvertSequence(first, 2, "d")], "f"), ArchiveJobJson.Default.ConvertJobParameters);
|
|
Assert.Equal("d", JsonSerializer.Deserialize(parameters, ArchiveJobJson.Default.ConvertJobParameters)!.Sequences.Single().Frames);
|
|
var older = $$"""{"archiveId":"{{fx.Archive.Id}}","target":"Jpeg","quality":80,"mode":"Replace","sequences":[{"id":"{{first}}","framesVersion":2}],"fingerprint":"f"}""";
|
|
Assert.Null(JsonSerializer.Deserialize(older, ArchiveJobJson.Default.ConvertJobParameters)!.Sequences.Single().Frames);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ConvertedFilesAreNamedAsFilingNamesThemAndNothingIsOverwritten()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
await fx.Service.UpdateSettingsAsync(fx.Archive.Id, s => s with { Templates = s.Templates with { File = "{name}_{n:00}" } }, fx.Ct);
|
|
var roof = await UploadedAsync(fx, 2);
|
|
var lake = await UploadedAsync(fx, 2, "upload-2", seed: 200);
|
|
Assert.SequenceEqual(["IMG_0001_01.JPG", "IMG_0002_02.JPG"], (await FramesAsync(fx, roof)).Select(f => f.FileName));
|
|
await fx.Service.SetSequenceInfoAsync(roof, "Roof", [], 0, null, fx.Ct);
|
|
await fx.Service.SetSequenceInfoAsync(lake, "Lake", [], 0, null, fx.Ct);
|
|
// Filing names a photo with the template of the moment: a conversion names its files as filing would now.
|
|
await fx.Service.UpdateSettingsAsync(fx.Archive.Id, s => s with { Templates = s.Templates with { File = "{title}_{n:00}" } }, fx.Ct);
|
|
// A file that is no frame holds the first name: it is never written over, the converted file takes the next.
|
|
var stray = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(roof))!.Folder).File("originals/Roof_01.dng");
|
|
File.WriteAllBytes(stray, [1, 2, 3]);
|
|
var engine = await fx.StartEngineAsync();
|
|
|
|
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, roof));
|
|
var replaced = await FramesAsync(fx, roof);
|
|
Assert.SequenceEqual(["Roof_01_2.dng", "Roof_02.dng"], replaced.Select(f => f.FileName));
|
|
Assert.SequenceEqual(["IMG_0001.JPG", "IMG_0002.JPG"], replaced.Select(f => f.OriginalFileName));
|
|
Assert.SequenceEqual(new byte[] { 1, 2, 3 }, File.ReadAllBytes(stray));
|
|
|
|
var job = await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, lake));
|
|
var made = Assert.NotNull(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, lake)));
|
|
var copies = await FramesAsync(fx, made);
|
|
Assert.SequenceEqual(["Lake (TIFF)_01.tif", "Lake (TIFF)_02.tif"], copies.Select(f => f.FileName));
|
|
Assert.SequenceEqual(["IMG_0001.JPG", "IMG_0002.JPG"], copies.Select(f => f.OriginalFileName));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AFolderMovedWhileTheJobWaitsForItsLeaseIsWhereTheConvertedFilesGo()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
|
|
var row = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var from = fx.Paths.Sequence(row.Folder).Folder;
|
|
Job job;
|
|
var planned = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
void OnProgress(JobProgressInfo info)
|
|
{
|
|
if (info.Kind == JobKind.Convert && info.Total == 2) planned.TrySetResult();
|
|
}
|
|
engine.Progress += OnProgress;
|
|
using (await LeaseFile.AcquireAsync(fx.Paths.LocksDirectory, $"seq-{id:N}", "editor", TimeSpan.FromSeconds(5), cancellationToken: fx.Ct))
|
|
{
|
|
job = await engine.EnqueueAsync(ArchiveJobs.Convert(fx.Archive, plan), cancellationToken: fx.Ct);
|
|
// Planned again where the index had the sequence then: the job now waits for its lease to convert it.
|
|
await planned.Task.WaitAsync(TimeSpan.FromSeconds(TestTiming.Budget(30)), fx.Ct);
|
|
// What a Reorganize step does under that lease meanwhile: the folder moves, and its index row follows.
|
|
var to = fx.Paths.Sequence("moved/Roof").Folder;
|
|
Directory.CreateDirectory(Path.GetDirectoryName(to)!);
|
|
Directory.Move(from, to);
|
|
Assert.True(await fx.Db.Sequences.RelocateAsync(id, "moved/Roof", null, fx.Ct));
|
|
}
|
|
engine.Progress -= OnProgress;
|
|
|
|
await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60));
|
|
var frames = await FramesAsync(fx, id);
|
|
Assert.True(frames.All(f => f.Format == ImageFileFormat.Tiff && f.RelativePath.StartsWith("moved/Roof/", StringComparison.Ordinal)));
|
|
Assert.False(Directory.Exists(from), "the folder the sequence left is never made again");
|
|
Assert.Equal("moved/Roof", (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Folder);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ASequenceAnotherIndexMovedIsLeftAsItIsUntilThisIndexFindsIt()
|
|
{
|
|
// The folder the sequence left holds another sequence now, or nothing.
|
|
foreach (var taken in new[] { true, false })
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
Guid? stranger = taken ? await UploadedAsync(fx, 2, "upload-2", seed: 200) : null;
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
|
|
var row = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var folder = fx.Paths.Sequence(row.Folder).Folder;
|
|
|
|
// Another index moves the folder away, then the other sequence into the folder it left; this index has scanned neither.
|
|
var (other, archive) = await fx.OtherWriterAsync();
|
|
var moves = stranger is not { } s ? [(id, "moved/{date}")] : new[] { (id, "moved/{date}"), (s, "parked/{date}"), (s, row.Folder) };
|
|
foreach (var (sequence, template) in moves)
|
|
await other.ApplyRenameAsync(await other.PlanRenameAsync(archive.Id, [sequence], template, null, fx.Ct), null, fx.Ct);
|
|
var before = taken ? ArchiveFixture.FilesOf(folder) : null;
|
|
|
|
var job = Guid.NewGuid();
|
|
var left = (await RunHereAsync(fx, plan, job)).Sequences.Single();
|
|
Assert.Equal((ConvertOutcome.Blocked, Loc.L("Errors.Archive.FolderMoved")), (left.Outcome, left.Error), taken.ToString());
|
|
if (taken) Assert.SequenceEqual(before!, ArchiveFixture.FilesOf(folder), "the other sequence's files are untouched");
|
|
else Assert.False(Directory.Exists(folder), "the folder it left is not created again");
|
|
|
|
// Once this index's scan found it there, the retry converts it where it is now.
|
|
await fx.ScanAsync();
|
|
Assert.Equal(ConvertOutcome.Converted, (await RunHereAsync(fx, plan, job)).Sequences.Single().Outcome, taken.ToString());
|
|
var moved = (await fx.Db.Sequences.GetAsync(id))!.Folder;
|
|
Assert.True(moved.StartsWith("moved/", StringComparison.Ordinal), moved);
|
|
Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Tiff && f.RelativePath.StartsWith(moved + "/", StringComparison.Ordinal)));
|
|
if (taken) Assert.SequenceEqual(before!, ArchiveFixture.FilesOf(folder), "the other sequence's files are still untouched");
|
|
else Assert.False(Directory.Exists(folder));
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public async Task AReorganizeTheArchiveWaitsForKeepsTheConversionOffItsSequences()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var originals = await FramesAsync(fx, id);
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
|
|
// A Reorganize paused by its user holds the archive: its plan waits for it.
|
|
await fx.Db.Jobs.EnqueueAsync(new Job
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
Kind = JobKind.Move,
|
|
TargetIds = [fx.Archive.Id],
|
|
State = JobState.Paused,
|
|
Created = DateTimeOffset.UtcNow,
|
|
});
|
|
Assert.SequenceEqual([ConvertBlock.Busy], (await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id)).Sequences.Single().Blocked);
|
|
|
|
var engine = await fx.StartEngineAsync();
|
|
var job = await RunAsync(fx, engine, plan, JobState.Failed);
|
|
Assert.Equal(Loc.Plural("Archive.Error.ConvertKept", 1), job.Error);
|
|
Assert.Contains(Loc.L("Archive.Log.ConvertBlocked", (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle), job.LogTail);
|
|
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256, f.RelativePath)), (await FramesAsync(fx, id)).Select(f => (f.Id, f.Sha256, f.RelativePath)));
|
|
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task PhotosVerifiedAfterThePlanAreNotReplacedWithoutAConfirmation()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
var engine = await fx.StartEngineAsync();
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var path = fx.Paths.Sequence(sequence.Folder).ManifestPath;
|
|
var verified = ManifestStore.ReadSequence(path);
|
|
// Never verified when the plan is made (adopted): the dry run says that photo stays as it is.
|
|
ManifestStore.WriteSequence(path, verified with { Frames = [verified.Frames[0], verified.Frames[1] with { Sha256 = null }, verified.Frames[2]] });
|
|
var plan = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.Replace, id);
|
|
Assert.Equal((2, ConvertSkip.Unhashed), (plan.Frames, plan.Sequences.Single().Skipped.Single().Reason));
|
|
// A verify then hashes it before the job runs, as ArchiveVerifier does: no new frames version.
|
|
ManifestStore.WriteSequence(path, verified);
|
|
|
|
// The job converts the two photos the plan listed, and leaves the third as it is, as the dry run said.
|
|
var job = await RunAsync(fx, engine, plan);
|
|
Assert.Contains(Loc.L("Archive.Log.Converted", sequence.DisplayTitle, 2), job.LogTail);
|
|
var frames = await FramesAsync(fx, id);
|
|
Assert.SequenceEqual(verified.Frames.Select(f => f.Id), frames.Select(f => f.Id));
|
|
Assert.Equal((verified.Frames[1].Sha256, ImageFileFormat.Jpeg), (frames[1].Sha256, frames[1].Format));
|
|
Assert.True(frames[0].Format == ImageFileFormat.Dng && frames[2].Format == ImageFileFormat.Dng);
|
|
Assert.Equal(2, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Frames);
|
|
Assert.False(StagedAnything(fx));
|
|
}
|
|
|
|
[Test]
|
|
public async Task PhotosThatComeInOnceTheSequenceIsPinnedStayAsTheyAreAndARetryConvertsTheRest()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
|
|
var title = (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle;
|
|
// An earlier attempt pinned the sequence's photos and stopped (stage B blocked, a pause); an upload came in since.
|
|
var jobId = Guid.CreateVersion7();
|
|
PinLog.Open(ConvertJournal.PinsPathFor(fx.Paths, jobId)).Append(id, plan.Sequences.Single());
|
|
var (path, size, sha) = fx.Stage(Start.AddSeconds(60), 999);
|
|
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, null, path, "IMG_0099.JPG", size, sha, null, null), fx.Ct);
|
|
var added = (await FramesAsync(fx, id)).Single(f => f.Sha256 == sha);
|
|
|
|
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Convert(fx.Archive, plan), "convert", jobId);
|
|
var done = await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60));
|
|
Assert.Contains(Loc.L("Archive.Log.Converted", title, 2), done.LogTail);
|
|
var frames = await FramesAsync(fx, id);
|
|
Assert.Equal(2, frames.Count(f => f.Format == ImageFileFormat.Tiff));
|
|
Assert.Equal((sha, ImageFileFormat.Jpeg), frames.Where(f => f.Id == added.Id).Select(f => (f.Sha256, f.Format)).Single());
|
|
Assert.Equal(2, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Frames);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AJobResumedAfterACrashInItsReplacementSettlesItAndConvertsTheWholeSequence()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var engine = await fx.StartEngineAsync();
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var folder = fx.Paths.Sequence(sequence.Folder);
|
|
var manifest = Sidecar(fx, sequence);
|
|
// The attempt that crashed: the photos pinned and staged, then its replacement cut short once every original had left
|
|
// for the trash, none of the converted files in yet.
|
|
var jobId = Guid.CreateVersion7();
|
|
PinLog.Open(ConvertJournal.PinsPathFor(fx.Paths, jobId)).Append(id, plan.Sequences.Single());
|
|
var log = StagedLog.Open(ConvertJournal.StagedPathFor(fx.Paths, jobId));
|
|
var entryFolder = Path.Combine(FrameTrash.Directory(fx.Paths), $"{id:N}-20261027120000");
|
|
var record = entryFolder + ".json";
|
|
List<SwapMove> outgoing = [];
|
|
List<SwapMove> incoming = [];
|
|
foreach (var frame in manifest.Frames)
|
|
{
|
|
var staged = Path.Combine(StagingOf(fx, jobId, id), frame.Id.ToString("N") + ".tif");
|
|
var file = FrameConverter.Convert(new ConvertSource(folder.File(frame.File), frame), new ConvertOptions(ConvertTarget.Tiff16), DecoderFactory.CreateServer(),
|
|
staged, cancellationToken: fx.Ct);
|
|
log.Append(id, manifest.Versions.Frames, new StagedConversion(frame.Id, frame.Sha256!, Path.GetFileName(staged), file.Size, file.Sha256, file.Width, file.Height));
|
|
outgoing.Add(new SwapMove(frame.File, fx.Paths.ToRelative(Path.Combine(entryFolder, "originals", Path.GetFileName(frame.File)))));
|
|
incoming.Add(new SwapMove(Path.ChangeExtension(frame.File, ".tif"), fx.Paths.ToRelative(staged)));
|
|
}
|
|
new FrameReplaceJournal(manifest.Versions.Frames, fx.Paths.ToRelative(record), fx.Paths.ToRelative(entryFolder), outgoing, incoming, false, null).Write(folder);
|
|
Directory.CreateDirectory(FrameTrash.Directory(fx.Paths));
|
|
File.WriteAllText(FrameRemovalJournal.PendingRecord(record), "{}");
|
|
foreach (var move in outgoing)
|
|
{
|
|
var trashed = fx.Paths.FromRelative(move.Other);
|
|
Directory.CreateDirectory(Path.GetDirectoryName(trashed)!);
|
|
File.Move(folder.File(move.Place), trashed);
|
|
}
|
|
|
|
// Not a sequence with nothing to convert, nor half of it converted: the originals come back, then all of it converts.
|
|
var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Convert(fx.Archive, plan), "convert", jobId);
|
|
var done = await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60));
|
|
Assert.Contains(Loc.L("Archive.Log.Converted", sequence.DisplayTitle, 2), done.LogTail);
|
|
Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Tiff));
|
|
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
|
|
Assert.Equal((TrashKind.Replaced, 2), (entry.Kind, entry.Frames));
|
|
Assert.False(Directory.Exists(entryFolder));
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheFollowUpsOfEachSequenceAreQueuedAsItEndsAndOnResumeWhenAPauseCameFirst()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var first = await UploadedAsync(fx, 2);
|
|
var second = await UploadedAsync(fx, 2, "upload-2", seed: 200);
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, first, second);
|
|
var events = new List<string>();
|
|
var callbacks = Callbacks(done: s => events.Add($"done {s.SequenceId}"), queue: r => events.AddRange(r.Select(q => $"{q.Kind} {q.Targets.Single()}")));
|
|
await RunHereAsync(fx, plan, Guid.CreateVersion7(), callbacks);
|
|
// Queued as each sequence is journaled, before the next starts: a pause, a yield or a crash later loses none of them.
|
|
Assert.Less(events.IndexOf($"done {first}"), events.IndexOf($"{JobKind.LightAnalyse} {first}"));
|
|
Assert.Less(events.IndexOf($"{JobKind.LightAnalyse} {first}"), events.IndexOf($"done {second}"));
|
|
Assert.Contains($"{JobKind.LightAnalyse} {second}", events);
|
|
|
|
// As a pause right after the first sequence's outcome was journaled leaves the job: its passes not queued yet.
|
|
var jobId = Guid.CreateVersion7();
|
|
var journal = new ConvertJournal { JobId = jobId };
|
|
journal.Note(new ConvertSequenceResult(first, "Roof", ConvertOutcome.Converted, 2), SequenceStatus.AnalyzedDeep | SequenceStatus.ThumbsBuilt);
|
|
journal.Note(new ConvertSequenceResult(second, "Lake", ConvertOutcome.Converted, 2), SequenceStatus.AnalyzedLight);
|
|
journal.Of(second)!.Queued = true;
|
|
journal.Write(fx.Paths);
|
|
List<ArchiveJobRequest> queued = [];
|
|
var resumed = await RunHereAsync(fx, plan, jobId, Callbacks(queue: queued.AddRange));
|
|
Assert.Equal(4, resumed.Frames);
|
|
Assert.SequenceEqual([(JobKind.LightAnalyse, first), (JobKind.DeepAnalyse, first), (JobKind.Thumbs, first)],
|
|
queued.Select(r => (r.Kind, r.Targets.Single())));
|
|
Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, jobId));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AConversionWaitingInTheQueueKeepsItsSequencesFromASecondConfirmation()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id);
|
|
var request = ArchiveJobs.Convert(fx.Archive, plan);
|
|
var job = new Job
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
Kind = request.Kind,
|
|
TargetIds = request.Targets,
|
|
ParametersJson = request.ParametersJson,
|
|
State = JobState.Queued,
|
|
Created = DateTimeOffset.UtcNow,
|
|
};
|
|
await fx.Db.Jobs.EnqueueAsync(job, fx.Ct);
|
|
|
|
// Another tab, or Apply pressed again after an answer was lost: the same copies are not queued twice.
|
|
var again = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id);
|
|
Assert.SequenceEqual([ConvertBlock.Busy], again.Sequences.Single().Blocked);
|
|
Assert.NotEqual(plan.Fingerprint(), again.Fingerprint());
|
|
Assert.Throws<ArgumentException>(() => ArchiveJobs.Convert(fx.Archive, again));
|
|
// The job itself plans again with its sequences free, and converts them.
|
|
var engine = await fx.StartEngineAsync();
|
|
engine.Wake();
|
|
await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60));
|
|
Assert.NotNull(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, id)));
|
|
Assert.Empty((await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id)).Sequences.Single().Blocked);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ANewSequenceIsOwnedAsItsSourceIsOrByWhoAskedForIt()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var owner = Guid.NewGuid();
|
|
var owned = await UploadedAsync(fx, 2, user: owner);
|
|
var unowned = await UploadedAsync(fx, 2, "upload-2", seed: 200);
|
|
Assert.Equal(owner, (await fx.Db.Sequences.GetAsync(owned))!.OwnerUserId);
|
|
var engine = await fx.StartEngineAsync();
|
|
var asker = Guid.NewGuid();
|
|
|
|
var job = await engine.EnqueueAsync(ArchiveJobs.Convert(fx.Archive, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, owned, unowned)), asker, fx.Ct);
|
|
await fx.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60));
|
|
foreach (var (source, expected) in new[] { (owned, owner), (unowned, asker) })
|
|
{
|
|
var made = (await fx.Db.Sequences.GetAsync((await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, source)))!.Value))!;
|
|
Assert.Equal(expected, made.OwnerUserId);
|
|
Assert.Equal(expected.ToString(), Sidecar(fx, made).CreatedBy);
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public async Task AStagedCopyThatVanishesFailsItsNewSequenceInsteadOfLeavingThePhotoOut()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id);
|
|
var first = Sidecar(fx, (await fx.Db.Sequences.GetAsync(id))!).Frames[0];
|
|
var jobId = Guid.CreateVersion7();
|
|
// A clean-up takes the first copy once both are staged, before they are filed.
|
|
var callbacks = Callbacks(progress: (n, total) =>
|
|
{
|
|
if (n == 2 && total == 2) File.Delete(Path.Combine(StagingOf(fx, jobId, id), first.Id.ToString("N") + ".tif"));
|
|
});
|
|
|
|
var failed = (await RunHereAsync(fx, plan, jobId, callbacks)).Sequences.Single();
|
|
Assert.Equal(ConvertOutcome.Failed, failed.Outcome);
|
|
// Why, in the catalog's words: the job log has no English exception text.
|
|
Assert.Equal(Loc.L("Archive.Convert.NotPlaced", Loc.L("Archive.Convert.Unstaged", first.File)), failed.Error);
|
|
Assert.Null(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(jobId, id)));
|
|
// The retry converts that photo again and makes the whole copy.
|
|
var made = (await RunHereAsync(fx, plan, jobId)).Sequences.Single();
|
|
Assert.Equal((ConvertOutcome.Created, 2, 0), (made.Outcome, made.Frames, made.LeftOut));
|
|
Assert.Count(2, await FramesAsync(fx, made.NewSequenceId!.Value));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ANewSequenceSaysWhichPhotosItLeftInTheSourceAlone()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
var engine = await fx.StartEngineAsync();
|
|
// Three TIFFs, then a JPEG uploaded: to JPEG, the TIFFs convert and the JPEG stays in the source alone.
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id));
|
|
var (path, size, sha) = fx.Stage(Start.AddSeconds(60), 999);
|
|
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, null, path, "IMG_0099.JPG", size, sha, null, null), fx.Ct);
|
|
|
|
var job = await RunAsync(fx, engine, await PlanAsync(fx, ConvertTarget.Jpeg, ConvertMode.NewSequence, id));
|
|
Assert.Contains(Loc.L("Archive.Log.ConvertedNew", sequence.DisplayTitle, 3), job.LogTail);
|
|
Assert.Contains(Loc.L("Archive.Log.ConvertLeftOut", sequence.DisplayTitle, 1), job.LogTail);
|
|
var made = Assert.NotNull(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(job.Id, id)));
|
|
Assert.Count(3, await FramesAsync(fx, made));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AVolumeThatFillsDuringTheJobStopsItBeforeTheNextSequenceAndTheNextPhoto()
|
|
{
|
|
long free = 1L << 50;
|
|
await using var fx = await FixtureAsync(o => o with { FreeSpace = _ => free, Parallelism = () => 1 });
|
|
var first = await UploadedAsync(fx, 2);
|
|
var second = await UploadedAsync(fx, 2, "upload-2", seed: 200);
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, first, second);
|
|
var jobId = Guid.CreateVersion7();
|
|
var originals = await FramesAsync(fx, second);
|
|
|
|
// Another writer fills the volume while the first sequence converts: the second does not start.
|
|
var full = await Assert.ThrowsAsync<IOException>(() => RunHereAsync(fx, plan, jobId, Callbacks(done: _ => free = 1_000)));
|
|
Assert.True(full.Message.StartsWith("The conversion needs", StringComparison.Ordinal), full.Message);
|
|
Assert.True((await FramesAsync(fx, first)).All(f => f.Format == ImageFileFormat.Tiff));
|
|
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256)), (await FramesAsync(fx, second)).Select(f => (f.Id, f.Sha256)));
|
|
Assert.False(Directory.Exists(StagingOf(fx, jobId, second)));
|
|
|
|
// Space again, until the first photo of the second sequence is staged: the next one does not start, what was staged stays.
|
|
free = 1L << 50;
|
|
await Assert.ThrowsAsync<IOException>(() => RunHereAsync(fx, plan, jobId, Callbacks(progress: (n, _) =>
|
|
{
|
|
if (n == 1) free = 1_000;
|
|
})));
|
|
Assert.Count(1, Directory.EnumerateFiles(StagingOf(fx, jobId, second)));
|
|
Assert.SequenceEqual(originals.Select(f => (f.Id, f.Sha256)), (await FramesAsync(fx, second)).Select(f => (f.Id, f.Sha256)));
|
|
|
|
free = 1L << 50;
|
|
var done = await RunHereAsync(fx, plan, jobId);
|
|
Assert.True(done.Sequences.All(s => s.Outcome == ConvertOutcome.Converted));
|
|
Assert.True((await FramesAsync(fx, second)).All(f => f.Format == ImageFileFormat.Tiff));
|
|
Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, jobId));
|
|
}
|
|
|
|
[Test]
|
|
public async Task MaintenanceKeepsWhatAWaitingConversionStagedAndClearsWhatAFinishedOneLeft()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var paused = Guid.CreateVersion7();
|
|
var finished = Guid.CreateVersion7();
|
|
var forgotten = Guid.CreateVersion7();
|
|
var stray = Guid.CreateVersion7();
|
|
foreach (var (id, state) in new[] { (paused, JobState.Paused), (finished, JobState.Done) })
|
|
await fx.Db.Jobs.EnqueueAsync(new Job { Id = id, Kind = JobKind.Convert, State = state, Created = DateTimeOffset.UtcNow }, fx.Ct);
|
|
string Staged(Guid job)
|
|
{
|
|
var file = Path.Combine(StagingOf(fx, job, Guid.NewGuid()), "frame.tif");
|
|
Directory.CreateDirectory(Path.GetDirectoryName(file)!);
|
|
File.WriteAllText(file, "x");
|
|
File.SetLastWriteTimeUtc(file, DateTime.UtcNow.AddDays(-3));
|
|
return file;
|
|
}
|
|
string Journal(Guid job)
|
|
{
|
|
new ConvertJournal { JobId = job }.Write(fx.Paths);
|
|
return ConvertJournal.PathFor(fx.Paths, job);
|
|
}
|
|
var (pausedFile, pausedJournal) = (Staged(paused), Journal(paused));
|
|
var (finishedFile, finishedJournal) = (Staged(finished), Journal(finished));
|
|
var (forgottenFile, forgottenJournal) = (Staged(forgotten), Journal(forgotten));
|
|
var strayFile = Staged(stray);
|
|
|
|
var dry = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: true, 0, _ => ValueTask.CompletedTask, fx.Ct);
|
|
Assert.SequenceEqual(new[] { finishedFile, finishedJournal, strayFile }.Select(fx.Paths.ToRelative).Order(), dry.TemporaryFiles.Order());
|
|
await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
|
|
// A paused job's staged conversions stay however old; a finished job's go with its journal. A job this index does not
|
|
// know (another host's on a shared archive, or forgotten by the history) may still resume where it is known: its
|
|
// staged conversions stay as long as its journal; without a journal no resume can use them, and they go once idle.
|
|
Assert.FileExists(pausedFile);
|
|
Assert.FileExists(pausedJournal);
|
|
Assert.False(Directory.Exists(Path.Combine(Staging(fx), MassConverter.StagingFolder(finished))));
|
|
Assert.FileMissing(finishedJournal);
|
|
Assert.FileExists(forgottenFile);
|
|
Assert.FileExists(forgottenJournal);
|
|
Assert.False(Directory.Exists(Path.Combine(Staging(fx), MassConverter.StagingFolder(stray))));
|
|
|
|
// Its journal untouched for as long as the history keeps jobs, but a staged file written since: both stay.
|
|
var longAgo = DateTime.UtcNow.AddDays(-CleanupSteps.DefaultHistoryDays - 1);
|
|
File.SetLastWriteTimeUtc(forgottenJournal, longAgo);
|
|
await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
|
|
Assert.FileExists(forgottenJournal);
|
|
Assert.FileExists(forgottenFile);
|
|
// Neither written for that long: they go together.
|
|
File.SetLastWriteTimeUtc(forgottenFile, longAgo);
|
|
await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
|
|
Assert.FileMissing(forgottenJournal);
|
|
Assert.False(Directory.Exists(Path.Combine(Staging(fx), MassConverter.StagingFolder(forgotten))));
|
|
Assert.FileExists(pausedJournal);
|
|
}
|
|
|
|
[Test]
|
|
public async Task APhotoAnotherProgramHoldsIsSaidSoAndNeverConvertedWithoutAPlanThatListsIt()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
var title = (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle;
|
|
var originals = await FramesAsync(fx, id);
|
|
var held = fx.Paths.FromRelative(originals[1].RelativePath);
|
|
|
|
// A sync client holds it while the dry run is made: not missing, and not planned, with why.
|
|
ConvertPlan plan;
|
|
using (Unreadable(held)) plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
|
|
var step = plan.Sequences.Single();
|
|
Assert.Equal((2, ConvertSkip.Inaccessible, 1), (step.Frames.Count, step.Skipped.Single().Reason, step.Skipped.Single().Frames));
|
|
Assert.DoesNotContain(originals[1].Id, step.Frames.Select(f => f.FrameId));
|
|
|
|
// Free again when the job runs: it converts the two photos the plan listed, not that one.
|
|
var done = (await RunHereAsync(fx, plan, Guid.CreateVersion7())).Sequences.Single();
|
|
Assert.Equal((ConvertOutcome.Converted, 2), (done.Outcome, done.Frames));
|
|
var frames = await FramesAsync(fx, id);
|
|
Assert.Equal((originals[1].Sha256, ImageFileFormat.Jpeg), (frames[1].Sha256, frames[1].Format));
|
|
Assert.True(frames[0].Format == ImageFileFormat.Tiff && frames[2].Format == ImageFileFormat.Tiff);
|
|
|
|
// Held again while the job that converts it comes to it: left as it is, saying why, and the retry converts it once free.
|
|
var rest = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
|
|
Assert.Equal(1, rest.Frames);
|
|
var jobId = Guid.CreateVersion7();
|
|
var lines = new List<string>();
|
|
ConvertSequenceResult left;
|
|
using (Unreadable(held))
|
|
left = (await RunHereAsync(fx, rest, jobId, Callbacks() with { Log = lines.Add })).Sequences.Single();
|
|
Assert.Equal((ConvertOutcome.Blocked, Loc.L("Archive.Convert.PhotoHeld")), (left.Outcome, left.Error));
|
|
Assert.SequenceEqual([Loc.L("Archive.Log.ConvertFailed", title, Loc.L("Archive.Convert.PhotoHeld"))], lines);
|
|
Assert.Equal(ImageFileFormat.Jpeg, (await FramesAsync(fx, id))[1].Format);
|
|
Assert.Equal(ConvertOutcome.Converted, (await RunHereAsync(fx, rest, jobId)).Sequences.Single().Outcome);
|
|
Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Tiff));
|
|
}
|
|
|
|
/// <summary>
|
|
/// Only a photo the job converts makes a sequence wait for its retry while another program holds it: one the plan left
|
|
/// out being held does not hide that a photo confirmed is gone, which no retry can mend.
|
|
/// </summary>
|
|
[Test]
|
|
public async Task APhotoHeldThatTheJobDoesNotConvertDoesNotHideAChange()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
var originals = await FramesAsync(fx, id);
|
|
var held = fx.Paths.FromRelative(originals[0].RelativePath);
|
|
ConvertPlan plan;
|
|
using (Unreadable(held)) plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
|
|
Assert.Equal(2, plan.Frames);
|
|
// A photo of the plan is deleted before the job comes to the sequence, and the one left out is held again then.
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var delete = await fx.Service.FrameTrash.PlanDeleteAsync(id, [originals[1].Id], sequence.FramesVersion, fx.Ct);
|
|
await fx.Service.FrameTrash.DeleteAsync(delete, "alberto", null, fx.Ct);
|
|
|
|
ConvertSequenceResult left;
|
|
using (Unreadable(held)) left = (await RunHereAsync(fx, plan, Guid.CreateVersion7())).Sequences.Single();
|
|
Assert.Equal((ConvertOutcome.Changed, (string?)null), (left.Outcome, left.Error));
|
|
Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Jpeg));
|
|
}
|
|
|
|
/// <summary>
|
|
/// Before each photo of stage A, the space is checked for it and for every photo handed to the work before it whose file
|
|
/// has not landed yet: a converted file lands whole, by a rename, before its photo's work verifies it, so the free space
|
|
/// measured holds the files landed and none of what the photos still converting will write. Near the end of a sequence
|
|
/// those are as many as anywhere else, never fewer; photos that are not there to start are never counted.
|
|
/// </summary>
|
|
[Test]
|
|
public void APhotoStartsOnlyWithTheSpaceForItAndThePhotosStillWriting()
|
|
{
|
|
const long photo = 100_000_000;
|
|
var staging = context.CreateTempDirectory("staging");
|
|
string Target(int i) => Path.Combine(staging, $"{i:00}.tif");
|
|
// The volume: what it had, less the files landed so far (those still converting are not on it yet).
|
|
var landed = 0L;
|
|
var volume = ConvertPlanner.Required(3 * photo);
|
|
var space = new MassConverter.SpaceWatch(() => volume - landed);
|
|
void Land(int i)
|
|
{
|
|
File.WriteAllBytes(Target(i), [1]);
|
|
landed += photo;
|
|
}
|
|
|
|
// The last photos of a sequence, four converting at once: none of the three started has landed, so the fourth, the
|
|
// last one, needs the space of all four, not its own alone.
|
|
for (var i = 0; i < 3; i++) Assert.Null(space.Start(photo, Target(i)));
|
|
Assert.Equal(3 * photo, space.Writing);
|
|
var refused = Assert.NotNull(space.Start(photo, Target(3)));
|
|
Assert.True(refused.StartsWith("The conversion needs", StringComparison.Ordinal), refused);
|
|
Assert.Equal(3 * photo, space.Writing);
|
|
|
|
// One lands, and its work verifies it: the volume holds it, so the watch counts it no more (never twice). The last
|
|
// photo still waits: the two others and it need more than what is left.
|
|
Land(0);
|
|
Assert.Equal(2 * photo, space.Writing);
|
|
Assert.NotNull(space.Start(photo, Target(3)));
|
|
space.Ended(Target(0));
|
|
Land(1);
|
|
Land(2);
|
|
Assert.Equal(0, space.Writing);
|
|
space.Ended(Target(1));
|
|
space.Ended(Target(2));
|
|
// All three landed: what is left, with the margin and the reserve, cannot take a fourth; the reserve is whole.
|
|
Assert.NotNull(space.Start(photo, Target(3)));
|
|
Assert.True(volume - landed >= ConvertPlanner.SpaceReserve);
|
|
|
|
// Room for one more: it starts, and a photo whose work ends without a file (it failed) counts no more either.
|
|
volume = landed + ConvertPlanner.Required(photo);
|
|
Assert.Null(space.Start(photo, Target(3)));
|
|
Assert.NotNull(space.Start(photo, Target(4)));
|
|
space.Ended(Target(3));
|
|
Assert.Null(space.Start(photo, Target(4)));
|
|
space.Ended(Target(4));
|
|
|
|
// A file a crash left where a photo converts is not its file: the photo counts until the new one replaces it.
|
|
File.WriteAllBytes(Target(5), [1, 2, 3]);
|
|
File.SetLastWriteTimeUtc(Target(5), new DateTime(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc));
|
|
Assert.Null(space.Start(photo, Target(5)));
|
|
Assert.Equal(photo, space.Writing);
|
|
File.WriteAllBytes(Target(5), new byte[10]);
|
|
Assert.Equal(0, space.Writing);
|
|
|
|
// Files staged over their estimate scale what is still writing too.
|
|
var scaled = new MassConverter.SpaceWatch(() => ConvertPlanner.Required(3 * photo));
|
|
scaled.Staged(2 * photo, photo);
|
|
Assert.Equal(2, scaled.Ratio);
|
|
Assert.Null(scaled.Start(photo, Target(6)));
|
|
Assert.NotNull(scaled.Start(photo, Target(7)));
|
|
scaled.Ended(Target(6));
|
|
Assert.Null(scaled.Start(photo, Target(7)));
|
|
// A volume that cannot be measured stops nothing.
|
|
Assert.Null(new MassConverter.SpaceWatch(() => null).Start(long.MaxValue / 4, Target(8)));
|
|
}
|
|
|
|
/// <summary>
|
|
/// A job whose dry run had the space never stops short at the end of a sequence: the last photo to start counts only
|
|
/// the photos that started and have not written yet, none that are not there.
|
|
/// </summary>
|
|
[Test]
|
|
public async Task TheLastPhotosToStartNeedOnlyTheSpaceOfThePhotosStillWriting()
|
|
{
|
|
long free = 1L << 50;
|
|
await using var fx = await FixtureAsync(o => o with { FreeSpace = _ => free, Parallelism = () => 4 });
|
|
var id = await UploadedAsync(fx, 1);
|
|
// Exactly what the dry run asks for: four photos may convert at once, but there is one.
|
|
free = ConvertPlanner.Required(ConvertPlanner.EstimateBytes(new ConvertOptions(ConvertTarget.Tiff16), 48, 32));
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
|
|
Assert.True(plan.Space.Enough);
|
|
Assert.Equal(free, plan.Space.Required);
|
|
|
|
var done = (await RunHereAsync(fx, plan, Guid.CreateVersion7())).Sequences.Single();
|
|
Assert.Equal((ConvertOutcome.Converted, 1), (done.Outcome, done.Frames));
|
|
}
|
|
|
|
/// <summary>
|
|
/// A sequence that fits converts to its last photo, four at once, while its converted files take the volume as they land:
|
|
/// at the end the photos still writing are counted, those landed (verified meanwhile) are in the free space measured, and
|
|
/// none is counted twice.
|
|
/// </summary>
|
|
[Test]
|
|
public async Task ASequenceThatFitsConvertsToItsLastPhotoWhileItsFilesFillTheVolume()
|
|
{
|
|
long volume = 1L << 50;
|
|
string? staging = null;
|
|
// The volume as the converted files land on it; an atomic write's temporary file is left out, being there only while
|
|
// its photo still counts.
|
|
long Landed() => staging is not null && Directory.Exists(staging)
|
|
? Directory.EnumerateFiles(staging).Where(f => !Path.GetFileName(f).StartsWith('.')).Sum(f => new FileInfo(f).Length)
|
|
: 0;
|
|
await using var fx = await FixtureAsync(o => o with { FreeSpace = _ => volume - Landed(), Parallelism = () => 4 });
|
|
var id = await UploadedAsync(fx, 12);
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
|
|
// What the twelve converted files take, each converted beforehand as the job converts it.
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var folder = fx.Paths.Sequence(sequence.Folder);
|
|
var manifest = Sidecar(fx, sequence);
|
|
var sizes = context.CreateTempDirectory("sizes");
|
|
var converted = manifest.Frames.Sum(f => FrameConverter.Convert(new ConvertSource(folder.File(f.File), f, manifest.Camera, manifest.Lens),
|
|
new ConvertOptions(ConvertTarget.Tiff16), DecoderFactory.CreateServer(), Path.Combine(sizes, f.Id.ToString("N") + ".tif"), cancellationToken: fx.Ct).Size);
|
|
// Just the space they need with the margin and the reserve, and never less than the dry run asks.
|
|
volume = ConvertPlanner.Required(Math.Max(converted, plan.BytesAfter));
|
|
var jobId = Guid.CreateVersion7();
|
|
staging = StagingOf(fx, jobId, id);
|
|
|
|
var done = (await RunHereAsync(fx, plan, jobId)).Sequences.Single();
|
|
Assert.Equal((ConvertOutcome.Converted, 12), (done.Outcome, done.Frames));
|
|
Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Tiff));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ASecondConversionOfTheSameSequencesIsRefusedWithOrWithoutADryRun()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id);
|
|
async Task Enqueue(ArchiveJobRequest request)
|
|
{
|
|
// Long enough for the other confirmation to check meanwhile, were it not held back.
|
|
await Task.Delay(50, fx.Ct);
|
|
await fx.Db.Jobs.EnqueueAsync(new Job
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
Kind = request.Kind,
|
|
TargetIds = request.Targets,
|
|
ParametersJson = request.ParametersJson,
|
|
State = JobState.Queued,
|
|
Created = DateTimeOffset.UtcNow,
|
|
}, fx.Ct);
|
|
}
|
|
async Task<bool> ConfirmAsync()
|
|
{
|
|
try
|
|
{
|
|
await fx.Service.Converter.QueueAsync(fx.Archive, plan, Enqueue, fx.Ct);
|
|
return true;
|
|
}
|
|
catch (ConversionQueuedException ex)
|
|
{
|
|
Assert.SequenceEqual([id], ex.SequenceIds);
|
|
Assert.Equal(Loc.L("Archive.Convert.AlreadyQueued"), ex.Message);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
// The Library's dialog and the Archive manager's confirm the same plan at once: one job.
|
|
var confirmed = await Task.WhenAll(ConfirmAsync(), ConfirmAsync());
|
|
Assert.SequenceEqual([false, true], confirmed.Order());
|
|
// And again later from that plan, made before the job was queued and never planned again.
|
|
Assert.False(await ConfirmAsync());
|
|
Assert.Count(1, (await fx.Db.Jobs.ListAsync(JobState.Queued, JobKind.Convert)).Items);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AConversionQueuedFirstHoldsItsSequencesFromADuplicateQueuedAfterIt()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id);
|
|
// Queued twice anyway (an earlier build, two processes on one index): the copies are made once.
|
|
var first = await QueuedAsync(fx, plan, DateTimeOffset.UtcNow.AddMinutes(-2));
|
|
var second = await QueuedAsync(fx, plan, DateTimeOffset.UtcNow.AddMinutes(-1));
|
|
|
|
var held = (await fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), second.Id, new ConvertRequester(null, null, second.Created), Callbacks(), fx.Ct))
|
|
.Sequences.Single();
|
|
Assert.Equal(ConvertOutcome.Blocked, held.Outcome);
|
|
Assert.Null(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(second.Id, id)));
|
|
// The one queued first is not held back by the one after it.
|
|
var made = (await fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), first.Id, new ConvertRequester(null, null, first.Created), Callbacks(), fx.Ct))
|
|
.Sequences.Single();
|
|
Assert.Equal((ConvertOutcome.Created, 2), (made.Outcome, made.Frames));
|
|
await fx.Db.Jobs.SetStateAsync(first.Id, JobState.Done, null, fx.Ct);
|
|
// The duplicate, tried again once the first is done, finds the copy made since it was queued: it makes no second one.
|
|
var again = (await fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), second.Id, new ConvertRequester(null, null, second.Created), Callbacks(), fx.Ct))
|
|
.Sequences.Single();
|
|
Assert.Equal(ConvertOutcome.Superseded, again.Outcome);
|
|
Assert.Null(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(second.Id, id)));
|
|
}
|
|
|
|
/// <summary>
|
|
/// A job's row names no photo, so it stays as small as the number of sequences however many photos convert: the photos
|
|
/// confirmed are written whole beside the journal, under the job's id, before it is queued, and go with the journal.
|
|
/// </summary>
|
|
[Test]
|
|
public async Task TheJobRowNamesNoPhotoAndThePhotosConfirmedWaitBesideItsJournal()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, id);
|
|
var frames = plan.Sequences.Single().Frames.Select(f => f.FrameId).ToList();
|
|
// Queued and not run yet: an engine that is not started.
|
|
await using var idle = new JobEngine(fx.Db, []);
|
|
Job? job = null;
|
|
await fx.Service.Converter.QueueAsync(fx.Archive, plan, async r =>
|
|
{
|
|
Assert.FileExists(ConfirmedPhotos.PathFor(fx.Paths, r.Id!.Value));
|
|
job = await idle.EnqueueAsync(r, cancellationToken: fx.Ct);
|
|
}, fx.Ct);
|
|
var queued = Assert.NotNull(await fx.Db.Jobs.GetAsync(job!.Id, fx.Ct));
|
|
Assert.SequenceEqual(frames, ConfirmedPhotos.Read(fx.Paths, queued.Id).Of(id)!);
|
|
Assert.Null(ConfirmedPhotos.Read(fx.Paths, queued.Id).Of(Guid.NewGuid()));
|
|
foreach (var frame in frames)
|
|
{
|
|
Assert.False(queued.ParametersJson!.Contains(frame.ToString(), StringComparison.OrdinalIgnoreCase), queued.ParametersJson);
|
|
Assert.False(queued.ParametersJson!.Contains(ConfirmedPhotos.Encode([frame])[..20], StringComparison.Ordinal), queued.ParametersJson);
|
|
}
|
|
Assert.Equal(plan.Sequences.Single().Digest(), Parameters(fx, plan).Sequences.Single().Frames);
|
|
// Lists of jobs read none of their parameters.
|
|
Assert.Null((await fx.Db.Jobs.ListBriefAsync(JobState.Queued, JobKind.Convert, 10, null, fx.Ct)).Items.Single().ParametersJson);
|
|
|
|
// A confirmation the queue refuses leaves nothing behind.
|
|
var other = await UploadedAsync(fx, 1, "upload-2", seed: 300);
|
|
var refused = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, other);
|
|
Guid? named = null;
|
|
await Assert.ThrowsAsync<InvalidOperationException>(() => fx.Service.Converter.QueueAsync(fx.Archive, refused, r =>
|
|
{
|
|
named = r.Id;
|
|
throw new InvalidOperationException("the index is away");
|
|
}, fx.Ct));
|
|
Assert.FileMissing(ConfirmedPhotos.PathFor(fx.Paths, named!.Value));
|
|
|
|
// The job runs, and once done with every sequence, its photos go with its journal.
|
|
var engine = await fx.StartEngineAsync();
|
|
engine.Wake();
|
|
await fx.WaitForAsync(queued.Id, JobState.Done, (int)TestTiming.Budget(60));
|
|
Assert.FileMissing(ConfirmedPhotos.PathFor(fx.Paths, queued.Id));
|
|
Assert.True((await FramesAsync(fx, id)).All(f => f.Format == ImageFileFormat.Tiff));
|
|
}
|
|
|
|
[Test]
|
|
public void ThePhotosConfirmedReadBackAsWrittenAndALineThatCannotBeReadNamesNothing()
|
|
{
|
|
Guid[] ids = [Guid.CreateVersion7(), Guid.CreateVersion7(), Guid.NewGuid()];
|
|
Assert.SequenceEqual(ids, ConfirmedPhotos.Decode(ConfirmedPhotos.Encode(ids)));
|
|
Assert.Empty(ConfirmedPhotos.Decode(ConfirmedPhotos.Encode([])));
|
|
Assert.Throws<FormatException>(() => ConfirmedPhotos.Decode(Convert.ToBase64String(new byte[15])));
|
|
|
|
var paths = new ArchivePaths(context.CreateTempDirectory("archive"));
|
|
var job = Guid.CreateVersion7();
|
|
var (good, bad) = (Guid.NewGuid(), Guid.NewGuid());
|
|
Directory.CreateDirectory(paths.JournalDirectory);
|
|
File.WriteAllLines(ConfirmedPhotos.PathFor(paths, job),
|
|
[
|
|
JsonSerializer.Serialize(new ConfirmedLine(good, ConfirmedPhotos.Encode(ids)), ArchiveJobJson.Default.ConfirmedLine),
|
|
JsonSerializer.Serialize(new ConfirmedLine(bad, "not ids"), ArchiveJobJson.Default.ConfirmedLine),
|
|
"{\"sequence\":",
|
|
]);
|
|
var read = ConfirmedPhotos.Read(paths, job);
|
|
Assert.SequenceEqual(ids, read.Of(good)!);
|
|
Assert.Null(read.Of(bad));
|
|
// None at all where nothing was written: every sequence is then held to its whole plan.
|
|
Assert.Null(ConfirmedPhotos.Read(paths, Guid.CreateVersion7()).Of(good));
|
|
}
|
|
|
|
/// <summary>
|
|
/// Without the photos confirmed (they could not be written when the job was queued, or an earlier build queued it), a
|
|
/// sequence converts only while its whole plan made again is the one confirmed: an upload since leaves it as it is.
|
|
/// </summary>
|
|
[Test]
|
|
public async Task WithoutThePhotosConfirmedASequenceIsHeldToItsWholePlan()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var still = await UploadedAsync(fx, 2);
|
|
var grown = await UploadedAsync(fx, 2, "upload-2", seed: 300);
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.Replace, still, grown);
|
|
var (path, size, sha) = fx.Stage(Start.AddSeconds(60), 999);
|
|
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, grown, null, null, path, "IMG_0099.JPG", size, sha, null, null), fx.Ct);
|
|
var before = await FramesAsync(fx, grown);
|
|
|
|
var result = await fx.Service.Converter.RunAsync(fx.Archive, Parameters(fx, plan), Guid.CreateVersion7(), new ConvertRequester(null, null), Callbacks(), fx.Ct);
|
|
Assert.SequenceEqual([ConvertOutcome.Converted, ConvertOutcome.Changed], result.Sequences.Select(s => s.Outcome));
|
|
Assert.SequenceEqual(before.Select(f => (f.Id, f.Sha256)), (await FramesAsync(fx, grown)).Select(f => (f.Id, f.Sha256)));
|
|
}
|
|
|
|
/// <summary>
|
|
/// A New sequence job that stopped once its copy held the photos, before its journal said the sequence was done (a crash,
|
|
/// a stop): retried after the source changed, it finishes that copy and makes no other; the copy is reported as made.
|
|
/// </summary>
|
|
[Test]
|
|
public async Task ARetryAfterACrashOnceTheCopyWasFiledFinishesThatCopyAndMakesNoOther()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 3);
|
|
await fx.Service.SetSequenceInfoAsync(id, "Roof", ["sunset"], 4, "From the roof", fx.Ct);
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id);
|
|
var jobId = Guid.CreateVersion7();
|
|
// What the attempt that stopped there left: the photos confirmed, pinned and staged, and the copy filed, the journal
|
|
// not saying more (it is written once the copy is finished).
|
|
var left = new Dictionary<string, byte[]>();
|
|
var callbacks = Callbacks(done: _ =>
|
|
{
|
|
foreach (var file in new[] { ConfirmedPhotos.PathFor(fx.Paths, jobId), ConvertJournal.PinsPathFor(fx.Paths, jobId), ConvertJournal.StagedPathFor(fx.Paths, jobId) })
|
|
left[file] = File.ReadAllBytes(file);
|
|
});
|
|
var first = (await RunHereAsync(fx, plan, jobId, callbacks)).Sequences.Single();
|
|
Assert.Equal(ConvertOutcome.Created, first.Outcome);
|
|
var copy = first.NewSequenceId!.Value;
|
|
await fx.Db.Sequences.EditAsync(copy, new SequenceEdit { Tags = ["other"], Rating = 1, Notes = "Scrambled" }, null, fx.Ct);
|
|
foreach (var (file, bytes) in left) File.WriteAllBytes(file, bytes);
|
|
// The source changed since: a photo of it deleted.
|
|
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
|
var delete = await fx.Service.FrameTrash.PlanDeleteAsync(id, [(await FramesAsync(fx, id))[1].Id], sequence.FramesVersion, fx.Ct);
|
|
await fx.Service.FrameTrash.DeleteAsync(delete, "alberto", null, fx.Ct);
|
|
var sequences = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count;
|
|
|
|
var retried = (await RunHereAsync(fx, plan, jobId)).Sequences.Single();
|
|
Assert.Equal((ConvertOutcome.Created, 3, (Guid?)copy, 0), (retried.Outcome, retried.Frames, retried.NewSequenceId, retried.LeftOut));
|
|
Assert.Equal(sequences, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count);
|
|
Assert.Count(3, await FramesAsync(fx, copy));
|
|
// Finished as stage B finishes a copy: the source's tags, rating and notes.
|
|
var finished = (await fx.Db.Sequences.GetAsync(copy))!;
|
|
Assert.Equal((4, "From the roof"), (finished.Rating, finished.Notes));
|
|
Assert.SequenceEqual(["sunset"], finished.Tags);
|
|
Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, jobId));
|
|
|
|
// Its journal names the copy as soon as the copy holds its photos: a copy deleted since is not made again.
|
|
var again = Guid.CreateVersion7();
|
|
var journal = new ConvertJournal { JobId = again };
|
|
journal.Made(id, Guid.CreateVersion7());
|
|
journal.Write(fx.Paths);
|
|
var gone = (await RunHereAsync(fx, plan, again)).Sequences.Single();
|
|
Assert.Equal(ConvertOutcome.Changed, gone.Outcome);
|
|
Assert.Null(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(again, id)));
|
|
Assert.Equal(sequences, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count);
|
|
}
|
|
|
|
/// <summary>The photos a New sequence job filed into its copy before it stopped take no space again: its retry finishes on a full volume.</summary>
|
|
[Test]
|
|
public async Task ARetryOnceTheCopyWasFiledNeedsNoSpaceForItsPhotosAgain()
|
|
{
|
|
long free = 1L << 50;
|
|
await using var fx = await FixtureAsync(o => o with { FreeSpace = _ => free });
|
|
var id = await UploadedAsync(fx, 2);
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id);
|
|
var jobId = Guid.CreateVersion7();
|
|
var left = new Dictionary<string, byte[]>();
|
|
var callbacks = Callbacks(done: _ =>
|
|
{
|
|
foreach (var file in new[] { ConfirmedPhotos.PathFor(fx.Paths, jobId), ConvertJournal.PinsPathFor(fx.Paths, jobId), ConvertJournal.StagedPathFor(fx.Paths, jobId) })
|
|
left[file] = File.ReadAllBytes(file);
|
|
});
|
|
var copy = (await RunHereAsync(fx, plan, jobId, callbacks)).Sequences.Single().NewSequenceId;
|
|
foreach (var (file, bytes) in left) File.WriteAllBytes(file, bytes);
|
|
free = 1_000;
|
|
|
|
var retried = (await RunHereAsync(fx, plan, jobId)).Sequences.Single();
|
|
Assert.Equal((ConvertOutcome.Created, copy), (retried.Outcome, retried.NewSequenceId));
|
|
Assert.Count(2, await FramesAsync(fx, copy!.Value));
|
|
}
|
|
|
|
/// <summary>
|
|
/// A conversion into new sequences retried after another conversion, confirmed later, copied the same source in the same
|
|
/// format: it makes no second copy, and is done with that sequence.
|
|
/// </summary>
|
|
[Test]
|
|
public async Task ARetriedConversionMakesNoSecondCopyOfASourceAnotherConversionCopiedSince()
|
|
{
|
|
await using var fx = await FixtureAsync();
|
|
var id = await UploadedAsync(fx, 2);
|
|
var title = (await fx.Db.Sequences.GetAsync(id))!.DisplayTitle;
|
|
var plan = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id);
|
|
// Job A finds the sequence busy (a render works on it), and fails.
|
|
var a = await QueuedAsync(fx, plan, DateTimeOffset.UtcNow.AddMinutes(-10));
|
|
var render = new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Render, TargetIds = [id], State = JobState.Running, Created = DateTimeOffset.UtcNow };
|
|
await fx.Db.Jobs.EnqueueAsync(render, fx.Ct);
|
|
Assert.Equal(ConvertOutcome.Blocked, (await RunHereAsync(fx, plan, a.Id, confirmed: a.Created)).Sequences.Single().Outcome);
|
|
await fx.Db.Jobs.SetStateAsync(render.Id, JobState.Done, null, fx.Ct);
|
|
await fx.Db.Jobs.SetStateAsync(a.Id, JobState.Failed, "kept", fx.Ct);
|
|
// The user plans again: job B makes the copy. A JPEG copy made meanwhile is another format, which does not count.
|
|
var again = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id);
|
|
var b = await QueuedAsync(fx, again, DateTimeOffset.UtcNow.AddMinutes(-5));
|
|
var made = (await RunHereAsync(fx, again, b.Id, confirmed: b.Created)).Sequences.Single();
|
|
Assert.Equal(ConvertOutcome.Created, made.Outcome);
|
|
await fx.Db.Jobs.SetStateAsync(b.Id, JobState.Done, null, fx.Ct);
|
|
var sequences = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count;
|
|
|
|
// Retry on A: done with the sequence, no second copy.
|
|
var lines = new List<string>();
|
|
var retried = (await RunHereAsync(fx, plan, a.Id, Callbacks() with { Log = lines.Add }, a.Created)).Sequences.Single();
|
|
Assert.Equal((ConvertOutcome.Superseded, true), (retried.Outcome, retried.Final));
|
|
Assert.SequenceEqual([Loc.L("Archive.Log.ConvertSuperseded", title)], lines);
|
|
Assert.Null(await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, MassConverter.SessionKey(a.Id, id)));
|
|
Assert.Equal(sequences, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count);
|
|
Assert.FileMissing(ConvertJournal.PathFor(fx.Paths, a.Id));
|
|
|
|
// A conversion confirmed after that copy was made is the user asking again: it makes its copy.
|
|
var latest = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, id);
|
|
var later = await QueuedAsync(fx, latest, DateTimeOffset.UtcNow);
|
|
Assert.Equal(ConvertOutcome.Created, (await RunHereAsync(fx, latest, later.Id, confirmed: later.Created)).Sequences.Single().Outcome);
|
|
|
|
// A copy in another format is no copy of this one: a TIFF conversion confirmed before a DNG copy was made makes its own.
|
|
var other = await UploadedAsync(fx, 2, "upload-2", seed: 300);
|
|
var tiff = await PlanAsync(fx, ConvertTarget.Tiff16, ConvertMode.NewSequence, other);
|
|
var confirmedFirst = DateTimeOffset.UtcNow.AddMinutes(-1);
|
|
var dng = await PlanAsync(fx, ConvertTarget.LinearDng, ConvertMode.NewSequence, other);
|
|
Assert.Equal(ConvertOutcome.Created, (await RunHereAsync(fx, dng, Guid.CreateVersion7(), confirmed: DateTimeOffset.UtcNow)).Sequences.Single().Outcome);
|
|
Assert.Equal(ConvertOutcome.Created, (await RunHereAsync(fx, tiff, Guid.CreateVersion7(), confirmed: confirmedFirst)).Sequences.Single().Outcome);
|
|
}
|
|
}
|