fix(convert): reserve the photos still writing, keep photo ids out of the job row, never copy a source twice
Stage A's space guard reserved, before each photo, only that photo and the ones after it, so at the end of a sequence it counted a single photo while up to twice the parallelism were still converting: a volume filling meanwhile could go into the reserve. It now reserves the photo and every photo started whose converted file has not landed yet (a file lands whole, by a rename, before it is verified), so nothing in flight is missed and nothing landed is counted twice, at the end of a sequence as anywhere. The confirmed photo ids (about 21 MB for a million photos) lived in the job's parameters, read by every dry run's busy check, by each sequence turn of the conversion and by the job lists. They are now written whole, under a job id chosen before the job is queued, into convert-<job>.photos beside the journal, and go with it; the row keeps each sequence's digest, so the job still converts exactly the photos confirmed (without the file, the whole plan made again must match). The busy check reads only ids, kinds and targets (and a verify's parameters), never the asking job's row; a conversion's turn reads only its creation time; the server's job list and dashboard and the engine's queue peeks read jobs without parameters. A retried New sequence conversion could make a second copy. The journal now names the copy as soon as its photos are filed; a retry finishes that copy whatever changed in the source since, never remakes one deleted since, and needs no space again for what it holds; and a source another conversion copied in the same format after this one was confirmed is done with, with no copy made (new outcome Superseded, with its job log line). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b471a33bfc
commit
8ea25228eb
14 files changed
+966
-201
No files matched your search
@@ -103,6 +103,56 @@ public abstract class JobRepositoryTests(TestContext context) : RepositoryTest(c
|
||||
return job;
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task HoldsNameTheTargetsOfRunningAndPausedJobsAndOnlyAVerifysParameters()
|
||||
{
|
||||
// What a dry run counts as busy, read at every one and at every sequence a conversion comes to: never the parameters
|
||||
// a conversion or an import keeps, nor the row of the job asking.
|
||||
var db = await OpenChronosAsync();
|
||||
Guid a = Guid.CreateVersion7(), b = Guid.CreateVersion7(), archive = Guid.CreateVersion7();
|
||||
var convert = await EnqueueAsync(db, TestData.Job(JobKind.Convert) with { TargetIds = [a, b], ParametersJson = "{\"fingerprint\": \"f\"}", State = JobState.Running });
|
||||
var verify = await EnqueueAsync(db, TestData.Job(JobKind.Verify) with { TargetIds = [archive], ParametersJson = "{\"sequenceIds\": []}", State = JobState.Paused });
|
||||
await EnqueueAsync(db, TestData.Job(JobKind.Thumbs) with { TargetIds = [a] });
|
||||
await EnqueueAsync(db, TestData.Job(JobKind.Scan) with { TargetIds = [archive], State = JobState.Done, Finished = TestData.Now });
|
||||
|
||||
var holds = await db.Jobs.ListHoldsAsync(cancellationToken: Ct);
|
||||
Assert.SequenceEqual(new[] { convert.Id, verify.Id }.Order(), holds.Select(h => h.Id).Order());
|
||||
var running = holds.Single(h => h.Id == convert.Id);
|
||||
Assert.Equal((JobKind.Convert, (string?)null), (running.Kind, running.ParametersJson));
|
||||
Assert.SequenceEqual([a, b], running.TargetIds);
|
||||
var paused = holds.Single(h => h.Id == verify.Id);
|
||||
Assert.Equal((JobKind.Verify, verify.ParametersJson), (paused.Kind, paused.ParametersJson));
|
||||
Assert.SequenceEqual([verify.Id], (await db.Jobs.ListHoldsAsync(convert.Id, Ct)).Select(h => h.Id));
|
||||
|
||||
Assert.Equal(convert.Created, await db.Jobs.GetCreatedAsync(convert.Id, Ct));
|
||||
Assert.Null(await db.Jobs.GetCreatedAsync(Guid.CreateVersion7(), Ct));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task BriefListsReadEveryFieldButTheParameters()
|
||||
{
|
||||
var db = await OpenChronosAsync();
|
||||
var job = TestData.Job(JobKind.Convert, JobPriority.User) with
|
||||
{
|
||||
TargetIds = [Guid.CreateVersion7(), Guid.CreateVersion7()],
|
||||
TargetLabel = "Harbour sunset",
|
||||
ParametersJson = "{\"fingerprint\": \"f\"}",
|
||||
LogTail = "line 1",
|
||||
CreatedBy = Guid.CreateVersion7(),
|
||||
ScheduleId = Guid.CreateVersion7(),
|
||||
};
|
||||
await db.Jobs.EnqueueAsync(job, Ct);
|
||||
|
||||
var brief = Assert.NotNull((await db.Jobs.ListBriefAsync(JobState.Queued, JobKind.Convert, 10, null, Ct)).Items.SingleOrDefault());
|
||||
Assert.SequenceEqual(job.TargetIds, brief.TargetIds);
|
||||
Assert.Equal(job with { TargetIds = brief.TargetIds, ParametersJson = null }, brief);
|
||||
Assert.Equal(job.ParametersJson, (await db.Jobs.ListAsync(JobState.Queued, JobKind.Convert, 10, null, Ct)).Items.Single().ParametersJson);
|
||||
Assert.Empty((await db.Jobs.ListBriefAsync(JobState.Running, null, 10, null, Ct)).Items);
|
||||
var next = Assert.NotNull((await db.Jobs.ListQueuedBriefAsync([JobKind.Convert], 10, Chronos.Data.Jobs.JobPriorityQueue.Off, Ct)).SingleOrDefault());
|
||||
Assert.Equal((job.Id, (string?)null), (next.Id, next.ParametersJson));
|
||||
Assert.Empty(await db.Jobs.ListQueuedBriefAsync([], 10, Chronos.Data.Jobs.JobPriorityQueue.Off, Ct));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ThePriorityQueueClaimsTheHigherKindsFirst()
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user