Stage A's space guard reserved, before each photo, only that photo and the ones after it, so at the end of a sequence it counted a single photo while up to twice the parallelism were still converting: a volume filling meanwhile could go into the reserve. It now reserves the photo and every photo started whose converted file has not landed yet (a file lands whole, by a rename, before it is verified), so nothing in flight is missed and nothing landed is counted twice, at the end of a sequence as anywhere. The confirmed photo ids (about 21 MB for a million photos) lived in the job's parameters, read by every dry run's busy check, by each sequence turn of the conversion and by the job lists. They are now written whole, under a job id chosen before the job is queued, into convert-<job>.photos beside the journal, and go with it; the row keeps each sequence's digest, so the job still converts exactly the photos confirmed (without the file, the whole plan made again must match). The busy check reads only ids, kinds and targets (and a verify's parameters), never the asking job's row; a conversion's turn reads only its creation time; the server's job list and dashboard and the engine's queue peeks read jobs without parameters. A retried New sequence conversion could make a second copy. The journal now names the copy as soon as its photos are filed; a retry finishes that copy whatever changed in the source since, never remakes one deleted since, and needs no space again for what it holds; and a source another conversion copied in the same format after this one was confirmed is done with, with no copy made (new outcome Superseded, with its job log line). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
105 lines
6.4 KiB
C#
105 lines
6.4 KiB
C#
using System.Text.Json;
|
||
using System.Text.Json.Serialization;
|
||
using Chronos.Core.Conversion;
|
||
using Chronos.Core.Import;
|
||
using Chronos.Core.Model;
|
||
|
||
namespace Chronos.Data.Archiving.Jobs;
|
||
|
||
/// <summary><see cref="JobKind.Scan"/>: target = the archive.</summary>
|
||
/// <param name="DeepAnalysis">The sequences the scan brings into the index (or whose frames it changes) also get a deep
|
||
/// analysis in the background, after their light one (Settings › Analysis on the desktop).</param>
|
||
public sealed record ScanParameters(bool Full, bool DeepAnalysis = false);
|
||
|
||
/// <summary><see cref="JobKind.Verify"/>: target = the archive; null sequences = all of them.</summary>
|
||
public sealed record VerifyParameters(IReadOnlyList<Guid>? SequenceIds);
|
||
|
||
/// <summary><see cref="JobKind.Thumbs"/> and <see cref="JobKind.Proxies"/>: targets = sequences.</summary>
|
||
public sealed record ImageJobParameters(bool Force);
|
||
|
||
/// <summary><see cref="JobKind.Maintenance"/>: target = the archive.</summary>
|
||
public sealed record MaintenanceParameters(bool DryRun);
|
||
|
||
/// <summary><see cref="JobKind.Import"/>: the exact plan the user confirmed.</summary>
|
||
public sealed record ImportJobParameters(Guid ArchiveId, ImportPlan Plan, Guid? CardId, bool Verify, Guid? UserId);
|
||
|
||
/// <summary><see cref="JobKind.Backup"/>: no target; the newest <paramref name="Keep"/> copies are kept.</summary>
|
||
public sealed record BackupParameters(int Keep);
|
||
|
||
/// <summary>
|
||
/// <see cref="JobKind.Cleanup"/>: target = the archive (or none); the clean-up steps to run, by name and in order, and
|
||
/// the age in days from which what they clean goes (0 keeps everything).
|
||
/// </summary>
|
||
public sealed record CleanupParameters(IReadOnlyList<string> Steps, int OlderThanDays);
|
||
|
||
/// <summary>
|
||
/// <see cref="JobKind.EmptyTrash"/>: target = the archive; the entries of the confirmed plan, each with its own fingerprint
|
||
/// (<see cref="SequenceTrash.EntryFingerprint"/>), and the plan's fingerprint (<see cref="TrashEmptyPlan.Fingerprint"/>).
|
||
/// </summary>
|
||
public sealed record EmptyTrashParameters(IReadOnlyList<EmptyTrashItem> Entries, string Fingerprint);
|
||
|
||
/// <summary>One entry of a confirmed emptying: its name and fingerprint when it was planned.</summary>
|
||
public sealed record EmptyTrashItem(string Name, string Fingerprint);
|
||
|
||
/// <summary>
|
||
/// <see cref="JobKind.Move"/> (Reorganize, owner review 2, R7): target = the archive; the templates (null keeps that part),
|
||
/// the sequences the confirmed plan changes in the order it was made for, and the plan's fingerprint
|
||
/// (<see cref="Core.Archiving.RenamePlan.Fingerprint"/>, which pins every file rename by its digest). No file list: the job
|
||
/// plans again and refuses when the plan is no longer the one confirmed.
|
||
/// </summary>
|
||
public sealed record MoveJobParameters(string? FolderTemplate, string? FileTemplate, string? TitleTemplate, IReadOnlyList<MoveSequence> Sequences, string Fingerprint);
|
||
|
||
/// <summary>A sequence of a <see cref="MoveJobParameters"/> (a job queued by an earlier build also names a frames version, which is ignored).</summary>
|
||
public sealed record MoveSequence(Guid Id);
|
||
|
||
/// <summary>
|
||
/// <see cref="JobKind.Convert"/> (owner review 2, R7): targets = the sequences converted, so the jobs that would touch them
|
||
/// wait; the archive, the target format and JPEG quality, the mode chosen for this run, the sequences of the confirmed plan
|
||
/// in its order, each with what it converts, and the plan's fingerprint (<see cref="ConvertPlan.Fingerprint"/>). A sequence
|
||
/// whose photos to convert are no longer those confirmed is left as it is and reported. Which photos those are is not
|
||
/// here but in the job's own file beside its journal (<see cref="ConfirmedPhotos"/>), so that the row stays as small as the
|
||
/// number of sequences, whatever the number of photos, for everything that reads jobs.
|
||
/// </summary>
|
||
public sealed record ConvertJobParameters(Guid ArchiveId, ConvertTarget Target, int Quality, ConvertMode Mode, IReadOnlyList<ConvertSequence> Sequences,
|
||
string Fingerprint);
|
||
|
||
/// <summary>
|
||
/// A sequence of a <see cref="ConvertJobParameters"/>: the frames version it was planned at, and the digest of the photos it
|
||
/// converts, each with its content (<see cref="ConvertSequencePlan.Digest"/>). Under the sequence's lease, before it
|
||
/// converts anything, the job takes the photos confirmed (<see cref="ConfirmedPhotos"/>) from its plan made again and goes
|
||
/// on only while they have that digest: photos that came in since (an upload while the job waited its turn) stay as they
|
||
/// are, and a photo the plan did not list (one another program held while it was made) is never converted. Without the
|
||
/// confirmed photos, the whole plan made again must have the digest. Null in a job an earlier build queued, which checks
|
||
/// the frames version only.
|
||
/// </summary>
|
||
public sealed record ConvertSequence(Guid Id, int FramesVersion, string? Frames = null);
|
||
|
||
/// <summary>
|
||
/// A job ready to enqueue; <paramref name="Id"/>, when set, is the id it is queued under (chosen first, for what the job
|
||
/// reads beside its journal: <see cref="ConfirmedPhotos"/>), otherwise the engine gives it one.
|
||
/// </summary>
|
||
public sealed record ArchiveJobRequest(JobKind Kind, IReadOnlyList<Guid> Targets, string? Label, string? ParametersJson, JobPriority Priority, Guid? Id = null);
|
||
|
||
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, UseStringEnumConverter = true)]
|
||
[JsonSerializable(typeof(ScanParameters))]
|
||
[JsonSerializable(typeof(VerifyParameters))]
|
||
[JsonSerializable(typeof(ImageJobParameters))]
|
||
[JsonSerializable(typeof(MaintenanceParameters))]
|
||
[JsonSerializable(typeof(ImportJobParameters))]
|
||
[JsonSerializable(typeof(BackupParameters))]
|
||
[JsonSerializable(typeof(CleanupParameters))]
|
||
[JsonSerializable(typeof(EmptyTrashParameters))]
|
||
[JsonSerializable(typeof(MoveJobParameters))]
|
||
[JsonSerializable(typeof(ConvertJobParameters))]
|
||
[JsonSerializable(typeof(ConvertJournal))]
|
||
[JsonSerializable(typeof(StagedLine))]
|
||
[JsonSerializable(typeof(PinnedSequence))]
|
||
[JsonSerializable(typeof(ConfirmedLine))]
|
||
[JsonSerializable(typeof(CardProfile))]
|
||
internal sealed partial class ArchiveJobJson : JsonSerializerContext
|
||
{
|
||
/// <summary>Parameters of a job, or <paramref name="fallback"/> when it has none.</summary>
|
||
public static T Read<T>(Job job, System.Text.Json.Serialization.Metadata.JsonTypeInfo<T> typeInfo, T fallback) =>
|
||
job.ParametersJson is { Length: > 0 } json ? JsonSerializer.Deserialize(json, typeInfo) ?? fallback : fallback;
|
||
}
|