feat: owner review 2 phase 4b: mass conversion after import (#13)
CI / Build and test (push) Failing after 3m1s
CI / PostgreSQL 16 integration (push) Skipped
CI / PostgreSQL 17 integration (push) Skipped
CI / Docker image (push) Skipped
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 10s
Release / Build, test and publish (push) Successful in 15m31s
Release / Server image (push) Successful in 1m13s

Convert photos already in the archive to JPEG, TIFF-16 or Linear DNG, from the panel's Library and the desktop's Archive and Library. Each run asks Replace (same frames, originals to the trash, recoverable) or New sequence. A dry run lists what converts, what is skipped and why, size before and after, and free space; the job converts exactly the confirmed photos, resumes, and commits all or nothing per sequence. A conversion never changes what Chronos reads from a photo. Replaced originals are erased only on the archive's own word that their conversion is gone. Desktop TIFFs that WIC showed black or as noise now go to the managed reader; the Edit picker no longer spins a core.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
This commit was merged in pull request #13.
This commit is contained in:
Alby96andClaude Opus 5.5 authored and Alby96 committed 2026-11-03 10:41:50 +01:00
1 parent 7e4a3b481d
commit efa87f68ce
354 files changed
+24530 -843

No files matched your search

+74 -7
View File
@@ -7,6 +7,12 @@ namespace Chronos.Data.Repositories;
/// <summary>Jobs per state; <see cref="DoneSince"/> and <see cref="FailedSince"/> count those finished since a given time.</summary>
public sealed record JobCounts(int Queued, int Running, int Paused, int DoneSince, int FailedSince);
/// <summary>
/// What a running or paused job works on (<see cref="JobRepository.ListHoldsAsync"/>): its targets and, for a verify, its
/// parameters, which name the sequences it holds; null for every other kind, whose parameters are not read.
/// </summary>
public sealed record JobHold(Guid Id, JobKind Kind, IReadOnlyList<Guid> TargetIds, string? ParametersJson);
/// <summary>
/// The persistent job queue (BRIEF §14.6): workers claim the oldest job of the highest priority atomically,
/// write checkpoints while running, and on startup requeue what a crash left running.
@@ -17,6 +23,14 @@ public sealed class JobRepository
"id, kind, target_ids, target_label, parameters_json, state, priority, progress_done, progress_total, items_per_second, " +
"eta_seconds, checkpoint, log_tail, created_by, node_id, attempts, error, created, started, finished, schedule_id";
/// <summary>
/// <see cref="Columns"/> as lists read them (<see cref="ListBriefAsync"/>): the parameters left out, null. A conversion's or
/// an import's name every sequence or file they work on; the job reads them, and so do the screens that act on that one job.
/// </summary>
private const string BriefColumns =
"id, kind, target_ids, target_label, CAST(NULL AS TEXT), state, priority, progress_done, progress_total, items_per_second, " +
"eta_seconds, checkpoint, log_tail, created_by, node_id, attempts, error, created, started, finished, schedule_id";
/// <summary>The states of a job that has not ended: queued, running, paused.</summary>
private static readonly string PendingStates =
string.Join(", ", new[] { JobState.Queued, JobState.Running, JobState.Paused }.Select(s => ((int)s).ToString(CultureInfo.InvariantCulture)));
@@ -37,6 +51,11 @@ public sealed class JobRepository
public Task<Job?> GetAsync(Guid id, CancellationToken cancellationToken = default) =>
_db.SingleAsync($"SELECT {Columns} FROM jobs WHERE id = $1", [id], Map, cancellationToken);
/// <summary>When the job <paramref name="id"/> was queued; null when there is no such job. Reads that column alone.</summary>
public async Task<DateTimeOffset?> GetCreatedAsync(Guid id, CancellationToken cancellationToken = default) =>
(await _db.ListAsync("SELECT created FROM jobs WHERE id = $1", [id], r => r.GetDateTimeOffset(0), cancellationToken).ConfigureAwait(false))
is [var created] ? created : null;
/// <summary>
/// Marks the next queued job (highest priority, then oldest) of the given kinds as running for
/// <paramref name="workerId"/> and returns it, or null when none waits. Concurrent claimers never get the same
@@ -93,7 +112,17 @@ public sealed class JobRepository
ListAsync(state, kind: null, limit, cursor, cancellationToken);
/// <summary>Jobs, newest first, optionally of one state and one kind.</summary>
public Task<Page<Job>> ListAsync(JobState? state, JobKind? kind, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default)
public Task<Page<Job>> ListAsync(JobState? state, JobKind? kind, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default) =>
PageAsync(Columns, state, kind, limit, cursor, cancellationToken);
/// <summary>
/// <see cref="ListAsync(JobState?, JobKind?, int, string?, CancellationToken)"/> as lists show jobs: every field but
/// <see cref="Job.ParametersJson"/>, which stays null (<see cref="GetAsync"/> reads a job whole).
/// </summary>
public Task<Page<Job>> ListBriefAsync(JobState? state, JobKind? kind, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default) =>
PageAsync(BriefColumns, state, kind, limit, cursor, cancellationToken);
private Task<Page<Job>> PageAsync(string columns, JobState? state, JobKind? kind, int limit, string? cursor, CancellationToken cancellationToken)
{
var p = new SqlParameters();
var where = new List<string>();
@@ -105,7 +134,7 @@ public sealed class JobRepository
where.Add(Cursor.After("created", "id", descending: true, value, id, p.Add));
}
var take = Cursor.ClampLimit(limit);
var sql = $"SELECT {Columns} FROM jobs{(where.Count > 0 ? " WHERE " + string.Join(" AND ", where) : string.Empty)} " +
var sql = $"SELECT {columns} FROM jobs{(where.Count > 0 ? " WHERE " + string.Join(" AND ", where) : string.Empty)} " +
$"ORDER BY created DESC, id DESC LIMIT {p.Add(take + 1)}";
return _db.RunAsync(async session =>
{
@@ -158,11 +187,22 @@ public sealed class JobRepository
ListQueuedAsync(kinds, limit, JobPriorityQueue.Off, cancellationToken);
/// <summary>Queued jobs of the given kinds in the claim order of <paramref name="queue"/>.</summary>
public Task<IReadOnlyList<Job>> ListQueuedAsync(IReadOnlyCollection<JobKind> kinds, int limit, JobPriorityQueue queue, CancellationToken cancellationToken = default)
public Task<IReadOnlyList<Job>> ListQueuedAsync(IReadOnlyCollection<JobKind> kinds, int limit, JobPriorityQueue queue, CancellationToken cancellationToken = default) =>
QueuedAsync(Columns, kinds, limit, queue, cancellationToken);
/// <summary>
/// <see cref="ListQueuedAsync(IReadOnlyCollection{JobKind}, int, JobPriorityQueue, CancellationToken)"/> without the
/// parameters (null), for a look at what would be claimed next: claiming a job reads it whole.
/// </summary>
public Task<IReadOnlyList<Job>> ListQueuedBriefAsync(IReadOnlyCollection<JobKind> kinds, int limit, JobPriorityQueue queue, CancellationToken cancellationToken = default) =>
QueuedAsync(BriefColumns, kinds, limit, queue, cancellationToken);
private Task<IReadOnlyList<Job>> QueuedAsync(string columns, IReadOnlyCollection<JobKind> kinds, int limit, JobPriorityQueue queue, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(kinds);
ArgumentNullException.ThrowIfNull(queue);
return kinds.Count == 0 ? Task.FromResult<IReadOnlyList<Job>>([]) : _db.ListAsync(
$"SELECT {Columns} FROM jobs WHERE state = {(int)JobState.Queued} AND kind IN ({KindList(kinds)}) ORDER BY {queue.OrderSql(kinds)}priority DESC, created, id LIMIT $1",
$"SELECT {columns} FROM jobs WHERE state = {(int)JobState.Queued} AND kind IN ({KindList(kinds)}) ORDER BY {queue.OrderSql(kinds)}priority DESC, created, id LIMIT $1",
[Cursor.ClampLimit(limit)], Map, cancellationToken);
}
@@ -195,13 +235,37 @@ public sealed class JobRepository
return await _db.ScalarAsync<long>(sql, p.ToArray(), cancellationToken).ConfigureAwait(false) > 0;
}
/// <summary>
/// What the running and paused jobs hold, but the job <paramref name="except"/> (the one asking): each one's id, kind and
/// targets, and only a verify's parameters (<see cref="JobHold"/>). Neither the other parameters nor the targets of
/// <paramref name="except"/> are read: a conversion asks at every sequence it comes to, and its own row names them all.
/// </summary>
public Task<IReadOnlyList<JobHold>> ListHoldsAsync(Guid? except = null, CancellationToken cancellationToken = default) =>
_db.ListAsync(
$"SELECT id, kind, target_ids, CASE WHEN kind = {(int)JobKind.Verify} THEN parameters_json END FROM jobs " +
$"WHERE state IN ({(int)JobState.Running}, {(int)JobState.Paused}){(except is null ? string.Empty : " AND id <> $1")} ORDER BY created, id",
except is { } self ? [self] : [],
r => new JobHold(r.GetGuid(0), (JobKind)r.GetInt32(1), Targets(r.GetString(2)), r.GetStringOrNull(3)), cancellationToken);
/// <summary>The targets of the queued, running and paused jobs of <paramref name="kind"/> (the newest 5000 such jobs).</summary>
public Task<HashSet<Guid>> ListPendingTargetsAsync(JobKind kind, CancellationToken cancellationToken = default) =>
PendingTargetsAsync(kind, null, cancellationToken);
/// <summary>
/// The targets of the queued, running and paused jobs of <paramref name="kind"/> created before <paramref name="createdBefore"/>
/// (the newest 5000 such jobs): those that came first.
/// </summary>
public Task<HashSet<Guid>> ListPendingTargetsAsync(JobKind kind, DateTimeOffset createdBefore, CancellationToken cancellationToken = default) =>
PendingTargetsAsync(kind, createdBefore, cancellationToken);
private Task<HashSet<Guid>> PendingTargetsAsync(JobKind kind, DateTimeOffset? createdBefore, CancellationToken cancellationToken) =>
_db.RunAsync(async session =>
{
var before = createdBefore is null ? string.Empty : " AND created < $2";
DbParam[] parameters = createdBefore is { } created ? [(int)kind, created] : [(int)kind];
var rows = await session.QueryListAsync(
$"SELECT target_ids FROM jobs WHERE kind = $1 AND state IN ({PendingStates}) ORDER BY created DESC, id DESC LIMIT 5000",
[(int)kind], r => r.GetString(0), cancellationToken).ConfigureAwait(false);
$"SELECT target_ids FROM jobs WHERE kind = $1 AND state IN ({PendingStates}){before} ORDER BY created DESC, id DESC LIMIT 5000",
parameters, r => r.GetString(0), cancellationToken).ConfigureAwait(false);
var ids = new HashSet<Guid>();
foreach (var text in rows)
{
@@ -294,11 +358,14 @@ public sealed class JobRepository
private static string KindList(IEnumerable<JobKind> kinds) => string.Join(", ", kinds.Select(k => ((int)k).ToString(CultureInfo.InvariantCulture)));
/// <summary>Target ids as the table stores them: comma-joined, empty for none.</summary>
private static Guid[] Targets(string ids) => ids.Length > 0 ? Array.ConvertAll(ids.Split(','), Guid.Parse) : [];
private static Job Map(DbRow r) => new()
{
Id = r.GetGuid(0),
Kind = (JobKind)r.GetInt32(1),
TargetIds = r.GetString(2) is { Length: > 0 } ids ? Array.ConvertAll(ids.Split(','), Guid.Parse) : [],
TargetIds = Targets(r.GetString(2)),
TargetLabel = r.GetStringOrNull(3),
ParametersJson = r.GetStringOrNull(4),
State = (JobState)r.GetInt32(5),
@@ -469,6 +469,50 @@ public sealed class SequenceRepository
}
}
/// <summary>
/// A digest of what a conversion of <paramref name="sequenceIds"/> (null: every sequence) plans from in the index of
/// <paramref name="archiveId"/> (owner review 2, R7): what a Reorganize's does (<see cref="GetStampAsync"/>: every sequence
/// and its folder, which decide what "every sequence" is and its order; of the sequences planned, their title, photo
/// count, frames version and how many of their photos are hashed), and of the photos of the sequences planned, what
/// decides whether each converts and the size it is estimated at: their format, size and pixel size, and the flags a
/// scan, a verify or a decode records without another frames version (missing, corrupt, read only from the preview,
/// undecodable). Another value once one of these changes. A change made on disk alone, which the index does not have
/// (a file gone before a scan finds it), leaves it as it was.
/// </summary>
public async Task<string> GetConvertStampAsync(Guid archiveId, IReadOnlyCollection<Guid>? sequenceIds, CancellationToken cancellationToken = default)
{
var stamp = await GetStampAsync(archiveId, sequenceIds, cancellationToken).ConfigureAwait(false);
// Per sequence planned: its photos, their bytes, formats and pixel sizes, and how many carry each flag that decides.
var columns = "count(*), COALESCE(SUM(f.size), 0), COALESCE(SUM(f.format), 0), COALESCE(SUM(f.width), 0), COALESCE(SUM(f.height), 0), " +
string.Join(", ", new[] { FrameFlags.Missing, FrameFlags.Corrupt, FrameFlags.PreviewOnly, FrameFlags.Undecodable }
.Select(flag => $"COALESCE(SUM(CASE WHEN (f.flags & {(int)flag}) <> 0 THEN 1 ELSE 0 END), 0)"));
static (Guid Id, string Photos) Read(DbRow r) =>
(r.GetGuid(0), string.Join('\u001f', Enumerable.Range(1, 9).Select(i => r.GetInt64(i).ToString(CultureInfo.InvariantCulture))));
var sums = new List<(Guid Id, string Photos)>();
if (sequenceIds is null)
{
sums.AddRange(await _db.ListAsync(
$"SELECT f.sequence_id, {columns} FROM frames f JOIN sequences s ON s.id = f.sequence_id WHERE s.archive_id = $1 GROUP BY f.sequence_id",
[archiveId], Read, cancellationToken).ConfigureAwait(false));
}
else
{
// 500 ids per statement stays below SQLite's historical 999-variable limit.
foreach (var chunk in sequenceIds.Distinct().Chunk(500))
{
var p = new SqlParameters();
var ids = string.Join(", ", chunk.Select(id => p.Add(id)));
sums.AddRange(await _db.ListAsync($"SELECT f.sequence_id, {columns} FROM frames f WHERE f.sequence_id IN ({ids}) GROUP BY f.sequence_id",
p.ToArray(), Read, cancellationToken).ConfigureAwait(false));
}
}
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
hash.AppendData(Encoding.UTF8.GetBytes(stamp));
foreach (var (id, photos) in sums.OrderBy(s => s.Id))
hash.AppendData(Encoding.UTF8.GetBytes(string.Create(CultureInfo.InvariantCulture, $"\n{id:N}\u001f{photos}")));
return Convert.ToHexStringLower(hash.GetHashAndReset());
}
/// <summary>
/// How many photos of each of <paramref name="sequenceIds"/> (null: every sequence of <paramref name="archiveId"/>) have
/// their SHA-256 in the index; a sequence without one is not listed.
@@ -521,6 +565,14 @@ public sealed class SequenceRepository
_db.ScalarAsync<Guid?>("SELECT sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key = $2",
[archiveId, sessionKey], cancellationToken);
/// <summary>
/// The session keys of an archive that match <paramref name="pattern"/> (a <c>LIKE</c> pattern, <c>\</c> its escape), each
/// with the sequence it resolves to, by key: the new sequences conversions made of one source, say.
/// </summary>
public Task<IReadOnlyList<(string Key, Guid SequenceId)>> ListSessionKeysAsync(Guid archiveId, string pattern, CancellationToken cancellationToken = default) =>
_db.ListAsync("SELECT session_key, sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key LIKE $2 ESCAPE '\\' ORDER BY session_key",
[archiveId, pattern], r => (r.GetString(0), r.GetGuid(1)), cancellationToken);
/// <summary>Binds a session key to a sequence unless it is already bound; returns the sequence the key resolves to.</summary>
public Task<Guid> MapSessionKeyAsync(Guid archiveId, string sessionKey, Guid sequenceId, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>