feat: owner review 2 phase 4b: mass conversion after import (#13)
CI / Build and test (push) Failing after 3m1s
CI / PostgreSQL 16 integration (push) Skipped
CI / PostgreSQL 17 integration (push) Skipped
CI / Docker image (push) Skipped
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 10s
Release / Build, test and publish (push) Successful in 15m31s
Release / Server image (push) Successful in 1m13s
CI / Build and test (push) Failing after 3m1s
CI / PostgreSQL 16 integration (push) Skipped
CI / PostgreSQL 17 integration (push) Skipped
CI / Docker image (push) Skipped
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 10s
Release / Build, test and publish (push) Successful in 15m31s
Release / Server image (push) Successful in 1m13s
Convert photos already in the archive to JPEG, TIFF-16 or Linear DNG, from the panel's Library and the desktop's Archive and Library. Each run asks Replace (same frames, originals to the trash, recoverable) or New sequence. A dry run lists what converts, what is skipped and why, size before and after, and free space; the job converts exactly the confirmed photos, resumes, and commits all or nothing per sequence. A conversion never changes what Chronos reads from a photo. Replaced originals are erased only on the archive's own word that their conversion is gone. Desktop TIFFs that WIC showed black or as noise now go to the managed reader; the Edit picker no longer spins a core. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
This commit was merged in pull request #13.
This commit is contained in:
1 parent
7e4a3b481d
commit
efa87f68ce
354 files changed
+24530
-843
No files matched your search
@@ -7,6 +7,12 @@ namespace Chronos.Data.Repositories;
|
||||
/// <summary>Jobs per state; <see cref="DoneSince"/> and <see cref="FailedSince"/> count those finished since a given time.</summary>
|
||||
public sealed record JobCounts(int Queued, int Running, int Paused, int DoneSince, int FailedSince);
|
||||
|
||||
/// <summary>
|
||||
/// What a running or paused job works on (<see cref="JobRepository.ListHoldsAsync"/>): its targets and, for a verify, its
|
||||
/// parameters, which name the sequences it holds; null for every other kind, whose parameters are not read.
|
||||
/// </summary>
|
||||
public sealed record JobHold(Guid Id, JobKind Kind, IReadOnlyList<Guid> TargetIds, string? ParametersJson);
|
||||
|
||||
/// <summary>
|
||||
/// The persistent job queue (BRIEF §14.6): workers claim the oldest job of the highest priority atomically,
|
||||
/// write checkpoints while running, and on startup requeue what a crash left running.
|
||||
@@ -17,6 +23,14 @@ public sealed class JobRepository
|
||||
"id, kind, target_ids, target_label, parameters_json, state, priority, progress_done, progress_total, items_per_second, " +
|
||||
"eta_seconds, checkpoint, log_tail, created_by, node_id, attempts, error, created, started, finished, schedule_id";
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="Columns"/> as lists read them (<see cref="ListBriefAsync"/>): the parameters left out, null. A conversion's or
|
||||
/// an import's name every sequence or file they work on; the job reads them, and so do the screens that act on that one job.
|
||||
/// </summary>
|
||||
private const string BriefColumns =
|
||||
"id, kind, target_ids, target_label, CAST(NULL AS TEXT), state, priority, progress_done, progress_total, items_per_second, " +
|
||||
"eta_seconds, checkpoint, log_tail, created_by, node_id, attempts, error, created, started, finished, schedule_id";
|
||||
|
||||
/// <summary>The states of a job that has not ended: queued, running, paused.</summary>
|
||||
private static readonly string PendingStates =
|
||||
string.Join(", ", new[] { JobState.Queued, JobState.Running, JobState.Paused }.Select(s => ((int)s).ToString(CultureInfo.InvariantCulture)));
|
||||
@@ -37,6 +51,11 @@ public sealed class JobRepository
|
||||
public Task<Job?> GetAsync(Guid id, CancellationToken cancellationToken = default) =>
|
||||
_db.SingleAsync($"SELECT {Columns} FROM jobs WHERE id = $1", [id], Map, cancellationToken);
|
||||
|
||||
/// <summary>When the job <paramref name="id"/> was queued; null when there is no such job. Reads that column alone.</summary>
|
||||
public async Task<DateTimeOffset?> GetCreatedAsync(Guid id, CancellationToken cancellationToken = default) =>
|
||||
(await _db.ListAsync("SELECT created FROM jobs WHERE id = $1", [id], r => r.GetDateTimeOffset(0), cancellationToken).ConfigureAwait(false))
|
||||
is [var created] ? created : null;
|
||||
|
||||
/// <summary>
|
||||
/// Marks the next queued job (highest priority, then oldest) of the given kinds as running for
|
||||
/// <paramref name="workerId"/> and returns it, or null when none waits. Concurrent claimers never get the same
|
||||
@@ -93,7 +112,17 @@ public sealed class JobRepository
|
||||
ListAsync(state, kind: null, limit, cursor, cancellationToken);
|
||||
|
||||
/// <summary>Jobs, newest first, optionally of one state and one kind.</summary>
|
||||
public Task<Page<Job>> ListAsync(JobState? state, JobKind? kind, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default)
|
||||
public Task<Page<Job>> ListAsync(JobState? state, JobKind? kind, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default) =>
|
||||
PageAsync(Columns, state, kind, limit, cursor, cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="ListAsync(JobState?, JobKind?, int, string?, CancellationToken)"/> as lists show jobs: every field but
|
||||
/// <see cref="Job.ParametersJson"/>, which stays null (<see cref="GetAsync"/> reads a job whole).
|
||||
/// </summary>
|
||||
public Task<Page<Job>> ListBriefAsync(JobState? state, JobKind? kind, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default) =>
|
||||
PageAsync(BriefColumns, state, kind, limit, cursor, cancellationToken);
|
||||
|
||||
private Task<Page<Job>> PageAsync(string columns, JobState? state, JobKind? kind, int limit, string? cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
var p = new SqlParameters();
|
||||
var where = new List<string>();
|
||||
@@ -105,7 +134,7 @@ public sealed class JobRepository
|
||||
where.Add(Cursor.After("created", "id", descending: true, value, id, p.Add));
|
||||
}
|
||||
var take = Cursor.ClampLimit(limit);
|
||||
var sql = $"SELECT {Columns} FROM jobs{(where.Count > 0 ? " WHERE " + string.Join(" AND ", where) : string.Empty)} " +
|
||||
var sql = $"SELECT {columns} FROM jobs{(where.Count > 0 ? " WHERE " + string.Join(" AND ", where) : string.Empty)} " +
|
||||
$"ORDER BY created DESC, id DESC LIMIT {p.Add(take + 1)}";
|
||||
return _db.RunAsync(async session =>
|
||||
{
|
||||
@@ -158,11 +187,22 @@ public sealed class JobRepository
|
||||
ListQueuedAsync(kinds, limit, JobPriorityQueue.Off, cancellationToken);
|
||||
|
||||
/// <summary>Queued jobs of the given kinds in the claim order of <paramref name="queue"/>.</summary>
|
||||
public Task<IReadOnlyList<Job>> ListQueuedAsync(IReadOnlyCollection<JobKind> kinds, int limit, JobPriorityQueue queue, CancellationToken cancellationToken = default)
|
||||
public Task<IReadOnlyList<Job>> ListQueuedAsync(IReadOnlyCollection<JobKind> kinds, int limit, JobPriorityQueue queue, CancellationToken cancellationToken = default) =>
|
||||
QueuedAsync(Columns, kinds, limit, queue, cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="ListQueuedAsync(IReadOnlyCollection{JobKind}, int, JobPriorityQueue, CancellationToken)"/> without the
|
||||
/// parameters (null), for a look at what would be claimed next: claiming a job reads it whole.
|
||||
/// </summary>
|
||||
public Task<IReadOnlyList<Job>> ListQueuedBriefAsync(IReadOnlyCollection<JobKind> kinds, int limit, JobPriorityQueue queue, CancellationToken cancellationToken = default) =>
|
||||
QueuedAsync(BriefColumns, kinds, limit, queue, cancellationToken);
|
||||
|
||||
private Task<IReadOnlyList<Job>> QueuedAsync(string columns, IReadOnlyCollection<JobKind> kinds, int limit, JobPriorityQueue queue, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(kinds);
|
||||
ArgumentNullException.ThrowIfNull(queue);
|
||||
return kinds.Count == 0 ? Task.FromResult<IReadOnlyList<Job>>([]) : _db.ListAsync(
|
||||
$"SELECT {Columns} FROM jobs WHERE state = {(int)JobState.Queued} AND kind IN ({KindList(kinds)}) ORDER BY {queue.OrderSql(kinds)}priority DESC, created, id LIMIT $1",
|
||||
$"SELECT {columns} FROM jobs WHERE state = {(int)JobState.Queued} AND kind IN ({KindList(kinds)}) ORDER BY {queue.OrderSql(kinds)}priority DESC, created, id LIMIT $1",
|
||||
[Cursor.ClampLimit(limit)], Map, cancellationToken);
|
||||
}
|
||||
|
||||
@@ -195,13 +235,37 @@ public sealed class JobRepository
|
||||
return await _db.ScalarAsync<long>(sql, p.ToArray(), cancellationToken).ConfigureAwait(false) > 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// What the running and paused jobs hold, but the job <paramref name="except"/> (the one asking): each one's id, kind and
|
||||
/// targets, and only a verify's parameters (<see cref="JobHold"/>). Neither the other parameters nor the targets of
|
||||
/// <paramref name="except"/> are read: a conversion asks at every sequence it comes to, and its own row names them all.
|
||||
/// </summary>
|
||||
public Task<IReadOnlyList<JobHold>> ListHoldsAsync(Guid? except = null, CancellationToken cancellationToken = default) =>
|
||||
_db.ListAsync(
|
||||
$"SELECT id, kind, target_ids, CASE WHEN kind = {(int)JobKind.Verify} THEN parameters_json END FROM jobs " +
|
||||
$"WHERE state IN ({(int)JobState.Running}, {(int)JobState.Paused}){(except is null ? string.Empty : " AND id <> $1")} ORDER BY created, id",
|
||||
except is { } self ? [self] : [],
|
||||
r => new JobHold(r.GetGuid(0), (JobKind)r.GetInt32(1), Targets(r.GetString(2)), r.GetStringOrNull(3)), cancellationToken);
|
||||
|
||||
/// <summary>The targets of the queued, running and paused jobs of <paramref name="kind"/> (the newest 5000 such jobs).</summary>
|
||||
public Task<HashSet<Guid>> ListPendingTargetsAsync(JobKind kind, CancellationToken cancellationToken = default) =>
|
||||
PendingTargetsAsync(kind, null, cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// The targets of the queued, running and paused jobs of <paramref name="kind"/> created before <paramref name="createdBefore"/>
|
||||
/// (the newest 5000 such jobs): those that came first.
|
||||
/// </summary>
|
||||
public Task<HashSet<Guid>> ListPendingTargetsAsync(JobKind kind, DateTimeOffset createdBefore, CancellationToken cancellationToken = default) =>
|
||||
PendingTargetsAsync(kind, createdBefore, cancellationToken);
|
||||
|
||||
private Task<HashSet<Guid>> PendingTargetsAsync(JobKind kind, DateTimeOffset? createdBefore, CancellationToken cancellationToken) =>
|
||||
_db.RunAsync(async session =>
|
||||
{
|
||||
var before = createdBefore is null ? string.Empty : " AND created < $2";
|
||||
DbParam[] parameters = createdBefore is { } created ? [(int)kind, created] : [(int)kind];
|
||||
var rows = await session.QueryListAsync(
|
||||
$"SELECT target_ids FROM jobs WHERE kind = $1 AND state IN ({PendingStates}) ORDER BY created DESC, id DESC LIMIT 5000",
|
||||
[(int)kind], r => r.GetString(0), cancellationToken).ConfigureAwait(false);
|
||||
$"SELECT target_ids FROM jobs WHERE kind = $1 AND state IN ({PendingStates}){before} ORDER BY created DESC, id DESC LIMIT 5000",
|
||||
parameters, r => r.GetString(0), cancellationToken).ConfigureAwait(false);
|
||||
var ids = new HashSet<Guid>();
|
||||
foreach (var text in rows)
|
||||
{
|
||||
@@ -294,11 +358,14 @@ public sealed class JobRepository
|
||||
|
||||
private static string KindList(IEnumerable<JobKind> kinds) => string.Join(", ", kinds.Select(k => ((int)k).ToString(CultureInfo.InvariantCulture)));
|
||||
|
||||
/// <summary>Target ids as the table stores them: comma-joined, empty for none.</summary>
|
||||
private static Guid[] Targets(string ids) => ids.Length > 0 ? Array.ConvertAll(ids.Split(','), Guid.Parse) : [];
|
||||
|
||||
private static Job Map(DbRow r) => new()
|
||||
{
|
||||
Id = r.GetGuid(0),
|
||||
Kind = (JobKind)r.GetInt32(1),
|
||||
TargetIds = r.GetString(2) is { Length: > 0 } ids ? Array.ConvertAll(ids.Split(','), Guid.Parse) : [],
|
||||
TargetIds = Targets(r.GetString(2)),
|
||||
TargetLabel = r.GetStringOrNull(3),
|
||||
ParametersJson = r.GetStringOrNull(4),
|
||||
State = (JobState)r.GetInt32(5),
|
||||
|
||||
@@ -469,6 +469,50 @@ public sealed class SequenceRepository
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A digest of what a conversion of <paramref name="sequenceIds"/> (null: every sequence) plans from in the index of
|
||||
/// <paramref name="archiveId"/> (owner review 2, R7): what a Reorganize's does (<see cref="GetStampAsync"/>: every sequence
|
||||
/// and its folder, which decide what "every sequence" is and its order; of the sequences planned, their title, photo
|
||||
/// count, frames version and how many of their photos are hashed), and of the photos of the sequences planned, what
|
||||
/// decides whether each converts and the size it is estimated at: their format, size and pixel size, and the flags a
|
||||
/// scan, a verify or a decode records without another frames version (missing, corrupt, read only from the preview,
|
||||
/// undecodable). Another value once one of these changes. A change made on disk alone, which the index does not have
|
||||
/// (a file gone before a scan finds it), leaves it as it was.
|
||||
/// </summary>
|
||||
public async Task<string> GetConvertStampAsync(Guid archiveId, IReadOnlyCollection<Guid>? sequenceIds, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var stamp = await GetStampAsync(archiveId, sequenceIds, cancellationToken).ConfigureAwait(false);
|
||||
// Per sequence planned: its photos, their bytes, formats and pixel sizes, and how many carry each flag that decides.
|
||||
var columns = "count(*), COALESCE(SUM(f.size), 0), COALESCE(SUM(f.format), 0), COALESCE(SUM(f.width), 0), COALESCE(SUM(f.height), 0), " +
|
||||
string.Join(", ", new[] { FrameFlags.Missing, FrameFlags.Corrupt, FrameFlags.PreviewOnly, FrameFlags.Undecodable }
|
||||
.Select(flag => $"COALESCE(SUM(CASE WHEN (f.flags & {(int)flag}) <> 0 THEN 1 ELSE 0 END), 0)"));
|
||||
static (Guid Id, string Photos) Read(DbRow r) =>
|
||||
(r.GetGuid(0), string.Join('\u001f', Enumerable.Range(1, 9).Select(i => r.GetInt64(i).ToString(CultureInfo.InvariantCulture))));
|
||||
var sums = new List<(Guid Id, string Photos)>();
|
||||
if (sequenceIds is null)
|
||||
{
|
||||
sums.AddRange(await _db.ListAsync(
|
||||
$"SELECT f.sequence_id, {columns} FROM frames f JOIN sequences s ON s.id = f.sequence_id WHERE s.archive_id = $1 GROUP BY f.sequence_id",
|
||||
[archiveId], Read, cancellationToken).ConfigureAwait(false));
|
||||
}
|
||||
else
|
||||
{
|
||||
// 500 ids per statement stays below SQLite's historical 999-variable limit.
|
||||
foreach (var chunk in sequenceIds.Distinct().Chunk(500))
|
||||
{
|
||||
var p = new SqlParameters();
|
||||
var ids = string.Join(", ", chunk.Select(id => p.Add(id)));
|
||||
sums.AddRange(await _db.ListAsync($"SELECT f.sequence_id, {columns} FROM frames f WHERE f.sequence_id IN ({ids}) GROUP BY f.sequence_id",
|
||||
p.ToArray(), Read, cancellationToken).ConfigureAwait(false));
|
||||
}
|
||||
}
|
||||
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
|
||||
hash.AppendData(Encoding.UTF8.GetBytes(stamp));
|
||||
foreach (var (id, photos) in sums.OrderBy(s => s.Id))
|
||||
hash.AppendData(Encoding.UTF8.GetBytes(string.Create(CultureInfo.InvariantCulture, $"\n{id:N}\u001f{photos}")));
|
||||
return Convert.ToHexStringLower(hash.GetHashAndReset());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// How many photos of each of <paramref name="sequenceIds"/> (null: every sequence of <paramref name="archiveId"/>) have
|
||||
/// their SHA-256 in the index; a sequence without one is not listed.
|
||||
@@ -521,6 +565,14 @@ public sealed class SequenceRepository
|
||||
_db.ScalarAsync<Guid?>("SELECT sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key = $2",
|
||||
[archiveId, sessionKey], cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// The session keys of an archive that match <paramref name="pattern"/> (a <c>LIKE</c> pattern, <c>\</c> its escape), each
|
||||
/// with the sequence it resolves to, by key: the new sequences conversions made of one source, say.
|
||||
/// </summary>
|
||||
public Task<IReadOnlyList<(string Key, Guid SequenceId)>> ListSessionKeysAsync(Guid archiveId, string pattern, CancellationToken cancellationToken = default) =>
|
||||
_db.ListAsync("SELECT session_key, sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key LIKE $2 ESCAPE '\\' ORDER BY session_key",
|
||||
[archiveId, pattern], r => (r.GetString(0), r.GetGuid(1)), cancellationToken);
|
||||
|
||||
/// <summary>Binds a session key to a sequence unless it is already bound; returns the sequence the key resolves to.</summary>
|
||||
public Task<Guid> MapSessionKeyAsync(Guid archiveId, string sessionKey, Guid sequenceId, CancellationToken cancellationToken = default) =>
|
||||
_db.InTransactionAsync(async session =>
|
||||
|
||||
Reference in new issue
Block a user