Files
Chronos/src/Chronos.Data/Repositories/SequenceRepository.cs
T
Alby96 8174852a2c fix(server): keep convert dry runs like Reorganize's, say what a new sequence lacks
The convert endpoint planned the whole selection inside the HTTP request, again for
every mode, quality keystroke and confirmation, with no limit: behind a proxy an
all-sequence conversion could never get past its dry run or its apply.

- ReorganizePlanCache becomes PlanCache, shared by Reorganize and Convert: one plan
  per key made apart from the request, two at once across both kinds, eight kept,
  15-minute timeout, 5-minute lifetime, 503 plan_dropped. Keys are typed records
  (ReorganizePlanKey, ConvertPlanKey) that also say which exceptions are refusals.
- A conversion plan is keyed by archive, a convert stamp of the index
  (SequenceRepository.GetConvertStampAsync: the Reorganize stamp plus, per planned
  sequence, the photos' count, sizes, formats, pixel sizes and missing, corrupt,
  preview-only and undecodable flags), the sequences and the target with the
  quality that counts. Not the mode: the request's mode and options are set on the
  kept plan, and the free space is measured again at every request through the
  engine. A plan made while the volume was away is not kept (503 archive_offline).
- New sequence holds the converted copies alone: the dry run says the skipped
  photos are not copied into it (Archive.Convert.NotCopied.*, Server.Convert.OnPcNew),
  the panel's totals, column and confirmation give the count, and so does the
  desktop's summary.
- Panel: the preview is asked in Replace's neutral words; a dry run cut off by a
  proxy is asked again while the totals show how long it takes, plan_dropped is
  retried once; an apply the server let go plans again, a changed one shows
  Admin.Convert.PlanChanged, and one with no answer is not resent (its job may be
  queued).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit f0b1e379ce7c08b6d3459ab25ca6214c8f366f6c)
2026-10-27 22:15:51 +01:00

751 lines
45 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using System.Globalization;
using System.Security.Cryptography;
using System.Text;
using Chronos.Core.Imaging;
using Chronos.Core.Model;
namespace Chronos.Data.Repositories;
/// <summary>Orderings of the Library; every one is keyset-paginated with the sequence id as tie-breaker.</summary>
public enum SequenceSort
{
/// <summary>Latest first capture first; sequences without captures last.</summary>
Newest,
Oldest,
/// <summary>Display title, case-insensitive.</summary>
Title,
FrameCount,
Size,
}
/// <summary>Library filters (all optional, combined with AND) plus sort and page.</summary>
public sealed record SequenceQuery
{
public Guid? ArchiveId { get; init; }
/// <summary>Calendar year of the first capture.</summary>
public int? Year { get; init; }
/// <summary>Exact camera model.</summary>
public string? Camera { get; init; }
/// <summary>Status flags that must all be set.</summary>
public SequenceStatus WithStatus { get; init; }
/// <summary>Status flags that must all be clear.</summary>
public SequenceStatus WithoutStatus { get; init; }
public string? Tag { get; init; }
/// <summary>True: with at least one render; false: without any; null: either.</summary>
public bool? Rendered { get; init; }
/// <summary>Sync states of which the sequence must be in one; null or empty: any.</summary>
public IReadOnlyCollection<SyncState>? SyncStates { get; init; }
/// <summary>Case-insensitive substring of the title, camera make/model or a tag.</summary>
public string? Text { get; init; }
public Guid? OwnerUserId { get; init; }
public SequenceSort Sort { get; init; } = SequenceSort.Newest;
/// <summary>Page size, 1–1000.</summary>
public int Limit { get; init; } = 100;
/// <summary><see cref="Page{T}.NextCursor"/> of the previous page, or null for the first page.</summary>
public string? Cursor { get; init; }
}
/// <summary>Figures of the Archive screen (BRIEF §12.5).</summary>
public sealed record ArchiveStats(
int Sequences,
long Photos,
long Bytes,
IReadOnlyList<YearBucket> ByYear,
IReadOnlyList<MonthBucket> ByMonth,
IReadOnlyList<CameraBucket> ByCamera,
IReadOnlyList<FormatBucket> ByFormat,
ArchiveHealth Health);
/// <summary>Fields a client may change on a sequence; null leaves a field unchanged, an empty title or notes clears it.</summary>
public sealed record SequenceEdit
{
public string? Title { get; init; }
public IReadOnlyList<string>? Tags { get; init; }
public int? Rating { get; init; }
public string? Notes { get; init; }
public string? SettingsJson { get; init; }
/// <summary>
/// The frame the Library shows for the sequence (the project's hero frame); <see cref="Guid.Empty"/> clears it, so
/// the first frame shows. A frame of another sequence is ignored.
/// </summary>
public Guid? HeroFrameId { get; init; }
}
public enum SequenceEditOutcome
{
Updated,
NotFound,
/// <summary>The settings version moved since the client read it.</summary>
Conflict,
}
/// <summary>Result of <see cref="SequenceRepository.EditAsync"/>: the updated sequence, or the stored one on conflict.</summary>
public sealed record SequenceEditResult(SequenceEditOutcome Outcome, Sequence? Sequence);
public sealed record YearBucket(int Year, int Sequences, long Photos, long Bytes);
/// <summary>One heatmap cell: sequences and photos whose first capture falls in the month.</summary>
public sealed record MonthBucket(int Year, int Month, int Sequences, long Photos);
public sealed record CameraBucket(string? Make, string? Model, int Sequences, long Photos, long Bytes);
public sealed record FormatBucket(ImageFileFormat Format, long Photos, long Bytes);
/// <summary>
/// A value a Library filter offers, as <see cref="SequenceQuery"/> takes it (a year, an exact camera model, a tag), and
/// how many sequences have it; <see cref="Make"/> is a camera model's make, for its label.
/// </summary>
public sealed record ArchiveFacet(string Value, int Sequences, string? Make = null);
/// <summary>The years of first capture, camera models and tags of an archive's sequences (owner review 2, R6: the panel's filters).</summary>
public sealed record ArchiveFacets(IReadOnlyList<ArchiveFacet> Years, IReadOnlyList<ArchiveFacet> Cameras, IReadOnlyList<ArchiveFacet> Tags);
/// <summary>Health counters: sequences needing attention and flagged frames.</summary>
public sealed record ArchiveHealth(
int Unverified,
int WithWarnings,
int WithErrors,
int NotAnalyzed,
long MissingFrames,
long CorruptFrames,
long DuplicateFrames);
/// <summary>Where a sequence lives in its archive.</summary>
public sealed record SequenceLocation(Guid Id, string Folder, string? Title);
/// <summary>Sequences (projects) with their tags; Library listing and Archive statistics.</summary>
public sealed class SequenceRepository
{
private const string Columns =
"s.id, s.archive_id, s.title, s.folder, s.camera_make, s.camera_model, s.camera_serial, s.lens, s.first_capture, s.last_capture, " +
"s.frame_count, s.median_interval, s.total_bytes, s.status, s.rendered_count, s.verified_at, s.hero_frame_id, s.rating, s.notes, " +
"s.settings_json, s.analysis_json, s.origin, s.sync_state, s.settings_version, s.frames_version, s.owner_user_id, s.warning_count, s.updated_at, " +
"s.preview_only_count";
/// <summary>rendered_count is owned by <see cref="RenderRepository"/> once the row exists.</summary>
private static readonly string UpsertSql = SqlDialect.Upsert("sequences",
[
"id", "archive_id", "title", "title_key", "search_text", "folder", "camera_make", "camera_model", "camera_serial", "lens",
"first_capture", "last_capture", "frame_count", "median_interval", "total_bytes", "status", "rendered_count", "verified_at",
"hero_frame_id", "rating", "notes", "settings_json", "analysis_json", "origin", "sync_state", "settings_version",
"frames_version", "owner_user_id", "warning_count", "updated_at", "preview_only_count",
], 1, ["rendered_count"]);
/// <summary>Ordinal of the tag list after <see cref="Columns"/>, and of the sort key a listing appends after it.</summary>
private const int TagsOrdinal = 29;
private const int SortKeyOrdinal = 30;
private readonly DbAccess _db;
private readonly string _select;
internal SequenceRepository(DbAccess db)
{
_db = db;
_select = SelectSql(string.Empty);
}
private string SelectSql(string extraColumns) =>
$"SELECT {Columns}, (SELECT {_db.Sql.StringAgg("t.name")} FROM sequence_tags st JOIN tags t ON t.id = st.tag_id " +
$"WHERE st.sequence_id = s.id) AS tags{extraColumns} FROM sequences s";
/// <summary>
/// Inserts the sequence, or replaces every stored field and the tag set with the record's (last writer wins);
/// only <see cref="Sequence.RenderedCount"/> of an existing row is kept, as renders maintain it. For concurrent
/// partial changes use <see cref="SetTagsAsync"/>, <see cref="UpdateStatusAsync"/> or <see cref="RefreshStatsAsync"/>.
/// </summary>
public Task UpsertAsync(Sequence sequence, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
await UpsertRowAsync(session, sequence, cancellationToken).ConfigureAwait(false);
await ReplaceTagsAsync(session, sequence.Id, sequence.Tags, cancellationToken).ConfigureAwait(false);
return true;
}, cancellationToken);
public Task<Sequence?> GetAsync(Guid id, CancellationToken cancellationToken = default) =>
_db.SingleAsync($"{_select} WHERE s.id = $1", [id], Map, cancellationToken);
/// <summary>
/// <see cref="GetAsync"/> inside a transaction, the row locked until it ends (PostgreSQL; SQLite's write lock already
/// serialises writers): a transaction that writes the whole row back from this read cannot undo an edit that another
/// one commits meanwhile, as <see cref="EditAsync(Guid, SequenceEdit, int?, int?, CancellationToken)"/> takes the same lock.
/// A transaction that writes frames of the sequence takes this lock, or writes the row, before the frames: with one
/// order no two of them wait for each other's rows (PostgreSQL would abort one with a deadlock).
/// </summary>
public Task<Sequence?> GetForUpdateAsync(Guid id, CancellationToken cancellationToken = default) =>
_db.SingleAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken);
/// <summary>Deletes the sequence with its frames, renders and tag links.</summary>
public async Task<bool> DeleteAsync(Guid id, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("DELETE FROM sequences WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>Replaces the tags of a sequence (trimmed, de-duplicated case-insensitively).</summary>
public async Task<bool> SetTagsAsync(Guid id, IEnumerable<string> tags, CancellationToken cancellationToken = default) =>
await _db.InTransactionAsync(async session =>
{
if (!await LockAsync(session, id, cancellationToken).ConfigureAwait(false)) return false;
await ReplaceTagsAsync(session, id, tags, cancellationToken).ConfigureAwait(false);
await session.ExecuteAsync("UPDATE sequences SET settings_version = settings_version + 1, updated_at = $2 WHERE id = $1",
[id, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false);
return true;
}, cancellationToken).ConfigureAwait(false);
/// <summary>Sets and clears status flags in one statement.</summary>
public async Task<bool> UpdateStatusAsync(Guid id, SequenceStatus set, SequenceStatus clear, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("UPDATE sequences SET status = (status | $2) & ~$3, updated_at = $4 WHERE id = $1",
[id, (int)set, (int)clear, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>
/// Records decode tiers written to the sequence's frames (<see cref="FrameRepository.UpdateTiersAsync"/>): the status
/// flags in <paramref name="set"/> and the preview-only count, in one statement. The frames, their versions and the
/// settings stay as they are, so an ingest or a settings edit committed meanwhile is never written back over.
/// </summary>
public async Task<bool> RecordTiersAsync(Guid id, SequenceStatus set, int previewOnlyCount, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("UPDATE sequences SET status = status | $2, preview_only_count = $3, updated_at = $4 WHERE id = $1",
[id, (int)set, previewOnlyCount, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>
/// Records a verify: status flags set and cleared and the verify time, in one statement; every other field stays as
/// it is (<see cref="RecordTiersAsync"/>).
/// </summary>
public async Task<bool> RecordVerifiedAsync(Guid id, SequenceStatus set, SequenceStatus clear, DateTimeOffset verifiedAt,
CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("UPDATE sequences SET status = (status | $2) & ~$3, verified_at = $4, updated_at = $5 WHERE id = $1",
[id, (int)set, (int)clear, verifiedAt, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>
/// Records an analysis: status flags set and cleared, the warning count and, for a deep analysis, the compact
/// summary (<paramref name="analysisJson"/> null keeps the stored one).
/// </summary>
public async Task<bool> RecordAnalysisAsync(Guid id, SequenceStatus set, SequenceStatus clear, int warningCount, string? analysisJson,
CancellationToken cancellationToken = default)
{
DbParam[] args = [id, (int)set, (int)clear, warningCount, DateTimeOffset.UtcNow];
var sql = "UPDATE sequences SET status = (status | $2) & ~$3, warning_count = $4, updated_at = $5";
if (analysisJson is not null)
{
sql += ", analysis_json = $6";
args = [.. args, DbParam.Json(analysisJson)];
}
return await _db.ExecuteAsync(sql + " WHERE id = $1", args, cancellationToken).ConfigureAwait(false) > 0;
}
/// <summary>Recomputes frame count, total bytes and first/last capture from the frames table.</summary>
public Task<Sequence?> RefreshStatsAsync(Guid id, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
var current = await session.QuerySingleOrDefaultAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken)
.ConfigureAwait(false);
if (current is null) return null;
var stats = await session.QuerySingleAsync(
"SELECT count(*), COALESCE(SUM(size), 0), MIN(captured_at), MAX(captured_at) FROM frames WHERE sequence_id = $1",
[id], cancellationToken).ConfigureAwait(false);
var updated = current with
{
FrameCount = (int)stats.GetInt64(0),
TotalBytes = stats.GetInt64(1),
FirstCapture = stats.GetDateTimeOrNull(2),
LastCapture = stats.GetDateTimeOrNull(3),
FramesVersion = current.FramesVersion + 1,
UpdatedAt = DateTimeOffset.UtcNow,
};
await UpsertRowAsync(session, updated, cancellationToken).ConfigureAwait(false);
return updated;
}, cancellationToken);
/// <summary>
/// Applies a client edit atomically and bumps <c>settings_version</c>. With <paramref name="expectedSettingsVersion"/>
/// the edit is refused (<see cref="SequenceEditOutcome.Conflict"/>, carrying the stored sequence) when someone else
/// changed the sequence first (BRIEF §14.5).
/// </summary>
public Task<SequenceEditResult> EditAsync(Guid id, SequenceEdit edit, int? expectedSettingsVersion, CancellationToken cancellationToken = default) =>
EditAsync(id, edit, expectedSettingsVersion, null, cancellationToken);
/// <summary>
/// <see cref="EditAsync(Guid, SequenceEdit, int?, CancellationToken)"/>, also refused when the sequence's frames
/// version is no longer <paramref name="expectedFramesVersion"/>: settings naming frames by position were written
/// against frames that changed since.
/// </summary>
public Task<SequenceEditResult> EditAsync(Guid id, SequenceEdit edit, int? expectedSettingsVersion, int? expectedFramesVersion,
CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
var current = await session.QuerySingleOrDefaultAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken)
.ConfigureAwait(false);
if (current is null) return new SequenceEditResult(SequenceEditOutcome.NotFound, null);
if ((expectedSettingsVersion is { } expected && expected != current.SettingsVersion)
|| (expectedFramesVersion is { } frames && frames != current.FramesVersion))
return new SequenceEditResult(SequenceEditOutcome.Conflict, current);
var hero = current.HeroFrameId;
if (edit.HeroFrameId == Guid.Empty) hero = null;
else if (edit.HeroFrameId is { } frame && await session.ScalarAsync<long>("SELECT count(*) FROM frames WHERE id = $1 AND sequence_id = $2",
[frame, id], cancellationToken).ConfigureAwait(false) > 0) hero = frame;
var updated = current with
{
Title = edit.Title is null ? current.Title : NullIfEmpty(edit.Title),
Notes = edit.Notes is null ? current.Notes : NullIfEmpty(edit.Notes),
Rating = edit.Rating ?? current.Rating,
SettingsJson = edit.SettingsJson ?? current.SettingsJson,
HeroFrameId = hero,
SettingsVersion = current.SettingsVersion + 1,
UpdatedAt = DateTimeOffset.UtcNow,
};
await UpsertRowAsync(session, updated, cancellationToken).ConfigureAwait(false);
if (edit.Tags is not null) await ReplaceTagsAsync(session, id, edit.Tags, cancellationToken).ConfigureAwait(false);
var stored = await session.QuerySingleAsync($"{_select} WHERE s.id = $1", [id], cancellationToken).ConfigureAwait(false);
return new SequenceEditResult(SequenceEditOutcome.Updated, Map(stored));
}, cancellationToken);
/// <summary>Gives every sequence owned by <paramref name="fromUserId"/> to <paramref name="toUserId"/>; returns how many.</summary>
public Task<int> ReassignOwnerAsync(Guid fromUserId, Guid toUserId, CancellationToken cancellationToken = default) =>
_db.ExecuteAsync("UPDATE sequences SET owner_user_id = $2 WHERE owner_user_id = $1", [fromUserId, toUserId], cancellationToken);
/// <summary>One page of sequences matching <paramref name="query"/>.</summary>
public Task<Page<Sequence>> ListAsync(SequenceQuery query, CancellationToken cancellationToken = default)
{
var p = new SqlParameters();
var where = Filter(query, p);
var (column, descending) = SortKey(query.Sort);
var scope = "seq:" + query.Sort;
if (query.Cursor is not null)
{
var (value, id) = Cursor.Decode(scope, query.Cursor);
where.Add(Cursor.After(column, "s.id", descending, value, id, p.Add));
}
var limit = Cursor.ClampLimit(query.Limit);
var direction = descending ? "DESC" : "ASC";
var sql = $"{SelectSql($", {column} AS sort_key")}{Where(where)} " +
$"ORDER BY {column} {direction} NULLS LAST, s.id {direction} LIMIT {p.Add(limit + 1)}";
return _db.RunAsync(async session =>
{
var rows = await session.QueryListAsync(sql, p.ToArray(), r => (Item: Map(r), Key: SortValue(query.Sort, r)), cancellationToken).ConfigureAwait(false);
string? next = null;
if (rows.Count > limit)
{
rows.RemoveAt(limit);
next = Cursor.Encode(scope, rows[^1].Key, rows[^1].Item.Id);
}
return new Page<Sequence>(rows.ConvertAll(r => r.Item), next);
}, cancellationToken);
}
/// <summary>Number of sequences matching the filters of <paramref name="query"/> (sort and page ignored).</summary>
public async Task<int> CountAsync(SequenceQuery query, CancellationToken cancellationToken = default)
{
var p = new SqlParameters();
var where = Filter(query, p);
return (int)await _db.ScalarAsync<long>($"SELECT count(*) FROM sequences s{Where(where)}", p.ToArray(), cancellationToken).ConfigureAwait(false);
}
/// <summary>Totals, distributions and health counters of one archive, or of all archives.</summary>
public Task<ArchiveStats> GetStatsAsync(Guid? archiveId = null, CancellationToken cancellationToken = default)
{
var sql = _db.Sql;
DbParam[] args = archiveId is { } a ? [a] : [];
var seqWhere = archiveId is null ? string.Empty : " WHERE s.archive_id = $1";
var seqAnd = archiveId is null ? " WHERE" : " WHERE s.archive_id = $1 AND";
var frameFrom = archiveId is null ? "frames f" : "frames f JOIN sequences s ON s.id = f.sequence_id WHERE s.archive_id = $1";
var year = sql.Year("s.first_capture");
return _db.RunAsync(async session =>
{
var totals = await session.QuerySingleAsync(
$"SELECT count(*), COALESCE(SUM(s.frame_count), 0), COALESCE(SUM(s.total_bytes), 0) FROM sequences s{seqWhere}",
args, cancellationToken).ConfigureAwait(false);
var byYear = await session.QueryListAsync(
$"SELECT {year}, count(*), COALESCE(SUM(s.frame_count), 0), COALESCE(SUM(s.total_bytes), 0) FROM sequences s" +
$"{seqAnd} s.first_capture IS NOT NULL GROUP BY 1 ORDER BY 1", args,
r => new YearBucket(r.GetInt32(0), r.GetInt32(1), r.GetInt64(2), r.GetInt64(3)), cancellationToken).ConfigureAwait(false);
var byMonth = await session.QueryListAsync(
$"SELECT {year}, {sql.Month("s.first_capture")}, count(*), COALESCE(SUM(s.frame_count), 0) FROM sequences s" +
$"{seqAnd} s.first_capture IS NOT NULL GROUP BY 1, 2 ORDER BY 1, 2", args,
r => new MonthBucket(r.GetInt32(0), r.GetInt32(1), r.GetInt32(2), r.GetInt64(3)), cancellationToken).ConfigureAwait(false);
var byCamera = await session.QueryListAsync(
$"SELECT s.camera_make, s.camera_model, count(*), COALESCE(SUM(s.frame_count), 0), COALESCE(SUM(s.total_bytes), 0) " +
$"FROM sequences s{seqWhere} GROUP BY s.camera_make, s.camera_model ORDER BY 4 DESC, 2 NULLS LAST, 1 NULLS LAST", args,
r => new CameraBucket(r.GetStringOrNull(0), r.GetStringOrNull(1), r.GetInt32(2), r.GetInt64(3), r.GetInt64(4)), cancellationToken).ConfigureAwait(false);
var byFormat = await session.QueryListAsync(
$"SELECT f.format, count(*), COALESCE(SUM(f.size), 0) FROM {frameFrom} GROUP BY f.format ORDER BY 2 DESC, 1", args,
r => new FormatBucket((ImageFileFormat)r.GetInt32(0), r.GetInt64(1), r.GetInt64(2)), cancellationToken).ConfigureAwait(false);
var sequenceHealth = await session.QuerySingleAsync(
"SELECT " +
"COALESCE(SUM(CASE WHEN s.verified_at IS NULL THEN 1 ELSE 0 END), 0), " +
$"COALESCE(SUM(CASE WHEN (s.status & {(int)SequenceStatus.HasWarnings}) <> 0 OR s.warning_count > 0 THEN 1 ELSE 0 END), 0), " +
$"COALESCE(SUM(CASE WHEN (s.status & {(int)SequenceStatus.HasErrors}) <> 0 THEN 1 ELSE 0 END), 0), " +
$"COALESCE(SUM(CASE WHEN (s.status & {(int)SequenceStatus.AnalyzedLight}) = 0 THEN 1 ELSE 0 END), 0) " +
$"FROM sequences s{seqWhere}", args, cancellationToken).ConfigureAwait(false);
var frameHealth = await session.QuerySingleAsync(
"SELECT " +
$"COALESCE(SUM(CASE WHEN (f.flags & {(int)FrameFlags.Missing}) <> 0 THEN 1 ELSE 0 END), 0), " +
$"COALESCE(SUM(CASE WHEN (f.flags & {(int)FrameFlags.Corrupt}) <> 0 THEN 1 ELSE 0 END), 0), " +
$"COALESCE(SUM(CASE WHEN (f.flags & {(int)FrameFlags.Duplicate}) <> 0 THEN 1 ELSE 0 END), 0) " +
$"FROM {frameFrom}", args, cancellationToken).ConfigureAwait(false);
return new ArchiveStats(
totals.GetInt32(0), totals.GetInt64(1), totals.GetInt64(2), byYear, byMonth, byCamera, byFormat,
new ArchiveHealth(
sequenceHealth.GetInt32(0), sequenceHealth.GetInt32(1), sequenceHealth.GetInt32(2), sequenceHealth.GetInt32(3),
frameHealth.GetInt64(0), frameHealth.GetInt64(1), frameHealth.GetInt64(2)));
}, cancellationToken);
}
/// <summary>
/// What the Library's filters offer for <paramref name="archiveId"/>: years of first capture (newest first), camera
/// models (most sequences first; one entry per model, as the camera filter matches the model alone) and the tags in use
/// (by name). Only sequence rows are read, never frames.
/// </summary>
public Task<ArchiveFacets> GetFacetsAsync(Guid archiveId, CancellationToken cancellationToken = default)
{
DbParam[] args = [archiveId];
var year = _db.Sql.Year("s.first_capture");
return _db.RunAsync(async session =>
{
var years = await session.QueryListAsync(
$"SELECT {year}, count(*) FROM sequences s WHERE s.archive_id = $1 AND s.first_capture IS NOT NULL GROUP BY 1 ORDER BY 1 DESC", args,
r => new ArchiveFacet(r.GetInt32(0).ToString(CultureInfo.InvariantCulture), r.GetInt32(1)), cancellationToken).ConfigureAwait(false);
var cameras = await session.QueryListAsync(
"SELECT s.camera_model, MIN(s.camera_make), count(*) FROM sequences s " +
"WHERE s.archive_id = $1 AND s.camera_model IS NOT NULL AND s.camera_model <> '' GROUP BY s.camera_model ORDER BY 3 DESC, 1", args,
r => new ArchiveFacet(r.GetString(0), r.GetInt32(2), r.GetStringOrNull(1)), cancellationToken).ConfigureAwait(false);
var tags = await session.QueryListAsync(
"SELECT MIN(t.name), count(*) FROM sequence_tags st JOIN tags t ON t.id = st.tag_id JOIN sequences s ON s.id = st.sequence_id " +
"WHERE s.archive_id = $1 GROUP BY t.name_key ORDER BY t.name_key", args,
r => new ArchiveFacet(r.GetString(0), r.GetInt32(1)), cancellationToken).ConfigureAwait(false);
return new ArchiveFacets(years, cameras, tags);
}, cancellationToken);
}
/// <summary>
/// A digest of what a Reorganize of <paramref name="sequenceIds"/> (null: every sequence) plans from in the index of
/// <paramref name="archiveId"/> (owner review 2, R7): every sequence of the archive and its folder, which decide the
/// folders that get " (2)" and what a moving folder carries; of the sequences planned, their title, first capture,
/// camera, lens, photo count, frames version and how many of their photos are hashed (only those get a new name, and a
/// verify hashes photos without another frames version); of the others inside their folders, the title. Another value
/// once one of these changes. Any other write (an upload into a sequence not planned, a verify that hashes no photo of
/// a planned sequence, a status, tag or note change) leaves it as it was, and so does a change made on disk alone, which
/// the index does not have.
/// </summary>
public async Task<string> GetStampAsync(Guid archiveId, IReadOnlyCollection<Guid>? sequenceIds, CancellationToken cancellationToken = default)
{
var rows = await _db.ListAsync(
"SELECT id, folder, title, first_capture, camera_make, camera_model, lens, frame_count, frames_version FROM sequences WHERE archive_id = $1 ORDER BY id",
[archiveId], r => (Id: r.GetGuid(0), Folder: r.GetString(1), Title: r.GetStringOrNull(2), Read: string.Create(CultureInfo.InvariantCulture,
$"{r.GetStringOrNull(2)}\u001f{r.GetDateTimeOrNull(3):O}\u001f{r.GetStringOrNull(4)}\u001f{r.GetStringOrNull(5)}\u001f{r.GetStringOrNull(6)}\u001f" +
$"{r.GetInt32(7)}\u001f{r.GetInt32(8)}")), cancellationToken).ConfigureAwait(false);
var selected = sequenceIds?.ToHashSet();
var hashed = await CountHashedAsync(archiveId, selected, cancellationToken).ConfigureAwait(false);
var planned = new HashSet<string>(rows.Where(r => selected?.Contains(r.Id) != false && r.Folder.Length > 0).Select(r => r.Folder),
StringComparer.OrdinalIgnoreCase);
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
foreach (var row in rows)
{
var read = selected?.Contains(row.Id) != false
? string.Create(CultureInfo.InvariantCulture, $"{row.Read}\u001f{hashed.GetValueOrDefault(row.Id)}")
: Inside(row.Folder) ? row.Title : null;
hash.AppendData(Encoding.UTF8.GetBytes(string.Create(CultureInfo.InvariantCulture, $"{row.Id:N}\u001f{row.Folder}\u001e{read}\n")));
}
return Convert.ToHexStringLower(hash.GetHashAndReset());
// Whether a folder lies inside a planned sequence's folder (RenamePlanner.IsInside), checking each folder around it.
bool Inside(string folder)
{
for (var slash = folder.LastIndexOf('/'); slash > 0; slash = folder.LastIndexOf('/', slash - 1))
{
if (planned.Contains(folder[..slash])) return true;
}
return false;
}
}
/// <summary>
/// A digest of what a conversion of <paramref name="sequenceIds"/> (null: every sequence) plans from in the index of
/// <paramref name="archiveId"/> (owner review 2, R7): what a Reorganize's does (<see cref="GetStampAsync"/>: every sequence
/// and its folder, which decide what "every sequence" is and its order; of the sequences planned, their title, photo
/// count, frames version and how many of their photos are hashed), and of the photos of the sequences planned, what
/// decides whether each converts and the size it is estimated at: their format, size and pixel size, and the flags a
/// scan, a verify or a decode records without another frames version (missing, corrupt, read only from the preview,
/// undecodable). Another value once one of these changes. A change made on disk alone, which the index does not have
/// (a file gone before a scan finds it), leaves it as it was.
/// </summary>
public async Task<string> GetConvertStampAsync(Guid archiveId, IReadOnlyCollection<Guid>? sequenceIds, CancellationToken cancellationToken = default)
{
var stamp = await GetStampAsync(archiveId, sequenceIds, cancellationToken).ConfigureAwait(false);
// Per sequence planned: its photos, their bytes, formats and pixel sizes, and how many carry each flag that decides.
var columns = "count(*), COALESCE(SUM(f.size), 0), COALESCE(SUM(f.format), 0), COALESCE(SUM(f.width), 0), COALESCE(SUM(f.height), 0), " +
string.Join(", ", new[] { FrameFlags.Missing, FrameFlags.Corrupt, FrameFlags.PreviewOnly, FrameFlags.Undecodable }
.Select(flag => $"COALESCE(SUM(CASE WHEN (f.flags & {(int)flag}) <> 0 THEN 1 ELSE 0 END), 0)"));
static (Guid Id, string Photos) Read(DbRow r) =>
(r.GetGuid(0), string.Join('\u001f', Enumerable.Range(1, 9).Select(i => r.GetInt64(i).ToString(CultureInfo.InvariantCulture))));
var sums = new List<(Guid Id, string Photos)>();
if (sequenceIds is null)
{
sums.AddRange(await _db.ListAsync(
$"SELECT f.sequence_id, {columns} FROM frames f JOIN sequences s ON s.id = f.sequence_id WHERE s.archive_id = $1 GROUP BY f.sequence_id",
[archiveId], Read, cancellationToken).ConfigureAwait(false));
}
else
{
// 500 ids per statement stays below SQLite's historical 999-variable limit.
foreach (var chunk in sequenceIds.Distinct().Chunk(500))
{
var p = new SqlParameters();
var ids = string.Join(", ", chunk.Select(id => p.Add(id)));
sums.AddRange(await _db.ListAsync($"SELECT f.sequence_id, {columns} FROM frames f WHERE f.sequence_id IN ({ids}) GROUP BY f.sequence_id",
p.ToArray(), Read, cancellationToken).ConfigureAwait(false));
}
}
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
hash.AppendData(Encoding.UTF8.GetBytes(stamp));
foreach (var (id, photos) in sums.OrderBy(s => s.Id))
hash.AppendData(Encoding.UTF8.GetBytes(string.Create(CultureInfo.InvariantCulture, $"\n{id:N}\u001f{photos}")));
return Convert.ToHexStringLower(hash.GetHashAndReset());
}
/// <summary>
/// How many photos of each of <paramref name="sequenceIds"/> (null: every sequence of <paramref name="archiveId"/>) have
/// their SHA-256 in the index; a sequence without one is not listed.
/// </summary>
private async Task<Dictionary<Guid, long>> CountHashedAsync(Guid archiveId, IReadOnlyCollection<Guid>? sequenceIds, CancellationToken cancellationToken)
{
var counts = new Dictionary<Guid, long>();
if (sequenceIds is null)
{
foreach (var (id, count) in await _db.ListAsync(
"SELECT f.sequence_id, count(*) FROM frames f JOIN sequences s ON s.id = f.sequence_id " +
"WHERE s.archive_id = $1 AND f.sha256 IS NOT NULL GROUP BY f.sequence_id",
[archiveId], r => (r.GetGuid(0), r.GetInt64(1)), cancellationToken).ConfigureAwait(false)) counts[id] = count;
return counts;
}
// 500 ids per statement stays below SQLite's historical 999-variable limit.
foreach (var chunk in sequenceIds.Chunk(500))
{
var p = new SqlParameters();
var ids = string.Join(", ", chunk.Select(id => p.Add(id)));
foreach (var (id, count) in await _db.ListAsync(
$"SELECT sequence_id, count(*) FROM frames WHERE sequence_id IN ({ids}) AND sha256 IS NOT NULL GROUP BY sequence_id",
p.ToArray(), r => (r.GetGuid(0), r.GetInt64(1)), cancellationToken).ConfigureAwait(false)) counts[id] = count;
}
return counts;
}
/// <summary>
/// The sequences of <paramref name="archiveId"/> a rolling verification checks next (owner review 2, R10): those never
/// verified first, then the longest unverified, at most <paramref name="limit"/>. With <paramref name="verifiedBefore"/>
/// only those never verified or last verified before it are listed. Sequences without frames are left out.
/// </summary>
public Task<IReadOnlyList<Guid>> ListVerifyCandidatesAsync(Guid archiveId, DateTimeOffset? verifiedBefore, int limit,
CancellationToken cancellationToken = default)
{
var p = new SqlParameters();
var where = $"archive_id = {p.Add(archiveId)} AND frame_count > 0";
if (verifiedBefore is { } before) where += $" AND (verified_at IS NULL OR verified_at < {p.Add(before)})";
return _db.ListAsync($"SELECT id FROM sequences WHERE {where} ORDER BY verified_at ASC NULLS FIRST, id LIMIT {p.Add(Math.Max(1, limit))}",
p.ToArray(), r => r.GetGuid(0), cancellationToken);
}
/// <summary>Folder and title of every sequence of an archive, by folder.</summary>
public Task<IReadOnlyList<SequenceLocation>> ListLocationsAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
_db.ListAsync("SELECT id, folder, title FROM sequences WHERE archive_id = $1 ORDER BY folder, id", [archiveId],
r => new SequenceLocation(r.GetGuid(0), r.GetString(1), r.GetStringOrNull(2)), cancellationToken);
/// <summary>The sequence created for a session key (automatic grouping of uploads and card sessions).</summary>
public Task<Guid?> FindBySessionKeyAsync(Guid archiveId, string sessionKey, CancellationToken cancellationToken = default) =>
_db.ScalarAsync<Guid?>("SELECT sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key = $2",
[archiveId, sessionKey], cancellationToken);
/// <summary>Binds a session key to a sequence unless it is already bound; returns the sequence the key resolves to.</summary>
public Task<Guid> MapSessionKeyAsync(Guid archiveId, string sessionKey, Guid sequenceId, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
await session.ExecuteAsync(
"INSERT INTO sequence_sessions (archive_id, session_key, sequence_id) VALUES ($1, $2, $3) ON CONFLICT (archive_id, session_key) DO NOTHING",
[archiveId, sessionKey, sequenceId], cancellationToken).ConfigureAwait(false);
return (await session.ScalarAsync<Guid?>("SELECT sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key = $2",
[archiveId, sessionKey], cancellationToken).ConfigureAwait(false))!.Value;
}, cancellationToken);
/// <summary>
/// Moves a sequence to another folder and, with <paramref name="title"/>, retitles it (mass rename; settings version
/// + 1 when the title changes); every other field stays as the locked row has it. False when it does not exist.
/// </summary>
/// <param name="title">The new title; null keeps the current one.</param>
public Task<bool> RelocateAsync(Guid id, string folder, string? title, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
var current = await session.QuerySingleOrDefaultAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken)
.ConfigureAwait(false);
if (current is null) return false;
var retitled = title is not null && current.Title != title;
await UpsertRowAsync(session, current with
{
Folder = folder,
Title = retitled ? title : current.Title,
SettingsVersion = retitled ? current.SettingsVersion + 1 : current.SettingsVersion,
UpdatedAt = DateTimeOffset.UtcNow,
}, cancellationToken).ConfigureAwait(false);
return true;
}, cancellationToken);
private static List<string> Filter(SequenceQuery query, SqlParameters p)
{
var where = new List<string>();
if (query.ArchiveId is { } archive) where.Add($"s.archive_id = {p.Add(archive)}");
if (query.Year is { } year)
where.Add($"s.first_capture >= {p.Add(new DateTime(year, 1, 1))} AND s.first_capture < {p.Add(new DateTime(year + 1, 1, 1))}");
if (query.Camera is { } camera) where.Add($"s.camera_model = {p.Add(camera)}");
if (query.WithStatus != SequenceStatus.None)
{
var mask = p.Add((int)query.WithStatus);
where.Add($"(s.status & {mask}) = {mask}");
}
if (query.WithoutStatus != SequenceStatus.None) where.Add($"(s.status & {p.Add((int)query.WithoutStatus)}) = 0");
if (query.Rendered is { } rendered) where.Add(rendered ? "s.rendered_count > 0" : "s.rendered_count = 0");
if (query.SyncStates is { Count: > 0 } states) where.Add($"s.sync_state IN ({string.Join(", ", states.Select(state => p.Add((int)state)))})");
if (query.Tag is { } tag)
where.Add("EXISTS (SELECT 1 FROM sequence_tags st JOIN tags t ON t.id = st.tag_id " +
$"WHERE st.sequence_id = s.id AND t.name_key = {p.Add(TagRepository.Key(tag))})");
if (!string.IsNullOrWhiteSpace(query.Text))
{
var pattern = p.Add(SqlDialect.ContainsPattern(query.Text.Trim().ToLowerInvariant()));
where.Add($"(s.search_text LIKE {pattern} ESCAPE '\\' OR EXISTS (SELECT 1 FROM sequence_tags st JOIN tags t ON t.id = st.tag_id " +
$"WHERE st.sequence_id = s.id AND t.name_key LIKE {pattern} ESCAPE '\\'))");
}
if (query.OwnerUserId is { } owner) where.Add($"s.owner_user_id = {p.Add(owner)}");
return where;
}
private static string Where(List<string> conditions) =>
conditions.Count == 0 ? string.Empty : " WHERE " + string.Join(" AND ", conditions);
private static (string Column, bool Descending) SortKey(SequenceSort sort) => sort switch
{
SequenceSort.Newest => ("s.first_capture", true),
SequenceSort.Oldest => ("s.first_capture", false),
SequenceSort.Title => ("s.title_key", false),
SequenceSort.FrameCount => ("s.frame_count", true),
SequenceSort.Size => ("s.total_bytes", true),
_ => throw new ArgumentOutOfRangeException(nameof(sort), sort, null),
};
private static object? SortValue(SequenceSort sort, DbRow row) => sort switch
{
SequenceSort.Newest or SequenceSort.Oldest => row.GetDateTimeOrNull(SortKeyOrdinal),
SequenceSort.Title => row.GetString(SortKeyOrdinal),
_ => row.GetInt64(SortKeyOrdinal),
};
private static Task<int> UpsertRowAsync(IDbSession session, Sequence s, CancellationToken cancellationToken)
{
var search = string.Join(' ', new[] { s.Title, s.CameraMake, s.CameraModel }.Where(v => !string.IsNullOrWhiteSpace(v))).ToLowerInvariant();
return session.ExecuteAsync(UpsertSql,
[
s.Id, s.ArchiveId, s.Title, s.DisplayTitle.ToLowerInvariant(), search, s.Folder, s.CameraMake, s.CameraModel, s.CameraSerial, s.Lens,
Wall(s.FirstCapture), Wall(s.LastCapture), s.FrameCount, s.MedianInterval, s.TotalBytes, (int)s.Status, s.RenderedCount, s.VerifiedAt,
s.HeroFrameId, s.Rating, s.Notes, DbParam.Json(s.SettingsJson), DbParam.Json(s.AnalysisJson), (int)s.Origin, (int)s.SyncState,
s.SettingsVersion, s.FramesVersion, s.OwnerUserId, s.WarningCount, s.UpdatedAt, s.PreviewOnlyCount,
], cancellationToken);
}
/// <summary>
/// Row-locks a sequence for a read-modify-write (PostgreSQL; SQLite's write lock already serialises writers).
/// False when it does not exist.
/// </summary>
internal static async Task<bool> LockAsync(IDbSession session, Guid id, CancellationToken cancellationToken) =>
await session.ScalarAsync<Guid?>($"SELECT id FROM sequences WHERE id = $1{SqlDialect.For(session.Dialect).ForUpdate}", [id], cancellationToken)
.ConfigureAwait(false) is not null;
/// <summary>
/// Replaces the tag links of a sequence whose row the caller has locked (or just upserted); a set the sequence already
/// carries (names compared case-insensitively) is left as it is. On PostgreSQL every tag written is locked until
/// commit, and the tags are written in the order of their keys whatever order the caller listed them in (a PATCH sends
/// them in any order): writers of two sequences that share tags then take those rows in one order, and never wait for
/// each other's (PostgreSQL would abort one with a deadlock, and nothing retries it).
/// </summary>
private async Task ReplaceTagsAsync(IDbSession session, Guid sequenceId, IEnumerable<string> tags, CancellationToken cancellationToken)
{
var wanted = TagRepository.NormalizeSet(tags);
var keys = wanted.Select(TagRepository.Key).ToList();
var stored = await session.QueryListAsync("SELECT t.name_key FROM sequence_tags st JOIN tags t ON t.id = st.tag_id WHERE st.sequence_id = $1",
[sequenceId], r => r.GetString(0), cancellationToken).ConfigureAwait(false);
if (stored.Count == keys.Count && stored.ToHashSet(StringComparer.Ordinal).SetEquals(keys)) return;
await session.ExecuteAsync("DELETE FROM sequence_tags WHERE sequence_id = $1", [sequenceId], cancellationToken).ConfigureAwait(false);
foreach (var tag in wanted.OrderBy(TagRepository.Key, StringComparer.Ordinal))
{
DbParam[] args = [Guid.CreateVersion7(), tag, TagRepository.Key(tag)];
Guid tagId;
if (_db.Sql.IsPostgres)
{
// The no-op update locks an existing tag, so a concurrent DeleteUnusedAsync cannot remove it before the link exists.
tagId = (await session.ScalarAsync<Guid?>(
"INSERT INTO tags (id, name, name_key) VALUES ($1, $2, $3) ON CONFLICT (name_key) DO UPDATE SET name_key = excluded.name_key RETURNING id",
args, cancellationToken).ConfigureAwait(false))!.Value;
}
else
{
await session.ExecuteAsync("INSERT INTO tags (id, name, name_key) VALUES ($1, $2, $3) ON CONFLICT (name_key) DO NOTHING",
args, cancellationToken).ConfigureAwait(false);
tagId = (await session.ScalarAsync<Guid?>("SELECT id FROM tags WHERE name_key = $1", [args[2]], cancellationToken).ConfigureAwait(false))!.Value;
}
await session.ExecuteAsync("INSERT INTO sequence_tags (sequence_id, tag_id) VALUES ($1, $2)", [sequenceId, tagId], cancellationToken)
.ConfigureAwait(false);
}
}
private static string? NullIfEmpty(string value) => string.IsNullOrWhiteSpace(value) ? null : value.Trim();
/// <summary>Camera wall-clock times are stored without zone, whatever kind the caller's value has.</summary>
internal static DateTime? Wall(DateTime? value) =>
value is { } v ? DateTime.SpecifyKind(v, DateTimeKind.Unspecified) : null;
private static Sequence Map(DbRow r)
{
var tags = r.GetStringOrNull(TagsOrdinal) is { Length: > 0 } list ? list.Split(SqlDialect.ListSeparator) : [];
Array.Sort(tags, StringComparer.OrdinalIgnoreCase);
return new Sequence
{
Id = r.GetGuid(0),
ArchiveId = r.GetGuid(1),
Title = r.GetStringOrNull(2),
Folder = r.GetString(3),
CameraMake = r.GetStringOrNull(4),
CameraModel = r.GetStringOrNull(5),
CameraSerial = r.GetStringOrNull(6),
Lens = r.GetStringOrNull(7),
FirstCapture = r.GetDateTimeOrNull(8),
LastCapture = r.GetDateTimeOrNull(9),
FrameCount = r.GetInt32(10),
MedianInterval = r.GetDoubleOrNull(11),
TotalBytes = r.GetInt64(12),
Status = (SequenceStatus)r.GetInt32(13),
RenderedCount = r.GetInt32(14),
VerifiedAt = r.GetDateTimeOffsetOrNull(15),
HeroFrameId = r.GetGuidOrNull(16),
Rating = r.GetInt32(17),
Notes = r.GetStringOrNull(18),
SettingsJson = r.GetStringOrNull(19),
AnalysisJson = r.GetStringOrNull(20),
Origin = (SequenceOrigin)r.GetInt32(21),
SyncState = (SyncState)r.GetInt32(22),
SettingsVersion = r.GetInt32(23),
FramesVersion = r.GetInt32(24),
OwnerUserId = r.GetGuidOrNull(25),
WarningCount = r.GetInt32(26),
UpdatedAt = r.GetDateTimeOffset(27),
PreviewOnlyCount = r.GetInt32(28),
Tags = tags,
};
}
}