refactor!: rename Titano to Chronos and move CI and releases to Gitea (#6)
CI / Build and test (push) Successful in 8m14s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 7s
Release / Build, test and publish (push) Successful in 13m19s
CI / PostgreSQL 16 integration (push) Successful in 4m49s
CI / PostgreSQL 17 integration (push) Successful in 5m28s
CI / Docker image (push) Successful in 53s
Release / Server image (push) Failing after 10s

The owner renamed the product and moved the repository to the self-hosted
Gitea (https://gitea.encke-hake.ts.net/BalboSoft/Chronos) for more control,
and asked for Actions and a first automatic release with the server package
and the Windows exe (docs/requests/2026-10-23-chronos-gitea.md).

- Chronos everywhere: a byte-exact rename of contents and paths, then a
  per-language review of the forms the old name's ending had dictated.
- Gitea instead of GitHub: the update check, licence, issue and support
  links read ChronosReleasesUrl; the update check takes the highest stable
  version, offers a pre-release build its final release and reports a
  repository it cannot see as a failed check. EULA, security policy and
  Gitea issue forms live at the repository root.
- Gitea Actions for the Linux act_runner (.gitea/workflows): CI, release,
  nightly image and performance gates in the .NET SDK container with git
  checkout and the system SQLite. The release cross-compiles the Windows
  programs, checks their icon, version and DPI manifest, packs installers
  and zips, publishes the Gitea release and pushes the server image
  (REGISTRY_TOKEN). Windows-only tests run in an opt-in job on a Windows
  runner (WINDOWS_RUNNER).
- New Chronos.Tools commands: zip, gitea-release, postgres-ci (replaces the
  docker-exec script, which Gitea's service containers do not allow; the
  throw-away server runs without durability) and pe-check.
- Timed tests take their budget from TestTiming (CHRONOS_TEST_TIME_FACTOR,
  3 on the shared runner; the weekly performance gates keep theirs).

BREAKING CHANGE: programs, settings, data folders, file types, environment
variables and the Docker image use the new name; nothing is migrated from
Titano, which never had a release.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
This commit was merged in pull request #6.
This commit is contained in:
Alby96andClaude Opus 5.5 authored and Alby96 committed 2026-10-23 12:45:18 +02:00
1 parent c99842cc86
commit f2b58522b9
1801 files changed
+11804 -10676

No files matched your search

@@ -0,0 +1,79 @@
using Chronos.Core.Archiving;
using Chronos.Core.Model;
namespace Chronos.Data.Repositories;
/// <summary>What the last scan saw in one sequence folder (sidecar and photo-directory stamps).</summary>
public sealed record FolderScanState(string Folder, Guid? SequenceId, FileStamp? Sidecar, FileStamp? Photos);
/// <summary>Archive roots (BRIEF §5, §7).</summary>
public sealed class ArchiveRepository
{
private const string Columns = "id, name, root_path, folder_template, file_template, schema_version, created, proxy_long_edge";
private const string FolderStateSelect = "SELECT folder, sequence_id, sidecar_size, sidecar_ticks, photos_ticks FROM archive_folders";
private static readonly string UpsertSql = SqlDialect.Upsert("archives",
["id", "name", "root_path", "folder_template", "file_template", "schema_version", "created", "proxy_long_edge"], 1);
private static readonly string FolderStateUpsertSql = SqlDialect.Upsert("archive_folders",
["archive_id", "folder", "sequence_id", "sidecar_size", "sidecar_ticks", "photos_ticks", "scanned_at"], 2);
private readonly DbAccess _db;
internal ArchiveRepository(DbAccess db) => _db = db;
public async Task UpsertAsync(Archive archive, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync(UpsertSql,
[archive.Id, archive.Name, archive.RootPath, archive.FolderTemplate, archive.FileTemplate, archive.SchemaVersion, archive.Created, archive.ProxyLongEdge],
cancellationToken).ConfigureAwait(false);
public Task<Archive?> GetAsync(Guid id, CancellationToken cancellationToken = default) =>
_db.SingleAsync($"SELECT {Columns} FROM archives WHERE id = $1", [id], Map, cancellationToken);
public Task<IReadOnlyList<Archive>> ListAsync(CancellationToken cancellationToken = default) =>
_db.ListAsync($"SELECT {Columns} FROM archives ORDER BY name, id", [], Map, cancellationToken);
/// <summary>Deletes the archive and everything indexed under it (the files on disk are untouched).</summary>
public async Task<bool> DeleteAsync(Guid id, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("DELETE FROM archives WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>Scan stamps of every folder of an archive, by folder.</summary>
public async Task<IReadOnlyDictionary<string, FolderScanState>> GetFolderStatesAsync(Guid archiveId, CancellationToken cancellationToken = default)
{
var rows = await _db.ListAsync($"{FolderStateSelect} WHERE archive_id = $1", [archiveId], MapFolderState, cancellationToken).ConfigureAwait(false);
return rows.ToDictionary(s => s.Folder, StringComparer.Ordinal);
}
/// <summary>Scan stamps of one folder of an archive; null when none are recorded.</summary>
public Task<FolderScanState?> GetFolderStateAsync(Guid archiveId, string folder, CancellationToken cancellationToken = default) =>
_db.SingleAsync($"{FolderStateSelect} WHERE archive_id = $1 AND folder = $2", [archiveId, folder], MapFolderState, cancellationToken);
public async Task SetFolderStateAsync(Guid archiveId, FolderScanState state, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync(FolderStateUpsertSql,
[archiveId, state.Folder, state.SequenceId, state.Sidecar?.Size, state.Sidecar?.ModifiedUtc.Ticks, state.Photos?.ModifiedUtc.Ticks, DateTimeOffset.UtcNow],
cancellationToken).ConfigureAwait(false);
public async Task<bool> DeleteFolderStateAsync(Guid archiveId, string folder, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("DELETE FROM archive_folders WHERE archive_id = $1 AND folder = $2", [archiveId, folder], cancellationToken)
.ConfigureAwait(false) > 0;
/// <summary>Forgets every scan stamp of an archive, so the next scan reads every sidecar (rebuild).</summary>
public Task<int> ClearFolderStatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
_db.ExecuteAsync("DELETE FROM archive_folders WHERE archive_id = $1", [archiveId], cancellationToken);
private static FolderScanState MapFolderState(DbRow r) => new(r.GetString(0), r.GetGuidOrNull(1),
r.GetInt64OrNull(3) is { } sidecar ? new FileStamp(r.GetInt64(2), new DateTime(sidecar, DateTimeKind.Utc)) : null,
r.GetInt64OrNull(4) is { } photos ? new FileStamp(0, new DateTime(photos, DateTimeKind.Utc)) : null);
private static Archive Map(DbRow r) => new()
{
Id = r.GetGuid(0),
Name = r.GetString(1),
RootPath = r.GetString(2),
FolderTemplate = r.GetString(3),
FileTemplate = r.GetString(4),
SchemaVersion = r.GetInt32(5),
Created = r.GetDateTimeOffset(6),
ProxyLongEdge = r.GetInt32(7),
};
}
@@ -0,0 +1,69 @@
using Chronos.Core.Model;
namespace Chronos.Data.Repositories;
/// <summary>Append-only record of who did what (BRIEF §5 AuditLog, §14.2).</summary>
public sealed class AuditRepository
{
private const string Columns = "id, at, user_id, username, action, target_kind, target_id, details, remote_address";
private readonly DbAccess _db;
internal AuditRepository(DbAccess db) => _db = db;
public async Task AppendAsync(AuditEntry entry, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync($"INSERT INTO audit_log ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)",
[entry.Id, entry.At, entry.UserId, entry.Username, entry.Action, entry.TargetKind, entry.TargetId, entry.Details, entry.RemoteAddress],
cancellationToken).ConfigureAwait(false);
/// <summary>Entries newest first, optionally of one user.</summary>
public Task<Page<AuditEntry>> ListAsync(Guid? userId = null, int limit = 100, string? cursor = null, CancellationToken cancellationToken = default) =>
ListAsync(userId, action: null, limit, cursor, cancellationToken);
/// <summary>Entries newest first, optionally of one user and one action.</summary>
public Task<Page<AuditEntry>> ListAsync(Guid? userId, string? action, int limit = 100, string? cursor = null, CancellationToken cancellationToken = default) =>
ListByActionsAsync(userId, action is null ? [] : [action], limit, cursor, cancellationToken);
/// <summary>Entries newest first, optionally of one user, of any of the actions (none: all), e.g. the sign-in log with failed attempts.</summary>
public Task<Page<AuditEntry>> ListByActionsAsync(Guid? userId, IReadOnlyCollection<string> actions, int limit = 100, string? cursor = null,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(actions);
var p = new SqlParameters();
var where = new List<string>();
if (userId is { } user) where.Add($"user_id = {p.Add(user)}");
if (actions.Count > 0) where.Add($"action IN ({string.Join(", ", actions.Select(a => p.Add(a)))})");
if (cursor is not null)
{
var (value, id) = Cursor.Decode("audit", cursor);
where.Add(Cursor.After("at", "id", descending: true, value, id, p.Add));
}
var take = Cursor.ClampLimit(limit);
var sql = $"SELECT {Columns} FROM audit_log{(where.Count > 0 ? " WHERE " + string.Join(" AND ", where) : string.Empty)} " +
$"ORDER BY at DESC, id DESC LIMIT {p.Add(take + 1)}";
return _db.RunAsync(async session =>
{
var entries = await session.QueryListAsync(sql, p.ToArray(), Map, cancellationToken).ConfigureAwait(false);
string? next = null;
if (entries.Count > take)
{
entries.RemoveAt(take);
next = Cursor.Encode("audit", entries[^1].At, entries[^1].Id);
}
return new Page<AuditEntry>(entries, next);
}, cancellationToken);
}
private static AuditEntry Map(DbRow r) => new()
{
Id = r.GetGuid(0),
At = r.GetDateTimeOffset(1),
UserId = r.GetGuidOrNull(2),
Username = r.GetStringOrNull(3),
Action = r.GetString(4),
TargetKind = r.GetStringOrNull(5),
TargetId = r.GetStringOrNull(6),
Details = r.GetStringOrNull(7),
RemoteAddress = r.GetStringOrNull(8),
};
}
+145
View File
@@ -0,0 +1,145 @@
using System.Buffers.Text;
using System.Globalization;
using System.Text;
namespace Chronos.Data.Repositories;
/// <summary>One page of a keyset-paginated listing; pass <see cref="NextCursor"/> back to get the following page.</summary>
public sealed record Page<T>(IReadOnlyList<T> Items, string? NextCursor);
/// <summary>
/// Where repository commands run: on a caller's session (so they join its transaction) or on a session
/// borrowed from the pool for the duration of one call.
/// </summary>
internal sealed class DbAccess
{
private readonly IDatabase? _database;
private readonly IDbSession? _session;
public DbAccess(IDatabase database)
{
_database = database;
Sql = SqlDialect.For(database.Dialect);
}
public DbAccess(IDbSession session)
{
_session = session;
Sql = SqlDialect.For(session.Dialect);
}
public SqlDialect Sql { get; }
public async Task<T> RunAsync<T>(Func<IDbSession, Task<T>> work, CancellationToken cancellationToken)
{
if (_session is not null) return await work(_session).ConfigureAwait(false);
var session = await _database!.OpenAsync(cancellationToken).ConfigureAwait(false);
await using (session.ConfigureAwait(false)) return await work(session).ConfigureAwait(false);
}
/// <summary>Runs <paramref name="work"/> atomically: inside the caller's transaction, or in a new one.</summary>
public Task<T> InTransactionAsync<T>(Func<IDbSession, Task<T>> work, CancellationToken cancellationToken) =>
RunAsync(async session =>
{
if (session.InTransaction) return await work(session).ConfigureAwait(false);
var transaction = await session.BeginAsync(cancellationToken).ConfigureAwait(false);
await using (transaction.ConfigureAwait(false))
{
var result = await work(session).ConfigureAwait(false);
await transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
return result;
}
}, cancellationToken);
public Task<int> ExecuteAsync(string sql, DbParam[] parameters, CancellationToken cancellationToken) =>
RunAsync(s => s.ExecuteAsync(sql, parameters, cancellationToken), cancellationToken);
public async Task<IReadOnlyList<T>> ListAsync<T>(string sql, DbParam[] parameters, Func<DbRow, T> map, CancellationToken cancellationToken) =>
await RunAsync(s => s.QueryListAsync(sql, parameters, map, cancellationToken), cancellationToken).ConfigureAwait(false);
public Task<T?> SingleAsync<T>(string sql, DbParam[] parameters, Func<DbRow, T> map, CancellationToken cancellationToken) =>
RunAsync(s => s.QuerySingleOrDefaultAsync(sql, parameters, map, cancellationToken), cancellationToken);
public Task<T?> ScalarAsync<T>(string sql, DbParam[] parameters, CancellationToken cancellationToken) =>
RunAsync(s => s.ScalarAsync<T>(sql, parameters, cancellationToken), cancellationToken);
}
/// <summary>
/// Opaque keyset cursors: the sort value and id of the last row of a page, tagged with the listing they belong
/// to so a cursor cannot be replayed against a different ordering.
/// </summary>
internal static class Cursor
{
public static string Encode(string scope, object? value, Guid id)
{
var payload = value switch
{
null => "n",
DateTime d => "t" + d.Ticks.ToString(CultureInfo.InvariantCulture),
DateTimeOffset o => "o" + o.UtcTicks.ToString(CultureInfo.InvariantCulture),
long l => "i" + l.ToString(CultureInfo.InvariantCulture),
int i => "i" + i.ToString(CultureInfo.InvariantCulture),
string s => "s" + s,
_ => throw new NotSupportedException($"Cursor values of type {value.GetType().Name} are not supported."),
};
return Base64Url.EncodeToString(Encoding.UTF8.GetBytes($"{scope}|{id:N}|{payload}"));
}
/// <summary>Decodes a cursor of <paramref name="scope"/>; anything else is a <see cref="FormatException"/>.</summary>
public static (object? Value, Guid Id) Decode(string scope, string cursor)
{
string text;
try
{
text = Encoding.UTF8.GetString(Base64Url.DecodeFromChars(cursor));
}
catch (FormatException ex)
{
throw new FormatException("Invalid paging cursor.", ex);
}
var parts = text.Split('|', 3);
if (parts.Length != 3 || parts[0] != scope || !Guid.TryParseExact(parts[1], "N", out var id) || parts[2].Length == 0)
throw new FormatException("The paging cursor does not belong to this listing.");
var payload = parts[2][1..];
object? value = parts[2][0] switch
{
'n' => null,
't' => new DateTime(long.Parse(payload, CultureInfo.InvariantCulture), DateTimeKind.Unspecified),
'o' => new DateTimeOffset(long.Parse(payload, CultureInfo.InvariantCulture), TimeSpan.Zero),
'i' => long.Parse(payload, CultureInfo.InvariantCulture),
's' => payload,
_ => throw new FormatException("Invalid paging cursor."),
};
return (value, id);
}
/// <summary>
/// Keyset condition "after (value, id)" for <c>ORDER BY column dir NULLS LAST, id dir</c>; adds its parameters
/// through <paramref name="parameter"/>, which returns the placeholder text.
/// </summary>
public static string After(string column, string idColumn, bool descending, object? value, Guid id, Func<DbParam, string> parameter)
{
var op = descending ? "<" : ">";
var idParam = parameter(id);
if (value is null) return $"({column} IS NULL AND {idColumn} {op} {idParam})";
var v = parameter(new DbParam(value));
return $"({column} {op} {v} OR ({column} = {v} AND {idColumn} {op} {idParam}) OR {column} IS NULL)";
}
public static int ClampLimit(int limit) => Math.Clamp(limit, 1, 1000);
}
/// <summary>Builds positional parameter lists alongside SQL text.</summary>
internal sealed class SqlParameters
{
private readonly List<DbParam> _values = [];
/// <summary>Adds a value and returns its placeholder (<c>$n</c>).</summary>
public string Add(DbParam value)
{
_values.Add(value);
return "$" + _values.Count.ToString(CultureInfo.InvariantCulture);
}
public DbParam[] ToArray() => [.. _values];
}
@@ -0,0 +1,374 @@
using System.Text;
using Chronos.Core.Imaging;
using Chronos.Core.Model;
namespace Chronos.Data.Repositories;
/// <summary>
/// Integrity fields a verify rewrites: <see cref="FrameFlags.Missing"/> and <see cref="FrameFlags.Corrupt"/> as found (other
/// bits of <paramref name="Flags"/> are ignored) and the hash of a newly hashed file.
/// </summary>
public readonly record struct FrameIntegrity(Guid FrameId, FrameFlags Flags, string? Sha256);
/// <summary>A frame and the sequence it belongs to.</summary>
public readonly record struct FrameRef(Guid FrameId, Guid SequenceId);
/// <summary>Where a frame stands in its sequence.</summary>
public readonly record struct FramePosition(Guid FrameId, int Index);
/// <summary>A frame whose content (SHA-256) also exists elsewhere in the archive.</summary>
public sealed record DuplicateFrame(string Sha256, Guid FrameId, Guid SequenceId, int Index, string RelativePath);
/// <summary>Light statistics of one frame, written by the analysis job.</summary>
public readonly record struct FrameLightStats(Guid FrameId, double? MeanLogLuminance, double? ClipLow, double? ClipHigh);
/// <summary>Decode-tier fields of one frame (BRIEF §8.2): its flags and the stored size of its largest embedded preview.</summary>
public readonly record struct FrameTierFields(Guid FrameId, FrameFlags Flags, int PreviewWidth, int PreviewHeight, ImageFileFormat Format);
/// <summary>Per-format totals and flagged-frame counts of one sequence.</summary>
public sealed record SequenceFrameStats(IReadOnlyList<FormatBucket> ByFormat, long Missing, long Corrupt, long Duplicate, long Excluded);
/// <summary>Frames of sequences: bulk import, paged listing, duplicate lookup and bulk updates.</summary>
public sealed class FrameRepository
{
private static readonly string[] ColumnNames =
[
"id", "sequence_id", "idx", "relative_path", "file_name", "original_file_name", "size", "sha256", "captured_at",
"utc_offset_minutes", "timestamp_source", "exposure_time", "f_number", "iso", "focal_length", "exposure_bias",
"white_balance", "orientation", "width", "height", "format", "preview_offset", "preview_length",
"mean_log_luminance", "clip_low", "clip_high", "flags", "preview_width", "preview_height",
];
private static readonly string Columns = string.Join(", ", ColumnNames);
private static readonly string InsertSql =
$"INSERT INTO frames ({Columns}) VALUES ({string.Join(", ", ColumnNames.Select((_, i) => "$" + (i + 1)))})";
/// <summary>Rows per statement of the batched PostgreSQL updates (4 or fewer parameters per row).</summary>
private const int Batch = 1000;
/// <summary>Rows per multi-row upsert: 30 x 29 parameters stays below SQLite's historical 999-variable limit.</summary>
private const int UpsertRows = 30;
/// <summary>Light statistics belong to the analysis job; a sync from the sidecar never clears them.</summary>
private static readonly string[] AnalysisColumns = ["mean_log_luminance", "clip_low", "clip_high"];
private readonly DbAccess _db;
internal FrameRepository(DbAccess db) => _db = db;
public async Task InsertAsync(Frame frame, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync(InsertSql, Array.ConvertAll(Values(frame), v => new DbParam(v)), cancellationToken).ConfigureAwait(false);
/// <summary>Inserts many frames at once (COPY on PostgreSQL, batched INSERTs in one transaction on SQLite).</summary>
public Task<long> BulkInsertAsync(IEnumerable<Frame> frames, CancellationToken cancellationToken = default) =>
_db.RunAsync(session => session.BulkInsertAsync("frames", ColumnNames, frames.Select(Values).ToAsyncEnumerable(), cancellationToken), cancellationToken);
public Task<Frame?> GetAsync(Guid id, CancellationToken cancellationToken = default) =>
_db.SingleAsync($"SELECT {Columns} FROM frames WHERE id = $1", [id], Map, cancellationToken);
/// <summary>Frames of a sequence by index, starting after <paramref name="afterIndex"/> (keyset paging).</summary>
public Task<IReadOnlyList<Frame>> ListBySequenceAsync(Guid sequenceId, int afterIndex = -1, int limit = 1000, CancellationToken cancellationToken = default) =>
_db.ListAsync($"SELECT {Columns} FROM frames WHERE sequence_id = $1 AND idx > $2 ORDER BY idx LIMIT $3",
[sequenceId, afterIndex, Cursor.ClampLimit(limit)], Map, cancellationToken);
/// <summary>Every frame with this content hash (duplicate detection, upload short-circuit).</summary>
public Task<IReadOnlyList<Frame>> FindBySha256Async(string sha256, CancellationToken cancellationToken = default) =>
_db.ListAsync($"SELECT {Columns} FROM frames WHERE sha256 = $1 ORDER BY sequence_id, idx", [sha256.ToLowerInvariant()], Map, cancellationToken);
/// <summary>Id and index of every frame of a sequence, in index order.</summary>
public Task<IReadOnlyList<FramePosition>> ListPositionsAsync(Guid sequenceId, CancellationToken cancellationToken = default) =>
_db.ListAsync("SELECT id, idx FROM frames WHERE sequence_id = $1 ORDER BY idx", [sequenceId], r => new FramePosition(r.GetGuid(0), r.GetInt32(1)),
cancellationToken);
/// <summary>
/// Writes analysis results for many frames in one transaction; returns the number of frames updated. The rows of their
/// sequences are locked first, as every writer of a sequence's frames does (<see cref="SequenceRepository.GetForUpdateAsync"/>).
/// </summary>
public Task<int> UpdateLightStatsAsync(IReadOnlyList<FrameLightStats> stats, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
var updated = 0;
if (_db.Sql.IsPostgres)
{
// The rows before any frame: a scan's sync holds its sequence's row while it rewrites every frame in its own
// order, so frames locked here first would wait for it while it waits for them, and PostgreSQL would abort
// one of the two. SQLite's write lock already serialises writers.
for (var start = 0; start < stats.Count; start += Batch)
{
var count = Math.Min(Batch, stats.Count - start);
var args = new DbParam[count];
var placeholders = new StringBuilder();
for (var i = 0; i < count; i++)
{
args[i] = stats[start + i].FrameId;
placeholders.Append(i > 0 ? ", $" : "$").Append(i + 1);
}
await session.ExecuteAsync(
$"SELECT id FROM sequences WHERE id IN (SELECT sequence_id FROM frames WHERE id IN ({placeholders})) ORDER BY id{_db.Sql.ForUpdate}",
args, cancellationToken).ConfigureAwait(false);
}
// One UPDATE … FROM (VALUES …) per batch instead of a round trip per frame.
for (var start = 0; start < stats.Count; start += Batch)
{
var count = Math.Min(Batch, stats.Count - start);
var sql = new StringBuilder("UPDATE frames AS f SET mean_log_luminance = v.m, clip_low = v.lo, clip_high = v.hi FROM (VALUES ");
var args = new DbParam[count * 4];
for (var i = 0; i < count; i++)
{
var s = stats[start + i];
sql.Append(i > 0 ? ", (" : "(").Append('$').Append(4 * i + 1).Append(", $").Append(4 * i + 2)
.Append(", $").Append(4 * i + 3).Append(", $").Append(4 * i + 4).Append(')');
args[4 * i] = s.FrameId;
args[4 * i + 1] = s.MeanLogLuminance;
args[4 * i + 2] = s.ClipLow;
args[4 * i + 3] = s.ClipHigh;
}
sql.Append(") AS v(id, m, lo, hi) WHERE f.id = v.id");
updated += await session.ExecuteAsync(sql.ToString(), args, cancellationToken).ConfigureAwait(false);
}
}
else
{
foreach (var s in stats)
{
updated += await session.ExecuteAsync(
"UPDATE frames SET mean_log_luminance = $2, clip_low = $3, clip_high = $4 WHERE id = $1",
[s.FrameId, s.MeanLogLuminance, s.ClipLow, s.ClipHigh], cancellationToken).ConfigureAwait(false);
}
}
return updated;
}, cancellationToken);
/// <summary>Sets and clears flags on many frames in one transaction; returns the number of frames updated.</summary>
public Task<int> UpdateFlagsAsync(IReadOnlyList<Guid> frameIds, FrameFlags set, FrameFlags clear, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
// 500 ids per statement stays below SQLite's historical 999-variable limit.
const int idsPerStatement = 500;
var updated = 0;
for (var start = 0; start < frameIds.Count; start += idsPerStatement)
{
var count = Math.Min(idsPerStatement, frameIds.Count - start);
var args = new DbParam[count + 2];
args[0] = (int)set;
args[1] = (int)clear;
var placeholders = new StringBuilder();
for (var i = 0; i < count; i++)
{
args[i + 2] = frameIds[start + i];
placeholders.Append(i > 0 ? ", $" : "$").Append(i + 3);
}
updated += await session.ExecuteAsync($"UPDATE frames SET flags = (flags | $1) & ~$2 WHERE id IN ({placeholders})",
args, cancellationToken).ConfigureAwait(false);
}
return updated;
}, cancellationToken);
/// <summary>Writes the decode-tier fields of many frames in one transaction; returns the number of frames updated.</summary>
public Task<int> UpdateTiersAsync(IReadOnlyList<FrameTierFields> frames, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
var updated = 0;
if (_db.Sql.IsPostgres)
{
// One UPDATE … FROM (VALUES …) per batch instead of a round trip per frame (a first analysis of a RAW sequence).
for (var start = 0; start < frames.Count; start += Batch)
{
var count = Math.Min(Batch, frames.Count - start);
var sql = new StringBuilder("UPDATE frames AS f SET flags = v.fl, preview_width = v.w, preview_height = v.h, format = v.fmt FROM (VALUES ");
var args = new DbParam[count * 5];
for (var i = 0; i < count; i++)
{
var f = frames[start + i];
sql.Append(i > 0 ? ", (" : "(").Append('$').Append(5 * i + 1).Append(", $").Append(5 * i + 2)
.Append(", $").Append(5 * i + 3).Append(", $").Append(5 * i + 4).Append(", $").Append(5 * i + 5).Append(')');
args[5 * i] = f.FrameId;
args[5 * i + 1] = (int)f.Flags;
args[5 * i + 2] = f.PreviewWidth;
args[5 * i + 3] = f.PreviewHeight;
args[5 * i + 4] = (int)f.Format;
}
sql.Append(") AS v(id, fl, w, h, fmt) WHERE f.id = v.id");
updated += await session.ExecuteAsync(sql.ToString(), args, cancellationToken).ConfigureAwait(false);
}
}
else
{
foreach (var f in frames)
{
updated += await session.ExecuteAsync("UPDATE frames SET flags = $2, preview_width = $3, preview_height = $4, format = $5 WHERE id = $1",
[f.FrameId, (int)f.Flags, f.PreviewWidth, f.PreviewHeight, (int)f.Format], cancellationToken).ConfigureAwait(false);
}
}
return updated;
}, cancellationToken);
/// <summary>
/// Makes the frames of a sequence exactly <paramref name="frames"/> (from its sidecar): rows are upserted by id,
/// rows not listed are deleted, light statistics of kept rows survive. Indexes may be renumbered freely.
/// </summary>
public Task<int> SyncSequenceAsync(Guid sequenceId, IReadOnlyList<Frame> frames, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
// Park current rows on negative indexes so renumbered rows never collide on (sequence_id, idx).
await session.ExecuteAsync("UPDATE frames SET idx = -idx - 1 WHERE sequence_id = $1", [sequenceId], cancellationToken).ConfigureAwait(false);
var updates = string.Join(", ", ColumnNames.Skip(1).Where(c => !AnalysisColumns.Contains(c)).Select(c => $"{c} = excluded.{c}"));
for (var start = 0; start < frames.Count; start += UpsertRows)
{
var count = Math.Min(UpsertRows, frames.Count - start);
var sql = new StringBuilder("INSERT INTO frames (").Append(Columns).Append(") VALUES ");
var args = new DbParam[count * ColumnNames.Length];
for (var i = 0; i < count; i++)
{
var values = Values(frames[start + i]);
sql.Append(i > 0 ? ", (" : "(");
for (var c = 0; c < values.Length; c++)
{
args[i * values.Length + c] = new DbParam(values[c]);
sql.Append(c > 0 ? ", $" : "$").Append(i * values.Length + c + 1);
}
sql.Append(')');
}
sql.Append(" ON CONFLICT (id) DO UPDATE SET ").Append(updates);
await session.ExecuteAsync(sql.ToString(), args, cancellationToken).ConfigureAwait(false);
}
return await session.ExecuteAsync("DELETE FROM frames WHERE sequence_id = $1 AND idx < 0", [sequenceId], cancellationToken).ConfigureAwait(false);
}, cancellationToken);
/// <summary>Number of frames of a sequence.</summary>
public async Task<int> CountBySequenceAsync(Guid sequenceId, CancellationToken cancellationToken = default) =>
(int)await _db.ScalarAsync<long>("SELECT count(*) FROM frames WHERE sequence_id = $1", [sequenceId], cancellationToken).ConfigureAwait(false);
/// <summary>Frames of an archive with this size and capture time (card import duplicate candidates).</summary>
public Task<IReadOnlyList<Frame>> FindBySizeAndCaptureAsync(Guid archiveId, long size, DateTime capturedAt, CancellationToken cancellationToken = default) =>
_db.ListAsync(
$"SELECT {string.Join(", ", ColumnNames.Select(c => "f." + c))} FROM frames f JOIN sequences s ON s.id = f.sequence_id " +
"WHERE s.archive_id = $1 AND f.size = $2 AND f.captured_at = $3",
[archiveId, size, SequenceRepository.Wall(capturedAt)], Map, cancellationToken);
/// <summary>Frames of an archive in this format captured from <paramref name="from"/> to <paramref name="to"/> (inclusive),
/// by capture time (a GoPro GPR's JPEG in another sequence).</summary>
public Task<IReadOnlyList<Frame>> ListByCaptureRangeAsync(Guid archiveId, ImageFileFormat format, DateTime from, DateTime to,
CancellationToken cancellationToken = default) =>
_db.ListAsync(
$"SELECT {string.Join(", ", ColumnNames.Select(c => "f." + c))} FROM frames f JOIN sequences s ON s.id = f.sequence_id " +
"WHERE s.archive_id = $1 AND f.format = $2 AND f.captured_at >= $3 AND f.captured_at <= $4 ORDER BY f.captured_at, f.id",
[archiveId, (int)format, SequenceRepository.Wall(from), SequenceRepository.Wall(to)], Map, cancellationToken);
/// <summary>Every frame of an archive whose SHA-256 occurs more than once, grouped by hash then sequence and index.</summary>
public Task<IReadOnlyList<DuplicateFrame>> ListDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
_db.ListAsync(
"SELECT f.sha256, f.id, f.sequence_id, f.idx, f.relative_path FROM frames f JOIN sequences s ON s.id = f.sequence_id " +
"WHERE s.archive_id = $1 AND f.sha256 IN (SELECT f2.sha256 FROM frames f2 JOIN sequences s2 ON s2.id = f2.sequence_id " +
"WHERE s2.archive_id = $1 AND f2.sha256 IS NOT NULL GROUP BY f2.sha256 HAVING count(*) > 1) " +
"ORDER BY f.sha256, s.first_capture, f.sequence_id, f.idx",
[archiveId], r => new DuplicateFrame(r.GetString(0), r.GetGuid(1), r.GetGuid(2), r.GetInt32(3), r.GetString(4)), cancellationToken);
/// <summary>
/// Sequences of an archive with frames stored as GPR whose file is not a .gpr: indexed before GoPro DNGs converted by
/// Adobe software counted as DNG (R4, owner review 1); their next light analysis probes them again.
/// </summary>
public Task<IReadOnlyList<Guid>> ListSequencesWithStaleGprAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
_db.ListAsync(
"SELECT DISTINCT f.sequence_id FROM frames f JOIN sequences s ON s.id = f.sequence_id " +
"WHERE s.archive_id = $1 AND f.format = $2 AND lower(f.file_name) NOT LIKE '%.gpr'",
[archiveId, (int)ImageFileFormat.Gpr], r => r.GetGuid(0), cancellationToken);
/// <summary>Frames of an archive that carry every bit of <paramref name="flags"/>.</summary>
public Task<IReadOnlyList<FrameRef>> ListFlaggedAsync(Guid archiveId, FrameFlags flags, CancellationToken cancellationToken = default) =>
_db.ListAsync(
"SELECT f.id, f.sequence_id FROM frames f JOIN sequences s ON s.id = f.sequence_id WHERE s.archive_id = $1 AND (f.flags & $2) = $2",
[archiveId, (int)flags], r => new FrameRef(r.GetGuid(0), r.GetGuid(1)), cancellationToken);
/// <summary>Rewrites the relative paths of a sequence's frames after its folder moved (mass rename).</summary>
/// <remarks>Only paths still under <paramref name="oldFolder"/> change, so repeating it after a crash is harmless.</remarks>
public Task<int> RebaseFolderAsync(Guid sequenceId, string oldFolder, string newFolder, CancellationToken cancellationToken = default)
{
// substr counts characters (code points) on both dialects; the tail starts with the '/' after the folder.
var prefix = oldFolder + "/";
var length = prefix.EnumerateRunes().Count();
return _db.ExecuteAsync(
"UPDATE frames SET relative_path = $2 || substr(relative_path, $3) WHERE sequence_id = $1 AND substr(relative_path, 1, $4) = $5",
[sequenceId, newFolder, length, length, prefix], cancellationToken);
}
/// <summary>
/// Writes what a verify found, in one transaction: the <see cref="FrameFlags.Missing"/> and <see cref="FrameFlags.Corrupt"/>
/// bits and new hashes. Every other flag stays as stored: decode tiers, duplicates and exclusions belong to other
/// writers, which may have changed them since the verify read the frame. Returns the number of frames updated.
/// </summary>
public Task<int> UpdateIntegrityAsync(IReadOnlyList<FrameIntegrity> frames, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
const FrameFlags integrity = FrameFlags.Missing | FrameFlags.Corrupt;
var updated = 0;
foreach (var f in frames)
{
updated += await session.ExecuteAsync("UPDATE frames SET flags = (flags & ~$4) | $2, sha256 = COALESCE($3, sha256) WHERE id = $1",
[f.FrameId, (int)(f.Flags & integrity), f.Sha256?.ToLowerInvariant(), (int)integrity], cancellationToken).ConfigureAwait(false);
}
return updated;
}, cancellationToken);
/// <summary>Formats and flagged-frame counters of one sequence.</summary>
public Task<SequenceFrameStats> GetSequenceStatsAsync(Guid sequenceId, CancellationToken cancellationToken = default) =>
_db.RunAsync(async session =>
{
var byFormat = await session.QueryListAsync(
"SELECT format, count(*), COALESCE(SUM(size), 0) FROM frames WHERE sequence_id = $1 GROUP BY format ORDER BY 2 DESC, 1",
[sequenceId], r => new FormatBucket((ImageFileFormat)r.GetInt32(0), r.GetInt64(1), r.GetInt64(2)), cancellationToken).ConfigureAwait(false);
var flags = await session.QuerySingleAsync(
"SELECT " +
$"COALESCE(SUM(CASE WHEN (flags & {(int)FrameFlags.Missing}) <> 0 THEN 1 ELSE 0 END), 0), " +
$"COALESCE(SUM(CASE WHEN (flags & {(int)FrameFlags.Corrupt}) <> 0 THEN 1 ELSE 0 END), 0), " +
$"COALESCE(SUM(CASE WHEN (flags & {(int)FrameFlags.Duplicate}) <> 0 THEN 1 ELSE 0 END), 0), " +
$"COALESCE(SUM(CASE WHEN (flags & {(int)FrameFlags.Excluded}) <> 0 THEN 1 ELSE 0 END), 0) " +
"FROM frames WHERE sequence_id = $1", [sequenceId], cancellationToken).ConfigureAwait(false);
return new SequenceFrameStats(byFormat, flags.GetInt64(0), flags.GetInt64(1), flags.GetInt64(2), flags.GetInt64(3));
}, cancellationToken);
/// <summary>Removes every frame of a sequence; returns how many.</summary>
public Task<int> DeleteBySequenceAsync(Guid sequenceId, CancellationToken cancellationToken = default) =>
_db.ExecuteAsync("DELETE FROM frames WHERE sequence_id = $1", [sequenceId], cancellationToken);
private static object?[] Values(Frame f) =>
[
f.Id, f.SequenceId, f.Index, f.RelativePath, f.FileName, f.OriginalFileName, f.Size, f.Sha256?.ToLowerInvariant(),
SequenceRepository.Wall(f.CapturedAt), f.UtcOffsetMinutes, (int)f.TimestampSource, f.ExposureTime, f.FNumber, f.Iso,
f.FocalLength, f.ExposureBias, f.WhiteBalance, f.Orientation, f.Width, f.Height, (int)f.Format, f.PreviewOffset,
f.PreviewLength, f.MeanLogLuminance, f.ClipLow, f.ClipHigh, (int)f.Flags, f.PreviewWidth, f.PreviewHeight,
];
private static Frame Map(DbRow r) => new()
{
Id = r.GetGuid(0),
SequenceId = r.GetGuid(1),
Index = r.GetInt32(2),
RelativePath = r.GetString(3),
FileName = r.GetString(4),
OriginalFileName = r.GetStringOrNull(5),
Size = r.GetInt64(6),
Sha256 = r.GetStringOrNull(7),
CapturedAt = r.GetDateTimeOrNull(8),
UtcOffsetMinutes = r.GetInt32OrNull(9),
TimestampSource = (TimestampSource)r.GetInt32(10),
ExposureTime = r.GetDoubleOrNull(11),
FNumber = r.GetDoubleOrNull(12),
Iso = r.GetInt32OrNull(13),
FocalLength = r.GetDoubleOrNull(14),
ExposureBias = r.GetDoubleOrNull(15),
WhiteBalance = r.GetInt32OrNull(16),
Orientation = r.GetInt32(17),
Width = r.GetInt32(18),
Height = r.GetInt32(19),
Format = (ImageFileFormat)r.GetInt32(20),
PreviewOffset = r.GetInt64OrNull(21),
PreviewLength = r.GetInt64OrNull(22),
MeanLogLuminance = r.GetDoubleOrNull(23),
ClipLow = r.GetDoubleOrNull(24),
ClipHigh = r.GetDoubleOrNull(25),
Flags = (FrameFlags)r.GetInt32(26),
PreviewWidth = r.GetInt32(27),
PreviewHeight = r.GetInt32(28),
};
}
@@ -0,0 +1,249 @@
using System.Globalization;
using Chronos.Core.Model;
using Chronos.Data.Jobs;
namespace Chronos.Data.Repositories;
/// <summary>Jobs per state; <see cref="DoneSince"/> and <see cref="FailedSince"/> count those finished since a given time.</summary>
public sealed record JobCounts(int Queued, int Running, int Paused, int DoneSince, int FailedSince);
/// <summary>
/// The persistent job queue (BRIEF §14.6): workers claim the oldest job of the highest priority atomically,
/// write checkpoints while running, and on startup requeue what a crash left running.
/// </summary>
public sealed class JobRepository
{
private const string Columns =
"id, kind, target_ids, target_label, parameters_json, state, priority, progress_done, progress_total, items_per_second, " +
"eta_seconds, checkpoint, log_tail, created_by, node_id, attempts, error, created, started, finished";
private readonly DbAccess _db;
internal JobRepository(DbAccess db) => _db = db;
public async Task EnqueueAsync(Job job, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync(
$"INSERT INTO jobs ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20)",
[
job.Id, (int)job.Kind, string.Join(',', job.TargetIds), job.TargetLabel, DbParam.Json(job.ParametersJson), (int)job.State,
(int)job.Priority, job.ProgressDone, job.ProgressTotal, job.ItemsPerSecond, job.Eta?.TotalSeconds, job.Checkpoint, job.LogTail,
job.CreatedBy, job.NodeId, job.Attempts, job.Error, job.Created, job.Started, job.Finished,
], cancellationToken).ConfigureAwait(false);
public Task<Job?> GetAsync(Guid id, CancellationToken cancellationToken = default) =>
_db.SingleAsync($"SELECT {Columns} FROM jobs WHERE id = $1", [id], Map, cancellationToken);
/// <summary>
/// Marks the next queued job (highest priority, then oldest) of the given kinds as running for
/// <paramref name="workerId"/> and returns it, or null when none waits. Concurrent claimers never get the same
/// job: PostgreSQL skips rows locked by other claimers, SQLite holds its write lock across select and update.
/// </summary>
public Task<Job?> ClaimNextAsync(string workerId, IReadOnlyCollection<JobKind>? kinds = null, CancellationToken cancellationToken = default) =>
ClaimNextAsync(workerId, kinds, JobPriorityQueue.Off, cancellationToken);
/// <summary>
/// <see cref="ClaimNextAsync(string, IReadOnlyCollection{JobKind}?, CancellationToken)"/> in the order of
/// <paramref name="queue"/>: while it is on, the kinds of the highest level first (<see cref="JobPriorityQueue.ClaimsBefore"/>).
/// </summary>
public Task<Job?> ClaimNextAsync(string workerId, IReadOnlyCollection<JobKind>? kinds, JobPriorityQueue queue, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(queue);
var kindFilter = kinds is { Count: > 0 } ? $" AND kind IN ({KindList(kinds)})" : string.Empty;
var pick = $"SELECT id FROM jobs WHERE state = {(int)JobState.Queued}{kindFilter} ORDER BY {queue.OrderSql(kinds is { Count: > 0 } ? kinds : Enum.GetValues<JobKind>())}" +
"priority DESC, created, id LIMIT 1";
var claim = $"UPDATE jobs SET state = {(int)JobState.Running}, node_id = $1, started = $2, finished = NULL, error = NULL, attempts = attempts + 1";
DbParam[] args = [workerId, DateTimeOffset.UtcNow];
if (_db.Sql.IsPostgres)
return _db.SingleAsync($"{claim} WHERE id = ({pick}{_db.Sql.ForUpdateSkipLocked}) RETURNING {Columns}", args, Map, cancellationToken);
return _db.InTransactionAsync(async session =>
{
var id = await session.ScalarAsync<Guid?>(pick, [], cancellationToken).ConfigureAwait(false);
if (id is null) return null;
await session.ExecuteAsync($"{claim} WHERE id = $3", [.. args, id], cancellationToken).ConfigureAwait(false);
return await session.QuerySingleOrDefaultAsync($"SELECT {Columns} FROM jobs WHERE id = $1", [id], Map, cancellationToken).ConfigureAwait(false);
}, cancellationToken);
}
/// <summary>
/// Checkpoint of a running job. Returns false when the job is no longer running (cancelled or paused meanwhile),
/// which tells the worker to stop.
/// </summary>
public async Task<bool> UpdateProgressAsync(Guid id, long done, long total, double? itemsPerSecond, TimeSpan? eta,
string? checkpoint, string? logTail, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync(
"UPDATE jobs SET progress_done = $2, progress_total = $3, items_per_second = $4, eta_seconds = $5, checkpoint = $6, log_tail = $7 " +
$"WHERE id = $1 AND state = {(int)JobState.Running}",
[id, done, total, itemsPerSecond, eta?.TotalSeconds, checkpoint, logTail], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>Moves a job to <paramref name="state"/>; done/failed/cancelled stamp the finish time, queued clears it for a retry.</summary>
public async Task<bool> SetStateAsync(Guid id, JobState state, string? error = null, CancellationToken cancellationToken = default)
{
var terminal = state is JobState.Done or JobState.Failed or JobState.Cancelled;
return await _db.ExecuteAsync("UPDATE jobs SET state = $2, error = $3, finished = $4 WHERE id = $1",
[id, (int)state, error, terminal ? DateTimeOffset.UtcNow : (DateTimeOffset?)null], cancellationToken).ConfigureAwait(false) > 0;
}
/// <summary>Jobs, newest first, optionally of one state.</summary>
public Task<Page<Job>> ListAsync(JobState? state = null, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default) =>
ListAsync(state, kind: null, limit, cursor, cancellationToken);
/// <summary>Jobs, newest first, optionally of one state and one kind.</summary>
public Task<Page<Job>> ListAsync(JobState? state, JobKind? kind, int limit = 50, string? cursor = null, CancellationToken cancellationToken = default)
{
var p = new SqlParameters();
var where = new List<string>();
if (state is { } s) where.Add($"state = {p.Add((int)s)}");
if (kind is { } k) where.Add($"kind = {p.Add((int)k)}");
if (cursor is not null)
{
var (value, id) = Cursor.Decode("jobs", cursor);
where.Add(Cursor.After("created", "id", descending: true, value, id, p.Add));
}
var take = Cursor.ClampLimit(limit);
var sql = $"SELECT {Columns} FROM jobs{(where.Count > 0 ? " WHERE " + string.Join(" AND ", where) : string.Empty)} " +
$"ORDER BY created DESC, id DESC LIMIT {p.Add(take + 1)}";
return _db.RunAsync(async session =>
{
var jobs = await session.QueryListAsync(sql, p.ToArray(), Map, cancellationToken).ConfigureAwait(false);
string? next = null;
if (jobs.Count > take)
{
jobs.RemoveAt(take);
next = Cursor.Encode("jobs", jobs[^1].Created, jobs[^1].Id);
}
return new Page<Job>(jobs, next);
}, cancellationToken);
}
/// <summary>Deletes finished jobs (done, failed, cancelled) that finished before <paramref name="finishedBefore"/>.</summary>
public Task<int> PurgeHistoryAsync(DateTimeOffset finishedBefore, CancellationToken cancellationToken = default) =>
_db.ExecuteAsync(
$"DELETE FROM jobs WHERE state IN ({(int)JobState.Done}, {(int)JobState.Failed}, {(int)JobState.Cancelled}) AND finished < $1",
[finishedBefore], cancellationToken);
/// <summary>
/// Crash recovery: puts running jobs back in the queue, keeping their checkpoints. With <paramref name="nodeId"/>
/// only that node's jobs are requeued, including those of its numbered workers (<c>node#0</c>, <c>node#1</c>, …);
/// other server instances keep theirs.
/// </summary>
public Task<int> RequeueRunningAsync(string? nodeId = null, CancellationToken cancellationToken = default) =>
_db.ExecuteAsync(
$"UPDATE jobs SET state = {(int)JobState.Queued}, node_id = NULL, started = NULL WHERE state = {(int)JobState.Running}" +
(nodeId is null ? string.Empty : " AND (node_id = $1 OR substr(node_id, 1, length($2)) = $2)"),
nodeId is null ? [] : [nodeId, nodeId + "#"], cancellationToken);
/// <summary>
/// Puts the job <paramref name="nodeId"/> runs back in the queue, keeping its checkpoint (a job that yielded to a higher
/// one in the priority queue); false when it no longer runs there: paused or cancelled meanwhile, that state stays.
/// </summary>
public async Task<bool> RequeueAsync(Guid id, string nodeId, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync(
$"UPDATE jobs SET state = {(int)JobState.Queued}, node_id = NULL, started = NULL WHERE id = $1 AND node_id = $2 AND state = {(int)JobState.Running}",
[id, nodeId], cancellationToken).ConfigureAwait(false) > 0;
public async Task<bool> SetPriorityAsync(Guid id, JobPriority priority, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("UPDATE jobs SET priority = $2 WHERE id = $1", [id, (int)priority], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>Queued jobs of one kind in claim order (highest priority, then oldest).</summary>
public Task<IReadOnlyList<Job>> ListQueuedAsync(JobKind kind, int limit = 50, CancellationToken cancellationToken = default) =>
ListQueuedAsync([kind], limit, cancellationToken);
/// <summary>Queued jobs of the given kinds in claim order (highest priority, then oldest).</summary>
public Task<IReadOnlyList<Job>> ListQueuedAsync(IReadOnlyCollection<JobKind> kinds, int limit = 50, CancellationToken cancellationToken = default) =>
ListQueuedAsync(kinds, limit, JobPriorityQueue.Off, cancellationToken);
/// <summary>Queued jobs of the given kinds in the claim order of <paramref name="queue"/>.</summary>
public Task<IReadOnlyList<Job>> ListQueuedAsync(IReadOnlyCollection<JobKind> kinds, int limit, JobPriorityQueue queue, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(queue);
return kinds.Count == 0 ? Task.FromResult<IReadOnlyList<Job>>([]) : _db.ListAsync(
$"SELECT {Columns} FROM jobs WHERE state = {(int)JobState.Queued} AND kind IN ({KindList(kinds)}) ORDER BY {queue.OrderSql(kinds)}priority DESC, created, id LIMIT $1",
[Cursor.ClampLimit(limit)], Map, cancellationToken);
}
/// <summary>Running jobs claimed by <paramref name="nodeId"/>.</summary>
public Task<IReadOnlyList<Job>> ListRunningByNodeAsync(string nodeId, CancellationToken cancellationToken = default) =>
_db.ListAsync($"SELECT {Columns} FROM jobs WHERE state = {(int)JobState.Running} AND node_id = $1 ORDER BY started, id",
[nodeId], Map, cancellationToken);
/// <summary>True when a job of <paramref name="kind"/> for exactly <paramref name="targetId"/> is waiting in the queue.</summary>
public async Task<bool> IsQueuedAsync(JobKind kind, Guid targetId, CancellationToken cancellationToken = default) =>
await _db.ScalarAsync<long>(
$"SELECT count(*) FROM jobs WHERE state = {(int)JobState.Queued} AND kind = $1 AND target_ids = $2",
[(int)kind, targetId.ToString()], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>
/// Claims one specific queued job for <paramref name="nodeId"/> (render nodes pick by capability); null when another
/// claimer was faster or the job is no longer queued.
/// </summary>
public Task<Job?> TryClaimAsync(Guid id, string nodeId, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
var claimed = await session.ExecuteAsync(
$"UPDATE jobs SET state = {(int)JobState.Running}, node_id = $2, started = $3, finished = NULL, error = NULL, attempts = attempts + 1 " +
$"WHERE id = $1 AND state = {(int)JobState.Queued}",
[id, nodeId, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false);
if (claimed == 0) return null;
return await session.QuerySingleOrDefaultAsync($"SELECT {Columns} FROM jobs WHERE id = $1", [id], Map, cancellationToken).ConfigureAwait(false);
}, cancellationToken);
/// <summary>Progress of a job a render node runs; false when the job is no longer running on that node.</summary>
public async Task<bool> UpdateNodeProgressAsync(Guid id, string nodeId, long done, long total, double? itemsPerSecond, TimeSpan? eta,
string? logTail, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync(
"UPDATE jobs SET progress_done = $3, progress_total = $4, items_per_second = $5, eta_seconds = $6, log_tail = $7 " +
$"WHERE id = $1 AND node_id = $2 AND state = {(int)JobState.Running}",
[id, nodeId, done, total, itemsPerSecond, eta?.TotalSeconds, logTail], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>Ends a job a render node runs (done or failed); false when it is no longer running on that node.</summary>
public async Task<bool> FinishNodeJobAsync(Guid id, string nodeId, JobState state, string? error, string? logTail, CancellationToken cancellationToken = default)
{
if (state is not (JobState.Done or JobState.Failed)) throw new ArgumentOutOfRangeException(nameof(state), state, "Only done or failed end a node job.");
return await _db.ExecuteAsync(
"UPDATE jobs SET state = $3, error = $4, finished = $5, log_tail = COALESCE($6, log_tail) " +
$"WHERE id = $1 AND node_id = $2 AND state = {(int)JobState.Running}",
[id, nodeId, (int)state, error, DateTimeOffset.UtcNow, logTail], cancellationToken).ConfigureAwait(false) > 0;
}
/// <summary>Queue and history counters for dashboards; finished counts are those since <paramref name="finishedSince"/>.</summary>
public Task<JobCounts> CountAsync(DateTimeOffset finishedSince, CancellationToken cancellationToken = default) =>
_db.RunAsync(async session =>
{
var row = await session.QuerySingleAsync(
"SELECT " +
$"COALESCE(SUM(CASE WHEN state = {(int)JobState.Queued} THEN 1 ELSE 0 END), 0), " +
$"COALESCE(SUM(CASE WHEN state = {(int)JobState.Running} THEN 1 ELSE 0 END), 0), " +
$"COALESCE(SUM(CASE WHEN state = {(int)JobState.Paused} THEN 1 ELSE 0 END), 0), " +
$"COALESCE(SUM(CASE WHEN state = {(int)JobState.Done} AND finished >= $1 THEN 1 ELSE 0 END), 0), " +
$"COALESCE(SUM(CASE WHEN state = {(int)JobState.Failed} AND finished >= $1 THEN 1 ELSE 0 END), 0) " +
"FROM jobs", [finishedSince], cancellationToken).ConfigureAwait(false);
return new JobCounts(row.GetInt32(0), row.GetInt32(1), row.GetInt32(2), row.GetInt32(3), row.GetInt32(4));
}, cancellationToken);
private static string KindList(IEnumerable<JobKind> kinds) => string.Join(", ", kinds.Select(k => ((int)k).ToString(CultureInfo.InvariantCulture)));
private static Job Map(DbRow r) => new()
{
Id = r.GetGuid(0),
Kind = (JobKind)r.GetInt32(1),
TargetIds = r.GetString(2) is { Length: > 0 } ids ? Array.ConvertAll(ids.Split(','), Guid.Parse) : [],
TargetLabel = r.GetStringOrNull(3),
ParametersJson = r.GetStringOrNull(4),
State = (JobState)r.GetInt32(5),
Priority = (JobPriority)r.GetInt32(6),
ProgressDone = r.GetInt64(7),
ProgressTotal = r.GetInt64(8),
ItemsPerSecond = r.GetDoubleOrNull(9),
Eta = JobEta.FromSeconds(r.GetDoubleOrNull(10)),
Checkpoint = r.GetStringOrNull(11),
LogTail = r.GetStringOrNull(12),
CreatedBy = r.GetGuidOrNull(13),
NodeId = r.GetStringOrNull(14),
Attempts = r.GetInt32(15),
Error = r.GetStringOrNull(16),
Created = r.GetDateTimeOffset(17),
Started = r.GetDateTimeOffsetOrNull(18),
Finished = r.GetDateTimeOffsetOrNull(19),
};
}
@@ -0,0 +1,144 @@
using Chronos.Core.Model;
namespace Chronos.Data.Repositories;
/// <summary>
/// Identity of a file on a card as seen at import: path on the card, size and modification time. Times are
/// truncated to whole microseconds, the precision the database keeps, so keys built from the file system match
/// stored ones.
/// </summary>
public readonly record struct ImportKey
{
public ImportKey(string path, long size, DateTimeOffset modified)
{
Path = path;
Size = size;
Modified = new DateTimeOffset(modified.UtcTicks - modified.UtcTicks % 10, TimeSpan.Zero);
}
public string Path { get; }
public long Size { get; }
public DateTimeOffset Modified { get; }
}
/// <summary>One file imported from a card and where it went.</summary>
public sealed record ImportRecord(Guid CardId, ImportKey Key, string? Sha256, Guid? FrameId, Guid? JobId, DateTimeOffset ImportedAt);
/// <summary>Known memory cards and their import history (BRIEF §13), used to skip files already imported.</summary>
public sealed class MediaCardRepository
{
private const string Columns =
"id, identity, volume_serial, label, capacity, nickname, camera_make, camera_model, camera_serial, profile_json, auto_import, first_seen, last_seen";
private static readonly string UpsertSql = SqlDialect.Upsert("media_cards",
[
"id", "identity", "volume_serial", "label", "capacity", "nickname", "camera_make", "camera_model", "camera_serial",
"profile_json", "auto_import", "first_seen", "last_seen",
], 1);
private static readonly string RecordSql = SqlDialect.Upsert("import_history",
["card_id", "path", "size", "mtime", "sha256", "frame_id", "job_id", "imported_at"], 4);
private readonly DbAccess _db;
internal MediaCardRepository(DbAccess db) => _db = db;
public async Task UpsertAsync(MediaCard card, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync(UpsertSql,
[
card.Id, card.Identity, (long)card.VolumeSerial, card.Label, card.Capacity, card.Nickname, card.CameraMake, card.CameraModel,
card.CameraSerial, DbParam.Json(card.ProfileJson), card.AutoImport, card.FirstSeen, card.LastSeen,
], cancellationToken).ConfigureAwait(false);
public Task<MediaCard?> GetAsync(Guid id, CancellationToken cancellationToken = default) =>
_db.SingleAsync($"SELECT {Columns} FROM media_cards WHERE id = $1", [id], Map, cancellationToken);
/// <summary>The card with this <c>{volumeSerial}-{label}-{capacity}</c> identity, if seen before.</summary>
public Task<MediaCard?> FindByIdentityAsync(string identity, CancellationToken cancellationToken = default) =>
_db.SingleAsync($"SELECT {Columns} FROM media_cards WHERE identity = $1", [identity], Map, cancellationToken);
/// <summary>Cards, most recently seen first.</summary>
public Task<IReadOnlyList<MediaCard>> ListAsync(CancellationToken cancellationToken = default) =>
_db.ListAsync($"SELECT {Columns} FROM media_cards ORDER BY last_seen DESC, id", [], Map, cancellationToken);
public async Task<bool> TouchAsync(Guid id, DateTimeOffset seen, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("UPDATE media_cards SET last_seen = $2 WHERE id = $1", [id, seen], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>Forgets a card and its import history.</summary>
public async Task<bool> DeleteAsync(Guid id, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("DELETE FROM media_cards WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>Records imported files in one transaction; re-recording a key updates its hash, frame and job.</summary>
public Task RecordImportsAsync(IEnumerable<ImportRecord> records, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
var count = 0;
foreach (var r in records)
{
count += await session.ExecuteAsync(RecordSql,
[r.CardId, r.Key.Path, r.Key.Size, r.Key.Modified, r.Sha256?.ToLowerInvariant(), r.FrameId, r.JobId, r.ImportedAt],
cancellationToken).ConfigureAwait(false);
}
return count;
}, cancellationToken);
/// <summary>Keys of every file already imported from the card, for skipping them on re-insert.</summary>
public async Task<IReadOnlySet<ImportKey>> GetImportedKeysAsync(Guid cardId, CancellationToken cancellationToken = default)
{
var keys = await _db.ListAsync("SELECT path, size, mtime FROM import_history WHERE card_id = $1", [cardId],
r => new ImportKey(r.GetString(0), r.GetInt64(1), r.GetDateTimeOffset(2)), cancellationToken).ConfigureAwait(false);
return keys.ToHashSet();
}
public async Task<bool> IsImportedAsync(Guid cardId, ImportKey key, CancellationToken cancellationToken = default) =>
await _db.ScalarAsync<int?>("SELECT 1 FROM import_history WHERE card_id = $1 AND path = $2 AND size = $3 AND mtime = $4",
[cardId, key.Path, key.Size, key.Modified], cancellationToken).ConfigureAwait(false) is not null;
/// <summary>Every recorded import from a card that has a hash and a frame (candidates for a verified card erase).</summary>
public Task<IReadOnlyList<ImportRecord>> ListVerifiableImportsAsync(Guid cardId, CancellationToken cancellationToken = default) =>
_db.ListAsync(
"SELECT card_id, path, size, mtime, sha256, frame_id, job_id, imported_at FROM import_history " +
"WHERE card_id = $1 AND sha256 IS NOT NULL AND frame_id IS NOT NULL ORDER BY path",
[cardId],
r => new ImportRecord(r.GetGuid(0), new ImportKey(r.GetString(1), r.GetInt64(2), r.GetDateTimeOffset(3)),
r.GetStringOrNull(4), r.GetGuidOrNull(5), r.GetGuidOrNull(6), r.GetDateTimeOffset(7)),
cancellationToken);
/// <summary>Most recent imports from a card.</summary>
public Task<IReadOnlyList<ImportRecord>> ListImportsAsync(Guid cardId, int limit = 1000, CancellationToken cancellationToken = default) =>
_db.ListAsync(
"SELECT card_id, path, size, mtime, sha256, frame_id, job_id, imported_at FROM import_history WHERE card_id = $1 " +
"ORDER BY imported_at DESC, path LIMIT $2",
[cardId, Cursor.ClampLimit(limit)],
r => new ImportRecord(r.GetGuid(0), new ImportKey(r.GetString(1), r.GetInt64(2), r.GetDateTimeOffset(3)),
r.GetStringOrNull(4), r.GetGuidOrNull(5), r.GetGuidOrNull(6), r.GetDateTimeOffset(7)),
cancellationToken);
/// <summary>The files one import job recorded from a card, by path (the import's verification result).</summary>
public Task<IReadOnlyList<ImportRecord>> ListImportsByJobAsync(Guid cardId, Guid jobId, CancellationToken cancellationToken = default) =>
_db.ListAsync(
"SELECT card_id, path, size, mtime, sha256, frame_id, job_id, imported_at FROM import_history WHERE card_id = $1 AND job_id = $2 ORDER BY path",
[cardId, jobId],
r => new ImportRecord(r.GetGuid(0), new ImportKey(r.GetString(1), r.GetInt64(2), r.GetDateTimeOffset(3)),
r.GetStringOrNull(4), r.GetGuidOrNull(5), r.GetGuidOrNull(6), r.GetDateTimeOffset(7)),
cancellationToken);
private static MediaCard Map(DbRow r) => new()
{
Id = r.GetGuid(0),
Identity = r.GetString(1),
VolumeSerial = checked((uint)r.GetInt64(2)),
Label = r.GetStringOrNull(3),
Capacity = r.GetInt64(4),
Nickname = r.GetStringOrNull(5),
CameraMake = r.GetStringOrNull(6),
CameraModel = r.GetStringOrNull(7),
CameraSerial = r.GetStringOrNull(8),
ProfileJson = r.GetStringOrNull(9),
AutoImport = r.GetBool(10),
FirstSeen = r.GetDateTimeOffset(11),
LastSeen = r.GetDateTimeOffset(12),
};
}
@@ -0,0 +1,58 @@
using Chronos.Core.Model;
namespace Chronos.Data.Repositories;
/// <summary>Clients offering their PC as render node (BRIEF §14.7).</summary>
public sealed class RenderNodeRepository
{
private const string Columns = "id, user_id, client_name, capabilities_json, last_heartbeat, state";
private static readonly string UpsertSql = SqlDialect.Upsert("render_nodes",
["id", "user_id", "client_name", "capabilities_json", "last_heartbeat", "state"], 1);
private readonly DbAccess _db;
internal RenderNodeRepository(DbAccess db) => _db = db;
/// <summary>Registers a node or refreshes its client name, capabilities and state.</summary>
public async Task RegisterAsync(RenderNode node, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync(UpsertSql,
[node.Id, node.UserId, node.ClientName, DbParam.Json(node.CapabilitiesJson), node.LastHeartbeat, node.State],
cancellationToken).ConfigureAwait(false);
public async Task<bool> HeartbeatAsync(Guid id, string state, DateTimeOffset at, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("UPDATE render_nodes SET last_heartbeat = $2, state = $3 WHERE id = $1", [id, at, state], cancellationToken)
.ConfigureAwait(false) > 0;
public Task<RenderNode?> GetAsync(Guid id, CancellationToken cancellationToken = default) =>
_db.SingleAsync($"SELECT {Columns} FROM render_nodes WHERE id = $1", [id], Map, cancellationToken);
public Task<IReadOnlyList<RenderNode>> ListAsync(CancellationToken cancellationToken = default) =>
_db.ListAsync($"SELECT {Columns} FROM render_nodes ORDER BY client_name, id", [], Map, cancellationToken);
/// <summary>Nodes whose last heartbeat is at or after <paramref name="since"/>, most recent first.</summary>
public Task<IReadOnlyList<RenderNode>> ListOnlineAsync(DateTimeOffset since, CancellationToken cancellationToken = default) =>
_db.ListAsync($"SELECT {Columns} FROM render_nodes WHERE last_heartbeat >= $1 ORDER BY last_heartbeat DESC, id", [since], Map, cancellationToken);
public async Task<bool> DeleteAsync(Guid id, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("DELETE FROM render_nodes WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>Changes the state only (a node that stopped heart-beating keeps its last heartbeat time).</summary>
public async Task<bool> SetStateAsync(Guid id, string state, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("UPDATE render_nodes SET state = $2 WHERE id = $1", [id, state], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>Nodes not in <paramref name="exceptState"/> whose last heartbeat is older than <paramref name="before"/>.</summary>
public Task<IReadOnlyList<RenderNode>> ListSilentAsync(DateTimeOffset before, string exceptState, CancellationToken cancellationToken = default) =>
_db.ListAsync($"SELECT {Columns} FROM render_nodes WHERE last_heartbeat < $1 AND state <> $2 ORDER BY last_heartbeat, id",
[before, exceptState], Map, cancellationToken);
private static RenderNode Map(DbRow r) => new()
{
Id = r.GetGuid(0),
UserId = r.GetGuid(1),
ClientName = r.GetString(2),
CapabilitiesJson = r.GetStringOrNull(3),
LastHeartbeat = r.GetDateTimeOffset(4),
State = r.GetString(5),
};
}
@@ -0,0 +1,73 @@
using Chronos.Core.Model;
namespace Chronos.Data.Repositories;
/// <summary>Exports of sequences; keeps <c>sequences.rendered_count</c> in step.</summary>
public sealed class RenderRepository
{
private const string Columns =
"id, sequence_id, preset_name, settings_json, output_path, codec, width, height, fps, duration_seconds, bytes, created, produced_by";
private const string RecountSql =
"UPDATE sequences SET rendered_count = (SELECT count(*) FROM renders WHERE sequence_id = $1) WHERE id = $1";
private readonly DbAccess _db;
internal RenderRepository(DbAccess db) => _db = db;
public Task InsertAsync(Render render, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
await LockSequenceAsync(session, render.SequenceId, cancellationToken).ConfigureAwait(false);
await session.ExecuteAsync($"INSERT INTO renders ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)",
[
render.Id, render.SequenceId, render.PresetName, DbParam.Json(render.SettingsJson), render.OutputPath, render.Codec,
render.Width, render.Height, render.Fps, render.DurationSeconds, render.Bytes, render.Created, render.ProducedBy,
], cancellationToken).ConfigureAwait(false);
return await session.ExecuteAsync(RecountSql, [render.SequenceId], cancellationToken).ConfigureAwait(false);
}, cancellationToken);
public Task<Render?> GetAsync(Guid id, CancellationToken cancellationToken = default) =>
_db.SingleAsync($"SELECT {Columns} FROM renders WHERE id = $1", [id], Map, cancellationToken);
/// <summary>Renders of a sequence, newest first.</summary>
public Task<IReadOnlyList<Render>> ListBySequenceAsync(Guid sequenceId, CancellationToken cancellationToken = default) =>
_db.ListAsync($"SELECT {Columns} FROM renders WHERE sequence_id = $1 ORDER BY created DESC, id DESC", [sequenceId], Map, cancellationToken);
public Task<bool> DeleteAsync(Guid id, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
var sequenceId = await session.ScalarAsync<Guid?>("SELECT sequence_id FROM renders WHERE id = $1", [id], cancellationToken).ConfigureAwait(false);
if (sequenceId is null) return false;
await LockSequenceAsync(session, sequenceId.Value, cancellationToken).ConfigureAwait(false);
if (await session.ExecuteAsync("DELETE FROM renders WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) == 0) return false;
await session.ExecuteAsync(RecountSql, [sequenceId], cancellationToken).ConfigureAwait(false);
return true;
}, cancellationToken);
/// <summary>
/// Serialises render changes per sequence on PostgreSQL, so the recount (a new statement, hence a new snapshot)
/// sees every committed render; SQLite's write lock already does this.
/// </summary>
private async Task LockSequenceAsync(IDbSession session, Guid sequenceId, CancellationToken cancellationToken)
{
if (_db.Sql.IsPostgres) await SequenceRepository.LockAsync(session, sequenceId, cancellationToken).ConfigureAwait(false);
}
private static Render Map(DbRow r) => new()
{
Id = r.GetGuid(0),
SequenceId = r.GetGuid(1),
PresetName = r.GetString(2),
SettingsJson = r.GetStringOrNull(3),
OutputPath = r.GetString(4),
Codec = r.GetStringOrNull(5),
Width = r.GetInt32(6),
Height = r.GetInt32(7),
Fps = r.GetDouble(8),
DurationSeconds = r.GetDouble(9),
Bytes = r.GetInt64(10),
Created = r.GetDateTimeOffset(11),
ProducedBy = r.GetStringOrNull(12),
};
}
@@ -0,0 +1,584 @@
using Chronos.Core.Imaging;
using Chronos.Core.Model;
namespace Chronos.Data.Repositories;
/// <summary>Orderings of the Library; every one is keyset-paginated with the sequence id as tie-breaker.</summary>
public enum SequenceSort
{
/// <summary>Latest first capture first; sequences without captures last.</summary>
Newest,
Oldest,
/// <summary>Display title, case-insensitive.</summary>
Title,
FrameCount,
Size,
}
/// <summary>Library filters (all optional, combined with AND) plus sort and page.</summary>
public sealed record SequenceQuery
{
public Guid? ArchiveId { get; init; }
/// <summary>Calendar year of the first capture.</summary>
public int? Year { get; init; }
/// <summary>Exact camera model.</summary>
public string? Camera { get; init; }
/// <summary>Status flags that must all be set.</summary>
public SequenceStatus WithStatus { get; init; }
/// <summary>Status flags that must all be clear.</summary>
public SequenceStatus WithoutStatus { get; init; }
public string? Tag { get; init; }
/// <summary>True: with at least one render; false: without any; null: either.</summary>
public bool? Rendered { get; init; }
/// <summary>Sync states of which the sequence must be in one; null or empty: any.</summary>
public IReadOnlyCollection<SyncState>? SyncStates { get; init; }
/// <summary>Case-insensitive substring of the title, camera make/model or a tag.</summary>
public string? Text { get; init; }
public Guid? OwnerUserId { get; init; }
public SequenceSort Sort { get; init; } = SequenceSort.Newest;
/// <summary>Page size, 1–1000.</summary>
public int Limit { get; init; } = 100;
/// <summary><see cref="Page{T}.NextCursor"/> of the previous page, or null for the first page.</summary>
public string? Cursor { get; init; }
}
/// <summary>Figures of the Archive screen (BRIEF §12.5).</summary>
public sealed record ArchiveStats(
int Sequences,
long Photos,
long Bytes,
IReadOnlyList<YearBucket> ByYear,
IReadOnlyList<MonthBucket> ByMonth,
IReadOnlyList<CameraBucket> ByCamera,
IReadOnlyList<FormatBucket> ByFormat,
ArchiveHealth Health);
/// <summary>Fields a client may change on a sequence; null leaves a field unchanged, an empty title or notes clears it.</summary>
public sealed record SequenceEdit
{
public string? Title { get; init; }
public IReadOnlyList<string>? Tags { get; init; }
public int? Rating { get; init; }
public string? Notes { get; init; }
public string? SettingsJson { get; init; }
/// <summary>
/// The frame the Library shows for the sequence (the project's hero frame); <see cref="Guid.Empty"/> clears it, so
/// the first frame shows. A frame of another sequence is ignored.
/// </summary>
public Guid? HeroFrameId { get; init; }
}
public enum SequenceEditOutcome
{
Updated,
NotFound,
/// <summary>The settings version moved since the client read it.</summary>
Conflict,
}
/// <summary>Result of <see cref="SequenceRepository.EditAsync"/>: the updated sequence, or the stored one on conflict.</summary>
public sealed record SequenceEditResult(SequenceEditOutcome Outcome, Sequence? Sequence);
public sealed record YearBucket(int Year, int Sequences, long Photos, long Bytes);
/// <summary>One heatmap cell: sequences and photos whose first capture falls in the month.</summary>
public sealed record MonthBucket(int Year, int Month, int Sequences, long Photos);
public sealed record CameraBucket(string? Make, string? Model, int Sequences, long Photos, long Bytes);
public sealed record FormatBucket(ImageFileFormat Format, long Photos, long Bytes);
/// <summary>Health counters: sequences needing attention and flagged frames.</summary>
public sealed record ArchiveHealth(
int Unverified,
int WithWarnings,
int WithErrors,
int NotAnalyzed,
long MissingFrames,
long CorruptFrames,
long DuplicateFrames);
/// <summary>Where a sequence lives in its archive.</summary>
public sealed record SequenceLocation(Guid Id, string Folder, string? Title);
/// <summary>Sequences (projects) with their tags; Library listing and Archive statistics.</summary>
public sealed class SequenceRepository
{
private const string Columns =
"s.id, s.archive_id, s.title, s.folder, s.camera_make, s.camera_model, s.camera_serial, s.lens, s.first_capture, s.last_capture, " +
"s.frame_count, s.median_interval, s.total_bytes, s.status, s.rendered_count, s.verified_at, s.hero_frame_id, s.rating, s.notes, " +
"s.settings_json, s.analysis_json, s.origin, s.sync_state, s.settings_version, s.frames_version, s.owner_user_id, s.warning_count, s.updated_at, " +
"s.preview_only_count";
/// <summary>rendered_count is owned by <see cref="RenderRepository"/> once the row exists.</summary>
private static readonly string UpsertSql = SqlDialect.Upsert("sequences",
[
"id", "archive_id", "title", "title_key", "search_text", "folder", "camera_make", "camera_model", "camera_serial", "lens",
"first_capture", "last_capture", "frame_count", "median_interval", "total_bytes", "status", "rendered_count", "verified_at",
"hero_frame_id", "rating", "notes", "settings_json", "analysis_json", "origin", "sync_state", "settings_version",
"frames_version", "owner_user_id", "warning_count", "updated_at", "preview_only_count",
], 1, ["rendered_count"]);
/// <summary>Ordinal of the tag list after <see cref="Columns"/>, and of the sort key a listing appends after it.</summary>
private const int TagsOrdinal = 29;
private const int SortKeyOrdinal = 30;
private readonly DbAccess _db;
private readonly string _select;
internal SequenceRepository(DbAccess db)
{
_db = db;
_select = SelectSql(string.Empty);
}
private string SelectSql(string extraColumns) =>
$"SELECT {Columns}, (SELECT {_db.Sql.StringAgg("t.name")} FROM sequence_tags st JOIN tags t ON t.id = st.tag_id " +
$"WHERE st.sequence_id = s.id) AS tags{extraColumns} FROM sequences s";
/// <summary>
/// Inserts the sequence, or replaces every stored field and the tag set with the record's (last writer wins);
/// only <see cref="Sequence.RenderedCount"/> of an existing row is kept, as renders maintain it. For concurrent
/// partial changes use <see cref="SetTagsAsync"/>, <see cref="UpdateStatusAsync"/> or <see cref="RefreshStatsAsync"/>.
/// </summary>
public Task UpsertAsync(Sequence sequence, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
await UpsertRowAsync(session, sequence, cancellationToken).ConfigureAwait(false);
await ReplaceTagsAsync(session, sequence.Id, sequence.Tags, cancellationToken).ConfigureAwait(false);
return true;
}, cancellationToken);
public Task<Sequence?> GetAsync(Guid id, CancellationToken cancellationToken = default) =>
_db.SingleAsync($"{_select} WHERE s.id = $1", [id], Map, cancellationToken);
/// <summary>
/// <see cref="GetAsync"/> inside a transaction, the row locked until it ends (PostgreSQL; SQLite's write lock already
/// serialises writers): a transaction that writes the whole row back from this read cannot undo an edit that another
/// one commits meanwhile, as <see cref="EditAsync(Guid, SequenceEdit, int?, int?, CancellationToken)"/> takes the same lock.
/// A transaction that writes frames of the sequence takes this lock, or writes the row, before the frames: with one
/// order no two of them wait for each other's rows (PostgreSQL would abort one with a deadlock).
/// </summary>
public Task<Sequence?> GetForUpdateAsync(Guid id, CancellationToken cancellationToken = default) =>
_db.SingleAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken);
/// <summary>Deletes the sequence with its frames, renders and tag links.</summary>
public async Task<bool> DeleteAsync(Guid id, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("DELETE FROM sequences WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>Replaces the tags of a sequence (trimmed, de-duplicated case-insensitively).</summary>
public async Task<bool> SetTagsAsync(Guid id, IEnumerable<string> tags, CancellationToken cancellationToken = default) =>
await _db.InTransactionAsync(async session =>
{
if (!await LockAsync(session, id, cancellationToken).ConfigureAwait(false)) return false;
await ReplaceTagsAsync(session, id, tags, cancellationToken).ConfigureAwait(false);
await session.ExecuteAsync("UPDATE sequences SET settings_version = settings_version + 1, updated_at = $2 WHERE id = $1",
[id, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false);
return true;
}, cancellationToken).ConfigureAwait(false);
/// <summary>Sets and clears status flags in one statement.</summary>
public async Task<bool> UpdateStatusAsync(Guid id, SequenceStatus set, SequenceStatus clear, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("UPDATE sequences SET status = (status | $2) & ~$3, updated_at = $4 WHERE id = $1",
[id, (int)set, (int)clear, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>
/// Records decode tiers written to the sequence's frames (<see cref="FrameRepository.UpdateTiersAsync"/>): the status
/// flags in <paramref name="set"/> and the preview-only count, in one statement. The frames, their versions and the
/// settings stay as they are, so an ingest or a settings edit committed meanwhile is never written back over.
/// </summary>
public async Task<bool> RecordTiersAsync(Guid id, SequenceStatus set, int previewOnlyCount, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("UPDATE sequences SET status = status | $2, preview_only_count = $3, updated_at = $4 WHERE id = $1",
[id, (int)set, previewOnlyCount, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>
/// Records a verify: status flags set and cleared and the verify time, in one statement; every other field stays as
/// it is (<see cref="RecordTiersAsync"/>).
/// </summary>
public async Task<bool> RecordVerifiedAsync(Guid id, SequenceStatus set, SequenceStatus clear, DateTimeOffset verifiedAt,
CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("UPDATE sequences SET status = (status | $2) & ~$3, verified_at = $4, updated_at = $5 WHERE id = $1",
[id, (int)set, (int)clear, verifiedAt, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>
/// Records an analysis: status flags set and cleared, the warning count and, for a deep analysis, the compact
/// summary (<paramref name="analysisJson"/> null keeps the stored one).
/// </summary>
public async Task<bool> RecordAnalysisAsync(Guid id, SequenceStatus set, SequenceStatus clear, int warningCount, string? analysisJson,
CancellationToken cancellationToken = default)
{
DbParam[] args = [id, (int)set, (int)clear, warningCount, DateTimeOffset.UtcNow];
var sql = "UPDATE sequences SET status = (status | $2) & ~$3, warning_count = $4, updated_at = $5";
if (analysisJson is not null)
{
sql += ", analysis_json = $6";
args = [.. args, DbParam.Json(analysisJson)];
}
return await _db.ExecuteAsync(sql + " WHERE id = $1", args, cancellationToken).ConfigureAwait(false) > 0;
}
/// <summary>Recomputes frame count, total bytes and first/last capture from the frames table.</summary>
public Task<Sequence?> RefreshStatsAsync(Guid id, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
var current = await session.QuerySingleOrDefaultAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken)
.ConfigureAwait(false);
if (current is null) return null;
var stats = await session.QuerySingleAsync(
"SELECT count(*), COALESCE(SUM(size), 0), MIN(captured_at), MAX(captured_at) FROM frames WHERE sequence_id = $1",
[id], cancellationToken).ConfigureAwait(false);
var updated = current with
{
FrameCount = (int)stats.GetInt64(0),
TotalBytes = stats.GetInt64(1),
FirstCapture = stats.GetDateTimeOrNull(2),
LastCapture = stats.GetDateTimeOrNull(3),
FramesVersion = current.FramesVersion + 1,
UpdatedAt = DateTimeOffset.UtcNow,
};
await UpsertRowAsync(session, updated, cancellationToken).ConfigureAwait(false);
return updated;
}, cancellationToken);
/// <summary>
/// Applies a client edit atomically and bumps <c>settings_version</c>. With <paramref name="expectedSettingsVersion"/>
/// the edit is refused (<see cref="SequenceEditOutcome.Conflict"/>, carrying the stored sequence) when someone else
/// changed the sequence first (BRIEF §14.5).
/// </summary>
public Task<SequenceEditResult> EditAsync(Guid id, SequenceEdit edit, int? expectedSettingsVersion, CancellationToken cancellationToken = default) =>
EditAsync(id, edit, expectedSettingsVersion, null, cancellationToken);
/// <summary>
/// <see cref="EditAsync(Guid, SequenceEdit, int?, CancellationToken)"/>, also refused when the sequence's frames
/// version is no longer <paramref name="expectedFramesVersion"/>: settings naming frames by position were written
/// against frames that changed since.
/// </summary>
public Task<SequenceEditResult> EditAsync(Guid id, SequenceEdit edit, int? expectedSettingsVersion, int? expectedFramesVersion,
CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
var current = await session.QuerySingleOrDefaultAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken)
.ConfigureAwait(false);
if (current is null) return new SequenceEditResult(SequenceEditOutcome.NotFound, null);
if ((expectedSettingsVersion is { } expected && expected != current.SettingsVersion)
|| (expectedFramesVersion is { } frames && frames != current.FramesVersion))
return new SequenceEditResult(SequenceEditOutcome.Conflict, current);
var hero = current.HeroFrameId;
if (edit.HeroFrameId == Guid.Empty) hero = null;
else if (edit.HeroFrameId is { } frame && await session.ScalarAsync<long>("SELECT count(*) FROM frames WHERE id = $1 AND sequence_id = $2",
[frame, id], cancellationToken).ConfigureAwait(false) > 0) hero = frame;
var updated = current with
{
Title = edit.Title is null ? current.Title : NullIfEmpty(edit.Title),
Notes = edit.Notes is null ? current.Notes : NullIfEmpty(edit.Notes),
Rating = edit.Rating ?? current.Rating,
SettingsJson = edit.SettingsJson ?? current.SettingsJson,
HeroFrameId = hero,
SettingsVersion = current.SettingsVersion + 1,
UpdatedAt = DateTimeOffset.UtcNow,
};
await UpsertRowAsync(session, updated, cancellationToken).ConfigureAwait(false);
if (edit.Tags is not null) await ReplaceTagsAsync(session, id, edit.Tags, cancellationToken).ConfigureAwait(false);
var stored = await session.QuerySingleAsync($"{_select} WHERE s.id = $1", [id], cancellationToken).ConfigureAwait(false);
return new SequenceEditResult(SequenceEditOutcome.Updated, Map(stored));
}, cancellationToken);
/// <summary>Gives every sequence owned by <paramref name="fromUserId"/> to <paramref name="toUserId"/>; returns how many.</summary>
public Task<int> ReassignOwnerAsync(Guid fromUserId, Guid toUserId, CancellationToken cancellationToken = default) =>
_db.ExecuteAsync("UPDATE sequences SET owner_user_id = $2 WHERE owner_user_id = $1", [fromUserId, toUserId], cancellationToken);
/// <summary>One page of sequences matching <paramref name="query"/>.</summary>
public Task<Page<Sequence>> ListAsync(SequenceQuery query, CancellationToken cancellationToken = default)
{
var p = new SqlParameters();
var where = Filter(query, p);
var (column, descending) = SortKey(query.Sort);
var scope = "seq:" + query.Sort;
if (query.Cursor is not null)
{
var (value, id) = Cursor.Decode(scope, query.Cursor);
where.Add(Cursor.After(column, "s.id", descending, value, id, p.Add));
}
var limit = Cursor.ClampLimit(query.Limit);
var direction = descending ? "DESC" : "ASC";
var sql = $"{SelectSql($", {column} AS sort_key")}{Where(where)} " +
$"ORDER BY {column} {direction} NULLS LAST, s.id {direction} LIMIT {p.Add(limit + 1)}";
return _db.RunAsync(async session =>
{
var rows = await session.QueryListAsync(sql, p.ToArray(), r => (Item: Map(r), Key: SortValue(query.Sort, r)), cancellationToken).ConfigureAwait(false);
string? next = null;
if (rows.Count > limit)
{
rows.RemoveAt(limit);
next = Cursor.Encode(scope, rows[^1].Key, rows[^1].Item.Id);
}
return new Page<Sequence>(rows.ConvertAll(r => r.Item), next);
}, cancellationToken);
}
/// <summary>Number of sequences matching the filters of <paramref name="query"/> (sort and page ignored).</summary>
public async Task<int> CountAsync(SequenceQuery query, CancellationToken cancellationToken = default)
{
var p = new SqlParameters();
var where = Filter(query, p);
return (int)await _db.ScalarAsync<long>($"SELECT count(*) FROM sequences s{Where(where)}", p.ToArray(), cancellationToken).ConfigureAwait(false);
}
/// <summary>Totals, distributions and health counters of one archive, or of all archives.</summary>
public Task<ArchiveStats> GetStatsAsync(Guid? archiveId = null, CancellationToken cancellationToken = default)
{
var sql = _db.Sql;
DbParam[] args = archiveId is { } a ? [a] : [];
var seqWhere = archiveId is null ? string.Empty : " WHERE s.archive_id = $1";
var seqAnd = archiveId is null ? " WHERE" : " WHERE s.archive_id = $1 AND";
var frameFrom = archiveId is null ? "frames f" : "frames f JOIN sequences s ON s.id = f.sequence_id WHERE s.archive_id = $1";
var year = sql.Year("s.first_capture");
return _db.RunAsync(async session =>
{
var totals = await session.QuerySingleAsync(
$"SELECT count(*), COALESCE(SUM(s.frame_count), 0), COALESCE(SUM(s.total_bytes), 0) FROM sequences s{seqWhere}",
args, cancellationToken).ConfigureAwait(false);
var byYear = await session.QueryListAsync(
$"SELECT {year}, count(*), COALESCE(SUM(s.frame_count), 0), COALESCE(SUM(s.total_bytes), 0) FROM sequences s" +
$"{seqAnd} s.first_capture IS NOT NULL GROUP BY 1 ORDER BY 1", args,
r => new YearBucket(r.GetInt32(0), r.GetInt32(1), r.GetInt64(2), r.GetInt64(3)), cancellationToken).ConfigureAwait(false);
var byMonth = await session.QueryListAsync(
$"SELECT {year}, {sql.Month("s.first_capture")}, count(*), COALESCE(SUM(s.frame_count), 0) FROM sequences s" +
$"{seqAnd} s.first_capture IS NOT NULL GROUP BY 1, 2 ORDER BY 1, 2", args,
r => new MonthBucket(r.GetInt32(0), r.GetInt32(1), r.GetInt32(2), r.GetInt64(3)), cancellationToken).ConfigureAwait(false);
var byCamera = await session.QueryListAsync(
$"SELECT s.camera_make, s.camera_model, count(*), COALESCE(SUM(s.frame_count), 0), COALESCE(SUM(s.total_bytes), 0) " +
$"FROM sequences s{seqWhere} GROUP BY s.camera_make, s.camera_model ORDER BY 4 DESC, 2 NULLS LAST, 1 NULLS LAST", args,
r => new CameraBucket(r.GetStringOrNull(0), r.GetStringOrNull(1), r.GetInt32(2), r.GetInt64(3), r.GetInt64(4)), cancellationToken).ConfigureAwait(false);
var byFormat = await session.QueryListAsync(
$"SELECT f.format, count(*), COALESCE(SUM(f.size), 0) FROM {frameFrom} GROUP BY f.format ORDER BY 2 DESC, 1", args,
r => new FormatBucket((ImageFileFormat)r.GetInt32(0), r.GetInt64(1), r.GetInt64(2)), cancellationToken).ConfigureAwait(false);
var sequenceHealth = await session.QuerySingleAsync(
"SELECT " +
"COALESCE(SUM(CASE WHEN s.verified_at IS NULL THEN 1 ELSE 0 END), 0), " +
$"COALESCE(SUM(CASE WHEN (s.status & {(int)SequenceStatus.HasWarnings}) <> 0 OR s.warning_count > 0 THEN 1 ELSE 0 END), 0), " +
$"COALESCE(SUM(CASE WHEN (s.status & {(int)SequenceStatus.HasErrors}) <> 0 THEN 1 ELSE 0 END), 0), " +
$"COALESCE(SUM(CASE WHEN (s.status & {(int)SequenceStatus.AnalyzedLight}) = 0 THEN 1 ELSE 0 END), 0) " +
$"FROM sequences s{seqWhere}", args, cancellationToken).ConfigureAwait(false);
var frameHealth = await session.QuerySingleAsync(
"SELECT " +
$"COALESCE(SUM(CASE WHEN (f.flags & {(int)FrameFlags.Missing}) <> 0 THEN 1 ELSE 0 END), 0), " +
$"COALESCE(SUM(CASE WHEN (f.flags & {(int)FrameFlags.Corrupt}) <> 0 THEN 1 ELSE 0 END), 0), " +
$"COALESCE(SUM(CASE WHEN (f.flags & {(int)FrameFlags.Duplicate}) <> 0 THEN 1 ELSE 0 END), 0) " +
$"FROM {frameFrom}", args, cancellationToken).ConfigureAwait(false);
return new ArchiveStats(
totals.GetInt32(0), totals.GetInt64(1), totals.GetInt64(2), byYear, byMonth, byCamera, byFormat,
new ArchiveHealth(
sequenceHealth.GetInt32(0), sequenceHealth.GetInt32(1), sequenceHealth.GetInt32(2), sequenceHealth.GetInt32(3),
frameHealth.GetInt64(0), frameHealth.GetInt64(1), frameHealth.GetInt64(2)));
}, cancellationToken);
}
/// <summary>Folder and title of every sequence of an archive, by folder.</summary>
public Task<IReadOnlyList<SequenceLocation>> ListLocationsAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
_db.ListAsync("SELECT id, folder, title FROM sequences WHERE archive_id = $1 ORDER BY folder, id", [archiveId],
r => new SequenceLocation(r.GetGuid(0), r.GetString(1), r.GetStringOrNull(2)), cancellationToken);
/// <summary>The sequence created for a session key (automatic grouping of uploads and card sessions).</summary>
public Task<Guid?> FindBySessionKeyAsync(Guid archiveId, string sessionKey, CancellationToken cancellationToken = default) =>
_db.ScalarAsync<Guid?>("SELECT sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key = $2",
[archiveId, sessionKey], cancellationToken);
/// <summary>Binds a session key to a sequence unless it is already bound; returns the sequence the key resolves to.</summary>
public Task<Guid> MapSessionKeyAsync(Guid archiveId, string sessionKey, Guid sequenceId, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
await session.ExecuteAsync(
"INSERT INTO sequence_sessions (archive_id, session_key, sequence_id) VALUES ($1, $2, $3) ON CONFLICT (archive_id, session_key) DO NOTHING",
[archiveId, sessionKey, sequenceId], cancellationToken).ConfigureAwait(false);
return (await session.ScalarAsync<Guid?>("SELECT sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key = $2",
[archiveId, sessionKey], cancellationToken).ConfigureAwait(false))!.Value;
}, cancellationToken);
/// <summary>
/// Moves a sequence to another folder and, with <paramref name="title"/>, retitles it (mass rename; settings version
/// + 1 when the title changes); every other field stays as the locked row has it. False when it does not exist.
/// </summary>
/// <param name="title">The new title; null keeps the current one.</param>
public Task<bool> RelocateAsync(Guid id, string folder, string? title, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
var current = await session.QuerySingleOrDefaultAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken)
.ConfigureAwait(false);
if (current is null) return false;
var retitled = title is not null && current.Title != title;
await UpsertRowAsync(session, current with
{
Folder = folder,
Title = retitled ? title : current.Title,
SettingsVersion = retitled ? current.SettingsVersion + 1 : current.SettingsVersion,
UpdatedAt = DateTimeOffset.UtcNow,
}, cancellationToken).ConfigureAwait(false);
return true;
}, cancellationToken);
private static List<string> Filter(SequenceQuery query, SqlParameters p)
{
var where = new List<string>();
if (query.ArchiveId is { } archive) where.Add($"s.archive_id = {p.Add(archive)}");
if (query.Year is { } year)
where.Add($"s.first_capture >= {p.Add(new DateTime(year, 1, 1))} AND s.first_capture < {p.Add(new DateTime(year + 1, 1, 1))}");
if (query.Camera is { } camera) where.Add($"s.camera_model = {p.Add(camera)}");
if (query.WithStatus != SequenceStatus.None)
{
var mask = p.Add((int)query.WithStatus);
where.Add($"(s.status & {mask}) = {mask}");
}
if (query.WithoutStatus != SequenceStatus.None) where.Add($"(s.status & {p.Add((int)query.WithoutStatus)}) = 0");
if (query.Rendered is { } rendered) where.Add(rendered ? "s.rendered_count > 0" : "s.rendered_count = 0");
if (query.SyncStates is { Count: > 0 } states) where.Add($"s.sync_state IN ({string.Join(", ", states.Select(state => p.Add((int)state)))})");
if (query.Tag is { } tag)
where.Add("EXISTS (SELECT 1 FROM sequence_tags st JOIN tags t ON t.id = st.tag_id " +
$"WHERE st.sequence_id = s.id AND t.name_key = {p.Add(TagRepository.Key(tag))})");
if (!string.IsNullOrWhiteSpace(query.Text))
{
var pattern = p.Add(SqlDialect.ContainsPattern(query.Text.Trim().ToLowerInvariant()));
where.Add($"(s.search_text LIKE {pattern} ESCAPE '\\' OR EXISTS (SELECT 1 FROM sequence_tags st JOIN tags t ON t.id = st.tag_id " +
$"WHERE st.sequence_id = s.id AND t.name_key LIKE {pattern} ESCAPE '\\'))");
}
if (query.OwnerUserId is { } owner) where.Add($"s.owner_user_id = {p.Add(owner)}");
return where;
}
private static string Where(List<string> conditions) =>
conditions.Count == 0 ? string.Empty : " WHERE " + string.Join(" AND ", conditions);
private static (string Column, bool Descending) SortKey(SequenceSort sort) => sort switch
{
SequenceSort.Newest => ("s.first_capture", true),
SequenceSort.Oldest => ("s.first_capture", false),
SequenceSort.Title => ("s.title_key", false),
SequenceSort.FrameCount => ("s.frame_count", true),
SequenceSort.Size => ("s.total_bytes", true),
_ => throw new ArgumentOutOfRangeException(nameof(sort), sort, null),
};
private static object? SortValue(SequenceSort sort, DbRow row) => sort switch
{
SequenceSort.Newest or SequenceSort.Oldest => row.GetDateTimeOrNull(SortKeyOrdinal),
SequenceSort.Title => row.GetString(SortKeyOrdinal),
_ => row.GetInt64(SortKeyOrdinal),
};
private static Task<int> UpsertRowAsync(IDbSession session, Sequence s, CancellationToken cancellationToken)
{
var search = string.Join(' ', new[] { s.Title, s.CameraMake, s.CameraModel }.Where(v => !string.IsNullOrWhiteSpace(v))).ToLowerInvariant();
return session.ExecuteAsync(UpsertSql,
[
s.Id, s.ArchiveId, s.Title, s.DisplayTitle.ToLowerInvariant(), search, s.Folder, s.CameraMake, s.CameraModel, s.CameraSerial, s.Lens,
Wall(s.FirstCapture), Wall(s.LastCapture), s.FrameCount, s.MedianInterval, s.TotalBytes, (int)s.Status, s.RenderedCount, s.VerifiedAt,
s.HeroFrameId, s.Rating, s.Notes, DbParam.Json(s.SettingsJson), DbParam.Json(s.AnalysisJson), (int)s.Origin, (int)s.SyncState,
s.SettingsVersion, s.FramesVersion, s.OwnerUserId, s.WarningCount, s.UpdatedAt, s.PreviewOnlyCount,
], cancellationToken);
}
/// <summary>
/// Row-locks a sequence for a read-modify-write (PostgreSQL; SQLite's write lock already serialises writers).
/// False when it does not exist.
/// </summary>
internal static async Task<bool> LockAsync(IDbSession session, Guid id, CancellationToken cancellationToken) =>
await session.ScalarAsync<Guid?>($"SELECT id FROM sequences WHERE id = $1{SqlDialect.For(session.Dialect).ForUpdate}", [id], cancellationToken)
.ConfigureAwait(false) is not null;
/// <summary>
/// Replaces the tag links of a sequence whose row the caller has locked (or just upserted); a set the sequence already
/// carries (names compared case-insensitively) is left as it is. On PostgreSQL every tag written is locked until
/// commit, and the tags are written in the order of their keys whatever order the caller listed them in (a PATCH sends
/// them in any order): writers of two sequences that share tags then take those rows in one order, and never wait for
/// each other's (PostgreSQL would abort one with a deadlock, and nothing retries it).
/// </summary>
private async Task ReplaceTagsAsync(IDbSession session, Guid sequenceId, IEnumerable<string> tags, CancellationToken cancellationToken)
{
var wanted = TagRepository.NormalizeSet(tags);
var keys = wanted.Select(TagRepository.Key).ToList();
var stored = await session.QueryListAsync("SELECT t.name_key FROM sequence_tags st JOIN tags t ON t.id = st.tag_id WHERE st.sequence_id = $1",
[sequenceId], r => r.GetString(0), cancellationToken).ConfigureAwait(false);
if (stored.Count == keys.Count && stored.ToHashSet(StringComparer.Ordinal).SetEquals(keys)) return;
await session.ExecuteAsync("DELETE FROM sequence_tags WHERE sequence_id = $1", [sequenceId], cancellationToken).ConfigureAwait(false);
foreach (var tag in wanted.OrderBy(TagRepository.Key, StringComparer.Ordinal))
{
DbParam[] args = [Guid.CreateVersion7(), tag, TagRepository.Key(tag)];
Guid tagId;
if (_db.Sql.IsPostgres)
{
// The no-op update locks an existing tag, so a concurrent DeleteUnusedAsync cannot remove it before the link exists.
tagId = (await session.ScalarAsync<Guid?>(
"INSERT INTO tags (id, name, name_key) VALUES ($1, $2, $3) ON CONFLICT (name_key) DO UPDATE SET name_key = excluded.name_key RETURNING id",
args, cancellationToken).ConfigureAwait(false))!.Value;
}
else
{
await session.ExecuteAsync("INSERT INTO tags (id, name, name_key) VALUES ($1, $2, $3) ON CONFLICT (name_key) DO NOTHING",
args, cancellationToken).ConfigureAwait(false);
tagId = (await session.ScalarAsync<Guid?>("SELECT id FROM tags WHERE name_key = $1", [args[2]], cancellationToken).ConfigureAwait(false))!.Value;
}
await session.ExecuteAsync("INSERT INTO sequence_tags (sequence_id, tag_id) VALUES ($1, $2)", [sequenceId, tagId], cancellationToken)
.ConfigureAwait(false);
}
}
private static string? NullIfEmpty(string value) => string.IsNullOrWhiteSpace(value) ? null : value.Trim();
/// <summary>Camera wall-clock times are stored without zone, whatever kind the caller's value has.</summary>
internal static DateTime? Wall(DateTime? value) =>
value is { } v ? DateTime.SpecifyKind(v, DateTimeKind.Unspecified) : null;
private static Sequence Map(DbRow r)
{
var tags = r.GetStringOrNull(TagsOrdinal) is { Length: > 0 } list ? list.Split(SqlDialect.ListSeparator) : [];
Array.Sort(tags, StringComparer.OrdinalIgnoreCase);
return new Sequence
{
Id = r.GetGuid(0),
ArchiveId = r.GetGuid(1),
Title = r.GetStringOrNull(2),
Folder = r.GetString(3),
CameraMake = r.GetStringOrNull(4),
CameraModel = r.GetStringOrNull(5),
CameraSerial = r.GetStringOrNull(6),
Lens = r.GetStringOrNull(7),
FirstCapture = r.GetDateTimeOrNull(8),
LastCapture = r.GetDateTimeOrNull(9),
FrameCount = r.GetInt32(10),
MedianInterval = r.GetDoubleOrNull(11),
TotalBytes = r.GetInt64(12),
Status = (SequenceStatus)r.GetInt32(13),
RenderedCount = r.GetInt32(14),
VerifiedAt = r.GetDateTimeOffsetOrNull(15),
HeroFrameId = r.GetGuidOrNull(16),
Rating = r.GetInt32(17),
Notes = r.GetStringOrNull(18),
SettingsJson = r.GetStringOrNull(19),
AnalysisJson = r.GetStringOrNull(20),
Origin = (SequenceOrigin)r.GetInt32(21),
SyncState = (SyncState)r.GetInt32(22),
SettingsVersion = r.GetInt32(23),
FramesVersion = r.GetInt32(24),
OwnerUserId = r.GetGuidOrNull(25),
WarningCount = r.GetInt32(26),
UpdatedAt = r.GetDateTimeOffset(27),
PreviewOnlyCount = r.GetInt32(28),
Tags = tags,
};
}
}
@@ -0,0 +1,93 @@
using System.Diagnostics.CodeAnalysis;
using System.Text.Json;
using System.Text.Json.Serialization.Metadata;
namespace Chronos.Data.Repositories;
/// <summary>Who a setting belongs to.</summary>
public enum SettingScope
{
/// <summary>The installation (client app or server); no scope id.</summary>
App,
/// <summary>One archive; the scope id is the archive id.</summary>
Archive,
/// <summary>One user; the scope id is the user id.</summary>
User,
}
/// <summary>Typed key/value settings stored as JSON (BRIEF §5 Setting).</summary>
public sealed class SettingsRepository
{
private static readonly string UpsertSql = SqlDialect.Upsert("settings", ["scope", "scope_id", "key", "value_json", "updated_at"], 3);
private readonly DbAccess _db;
internal SettingsRepository(DbAccess db) => _db = db;
/// <summary>The raw JSON of a setting, or null when unset.</summary>
public Task<string?> GetJsonAsync(SettingScope scope, Guid? scopeId, string key, CancellationToken cancellationToken = default) =>
_db.ScalarAsync<string>("SELECT value_json FROM settings WHERE scope = $1 AND scope_id = $2 AND key = $3",
[ScopeName(scope), ScopeId(scope, scopeId), key], cancellationToken);
/// <summary>Stores a JSON value; malformed JSON is rejected on both dialects.</summary>
public async Task SetJsonAsync(SettingScope scope, Guid? scopeId, string key, string json, CancellationToken cancellationToken = default)
{
using (JsonDocument.Parse(json))
{
}
await _db.ExecuteAsync(UpsertSql, [ScopeName(scope), ScopeId(scope, scopeId), key, DbParam.Json(json), DateTimeOffset.UtcNow], cancellationToken)
.ConfigureAwait(false);
}
/// <summary>Reads a setting with source-generated metadata (trim/AOT safe); <c>default</c> when unset.</summary>
public async Task<T?> GetAsync<T>(SettingScope scope, Guid? scopeId, string key, JsonTypeInfo<T> typeInfo, CancellationToken cancellationToken = default) =>
await GetJsonAsync(scope, scopeId, key, cancellationToken).ConfigureAwait(false) is { } json
? JsonSerializer.Deserialize(json, typeInfo)
: default;
public Task SetAsync<T>(SettingScope scope, Guid? scopeId, string key, T value, JsonTypeInfo<T> typeInfo, CancellationToken cancellationToken = default) =>
SetJsonAsync(scope, scopeId, key, JsonSerializer.Serialize(value, typeInfo), cancellationToken);
/// <summary>Reads a setting through reflection-based serialization; <c>default</c> when unset.</summary>
[RequiresUnreferencedCode("Reflection-based JSON; use the JsonTypeInfo overload in trimmed builds.")]
[RequiresDynamicCode("Reflection-based JSON; use the JsonTypeInfo overload in AOT builds.")]
public async Task<T?> GetAsync<T>(SettingScope scope, Guid? scopeId, string key, JsonSerializerOptions? options = null, CancellationToken cancellationToken = default) =>
await GetJsonAsync(scope, scopeId, key, cancellationToken).ConfigureAwait(false) is { } json
? JsonSerializer.Deserialize<T>(json, options)
: default;
[RequiresUnreferencedCode("Reflection-based JSON; use the JsonTypeInfo overload in trimmed builds.")]
[RequiresDynamicCode("Reflection-based JSON; use the JsonTypeInfo overload in AOT builds.")]
public Task SetAsync<T>(SettingScope scope, Guid? scopeId, string key, T value, JsonSerializerOptions? options = null, CancellationToken cancellationToken = default) =>
SetJsonAsync(scope, scopeId, key, JsonSerializer.Serialize(value, options), cancellationToken);
public async Task<bool> DeleteAsync(SettingScope scope, Guid? scopeId, string key, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("DELETE FROM settings WHERE scope = $1 AND scope_id = $2 AND key = $3",
[ScopeName(scope), ScopeId(scope, scopeId), key], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>Every setting of a scope as key → JSON.</summary>
public async Task<IReadOnlyDictionary<string, string>> ListAsync(SettingScope scope, Guid? scopeId, CancellationToken cancellationToken = default)
{
var rows = await _db.ListAsync("SELECT key, value_json FROM settings WHERE scope = $1 AND scope_id = $2 ORDER BY key",
[ScopeName(scope), ScopeId(scope, scopeId)], r => KeyValuePair.Create(r.GetString(0), r.GetString(1)), cancellationToken).ConfigureAwait(false);
return new Dictionary<string, string>(rows, StringComparer.Ordinal);
}
private static string ScopeName(SettingScope scope) => scope switch
{
SettingScope.App => "app",
SettingScope.Archive => "archive",
SettingScope.User => "user",
_ => throw new ArgumentOutOfRangeException(nameof(scope), scope, null),
};
private static string ScopeId(SettingScope scope, Guid? scopeId) => (scope, scopeId) switch
{
(SettingScope.App, null) => string.Empty,
(SettingScope.App, _) => throw new ArgumentException("App settings have no scope id.", nameof(scopeId)),
(_, { } id) => TextValues.FormatGuid(id),
_ => throw new ArgumentException($"{scope} settings need a scope id.", nameof(scopeId)),
};
}
@@ -0,0 +1,97 @@
using System.Text;
namespace Chronos.Data.Repositories;
/// <summary>A tag and how many sequences carry it.</summary>
public sealed record TagInfo(Guid Id, string Name, int SequenceCount);
/// <summary>
/// Normalised tags (BRIEF §5 Tag/SequenceTag): one row per case-insensitive name, linked to sequences through
/// <c>sequence_tags</c>. The first spelling of a name is kept.
/// </summary>
public sealed class TagRepository
{
/// <summary>Longest tag name accepted, in characters.</summary>
public const int MaxLength = 64;
private readonly DbAccess _db;
internal TagRepository(DbAccess db) => _db = db;
/// <summary>All tags by name with their usage counts.</summary>
public Task<IReadOnlyList<TagInfo>> ListAsync(CancellationToken cancellationToken = default) =>
_db.ListAsync(
"SELECT t.id, t.name, (SELECT count(*) FROM sequence_tags st WHERE st.tag_id = t.id) FROM tags t ORDER BY t.name_key",
[], r => new TagInfo(r.GetGuid(0), r.GetString(1), r.GetInt32(2)), cancellationToken);
/// <summary>Renames a tag; a name already used by another tag violates the unique key (<see cref="DatabaseException.IsUniqueViolation"/>).</summary>
public async Task<bool> RenameAsync(Guid id, string name, CancellationToken cancellationToken = default)
{
var normalized = Normalize(name);
return await _db.ExecuteAsync("UPDATE tags SET name = $2, name_key = $3 WHERE id = $1",
[id, normalized, Key(normalized)], cancellationToken).ConfigureAwait(false) > 0;
}
/// <summary>Deletes a tag and removes it from every sequence.</summary>
public async Task<bool> DeleteAsync(Guid id, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("DELETE FROM tags WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>
/// Deletes tags no sequence uses any more; returns how many. Candidates are locked first (skipping tags a
/// concurrent tagger holds) and re-checked by a later statement, whose fresh snapshot sees every link committed
/// before the lock, so a tag being attached at the same moment survives.
/// </summary>
public Task<int> DeleteUnusedAsync(CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
const string unused = "NOT EXISTS (SELECT 1 FROM sequence_tags st WHERE st.tag_id = tags.id)";
var candidates = await session.QueryListAsync($"SELECT id FROM tags WHERE {unused}{_db.Sql.ForUpdateSkipLocked}", [],
r => r.GetGuid(0), cancellationToken).ConfigureAwait(false);
var deleted = 0;
foreach (var chunk in candidates.Chunk(500))
{
var p = new SqlParameters();
var ids = string.Join(", ", chunk.Select(id => p.Add(id)));
deleted += await session.ExecuteAsync($"DELETE FROM tags WHERE id IN ({ids}) AND {unused}", p.ToArray(), cancellationToken)
.ConfigureAwait(false);
}
return deleted;
}, cancellationToken);
/// <summary>Trims, collapses inner whitespace and validates a tag name.</summary>
public static string Normalize(string name)
{
var sb = new StringBuilder(name.Length);
foreach (var c in name.Trim())
{
if (char.IsControl(c)) throw new ArgumentException("Tags cannot contain control characters.", nameof(name));
if (char.IsWhiteSpace(c))
{
if (sb.Length > 0 && sb[^1] != ' ') sb.Append(' ');
}
else
{
sb.Append(c);
}
}
if (sb.Length == 0) throw new ArgumentException("A tag cannot be empty.", nameof(name));
if (sb.Length > MaxLength) throw new ArgumentException($"A tag is at most {MaxLength} characters.", nameof(name));
return sb.ToString();
}
/// <summary>The case-insensitive identity of a tag name.</summary>
public static string Key(string name) => Normalize(name).ToLowerInvariant();
/// <summary>Normalised tags without case-insensitive duplicates, first spelling kept.</summary>
internal static List<string> NormalizeSet(IEnumerable<string> tags)
{
var seen = new HashSet<string>(StringComparer.Ordinal);
var result = new List<string>();
foreach (var tag in tags)
{
var normalized = Normalize(tag);
if (seen.Add(normalized.ToLowerInvariant())) result.Add(normalized);
}
return result;
}
}
@@ -0,0 +1,73 @@
using Chronos.Core.Model;
namespace Chronos.Data.Repositories;
/// <summary>API session tokens, stored by SHA-256 hash, with sliding expiry and revocation (BRIEF §14.2).</summary>
public sealed class TokenRepository
{
private const string Columns = "token_hash, user_id, client_name, created, expires, last_used, revoked";
private readonly DbAccess _db;
internal TokenRepository(DbAccess db) => _db = db;
public async Task InsertAsync(SessionToken token, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync($"INSERT INTO tokens ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7)",
[token.TokenHash, token.UserId, token.ClientName, token.Created, token.Expires, token.LastUsed, token.Revoked],
cancellationToken).ConfigureAwait(false);
/// <summary>The token with this hash, revoked or expired ones included (the caller decides).</summary>
public Task<SessionToken?> GetAsync(string tokenHash, CancellationToken cancellationToken = default) =>
_db.SingleAsync($"SELECT {Columns} FROM tokens WHERE token_hash = $1", [tokenHash], Map, cancellationToken);
/// <summary>
/// Sliding expiry: records use at <paramref name="now"/> and moves the expiry to <paramref name="expires"/>.
/// Returns false for unknown, revoked or already expired tokens.
/// </summary>
public async Task<bool> TouchAsync(string tokenHash, DateTimeOffset now, DateTimeOffset expires, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync(
"UPDATE tokens SET last_used = $2, expires = $3 WHERE token_hash = $1 AND revoked = FALSE AND expires > $2",
[tokenHash, now, expires], cancellationToken).ConfigureAwait(false) > 0;
public async Task<bool> RevokeAsync(string tokenHash, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("UPDATE tokens SET revoked = TRUE WHERE token_hash = $1 AND revoked = FALSE", [tokenHash], cancellationToken)
.ConfigureAwait(false) > 0;
/// <summary>Revokes every live token of a user (password change, account disabled).</summary>
public Task<int> RevokeAllAsync(Guid userId, CancellationToken cancellationToken = default) =>
_db.ExecuteAsync("UPDATE tokens SET revoked = TRUE WHERE user_id = $1 AND revoked = FALSE", [userId], cancellationToken);
/// <summary>Revokes every live token of a user except <paramref name="keepTokenHash"/> (password changed in that session).</summary>
public Task<int> RevokeAllExceptAsync(Guid userId, string keepTokenHash, CancellationToken cancellationToken = default) =>
_db.ExecuteAsync("UPDATE tokens SET revoked = TRUE WHERE user_id = $1 AND token_hash <> $2 AND revoked = FALSE",
[userId, keepTokenHash], cancellationToken);
/// <summary>The most recently used token of every user (last seen time and client), keyed by user id.</summary>
public async Task<IReadOnlyDictionary<Guid, SessionToken>> LatestByUserAsync(CancellationToken cancellationToken = default)
{
var tokens = await _db.ListAsync(
$"SELECT {Columns} FROM tokens t WHERE last_used = (SELECT MAX(last_used) FROM tokens l WHERE l.user_id = t.user_id)",
[], Map, cancellationToken).ConfigureAwait(false);
var latest = new Dictionary<Guid, SessionToken>();
foreach (var token in tokens) latest[token.UserId] = token;
return latest;
}
public Task<IReadOnlyList<SessionToken>> ListByUserAsync(Guid userId, CancellationToken cancellationToken = default) =>
_db.ListAsync($"SELECT {Columns} FROM tokens WHERE user_id = $1 ORDER BY last_used DESC", [userId], Map, cancellationToken);
/// <summary>Deletes tokens that expired by <paramref name="now"/> or were revoked.</summary>
public Task<int> PurgeAsync(DateTimeOffset now, CancellationToken cancellationToken = default) =>
_db.ExecuteAsync("DELETE FROM tokens WHERE expires <= $1 OR revoked = TRUE", [now], cancellationToken);
private static SessionToken Map(DbRow r) => new()
{
TokenHash = r.GetString(0),
UserId = r.GetGuid(1),
ClientName = r.GetStringOrNull(2),
Created = r.GetDateTimeOffset(3),
Expires = r.GetDateTimeOffset(4),
LastUsed = r.GetDateTimeOffset(5),
Revoked = r.GetBool(6),
};
}
@@ -0,0 +1,201 @@
namespace Chronos.Data.Repositories;
/// <summary>Life cycle of a resumable upload (BRIEF §14.4).</summary>
public enum UploadState
{
/// <summary>Chunks are arriving.</summary>
Receiving,
/// <summary>All chunks are in; the server is verifying the hash and moving the file into the archive.</summary>
Committing,
/// <summary>The file is a frame of the archive (<see cref="Upload.FrameId"/>).</summary>
Committed,
Failed,
Cancelled,
}
/// <summary>A chunked upload and its received-chunk bitmap (bit n of byte n/8, least significant first).</summary>
public sealed record Upload
{
public required Guid Id { get; init; }
public required Guid UserId { get; init; }
public Guid? SequenceId { get; init; }
public required string FileName { get; init; }
public long Size { get; init; }
/// <summary>Lower-case hex SHA-256 announced by the client.</summary>
public required string Sha256 { get; init; }
public DateTime? CapturedAt { get; init; }
public required int ChunkSize { get; init; }
public byte[] Received { get; init; } = [];
public int ReceivedCount { get; init; }
public UploadState State { get; init; }
public Guid? FrameId { get; init; }
public DateTimeOffset Created { get; init; }
public DateTimeOffset UpdatedAt { get; init; }
/// <summary>Automatic grouping key of a frame upload without a target sequence (BRIEF §14.4).</summary>
public string? SessionKey { get; init; }
/// <summary>The render job whose output this upload carries; null for frame uploads.</summary>
public Guid? JobId { get; init; }
/// <summary>Metadata of the render a render-result upload becomes (JSON).</summary>
public string? RenderJson { get; init; }
public int ChunkCount => (int)((Size + ChunkSize - 1) / ChunkSize);
public bool IsComplete => ReceivedCount >= ChunkCount;
public bool HasChunk(int index) =>
index >= 0 && index / 8 < Received.Length && (Received[index / 8] & (1 << (index % 8))) != 0;
}
/// <summary>Committed uploads of one UTC day.</summary>
public sealed record UploadDay(DateOnly Day, long Count, long Bytes);
/// <summary>Resumable uploads: creation, per-chunk bookkeeping, state changes and cleanup.</summary>
public sealed class UploadRepository
{
private const string Columns =
"id, user_id, sequence_id, file_name, size, sha256, captured_at, chunk_size, received, received_count, state, frame_id, created, updated_at, " +
"session_key, job_id, render_json";
private readonly DbAccess _db;
internal UploadRepository(DbAccess db) => _db = db;
/// <summary>Stores a new upload with an empty bitmap sized for its chunk count.</summary>
public async Task<Upload> CreateAsync(Upload upload, CancellationToken cancellationToken = default)
{
ArgumentOutOfRangeException.ThrowIfLessThan(upload.ChunkSize, 1);
ArgumentOutOfRangeException.ThrowIfNegative(upload.Size);
var stored = upload with { Received = new byte[(upload.ChunkCount + 7) / 8], ReceivedCount = 0, Sha256 = upload.Sha256.ToLowerInvariant() };
await _db.ExecuteAsync(
$"INSERT INTO uploads ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)",
[
stored.Id, stored.UserId, stored.SequenceId, stored.FileName, stored.Size, stored.Sha256, SequenceRepository.Wall(stored.CapturedAt),
stored.ChunkSize, stored.Received, 0, (int)stored.State, stored.FrameId, stored.Created, stored.UpdatedAt,
stored.SessionKey, stored.JobId, DbParam.Json(stored.RenderJson),
], cancellationToken).ConfigureAwait(false);
return stored;
}
public Task<Upload?> GetAsync(Guid id, CancellationToken cancellationToken = default) =>
_db.SingleAsync($"SELECT {Columns} FROM uploads WHERE id = $1", [id], Map, cancellationToken);
/// <summary>A frame upload of the same content by the same user that is still receiving, so a client can resume it.</summary>
public Task<Upload?> FindReceivingAsync(Guid userId, string sha256, CancellationToken cancellationToken = default) =>
_db.SingleAsync(
$"SELECT {Columns} FROM uploads WHERE user_id = $1 AND sha256 = $2 AND state = {(int)UploadState.Receiving} AND job_id IS NULL " +
"ORDER BY created DESC LIMIT 1",
[userId, sha256.ToLowerInvariant()], Map, cancellationToken);
/// <summary>
/// Marks chunk <paramref name="index"/> as received (idempotent; concurrent chunks of one upload serialise on
/// its row) and returns the updated upload, or null when it does not exist.
/// </summary>
public Task<Upload?> MarkChunkReceivedAsync(Guid id, int index, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
var upload = await session.QuerySingleOrDefaultAsync($"SELECT {Columns} FROM uploads WHERE id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken)
.ConfigureAwait(false);
if (upload is null) return null;
ArgumentOutOfRangeException.ThrowIfNegative(index);
ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(index, upload.ChunkCount);
if (upload.HasChunk(index)) return upload;
var bitmap = (byte[])upload.Received.Clone();
bitmap[index / 8] |= (byte)(1 << (index % 8));
var updated = upload with { Received = bitmap, ReceivedCount = upload.ReceivedCount + 1, UpdatedAt = DateTimeOffset.UtcNow };
await session.ExecuteAsync("UPDATE uploads SET received = $2, received_count = $3, updated_at = $4 WHERE id = $1",
[id, bitmap, updated.ReceivedCount, updated.UpdatedAt], cancellationToken).ConfigureAwait(false);
return updated;
}, cancellationToken);
/// <summary>Moves an upload to <paramref name="state"/>, recording the frame it became when committed.</summary>
public async Task<bool> SetStateAsync(Guid id, UploadState state, Guid? frameId = null, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("UPDATE uploads SET state = $2, frame_id = COALESCE($3, frame_id), updated_at = $4 WHERE id = $1",
[id, (int)state, frameId, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0;
public Task<IReadOnlyList<Upload>> ListByUserAsync(Guid userId, CancellationToken cancellationToken = default) =>
_db.ListAsync($"SELECT {Columns} FROM uploads WHERE user_id = $1 ORDER BY created DESC", [userId], Map, cancellationToken);
/// <summary>Uploads still receiving but untouched since <paramref name="updatedBefore"/> (abandoned; their part files can go).</summary>
public Task<IReadOnlyList<Upload>> ListStaleAsync(DateTimeOffset updatedBefore, CancellationToken cancellationToken = default) =>
_db.ListAsync($"SELECT {Columns} FROM uploads WHERE state = {(int)UploadState.Receiving} AND updated_at < $1 ORDER BY updated_at",
[updatedBefore], Map, cancellationToken);
public async Task<bool> DeleteAsync(Guid id, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("DELETE FROM uploads WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>
/// Moves an upload from <paramref name="from"/> to <paramref name="to"/> only if it is still in <paramref name="from"/>,
/// so two concurrent completions cannot both commit it. Returns false when the state had changed.
/// </summary>
public async Task<bool> TryTransitionAsync(Guid id, UploadState from, UploadState to, Guid? frameId = null, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("UPDATE uploads SET state = $3, frame_id = COALESCE($4, frame_id), updated_at = $5 WHERE id = $1 AND state = $2",
[id, (int)from, (int)to, frameId, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>The newest upload carrying the output of render job <paramref name="jobId"/>.</summary>
public Task<Upload?> FindByJobAsync(Guid jobId, CancellationToken cancellationToken = default) =>
_db.SingleAsync($"SELECT {Columns} FROM uploads WHERE job_id = $1 ORDER BY created DESC, id DESC LIMIT 1", [jobId], Map, cancellationToken);
/// <summary>Every upload in <paramref name="state"/>, oldest change first.</summary>
public Task<IReadOnlyList<Upload>> ListByStateAsync(UploadState state, CancellationToken cancellationToken = default) =>
_db.ListAsync($"SELECT {Columns} FROM uploads WHERE state = $1 ORDER BY updated_at, id", [(int)state], Map, cancellationToken);
/// <summary>Bytes a user has announced in frame uploads that are not committed yet (reserved against the quota).</summary>
public async Task<long> PendingBytesAsync(Guid userId, CancellationToken cancellationToken = default) =>
await _db.ScalarAsync<long>(
$"SELECT COALESCE(SUM(size), 0) FROM uploads WHERE user_id = $1 AND job_id IS NULL AND state IN ({(int)UploadState.Receiving}, {(int)UploadState.Committing})",
[userId], cancellationToken).ConfigureAwait(false);
/// <summary>Committed frame uploads per UTC day since <paramref name="since"/>, oldest day first.</summary>
public Task<IReadOnlyList<UploadDay>> CommittedPerDayAsync(DateTimeOffset since, CancellationToken cancellationToken = default)
{
var day = _db.Sql.IsPostgres ? "to_char(updated_at AT TIME ZONE 'UTC', 'YYYY-MM-DD')" : "substr(updated_at, 1, 10)";
return _db.ListAsync(
$"SELECT {day}, count(*), COALESCE(SUM(size), 0) FROM uploads WHERE state = {(int)UploadState.Committed} AND job_id IS NULL AND updated_at >= $1 " +
"GROUP BY 1 ORDER BY 1",
[since], r => new UploadDay(DateOnly.ParseExact(r.GetString(0), "yyyy-MM-dd", System.Globalization.CultureInfo.InvariantCulture), r.GetInt64(1), r.GetInt64(2)),
cancellationToken);
}
/// <summary>Committed frame uploads per user since <paramref name="since"/>.</summary>
public async Task<IReadOnlyDictionary<Guid, long>> CommittedCountByUserAsync(DateTimeOffset since, CancellationToken cancellationToken = default)
{
var rows = await _db.ListAsync(
$"SELECT user_id, count(*) FROM uploads WHERE state = {(int)UploadState.Committed} AND job_id IS NULL AND updated_at >= $1 GROUP BY user_id",
[since], r => KeyValuePair.Create(r.GetGuid(0), r.GetInt64(1)), cancellationToken).ConfigureAwait(false);
return new Dictionary<Guid, long>(rows);
}
/// <summary>Deletes committed, failed and cancelled uploads last changed before <paramref name="updatedBefore"/>; returns how many.</summary>
public Task<int> PurgeFinishedAsync(DateTimeOffset updatedBefore, CancellationToken cancellationToken = default) =>
_db.ExecuteAsync(
$"DELETE FROM uploads WHERE state IN ({(int)UploadState.Committed}, {(int)UploadState.Failed}, {(int)UploadState.Cancelled}) AND updated_at < $1",
[updatedBefore], cancellationToken);
private static Upload Map(DbRow r) => new()
{
Id = r.GetGuid(0),
UserId = r.GetGuid(1),
SequenceId = r.GetGuidOrNull(2),
FileName = r.GetString(3),
Size = r.GetInt64(4),
Sha256 = r.GetString(5),
CapturedAt = r.GetDateTimeOrNull(6),
ChunkSize = r.GetInt32(7),
Received = r.GetBytes(8),
ReceivedCount = r.GetInt32(9),
State = (UploadState)r.GetInt32(10),
FrameId = r.GetGuidOrNull(11),
Created = r.GetDateTimeOffset(12),
UpdatedAt = r.GetDateTimeOffset(13),
SessionKey = r.GetStringOrNull(14),
JobId = r.GetGuidOrNull(15),
RenderJson = r.GetStringOrNull(16),
};
}
@@ -0,0 +1,75 @@
using Chronos.Core.Model;
namespace Chronos.Data.Repositories;
/// <summary>Server accounts. User names are unique case-insensitively.</summary>
public sealed class UserRepository
{
private const string Columns =
"id, username, display_name, password_hash, role, permissions, quota_bytes, used_bytes, disabled, must_change_password, created, last_login";
private readonly DbAccess _db;
internal UserRepository(DbAccess db) => _db = db;
/// <summary>Adds a user; a taken name raises a <see cref="DatabaseException"/> with <see cref="DatabaseException.IsUniqueViolation"/>.</summary>
public async Task CreateAsync(User user, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync(
"INSERT INTO users (id, username, username_key, display_name, password_hash, role, permissions, quota_bytes, used_bytes, " +
"disabled, must_change_password, created, last_login) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)",
[
user.Id, user.Username, Key(user.Username), user.DisplayName, user.PasswordHash, (int)user.Role, (long)user.Permissions,
user.QuotaBytes, user.UsedBytes, user.Disabled, user.MustChangePassword, user.Created, user.LastLogin,
], cancellationToken).ConfigureAwait(false);
/// <summary>Saves every field except id and creation time.</summary>
public async Task<bool> UpdateAsync(User user, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync(
"UPDATE users SET username = $2, username_key = $3, display_name = $4, password_hash = $5, role = $6, permissions = $7, " +
"quota_bytes = $8, used_bytes = $9, disabled = $10, must_change_password = $11, last_login = $12 WHERE id = $1",
[
user.Id, user.Username, Key(user.Username), user.DisplayName, user.PasswordHash, (int)user.Role, (long)user.Permissions,
user.QuotaBytes, user.UsedBytes, user.Disabled, user.MustChangePassword, user.LastLogin,
], cancellationToken).ConfigureAwait(false) > 0;
public Task<User?> GetAsync(Guid id, CancellationToken cancellationToken = default) =>
_db.SingleAsync($"SELECT {Columns} FROM users WHERE id = $1", [id], Map, cancellationToken);
public Task<User?> GetByUsernameAsync(string username, CancellationToken cancellationToken = default) =>
_db.SingleAsync($"SELECT {Columns} FROM users WHERE username_key = $1", [Key(username)], Map, cancellationToken);
public Task<IReadOnlyList<User>> ListAsync(CancellationToken cancellationToken = default) =>
_db.ListAsync($"SELECT {Columns} FROM users ORDER BY username_key", [], Map, cancellationToken);
public async Task<int> CountAsync(CancellationToken cancellationToken = default) =>
(int)await _db.ScalarAsync<long>("SELECT count(*) FROM users", [], cancellationToken).ConfigureAwait(false);
/// <summary>Deletes the user with their tokens, render nodes and uploads.</summary>
public async Task<bool> DeleteAsync(Guid id, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("DELETE FROM users WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) > 0;
public async Task<bool> RecordLoginAsync(Guid id, DateTimeOffset at, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("UPDATE users SET last_login = $2 WHERE id = $1", [id, at], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>Adds <paramref name="delta"/> (negative to release) to the user's quota usage atomically.</summary>
public async Task<bool> AddUsedBytesAsync(Guid id, long delta, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("UPDATE users SET used_bytes = used_bytes + $2 WHERE id = $1", [id, delta], cancellationToken).ConfigureAwait(false) > 0;
private static string Key(string username) => username.Trim().ToLowerInvariant();
private static User Map(DbRow r) => new()
{
Id = r.GetGuid(0),
Username = r.GetString(1),
DisplayName = r.GetStringOrNull(2),
PasswordHash = r.GetString(3),
Role = (UserRole)r.GetInt32(4),
Permissions = (Permissions)r.GetInt64(5),
QuotaBytes = r.GetInt64(6),
UsedBytes = r.GetInt64(7),
Disabled = r.GetBool(8),
MustChangePassword = r.GetBool(9),
Created = r.GetDateTimeOffset(10),
LastLogin = r.GetDateTimeOffsetOrNull(11),
};
}