using System.Globalization; using System.Security.Cryptography; using System.Text; using Chronos.Core.Imaging; using Chronos.Core.Model; namespace Chronos.Data.Repositories; /// Orderings of the Library; every one is keyset-paginated with the sequence id as tie-breaker. public enum SequenceSort { /// Latest first capture first; sequences without captures last. Newest, Oldest, /// Display title, case-insensitive. Title, FrameCount, Size, } /// Library filters (all optional, combined with AND) plus sort and page. public sealed record SequenceQuery { public Guid? ArchiveId { get; init; } /// Calendar year of the first capture. public int? Year { get; init; } /// Exact camera model. public string? Camera { get; init; } /// Status flags that must all be set. public SequenceStatus WithStatus { get; init; } /// Status flags that must all be clear. public SequenceStatus WithoutStatus { get; init; } public string? Tag { get; init; } /// True: with at least one render; false: without any; null: either. public bool? Rendered { get; init; } /// Sync states of which the sequence must be in one; null or empty: any. public IReadOnlyCollection? SyncStates { get; init; } /// Case-insensitive substring of the title, camera make/model or a tag. public string? Text { get; init; } public Guid? OwnerUserId { get; init; } public SequenceSort Sort { get; init; } = SequenceSort.Newest; /// Page size, 1–1000. public int Limit { get; init; } = 100; /// of the previous page, or null for the first page. public string? Cursor { get; init; } } /// Figures of the Archive screen (BRIEF §12.5). public sealed record ArchiveStats( int Sequences, long Photos, long Bytes, IReadOnlyList ByYear, IReadOnlyList ByMonth, IReadOnlyList ByCamera, IReadOnlyList ByFormat, ArchiveHealth Health); /// Fields a client may change on a sequence; null leaves a field unchanged, an empty title or notes clears it. public sealed record SequenceEdit { public string? Title { get; init; } public IReadOnlyList? Tags { get; init; } public int? Rating { get; init; } public string? Notes { get; init; } public string? SettingsJson { get; init; } /// /// The frame the Library shows for the sequence (the project's hero frame); clears it, so /// the first frame shows. A frame of another sequence is ignored. /// public Guid? HeroFrameId { get; init; } } public enum SequenceEditOutcome { Updated, NotFound, /// The settings version moved since the client read it. Conflict, } /// Result of : the updated sequence, or the stored one on conflict. public sealed record SequenceEditResult(SequenceEditOutcome Outcome, Sequence? Sequence); public sealed record YearBucket(int Year, int Sequences, long Photos, long Bytes); /// One heatmap cell: sequences and photos whose first capture falls in the month. public sealed record MonthBucket(int Year, int Month, int Sequences, long Photos); public sealed record CameraBucket(string? Make, string? Model, int Sequences, long Photos, long Bytes); public sealed record FormatBucket(ImageFileFormat Format, long Photos, long Bytes); /// /// A value a Library filter offers, as takes it (a year, an exact camera model, a tag), and /// how many sequences have it; is a camera model's make, for its label. /// public sealed record ArchiveFacet(string Value, int Sequences, string? Make = null); /// The years of first capture, camera models and tags of an archive's sequences (owner review 2, R6: the panel's filters). public sealed record ArchiveFacets(IReadOnlyList Years, IReadOnlyList Cameras, IReadOnlyList Tags); /// Health counters: sequences needing attention and flagged frames. public sealed record ArchiveHealth( int Unverified, int WithWarnings, int WithErrors, int NotAnalyzed, long MissingFrames, long CorruptFrames, long DuplicateFrames); /// Where a sequence lives in its archive. public sealed record SequenceLocation(Guid Id, string Folder, string? Title); /// Sequences (projects) with their tags; Library listing and Archive statistics. public sealed class SequenceRepository { private const string Columns = "s.id, s.archive_id, s.title, s.folder, s.camera_make, s.camera_model, s.camera_serial, s.lens, s.first_capture, s.last_capture, " + "s.frame_count, s.median_interval, s.total_bytes, s.status, s.rendered_count, s.verified_at, s.hero_frame_id, s.rating, s.notes, " + "s.settings_json, s.analysis_json, s.origin, s.sync_state, s.settings_version, s.frames_version, s.owner_user_id, s.warning_count, s.updated_at, " + "s.preview_only_count"; /// rendered_count is owned by once the row exists. private static readonly string UpsertSql = SqlDialect.Upsert("sequences", [ "id", "archive_id", "title", "title_key", "search_text", "folder", "camera_make", "camera_model", "camera_serial", "lens", "first_capture", "last_capture", "frame_count", "median_interval", "total_bytes", "status", "rendered_count", "verified_at", "hero_frame_id", "rating", "notes", "settings_json", "analysis_json", "origin", "sync_state", "settings_version", "frames_version", "owner_user_id", "warning_count", "updated_at", "preview_only_count", ], 1, ["rendered_count"]); /// Ordinal of the tag list after , and of the sort key a listing appends after it. private const int TagsOrdinal = 29; private const int SortKeyOrdinal = 30; private readonly DbAccess _db; private readonly string _select; internal SequenceRepository(DbAccess db) { _db = db; _select = SelectSql(string.Empty); } private string SelectSql(string extraColumns) => $"SELECT {Columns}, (SELECT {_db.Sql.StringAgg("t.name")} FROM sequence_tags st JOIN tags t ON t.id = st.tag_id " + $"WHERE st.sequence_id = s.id) AS tags{extraColumns} FROM sequences s"; /// /// Inserts the sequence, or replaces every stored field and the tag set with the record's (last writer wins); /// only of an existing row is kept, as renders maintain it. For concurrent /// partial changes use , or . /// public Task UpsertAsync(Sequence sequence, CancellationToken cancellationToken = default) => _db.InTransactionAsync(async session => { await UpsertRowAsync(session, sequence, cancellationToken).ConfigureAwait(false); await ReplaceTagsAsync(session, sequence.Id, sequence.Tags, cancellationToken).ConfigureAwait(false); return true; }, cancellationToken); public Task GetAsync(Guid id, CancellationToken cancellationToken = default) => _db.SingleAsync($"{_select} WHERE s.id = $1", [id], Map, cancellationToken); /// /// inside a transaction, the row locked until it ends (PostgreSQL; SQLite's write lock already /// serialises writers): a transaction that writes the whole row back from this read cannot undo an edit that another /// one commits meanwhile, as takes the same lock. /// A transaction that writes frames of the sequence takes this lock, or writes the row, before the frames: with one /// order no two of them wait for each other's rows (PostgreSQL would abort one with a deadlock). /// public Task GetForUpdateAsync(Guid id, CancellationToken cancellationToken = default) => _db.SingleAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken); /// Deletes the sequence with its frames, renders and tag links. public async Task DeleteAsync(Guid id, CancellationToken cancellationToken = default) => await _db.ExecuteAsync("DELETE FROM sequences WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) > 0; /// Replaces the tags of a sequence (trimmed, de-duplicated case-insensitively). public async Task SetTagsAsync(Guid id, IEnumerable tags, CancellationToken cancellationToken = default) => await _db.InTransactionAsync(async session => { if (!await LockAsync(session, id, cancellationToken).ConfigureAwait(false)) return false; await ReplaceTagsAsync(session, id, tags, cancellationToken).ConfigureAwait(false); await session.ExecuteAsync("UPDATE sequences SET settings_version = settings_version + 1, updated_at = $2 WHERE id = $1", [id, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false); return true; }, cancellationToken).ConfigureAwait(false); /// Sets and clears status flags in one statement. public async Task UpdateStatusAsync(Guid id, SequenceStatus set, SequenceStatus clear, CancellationToken cancellationToken = default) => await _db.ExecuteAsync("UPDATE sequences SET status = (status | $2) & ~$3, updated_at = $4 WHERE id = $1", [id, (int)set, (int)clear, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0; /// /// Records decode tiers written to the sequence's frames (): the status /// flags in and the preview-only count, in one statement. The frames, their versions and the /// settings stay as they are, so an ingest or a settings edit committed meanwhile is never written back over. /// public async Task RecordTiersAsync(Guid id, SequenceStatus set, int previewOnlyCount, CancellationToken cancellationToken = default) => await _db.ExecuteAsync("UPDATE sequences SET status = status | $2, preview_only_count = $3, updated_at = $4 WHERE id = $1", [id, (int)set, previewOnlyCount, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0; /// /// Records a verify: status flags set and cleared and the verify time, in one statement; every other field stays as /// it is (). /// public async Task RecordVerifiedAsync(Guid id, SequenceStatus set, SequenceStatus clear, DateTimeOffset verifiedAt, CancellationToken cancellationToken = default) => await _db.ExecuteAsync("UPDATE sequences SET status = (status | $2) & ~$3, verified_at = $4, updated_at = $5 WHERE id = $1", [id, (int)set, (int)clear, verifiedAt, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0; /// /// Records an analysis: status flags set and cleared, the warning count and, for a deep analysis, the compact /// summary ( null keeps the stored one). /// public async Task RecordAnalysisAsync(Guid id, SequenceStatus set, SequenceStatus clear, int warningCount, string? analysisJson, CancellationToken cancellationToken = default) { DbParam[] args = [id, (int)set, (int)clear, warningCount, DateTimeOffset.UtcNow]; var sql = "UPDATE sequences SET status = (status | $2) & ~$3, warning_count = $4, updated_at = $5"; if (analysisJson is not null) { sql += ", analysis_json = $6"; args = [.. args, DbParam.Json(analysisJson)]; } return await _db.ExecuteAsync(sql + " WHERE id = $1", args, cancellationToken).ConfigureAwait(false) > 0; } /// Recomputes frame count, total bytes and first/last capture from the frames table. public Task RefreshStatsAsync(Guid id, CancellationToken cancellationToken = default) => _db.InTransactionAsync(async session => { var current = await session.QuerySingleOrDefaultAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken) .ConfigureAwait(false); if (current is null) return null; var stats = await session.QuerySingleAsync( "SELECT count(*), COALESCE(SUM(size), 0), MIN(captured_at), MAX(captured_at) FROM frames WHERE sequence_id = $1", [id], cancellationToken).ConfigureAwait(false); var updated = current with { FrameCount = (int)stats.GetInt64(0), TotalBytes = stats.GetInt64(1), FirstCapture = stats.GetDateTimeOrNull(2), LastCapture = stats.GetDateTimeOrNull(3), FramesVersion = current.FramesVersion + 1, UpdatedAt = DateTimeOffset.UtcNow, }; await UpsertRowAsync(session, updated, cancellationToken).ConfigureAwait(false); return updated; }, cancellationToken); /// /// Applies a client edit atomically and bumps settings_version. With /// the edit is refused (, carrying the stored sequence) when someone else /// changed the sequence first (BRIEF §14.5). /// public Task EditAsync(Guid id, SequenceEdit edit, int? expectedSettingsVersion, CancellationToken cancellationToken = default) => EditAsync(id, edit, expectedSettingsVersion, null, cancellationToken); /// /// , also refused when the sequence's frames /// version is no longer : settings naming frames by position were written /// against frames that changed since. /// public Task EditAsync(Guid id, SequenceEdit edit, int? expectedSettingsVersion, int? expectedFramesVersion, CancellationToken cancellationToken = default) => _db.InTransactionAsync(async session => { var current = await session.QuerySingleOrDefaultAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken) .ConfigureAwait(false); if (current is null) return new SequenceEditResult(SequenceEditOutcome.NotFound, null); if ((expectedSettingsVersion is { } expected && expected != current.SettingsVersion) || (expectedFramesVersion is { } frames && frames != current.FramesVersion)) return new SequenceEditResult(SequenceEditOutcome.Conflict, current); var hero = current.HeroFrameId; if (edit.HeroFrameId == Guid.Empty) hero = null; else if (edit.HeroFrameId is { } frame && await session.ScalarAsync("SELECT count(*) FROM frames WHERE id = $1 AND sequence_id = $2", [frame, id], cancellationToken).ConfigureAwait(false) > 0) hero = frame; var updated = current with { Title = edit.Title is null ? current.Title : NullIfEmpty(edit.Title), Notes = edit.Notes is null ? current.Notes : NullIfEmpty(edit.Notes), Rating = edit.Rating ?? current.Rating, SettingsJson = edit.SettingsJson ?? current.SettingsJson, HeroFrameId = hero, SettingsVersion = current.SettingsVersion + 1, UpdatedAt = DateTimeOffset.UtcNow, }; await UpsertRowAsync(session, updated, cancellationToken).ConfigureAwait(false); if (edit.Tags is not null) await ReplaceTagsAsync(session, id, edit.Tags, cancellationToken).ConfigureAwait(false); var stored = await session.QuerySingleAsync($"{_select} WHERE s.id = $1", [id], cancellationToken).ConfigureAwait(false); return new SequenceEditResult(SequenceEditOutcome.Updated, Map(stored)); }, cancellationToken); /// Gives every sequence owned by to ; returns how many. public Task ReassignOwnerAsync(Guid fromUserId, Guid toUserId, CancellationToken cancellationToken = default) => _db.ExecuteAsync("UPDATE sequences SET owner_user_id = $2 WHERE owner_user_id = $1", [fromUserId, toUserId], cancellationToken); /// One page of sequences matching . public Task> ListAsync(SequenceQuery query, CancellationToken cancellationToken = default) { var p = new SqlParameters(); var where = Filter(query, p); var (column, descending) = SortKey(query.Sort); var scope = "seq:" + query.Sort; if (query.Cursor is not null) { var (value, id) = Cursor.Decode(scope, query.Cursor); where.Add(Cursor.After(column, "s.id", descending, value, id, p.Add)); } var limit = Cursor.ClampLimit(query.Limit); var direction = descending ? "DESC" : "ASC"; var sql = $"{SelectSql($", {column} AS sort_key")}{Where(where)} " + $"ORDER BY {column} {direction} NULLS LAST, s.id {direction} LIMIT {p.Add(limit + 1)}"; return _db.RunAsync(async session => { var rows = await session.QueryListAsync(sql, p.ToArray(), r => (Item: Map(r), Key: SortValue(query.Sort, r)), cancellationToken).ConfigureAwait(false); string? next = null; if (rows.Count > limit) { rows.RemoveAt(limit); next = Cursor.Encode(scope, rows[^1].Key, rows[^1].Item.Id); } return new Page(rows.ConvertAll(r => r.Item), next); }, cancellationToken); } /// Number of sequences matching the filters of (sort and page ignored). public async Task CountAsync(SequenceQuery query, CancellationToken cancellationToken = default) { var p = new SqlParameters(); var where = Filter(query, p); return (int)await _db.ScalarAsync($"SELECT count(*) FROM sequences s{Where(where)}", p.ToArray(), cancellationToken).ConfigureAwait(false); } /// Totals, distributions and health counters of one archive, or of all archives. public Task GetStatsAsync(Guid? archiveId = null, CancellationToken cancellationToken = default) { var sql = _db.Sql; DbParam[] args = archiveId is { } a ? [a] : []; var seqWhere = archiveId is null ? string.Empty : " WHERE s.archive_id = $1"; var seqAnd = archiveId is null ? " WHERE" : " WHERE s.archive_id = $1 AND"; var frameFrom = archiveId is null ? "frames f" : "frames f JOIN sequences s ON s.id = f.sequence_id WHERE s.archive_id = $1"; var year = sql.Year("s.first_capture"); return _db.RunAsync(async session => { var totals = await session.QuerySingleAsync( $"SELECT count(*), COALESCE(SUM(s.frame_count), 0), COALESCE(SUM(s.total_bytes), 0) FROM sequences s{seqWhere}", args, cancellationToken).ConfigureAwait(false); var byYear = await session.QueryListAsync( $"SELECT {year}, count(*), COALESCE(SUM(s.frame_count), 0), COALESCE(SUM(s.total_bytes), 0) FROM sequences s" + $"{seqAnd} s.first_capture IS NOT NULL GROUP BY 1 ORDER BY 1", args, r => new YearBucket(r.GetInt32(0), r.GetInt32(1), r.GetInt64(2), r.GetInt64(3)), cancellationToken).ConfigureAwait(false); var byMonth = await session.QueryListAsync( $"SELECT {year}, {sql.Month("s.first_capture")}, count(*), COALESCE(SUM(s.frame_count), 0) FROM sequences s" + $"{seqAnd} s.first_capture IS NOT NULL GROUP BY 1, 2 ORDER BY 1, 2", args, r => new MonthBucket(r.GetInt32(0), r.GetInt32(1), r.GetInt32(2), r.GetInt64(3)), cancellationToken).ConfigureAwait(false); var byCamera = await session.QueryListAsync( $"SELECT s.camera_make, s.camera_model, count(*), COALESCE(SUM(s.frame_count), 0), COALESCE(SUM(s.total_bytes), 0) " + $"FROM sequences s{seqWhere} GROUP BY s.camera_make, s.camera_model ORDER BY 4 DESC, 2 NULLS LAST, 1 NULLS LAST", args, r => new CameraBucket(r.GetStringOrNull(0), r.GetStringOrNull(1), r.GetInt32(2), r.GetInt64(3), r.GetInt64(4)), cancellationToken).ConfigureAwait(false); var byFormat = await session.QueryListAsync( $"SELECT f.format, count(*), COALESCE(SUM(f.size), 0) FROM {frameFrom} GROUP BY f.format ORDER BY 2 DESC, 1", args, r => new FormatBucket((ImageFileFormat)r.GetInt32(0), r.GetInt64(1), r.GetInt64(2)), cancellationToken).ConfigureAwait(false); var sequenceHealth = await session.QuerySingleAsync( "SELECT " + "COALESCE(SUM(CASE WHEN s.verified_at IS NULL THEN 1 ELSE 0 END), 0), " + $"COALESCE(SUM(CASE WHEN (s.status & {(int)SequenceStatus.HasWarnings}) <> 0 OR s.warning_count > 0 THEN 1 ELSE 0 END), 0), " + $"COALESCE(SUM(CASE WHEN (s.status & {(int)SequenceStatus.HasErrors}) <> 0 THEN 1 ELSE 0 END), 0), " + $"COALESCE(SUM(CASE WHEN (s.status & {(int)SequenceStatus.AnalyzedLight}) = 0 THEN 1 ELSE 0 END), 0) " + $"FROM sequences s{seqWhere}", args, cancellationToken).ConfigureAwait(false); var frameHealth = await session.QuerySingleAsync( "SELECT " + $"COALESCE(SUM(CASE WHEN (f.flags & {(int)FrameFlags.Missing}) <> 0 THEN 1 ELSE 0 END), 0), " + $"COALESCE(SUM(CASE WHEN (f.flags & {(int)FrameFlags.Corrupt}) <> 0 THEN 1 ELSE 0 END), 0), " + $"COALESCE(SUM(CASE WHEN (f.flags & {(int)FrameFlags.Duplicate}) <> 0 THEN 1 ELSE 0 END), 0) " + $"FROM {frameFrom}", args, cancellationToken).ConfigureAwait(false); return new ArchiveStats( totals.GetInt32(0), totals.GetInt64(1), totals.GetInt64(2), byYear, byMonth, byCamera, byFormat, new ArchiveHealth( sequenceHealth.GetInt32(0), sequenceHealth.GetInt32(1), sequenceHealth.GetInt32(2), sequenceHealth.GetInt32(3), frameHealth.GetInt64(0), frameHealth.GetInt64(1), frameHealth.GetInt64(2))); }, cancellationToken); } /// /// What the Library's filters offer for : years of first capture (newest first), camera /// models (most sequences first; one entry per model, as the camera filter matches the model alone) and the tags in use /// (by name). Only sequence rows are read, never frames. /// public Task GetFacetsAsync(Guid archiveId, CancellationToken cancellationToken = default) { DbParam[] args = [archiveId]; var year = _db.Sql.Year("s.first_capture"); return _db.RunAsync(async session => { var years = await session.QueryListAsync( $"SELECT {year}, count(*) FROM sequences s WHERE s.archive_id = $1 AND s.first_capture IS NOT NULL GROUP BY 1 ORDER BY 1 DESC", args, r => new ArchiveFacet(r.GetInt32(0).ToString(CultureInfo.InvariantCulture), r.GetInt32(1)), cancellationToken).ConfigureAwait(false); var cameras = await session.QueryListAsync( "SELECT s.camera_model, MIN(s.camera_make), count(*) FROM sequences s " + "WHERE s.archive_id = $1 AND s.camera_model IS NOT NULL AND s.camera_model <> '' GROUP BY s.camera_model ORDER BY 3 DESC, 1", args, r => new ArchiveFacet(r.GetString(0), r.GetInt32(2), r.GetStringOrNull(1)), cancellationToken).ConfigureAwait(false); var tags = await session.QueryListAsync( "SELECT MIN(t.name), count(*) FROM sequence_tags st JOIN tags t ON t.id = st.tag_id JOIN sequences s ON s.id = st.sequence_id " + "WHERE s.archive_id = $1 GROUP BY t.name_key ORDER BY t.name_key", args, r => new ArchiveFacet(r.GetString(0), r.GetInt32(1)), cancellationToken).ConfigureAwait(false); return new ArchiveFacets(years, cameras, tags); }, cancellationToken); } /// /// A digest of what a Reorganize of (null: every sequence) plans from in the index of /// (owner review 2, R7): every sequence of the archive and its folder, which decide the /// folders that get " (2)" and what a moving folder carries; of the sequences planned, their title, first capture, /// camera, lens, photo count, frames version and how many of their photos are hashed (only those get a new name, and a /// verify hashes photos without another frames version); of the others inside their folders, the title. Another value /// once one of these changes. Any other write (an upload into a sequence not planned, a verify that hashes no photo of /// a planned sequence, a status, tag or note change) leaves it as it was, and so does a change made on disk alone, which /// the index does not have. /// public async Task GetStampAsync(Guid archiveId, IReadOnlyCollection? sequenceIds, CancellationToken cancellationToken = default) { var rows = await _db.ListAsync( "SELECT id, folder, title, first_capture, camera_make, camera_model, lens, frame_count, frames_version FROM sequences WHERE archive_id = $1 ORDER BY id", [archiveId], r => (Id: r.GetGuid(0), Folder: r.GetString(1), Title: r.GetStringOrNull(2), Read: string.Create(CultureInfo.InvariantCulture, $"{r.GetStringOrNull(2)}\u001f{r.GetDateTimeOrNull(3):O}\u001f{r.GetStringOrNull(4)}\u001f{r.GetStringOrNull(5)}\u001f{r.GetStringOrNull(6)}\u001f" + $"{r.GetInt32(7)}\u001f{r.GetInt32(8)}")), cancellationToken).ConfigureAwait(false); var selected = sequenceIds?.ToHashSet(); var hashed = await CountHashedAsync(archiveId, selected, cancellationToken).ConfigureAwait(false); var planned = new HashSet(rows.Where(r => selected?.Contains(r.Id) != false && r.Folder.Length > 0).Select(r => r.Folder), StringComparer.OrdinalIgnoreCase); using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); foreach (var row in rows) { var read = selected?.Contains(row.Id) != false ? string.Create(CultureInfo.InvariantCulture, $"{row.Read}\u001f{hashed.GetValueOrDefault(row.Id)}") : Inside(row.Folder) ? row.Title : null; hash.AppendData(Encoding.UTF8.GetBytes(string.Create(CultureInfo.InvariantCulture, $"{row.Id:N}\u001f{row.Folder}\u001e{read}\n"))); } return Convert.ToHexStringLower(hash.GetHashAndReset()); // Whether a folder lies inside a planned sequence's folder (RenamePlanner.IsInside), checking each folder around it. bool Inside(string folder) { for (var slash = folder.LastIndexOf('/'); slash > 0; slash = folder.LastIndexOf('/', slash - 1)) { if (planned.Contains(folder[..slash])) return true; } return false; } } /// /// A digest of what a conversion of (null: every sequence) plans from in the index of /// (owner review 2, R7): what a Reorganize's does (: every sequence /// and its folder, which decide what "every sequence" is and its order; of the sequences planned, their title, photo /// count, frames version and how many of their photos are hashed), and of the photos of the sequences planned, what /// decides whether each converts and the size it is estimated at: their format, size and pixel size, and the flags a /// scan, a verify or a decode records without another frames version (missing, corrupt, read only from the preview, /// undecodable). Another value once one of these changes. A change made on disk alone, which the index does not have /// (a file gone before a scan finds it), leaves it as it was. /// public async Task GetConvertStampAsync(Guid archiveId, IReadOnlyCollection? sequenceIds, CancellationToken cancellationToken = default) { var stamp = await GetStampAsync(archiveId, sequenceIds, cancellationToken).ConfigureAwait(false); // Per sequence planned: its photos, their bytes, formats and pixel sizes, and how many carry each flag that decides. var columns = "count(*), COALESCE(SUM(f.size), 0), COALESCE(SUM(f.format), 0), COALESCE(SUM(f.width), 0), COALESCE(SUM(f.height), 0), " + string.Join(", ", new[] { FrameFlags.Missing, FrameFlags.Corrupt, FrameFlags.PreviewOnly, FrameFlags.Undecodable } .Select(flag => $"COALESCE(SUM(CASE WHEN (f.flags & {(int)flag}) <> 0 THEN 1 ELSE 0 END), 0)")); static (Guid Id, string Photos) Read(DbRow r) => (r.GetGuid(0), string.Join('\u001f', Enumerable.Range(1, 9).Select(i => r.GetInt64(i).ToString(CultureInfo.InvariantCulture)))); var sums = new List<(Guid Id, string Photos)>(); if (sequenceIds is null) { sums.AddRange(await _db.ListAsync( $"SELECT f.sequence_id, {columns} FROM frames f JOIN sequences s ON s.id = f.sequence_id WHERE s.archive_id = $1 GROUP BY f.sequence_id", [archiveId], Read, cancellationToken).ConfigureAwait(false)); } else { // 500 ids per statement stays below SQLite's historical 999-variable limit. foreach (var chunk in sequenceIds.Distinct().Chunk(500)) { var p = new SqlParameters(); var ids = string.Join(", ", chunk.Select(id => p.Add(id))); sums.AddRange(await _db.ListAsync($"SELECT f.sequence_id, {columns} FROM frames f WHERE f.sequence_id IN ({ids}) GROUP BY f.sequence_id", p.ToArray(), Read, cancellationToken).ConfigureAwait(false)); } } using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); hash.AppendData(Encoding.UTF8.GetBytes(stamp)); foreach (var (id, photos) in sums.OrderBy(s => s.Id)) hash.AppendData(Encoding.UTF8.GetBytes(string.Create(CultureInfo.InvariantCulture, $"\n{id:N}\u001f{photos}"))); return Convert.ToHexStringLower(hash.GetHashAndReset()); } /// /// How many photos of each of (null: every sequence of ) have /// their SHA-256 in the index; a sequence without one is not listed. /// private async Task> CountHashedAsync(Guid archiveId, IReadOnlyCollection? sequenceIds, CancellationToken cancellationToken) { var counts = new Dictionary(); if (sequenceIds is null) { foreach (var (id, count) in await _db.ListAsync( "SELECT f.sequence_id, count(*) FROM frames f JOIN sequences s ON s.id = f.sequence_id " + "WHERE s.archive_id = $1 AND f.sha256 IS NOT NULL GROUP BY f.sequence_id", [archiveId], r => (r.GetGuid(0), r.GetInt64(1)), cancellationToken).ConfigureAwait(false)) counts[id] = count; return counts; } // 500 ids per statement stays below SQLite's historical 999-variable limit. foreach (var chunk in sequenceIds.Chunk(500)) { var p = new SqlParameters(); var ids = string.Join(", ", chunk.Select(id => p.Add(id))); foreach (var (id, count) in await _db.ListAsync( $"SELECT sequence_id, count(*) FROM frames WHERE sequence_id IN ({ids}) AND sha256 IS NOT NULL GROUP BY sequence_id", p.ToArray(), r => (r.GetGuid(0), r.GetInt64(1)), cancellationToken).ConfigureAwait(false)) counts[id] = count; } return counts; } /// /// The sequences of a rolling verification checks next (owner review 2, R10): those never /// verified first, then the longest unverified, at most . With /// only those never verified or last verified before it are listed. Sequences without frames are left out. /// public Task> ListVerifyCandidatesAsync(Guid archiveId, DateTimeOffset? verifiedBefore, int limit, CancellationToken cancellationToken = default) { var p = new SqlParameters(); var where = $"archive_id = {p.Add(archiveId)} AND frame_count > 0"; if (verifiedBefore is { } before) where += $" AND (verified_at IS NULL OR verified_at < {p.Add(before)})"; return _db.ListAsync($"SELECT id FROM sequences WHERE {where} ORDER BY verified_at ASC NULLS FIRST, id LIMIT {p.Add(Math.Max(1, limit))}", p.ToArray(), r => r.GetGuid(0), cancellationToken); } /// Folder and title of every sequence of an archive, by folder. public Task> ListLocationsAsync(Guid archiveId, CancellationToken cancellationToken = default) => _db.ListAsync("SELECT id, folder, title FROM sequences WHERE archive_id = $1 ORDER BY folder, id", [archiveId], r => new SequenceLocation(r.GetGuid(0), r.GetString(1), r.GetStringOrNull(2)), cancellationToken); /// The sequence created for a session key (automatic grouping of uploads and card sessions). public Task FindBySessionKeyAsync(Guid archiveId, string sessionKey, CancellationToken cancellationToken = default) => _db.ScalarAsync("SELECT sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key = $2", [archiveId, sessionKey], cancellationToken); /// Binds a session key to a sequence unless it is already bound; returns the sequence the key resolves to. public Task MapSessionKeyAsync(Guid archiveId, string sessionKey, Guid sequenceId, CancellationToken cancellationToken = default) => _db.InTransactionAsync(async session => { await session.ExecuteAsync( "INSERT INTO sequence_sessions (archive_id, session_key, sequence_id) VALUES ($1, $2, $3) ON CONFLICT (archive_id, session_key) DO NOTHING", [archiveId, sessionKey, sequenceId], cancellationToken).ConfigureAwait(false); return (await session.ScalarAsync("SELECT sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key = $2", [archiveId, sessionKey], cancellationToken).ConfigureAwait(false))!.Value; }, cancellationToken); /// /// Moves a sequence to another folder and, with , retitles it (mass rename; settings version /// + 1 when the title changes); every other field stays as the locked row has it. False when it does not exist. /// /// The new title; null keeps the current one. public Task RelocateAsync(Guid id, string folder, string? title, CancellationToken cancellationToken = default) => _db.InTransactionAsync(async session => { var current = await session.QuerySingleOrDefaultAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken) .ConfigureAwait(false); if (current is null) return false; var retitled = title is not null && current.Title != title; await UpsertRowAsync(session, current with { Folder = folder, Title = retitled ? title : current.Title, SettingsVersion = retitled ? current.SettingsVersion + 1 : current.SettingsVersion, UpdatedAt = DateTimeOffset.UtcNow, }, cancellationToken).ConfigureAwait(false); return true; }, cancellationToken); private static List Filter(SequenceQuery query, SqlParameters p) { var where = new List(); if (query.ArchiveId is { } archive) where.Add($"s.archive_id = {p.Add(archive)}"); if (query.Year is { } year) where.Add($"s.first_capture >= {p.Add(new DateTime(year, 1, 1))} AND s.first_capture < {p.Add(new DateTime(year + 1, 1, 1))}"); if (query.Camera is { } camera) where.Add($"s.camera_model = {p.Add(camera)}"); if (query.WithStatus != SequenceStatus.None) { var mask = p.Add((int)query.WithStatus); where.Add($"(s.status & {mask}) = {mask}"); } if (query.WithoutStatus != SequenceStatus.None) where.Add($"(s.status & {p.Add((int)query.WithoutStatus)}) = 0"); if (query.Rendered is { } rendered) where.Add(rendered ? "s.rendered_count > 0" : "s.rendered_count = 0"); if (query.SyncStates is { Count: > 0 } states) where.Add($"s.sync_state IN ({string.Join(", ", states.Select(state => p.Add((int)state)))})"); if (query.Tag is { } tag) where.Add("EXISTS (SELECT 1 FROM sequence_tags st JOIN tags t ON t.id = st.tag_id " + $"WHERE st.sequence_id = s.id AND t.name_key = {p.Add(TagRepository.Key(tag))})"); if (!string.IsNullOrWhiteSpace(query.Text)) { var pattern = p.Add(SqlDialect.ContainsPattern(query.Text.Trim().ToLowerInvariant())); where.Add($"(s.search_text LIKE {pattern} ESCAPE '\\' OR EXISTS (SELECT 1 FROM sequence_tags st JOIN tags t ON t.id = st.tag_id " + $"WHERE st.sequence_id = s.id AND t.name_key LIKE {pattern} ESCAPE '\\'))"); } if (query.OwnerUserId is { } owner) where.Add($"s.owner_user_id = {p.Add(owner)}"); return where; } private static string Where(List conditions) => conditions.Count == 0 ? string.Empty : " WHERE " + string.Join(" AND ", conditions); private static (string Column, bool Descending) SortKey(SequenceSort sort) => sort switch { SequenceSort.Newest => ("s.first_capture", true), SequenceSort.Oldest => ("s.first_capture", false), SequenceSort.Title => ("s.title_key", false), SequenceSort.FrameCount => ("s.frame_count", true), SequenceSort.Size => ("s.total_bytes", true), _ => throw new ArgumentOutOfRangeException(nameof(sort), sort, null), }; private static object? SortValue(SequenceSort sort, DbRow row) => sort switch { SequenceSort.Newest or SequenceSort.Oldest => row.GetDateTimeOrNull(SortKeyOrdinal), SequenceSort.Title => row.GetString(SortKeyOrdinal), _ => row.GetInt64(SortKeyOrdinal), }; private static Task UpsertRowAsync(IDbSession session, Sequence s, CancellationToken cancellationToken) { var search = string.Join(' ', new[] { s.Title, s.CameraMake, s.CameraModel }.Where(v => !string.IsNullOrWhiteSpace(v))).ToLowerInvariant(); return session.ExecuteAsync(UpsertSql, [ s.Id, s.ArchiveId, s.Title, s.DisplayTitle.ToLowerInvariant(), search, s.Folder, s.CameraMake, s.CameraModel, s.CameraSerial, s.Lens, Wall(s.FirstCapture), Wall(s.LastCapture), s.FrameCount, s.MedianInterval, s.TotalBytes, (int)s.Status, s.RenderedCount, s.VerifiedAt, s.HeroFrameId, s.Rating, s.Notes, DbParam.Json(s.SettingsJson), DbParam.Json(s.AnalysisJson), (int)s.Origin, (int)s.SyncState, s.SettingsVersion, s.FramesVersion, s.OwnerUserId, s.WarningCount, s.UpdatedAt, s.PreviewOnlyCount, ], cancellationToken); } /// /// Row-locks a sequence for a read-modify-write (PostgreSQL; SQLite's write lock already serialises writers). /// False when it does not exist. /// internal static async Task LockAsync(IDbSession session, Guid id, CancellationToken cancellationToken) => await session.ScalarAsync($"SELECT id FROM sequences WHERE id = $1{SqlDialect.For(session.Dialect).ForUpdate}", [id], cancellationToken) .ConfigureAwait(false) is not null; /// /// Replaces the tag links of a sequence whose row the caller has locked (or just upserted); a set the sequence already /// carries (names compared case-insensitively) is left as it is. On PostgreSQL every tag written is locked until /// commit, and the tags are written in the order of their keys whatever order the caller listed them in (a PATCH sends /// them in any order): writers of two sequences that share tags then take those rows in one order, and never wait for /// each other's (PostgreSQL would abort one with a deadlock, and nothing retries it). /// private async Task ReplaceTagsAsync(IDbSession session, Guid sequenceId, IEnumerable tags, CancellationToken cancellationToken) { var wanted = TagRepository.NormalizeSet(tags); var keys = wanted.Select(TagRepository.Key).ToList(); var stored = await session.QueryListAsync("SELECT t.name_key FROM sequence_tags st JOIN tags t ON t.id = st.tag_id WHERE st.sequence_id = $1", [sequenceId], r => r.GetString(0), cancellationToken).ConfigureAwait(false); if (stored.Count == keys.Count && stored.ToHashSet(StringComparer.Ordinal).SetEquals(keys)) return; await session.ExecuteAsync("DELETE FROM sequence_tags WHERE sequence_id = $1", [sequenceId], cancellationToken).ConfigureAwait(false); foreach (var tag in wanted.OrderBy(TagRepository.Key, StringComparer.Ordinal)) { DbParam[] args = [Guid.CreateVersion7(), tag, TagRepository.Key(tag)]; Guid tagId; if (_db.Sql.IsPostgres) { // The no-op update locks an existing tag, so a concurrent DeleteUnusedAsync cannot remove it before the link exists. tagId = (await session.ScalarAsync( "INSERT INTO tags (id, name, name_key) VALUES ($1, $2, $3) ON CONFLICT (name_key) DO UPDATE SET name_key = excluded.name_key RETURNING id", args, cancellationToken).ConfigureAwait(false))!.Value; } else { await session.ExecuteAsync("INSERT INTO tags (id, name, name_key) VALUES ($1, $2, $3) ON CONFLICT (name_key) DO NOTHING", args, cancellationToken).ConfigureAwait(false); tagId = (await session.ScalarAsync("SELECT id FROM tags WHERE name_key = $1", [args[2]], cancellationToken).ConfigureAwait(false))!.Value; } await session.ExecuteAsync("INSERT INTO sequence_tags (sequence_id, tag_id) VALUES ($1, $2)", [sequenceId, tagId], cancellationToken) .ConfigureAwait(false); } } private static string? NullIfEmpty(string value) => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); /// Camera wall-clock times are stored without zone, whatever kind the caller's value has. internal static DateTime? Wall(DateTime? value) => value is { } v ? DateTime.SpecifyKind(v, DateTimeKind.Unspecified) : null; private static Sequence Map(DbRow r) { var tags = r.GetStringOrNull(TagsOrdinal) is { Length: > 0 } list ? list.Split(SqlDialect.ListSeparator) : []; Array.Sort(tags, StringComparer.OrdinalIgnoreCase); return new Sequence { Id = r.GetGuid(0), ArchiveId = r.GetGuid(1), Title = r.GetStringOrNull(2), Folder = r.GetString(3), CameraMake = r.GetStringOrNull(4), CameraModel = r.GetStringOrNull(5), CameraSerial = r.GetStringOrNull(6), Lens = r.GetStringOrNull(7), FirstCapture = r.GetDateTimeOrNull(8), LastCapture = r.GetDateTimeOrNull(9), FrameCount = r.GetInt32(10), MedianInterval = r.GetDoubleOrNull(11), TotalBytes = r.GetInt64(12), Status = (SequenceStatus)r.GetInt32(13), RenderedCount = r.GetInt32(14), VerifiedAt = r.GetDateTimeOffsetOrNull(15), HeroFrameId = r.GetGuidOrNull(16), Rating = r.GetInt32(17), Notes = r.GetStringOrNull(18), SettingsJson = r.GetStringOrNull(19), AnalysisJson = r.GetStringOrNull(20), Origin = (SequenceOrigin)r.GetInt32(21), SyncState = (SyncState)r.GetInt32(22), SettingsVersion = r.GetInt32(23), FramesVersion = r.GetInt32(24), OwnerUserId = r.GetGuidOrNull(25), WarningCount = r.GetInt32(26), UpdatedAt = r.GetDateTimeOffset(27), PreviewOnlyCount = r.GetInt32(28), Tags = tags, }; } }