diff --git a/src/Titano.Data/Migrations/postgres/0002_archive_index.sql b/src/Titano.Data/Migrations/postgres/0002_archive_index.sql new file mode 100644 index 00000000..b872e6d2 --- /dev/null +++ b/src/Titano.Data/Migrations/postgres/0002_archive_index.sql @@ -0,0 +1,23 @@ +-- Archive index state (BRIEF §7, §14.4). Mirrors sqlite/0002_archive_index.sql. + +-- Automatic grouping: a session key (upload session, card session) resolves to the sequence created for it. +CREATE TABLE sequence_sessions ( + archive_id uuid NOT NULL REFERENCES archives (id) ON DELETE CASCADE, + session_key text NOT NULL, + sequence_id uuid NOT NULL REFERENCES sequences (id) ON DELETE CASCADE, + PRIMARY KEY (archive_id, session_key) +); +CREATE INDEX ix_sequence_sessions_sequence ON sequence_sessions (sequence_id); + +-- What the last scan saw in each sequence folder, so incremental scans skip unchanged folders by size and time. +-- Times are UTC ticks (100 ns), compared exactly with the file system's. +CREATE TABLE archive_folders ( + archive_id uuid NOT NULL REFERENCES archives (id) ON DELETE CASCADE, + folder text COLLATE "C" NOT NULL, + sequence_id uuid REFERENCES sequences (id) ON DELETE SET NULL, + sidecar_size bigint, + sidecar_ticks bigint, + photos_ticks bigint, + scanned_at timestamptz NOT NULL, + PRIMARY KEY (archive_id, folder) +); diff --git a/src/Titano.Data/Migrations/sqlite/0002_archive_index.sql b/src/Titano.Data/Migrations/sqlite/0002_archive_index.sql new file mode 100644 index 00000000..bace37eb --- /dev/null +++ b/src/Titano.Data/Migrations/sqlite/0002_archive_index.sql @@ -0,0 +1,23 @@ +-- Archive index state (BRIEF §7, §14.4). Mirrors postgres/0002_archive_index.sql. + +-- Automatic grouping: a session key (upload session, card session) resolves to the sequence created for it. +CREATE TABLE sequence_sessions ( + archive_id TEXT NOT NULL REFERENCES archives (id) ON DELETE CASCADE, + session_key TEXT NOT NULL, + sequence_id TEXT NOT NULL REFERENCES sequences (id) ON DELETE CASCADE, + PRIMARY KEY (archive_id, session_key) +); +CREATE INDEX ix_sequence_sessions_sequence ON sequence_sessions (sequence_id); + +-- What the last scan saw in each sequence folder, so incremental scans skip unchanged folders by size and time. +-- Times are UTC ticks (100 ns), compared exactly with the file system's. +CREATE TABLE archive_folders ( + archive_id TEXT NOT NULL REFERENCES archives (id) ON DELETE CASCADE, + folder TEXT NOT NULL, + sequence_id TEXT REFERENCES sequences (id) ON DELETE SET NULL, + sidecar_size INTEGER, + sidecar_ticks INTEGER, + photos_ticks INTEGER, + scanned_at TEXT NOT NULL, + PRIMARY KEY (archive_id, folder) +); diff --git a/src/Titano.Data/Repositories/ArchiveRepository.cs b/src/Titano.Data/Repositories/ArchiveRepository.cs index 717eb4f9..dec57a0a 100644 --- a/src/Titano.Data/Repositories/ArchiveRepository.cs +++ b/src/Titano.Data/Repositories/ArchiveRepository.cs @@ -1,7 +1,11 @@ +using Titano.Core.Archiving; using Titano.Core.Model; namespace Titano.Data.Repositories; +/// What the last scan saw in one sequence folder (sidecar and photo-directory stamps). +public sealed record FolderScanState(string Folder, Guid? SequenceId, FileStamp? Sidecar, FileStamp? Photos); + /// Archive roots (BRIEF §5, §7). public sealed class ArchiveRepository { @@ -10,6 +14,9 @@ public sealed class ArchiveRepository private static readonly string UpsertSql = SqlDialect.Upsert("archives", ["id", "name", "root_path", "folder_template", "file_template", "schema_version", "created", "proxy_long_edge"], 1); + private static readonly string FolderStateUpsertSql = SqlDialect.Upsert("archive_folders", + ["archive_id", "folder", "sequence_id", "sidecar_size", "sidecar_ticks", "photos_ticks", "scanned_at"], 2); + private readonly DbAccess _db; internal ArchiveRepository(DbAccess db) => _db = db; @@ -29,6 +36,31 @@ public sealed class ArchiveRepository public async Task DeleteAsync(Guid id, CancellationToken cancellationToken = default) => await _db.ExecuteAsync("DELETE FROM archives WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) > 0; + /// Scan stamps of every folder of an archive, by folder. + public async Task> GetFolderStatesAsync(Guid archiveId, CancellationToken cancellationToken = default) + { + var rows = await _db.ListAsync( + "SELECT folder, sequence_id, sidecar_size, sidecar_ticks, photos_ticks FROM archive_folders WHERE archive_id = $1", [archiveId], + r => new FolderScanState(r.GetString(0), r.GetGuidOrNull(1), + r.GetInt64OrNull(3) is { } sidecar ? new FileStamp(r.GetInt64(2), new DateTime(sidecar, DateTimeKind.Utc)) : null, + r.GetInt64OrNull(4) is { } photos ? new FileStamp(0, new DateTime(photos, DateTimeKind.Utc)) : null), + cancellationToken).ConfigureAwait(false); + return rows.ToDictionary(s => s.Folder, StringComparer.Ordinal); + } + + public async Task SetFolderStateAsync(Guid archiveId, FolderScanState state, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync(FolderStateUpsertSql, + [archiveId, state.Folder, state.SequenceId, state.Sidecar?.Size, state.Sidecar?.ModifiedUtc.Ticks, state.Photos?.ModifiedUtc.Ticks, DateTimeOffset.UtcNow], + cancellationToken).ConfigureAwait(false); + + public async Task DeleteFolderStateAsync(Guid archiveId, string folder, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync("DELETE FROM archive_folders WHERE archive_id = $1 AND folder = $2", [archiveId, folder], cancellationToken) + .ConfigureAwait(false) > 0; + + /// Forgets every scan stamp of an archive, so the next scan reads every sidecar (rebuild). + public Task ClearFolderStatesAsync(Guid archiveId, CancellationToken cancellationToken = default) => + _db.ExecuteAsync("DELETE FROM archive_folders WHERE archive_id = $1", [archiveId], cancellationToken); + private static Archive Map(DbRow r) => new() { Id = r.GetGuid(0), diff --git a/src/Titano.Data/Repositories/FrameRepository.cs b/src/Titano.Data/Repositories/FrameRepository.cs index 30350e5e..ae064902 100644 --- a/src/Titano.Data/Repositories/FrameRepository.cs +++ b/src/Titano.Data/Repositories/FrameRepository.cs @@ -4,6 +4,15 @@ using Titano.Core.Model; namespace Titano.Data.Repositories; +/// Integrity fields a verify rewrites: the flags and the hash of a newly hashed file. +public readonly record struct FrameIntegrity(Guid FrameId, FrameFlags Flags, string? Sha256); + +/// A frame and the sequence it belongs to. +public readonly record struct FrameRef(Guid FrameId, Guid SequenceId); + +/// A frame whose content (SHA-256) also exists elsewhere in the archive. +public sealed record DuplicateFrame(string Sha256, Guid FrameId, Guid SequenceId, int Index, string RelativePath); + /// Light statistics of one frame, written by the analysis job. public readonly record struct FrameLightStats(Guid FrameId, double? MeanLogLuminance, double? ClipLow, double? ClipHigh); @@ -25,6 +34,12 @@ public sealed class FrameRepository /// Rows per statement of the batched PostgreSQL updates (4 or fewer parameters per row). private const int Batch = 1000; + /// Rows per multi-row upsert: 30 x 27 parameters stays below SQLite's historical 999-variable limit. + private const int UpsertRows = 30; + + /// Light statistics belong to the analysis job; a sync from the sidecar never clears them. + private static readonly string[] AnalysisColumns = ["mean_log_luminance", "clip_low", "clip_high"]; + private readonly DbAccess _db; internal FrameRepository(DbAccess db) => _db = db; @@ -112,6 +127,89 @@ public sealed class FrameRepository return updated; }, cancellationToken); + /// + /// Makes the frames of a sequence exactly (from its sidecar): rows are upserted by id, + /// rows not listed are deleted, light statistics of kept rows survive. Indexes may be renumbered freely. + /// + public Task SyncSequenceAsync(Guid sequenceId, IReadOnlyList frames, CancellationToken cancellationToken = default) => + _db.InTransactionAsync(async session => + { + // Park current rows on negative indexes so renumbered rows never collide on (sequence_id, idx). + await session.ExecuteAsync("UPDATE frames SET idx = -idx - 1 WHERE sequence_id = $1", [sequenceId], cancellationToken).ConfigureAwait(false); + var updates = string.Join(", ", ColumnNames.Skip(1).Where(c => !AnalysisColumns.Contains(c)).Select(c => $"{c} = excluded.{c}")); + for (var start = 0; start < frames.Count; start += UpsertRows) + { + var count = Math.Min(UpsertRows, frames.Count - start); + var sql = new StringBuilder("INSERT INTO frames (").Append(Columns).Append(") VALUES "); + var args = new DbParam[count * ColumnNames.Length]; + for (var i = 0; i < count; i++) + { + var values = Values(frames[start + i]); + sql.Append(i > 0 ? ", (" : "("); + for (var c = 0; c < values.Length; c++) + { + args[i * values.Length + c] = new DbParam(values[c]); + sql.Append(c > 0 ? ", $" : "$").Append(i * values.Length + c + 1); + } + sql.Append(')'); + } + sql.Append(" ON CONFLICT (id) DO UPDATE SET ").Append(updates); + await session.ExecuteAsync(sql.ToString(), args, cancellationToken).ConfigureAwait(false); + } + return await session.ExecuteAsync("DELETE FROM frames WHERE sequence_id = $1 AND idx < 0", [sequenceId], cancellationToken).ConfigureAwait(false); + }, cancellationToken); + + /// Number of frames of a sequence. + public async Task CountBySequenceAsync(Guid sequenceId, CancellationToken cancellationToken = default) => + (int)await _db.ScalarAsync("SELECT count(*) FROM frames WHERE sequence_id = $1", [sequenceId], cancellationToken).ConfigureAwait(false); + + /// Frames of an archive with this size and capture time (card import duplicate candidates). + public Task> FindBySizeAndCaptureAsync(Guid archiveId, long size, DateTime capturedAt, CancellationToken cancellationToken = default) => + _db.ListAsync( + $"SELECT {string.Join(", ", ColumnNames.Select(c => "f." + c))} FROM frames f JOIN sequences s ON s.id = f.sequence_id " + + "WHERE s.archive_id = $1 AND f.size = $2 AND f.captured_at = $3", + [archiveId, size, SequenceRepository.Wall(capturedAt)], Map, cancellationToken); + + /// Every frame of an archive whose SHA-256 occurs more than once, grouped by hash then sequence and index. + public Task> ListDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) => + _db.ListAsync( + "SELECT f.sha256, f.id, f.sequence_id, f.idx, f.relative_path FROM frames f JOIN sequences s ON s.id = f.sequence_id " + + "WHERE s.archive_id = $1 AND f.sha256 IN (SELECT f2.sha256 FROM frames f2 JOIN sequences s2 ON s2.id = f2.sequence_id " + + "WHERE s2.archive_id = $1 AND f2.sha256 IS NOT NULL GROUP BY f2.sha256 HAVING count(*) > 1) " + + "ORDER BY f.sha256, s.first_capture, f.sequence_id, f.idx", + [archiveId], r => new DuplicateFrame(r.GetString(0), r.GetGuid(1), r.GetGuid(2), r.GetInt32(3), r.GetString(4)), cancellationToken); + + /// Frames of an archive that carry every bit of . + public Task> ListFlaggedAsync(Guid archiveId, FrameFlags flags, CancellationToken cancellationToken = default) => + _db.ListAsync( + "SELECT f.id, f.sequence_id FROM frames f JOIN sequences s ON s.id = f.sequence_id WHERE s.archive_id = $1 AND (f.flags & $2) = $2", + [archiveId, (int)flags], r => new FrameRef(r.GetGuid(0), r.GetGuid(1)), cancellationToken); + + /// Rewrites the relative paths of a sequence's frames after its folder moved (mass rename). + /// Only paths still under change, so repeating it after a crash is harmless. + public Task RebaseFolderAsync(Guid sequenceId, string oldFolder, string newFolder, CancellationToken cancellationToken = default) + { + // substr counts characters (code points) on both dialects; the tail starts with the '/' after the folder. + var prefix = oldFolder + "/"; + var length = prefix.EnumerateRunes().Count(); + return _db.ExecuteAsync( + "UPDATE frames SET relative_path = $2 || substr(relative_path, $3) WHERE sequence_id = $1 AND substr(relative_path, 1, $4) = $5", + [sequenceId, newFolder, length, length, prefix], cancellationToken); + } + + /// Writes flags and hashes found by a verify, in one transaction; returns the number of frames updated. + public Task UpdateIntegrityAsync(IReadOnlyList frames, CancellationToken cancellationToken = default) => + _db.InTransactionAsync(async session => + { + var updated = 0; + foreach (var f in frames) + { + updated += await session.ExecuteAsync("UPDATE frames SET flags = $2, sha256 = COALESCE($3, sha256) WHERE id = $1", + [f.FrameId, (int)f.Flags, f.Sha256?.ToLowerInvariant()], cancellationToken).ConfigureAwait(false); + } + return updated; + }, cancellationToken); + /// Removes every frame of a sequence; returns how many. public Task DeleteBySequenceAsync(Guid sequenceId, CancellationToken cancellationToken = default) => _db.ExecuteAsync("DELETE FROM frames WHERE sequence_id = $1", [sequenceId], cancellationToken); diff --git a/src/Titano.Data/Repositories/MediaCardRepository.cs b/src/Titano.Data/Repositories/MediaCardRepository.cs index ab95cc5f..d42c6971 100644 --- a/src/Titano.Data/Repositories/MediaCardRepository.cs +++ b/src/Titano.Data/Repositories/MediaCardRepository.cs @@ -96,6 +96,16 @@ public sealed class MediaCardRepository await _db.ScalarAsync("SELECT 1 FROM import_history WHERE card_id = $1 AND path = $2 AND size = $3 AND mtime = $4", [cardId, key.Path, key.Size, key.Modified], cancellationToken).ConfigureAwait(false) is not null; + /// Every recorded import from a card that has a hash and a frame (candidates for a verified card erase). + public Task> ListVerifiableImportsAsync(Guid cardId, CancellationToken cancellationToken = default) => + _db.ListAsync( + "SELECT card_id, path, size, mtime, sha256, frame_id, job_id, imported_at FROM import_history " + + "WHERE card_id = $1 AND sha256 IS NOT NULL AND frame_id IS NOT NULL ORDER BY path", + [cardId], + r => new ImportRecord(r.GetGuid(0), new ImportKey(r.GetString(1), r.GetInt64(2), r.GetDateTimeOffset(3)), + r.GetStringOrNull(4), r.GetGuidOrNull(5), r.GetGuidOrNull(6), r.GetDateTimeOffset(7)), + cancellationToken); + /// Most recent imports from a card. public Task> ListImportsAsync(Guid cardId, int limit = 1000, CancellationToken cancellationToken = default) => _db.ListAsync( diff --git a/src/Titano.Data/Repositories/SequenceRepository.cs b/src/Titano.Data/Repositories/SequenceRepository.cs index 4912cec5..eb566ded 100644 --- a/src/Titano.Data/Repositories/SequenceRepository.cs +++ b/src/Titano.Data/Repositories/SequenceRepository.cs @@ -79,6 +79,9 @@ public sealed record ArchiveHealth( long CorruptFrames, long DuplicateFrames); +/// Where a sequence lives in its archive. +public sealed record SequenceLocation(Guid Id, string Folder, string? Title); + /// Sequences (projects) with their tags; Library listing and Archive statistics. public sealed class SequenceRepository { @@ -259,6 +262,45 @@ public sealed class SequenceRepository }, cancellationToken); } + /// Folder and title of every sequence of an archive, by folder. + public Task> ListLocationsAsync(Guid archiveId, CancellationToken cancellationToken = default) => + _db.ListAsync("SELECT id, folder, title FROM sequences WHERE archive_id = $1 ORDER BY folder, id", [archiveId], + r => new SequenceLocation(r.GetGuid(0), r.GetString(1), r.GetStringOrNull(2)), cancellationToken); + + /// The sequence created for a session key (automatic grouping of uploads and card sessions). + public Task FindBySessionKeyAsync(Guid archiveId, string sessionKey, CancellationToken cancellationToken = default) => + _db.ScalarAsync("SELECT sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key = $2", + [archiveId, sessionKey], cancellationToken); + + /// Binds a session key to a sequence unless it is already bound; returns the sequence the key resolves to. + public Task MapSessionKeyAsync(Guid archiveId, string sessionKey, Guid sequenceId, CancellationToken cancellationToken = default) => + _db.InTransactionAsync(async session => + { + await session.ExecuteAsync( + "INSERT INTO sequence_sessions (archive_id, session_key, sequence_id) VALUES ($1, $2, $3) ON CONFLICT (archive_id, session_key) DO NOTHING", + [archiveId, sessionKey, sequenceId], cancellationToken).ConfigureAwait(false); + return (await session.ScalarAsync("SELECT sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key = $2", + [archiveId, sessionKey], cancellationToken).ConfigureAwait(false))!.Value; + }, cancellationToken); + + /// Moves a sequence to another folder and/or title (mass rename); false when it does not exist. + public Task RelocateAsync(Guid id, string folder, string? title, CancellationToken cancellationToken = default) => + _db.InTransactionAsync(async session => + { + var current = await session.QuerySingleOrDefaultAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken) + .ConfigureAwait(false); + if (current is null) return false; + var titleChanged = current.Title != title; + await UpsertRowAsync(session, current with + { + Folder = folder, + Title = title, + SettingsVersion = titleChanged ? current.SettingsVersion + 1 : current.SettingsVersion, + UpdatedAt = DateTimeOffset.UtcNow, + }, cancellationToken).ConfigureAwait(false); + return true; + }, cancellationToken); + private static List Filter(SequenceQuery query, SqlParameters p) { var where = new List();