diff --git a/src/Titano.Data/Migrations/postgres/0002_archive_index.sql b/src/Titano.Data/Migrations/postgres/0002_archive_index.sql
new file mode 100644
index 00000000..b872e6d2
--- /dev/null
+++ b/src/Titano.Data/Migrations/postgres/0002_archive_index.sql
@@ -0,0 +1,23 @@
+-- Archive index state (BRIEF §7, §14.4). Mirrors sqlite/0002_archive_index.sql.
+
+-- Automatic grouping: a session key (upload session, card session) resolves to the sequence created for it.
+CREATE TABLE sequence_sessions (
+ archive_id uuid NOT NULL REFERENCES archives (id) ON DELETE CASCADE,
+ session_key text NOT NULL,
+ sequence_id uuid NOT NULL REFERENCES sequences (id) ON DELETE CASCADE,
+ PRIMARY KEY (archive_id, session_key)
+);
+CREATE INDEX ix_sequence_sessions_sequence ON sequence_sessions (sequence_id);
+
+-- What the last scan saw in each sequence folder, so incremental scans skip unchanged folders by size and time.
+-- Times are UTC ticks (100 ns), compared exactly with the file system's.
+CREATE TABLE archive_folders (
+ archive_id uuid NOT NULL REFERENCES archives (id) ON DELETE CASCADE,
+ folder text COLLATE "C" NOT NULL,
+ sequence_id uuid REFERENCES sequences (id) ON DELETE SET NULL,
+ sidecar_size bigint,
+ sidecar_ticks bigint,
+ photos_ticks bigint,
+ scanned_at timestamptz NOT NULL,
+ PRIMARY KEY (archive_id, folder)
+);
diff --git a/src/Titano.Data/Migrations/sqlite/0002_archive_index.sql b/src/Titano.Data/Migrations/sqlite/0002_archive_index.sql
new file mode 100644
index 00000000..bace37eb
--- /dev/null
+++ b/src/Titano.Data/Migrations/sqlite/0002_archive_index.sql
@@ -0,0 +1,23 @@
+-- Archive index state (BRIEF §7, §14.4). Mirrors postgres/0002_archive_index.sql.
+
+-- Automatic grouping: a session key (upload session, card session) resolves to the sequence created for it.
+CREATE TABLE sequence_sessions (
+ archive_id TEXT NOT NULL REFERENCES archives (id) ON DELETE CASCADE,
+ session_key TEXT NOT NULL,
+ sequence_id TEXT NOT NULL REFERENCES sequences (id) ON DELETE CASCADE,
+ PRIMARY KEY (archive_id, session_key)
+);
+CREATE INDEX ix_sequence_sessions_sequence ON sequence_sessions (sequence_id);
+
+-- What the last scan saw in each sequence folder, so incremental scans skip unchanged folders by size and time.
+-- Times are UTC ticks (100 ns), compared exactly with the file system's.
+CREATE TABLE archive_folders (
+ archive_id TEXT NOT NULL REFERENCES archives (id) ON DELETE CASCADE,
+ folder TEXT NOT NULL,
+ sequence_id TEXT REFERENCES sequences (id) ON DELETE SET NULL,
+ sidecar_size INTEGER,
+ sidecar_ticks INTEGER,
+ photos_ticks INTEGER,
+ scanned_at TEXT NOT NULL,
+ PRIMARY KEY (archive_id, folder)
+);
diff --git a/src/Titano.Data/Repositories/ArchiveRepository.cs b/src/Titano.Data/Repositories/ArchiveRepository.cs
index 717eb4f9..dec57a0a 100644
--- a/src/Titano.Data/Repositories/ArchiveRepository.cs
+++ b/src/Titano.Data/Repositories/ArchiveRepository.cs
@@ -1,7 +1,11 @@
+using Titano.Core.Archiving;
using Titano.Core.Model;
namespace Titano.Data.Repositories;
+/// What the last scan saw in one sequence folder (sidecar and photo-directory stamps).
+public sealed record FolderScanState(string Folder, Guid? SequenceId, FileStamp? Sidecar, FileStamp? Photos);
+
/// Archive roots (BRIEF §5, §7).
public sealed class ArchiveRepository
{
@@ -10,6 +14,9 @@ public sealed class ArchiveRepository
private static readonly string UpsertSql = SqlDialect.Upsert("archives",
["id", "name", "root_path", "folder_template", "file_template", "schema_version", "created", "proxy_long_edge"], 1);
+ private static readonly string FolderStateUpsertSql = SqlDialect.Upsert("archive_folders",
+ ["archive_id", "folder", "sequence_id", "sidecar_size", "sidecar_ticks", "photos_ticks", "scanned_at"], 2);
+
private readonly DbAccess _db;
internal ArchiveRepository(DbAccess db) => _db = db;
@@ -29,6 +36,31 @@ public sealed class ArchiveRepository
public async Task DeleteAsync(Guid id, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("DELETE FROM archives WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) > 0;
+ /// Scan stamps of every folder of an archive, by folder.
+ public async Task> GetFolderStatesAsync(Guid archiveId, CancellationToken cancellationToken = default)
+ {
+ var rows = await _db.ListAsync(
+ "SELECT folder, sequence_id, sidecar_size, sidecar_ticks, photos_ticks FROM archive_folders WHERE archive_id = $1", [archiveId],
+ r => new FolderScanState(r.GetString(0), r.GetGuidOrNull(1),
+ r.GetInt64OrNull(3) is { } sidecar ? new FileStamp(r.GetInt64(2), new DateTime(sidecar, DateTimeKind.Utc)) : null,
+ r.GetInt64OrNull(4) is { } photos ? new FileStamp(0, new DateTime(photos, DateTimeKind.Utc)) : null),
+ cancellationToken).ConfigureAwait(false);
+ return rows.ToDictionary(s => s.Folder, StringComparer.Ordinal);
+ }
+
+ public async Task SetFolderStateAsync(Guid archiveId, FolderScanState state, CancellationToken cancellationToken = default) =>
+ await _db.ExecuteAsync(FolderStateUpsertSql,
+ [archiveId, state.Folder, state.SequenceId, state.Sidecar?.Size, state.Sidecar?.ModifiedUtc.Ticks, state.Photos?.ModifiedUtc.Ticks, DateTimeOffset.UtcNow],
+ cancellationToken).ConfigureAwait(false);
+
+ public async Task DeleteFolderStateAsync(Guid archiveId, string folder, CancellationToken cancellationToken = default) =>
+ await _db.ExecuteAsync("DELETE FROM archive_folders WHERE archive_id = $1 AND folder = $2", [archiveId, folder], cancellationToken)
+ .ConfigureAwait(false) > 0;
+
+ /// Forgets every scan stamp of an archive, so the next scan reads every sidecar (rebuild).
+ public Task ClearFolderStatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
+ _db.ExecuteAsync("DELETE FROM archive_folders WHERE archive_id = $1", [archiveId], cancellationToken);
+
private static Archive Map(DbRow r) => new()
{
Id = r.GetGuid(0),
diff --git a/src/Titano.Data/Repositories/FrameRepository.cs b/src/Titano.Data/Repositories/FrameRepository.cs
index 30350e5e..ae064902 100644
--- a/src/Titano.Data/Repositories/FrameRepository.cs
+++ b/src/Titano.Data/Repositories/FrameRepository.cs
@@ -4,6 +4,15 @@ using Titano.Core.Model;
namespace Titano.Data.Repositories;
+/// Integrity fields a verify rewrites: the flags and the hash of a newly hashed file.
+public readonly record struct FrameIntegrity(Guid FrameId, FrameFlags Flags, string? Sha256);
+
+/// A frame and the sequence it belongs to.
+public readonly record struct FrameRef(Guid FrameId, Guid SequenceId);
+
+/// A frame whose content (SHA-256) also exists elsewhere in the archive.
+public sealed record DuplicateFrame(string Sha256, Guid FrameId, Guid SequenceId, int Index, string RelativePath);
+
/// Light statistics of one frame, written by the analysis job.
public readonly record struct FrameLightStats(Guid FrameId, double? MeanLogLuminance, double? ClipLow, double? ClipHigh);
@@ -25,6 +34,12 @@ public sealed class FrameRepository
/// Rows per statement of the batched PostgreSQL updates (4 or fewer parameters per row).
private const int Batch = 1000;
+ /// Rows per multi-row upsert: 30 x 27 parameters stays below SQLite's historical 999-variable limit.
+ private const int UpsertRows = 30;
+
+ /// Light statistics belong to the analysis job; a sync from the sidecar never clears them.
+ private static readonly string[] AnalysisColumns = ["mean_log_luminance", "clip_low", "clip_high"];
+
private readonly DbAccess _db;
internal FrameRepository(DbAccess db) => _db = db;
@@ -112,6 +127,89 @@ public sealed class FrameRepository
return updated;
}, cancellationToken);
+ ///
+ /// Makes the frames of a sequence exactly (from its sidecar): rows are upserted by id,
+ /// rows not listed are deleted, light statistics of kept rows survive. Indexes may be renumbered freely.
+ ///
+ public Task SyncSequenceAsync(Guid sequenceId, IReadOnlyList frames, CancellationToken cancellationToken = default) =>
+ _db.InTransactionAsync(async session =>
+ {
+ // Park current rows on negative indexes so renumbered rows never collide on (sequence_id, idx).
+ await session.ExecuteAsync("UPDATE frames SET idx = -idx - 1 WHERE sequence_id = $1", [sequenceId], cancellationToken).ConfigureAwait(false);
+ var updates = string.Join(", ", ColumnNames.Skip(1).Where(c => !AnalysisColumns.Contains(c)).Select(c => $"{c} = excluded.{c}"));
+ for (var start = 0; start < frames.Count; start += UpsertRows)
+ {
+ var count = Math.Min(UpsertRows, frames.Count - start);
+ var sql = new StringBuilder("INSERT INTO frames (").Append(Columns).Append(") VALUES ");
+ var args = new DbParam[count * ColumnNames.Length];
+ for (var i = 0; i < count; i++)
+ {
+ var values = Values(frames[start + i]);
+ sql.Append(i > 0 ? ", (" : "(");
+ for (var c = 0; c < values.Length; c++)
+ {
+ args[i * values.Length + c] = new DbParam(values[c]);
+ sql.Append(c > 0 ? ", $" : "$").Append(i * values.Length + c + 1);
+ }
+ sql.Append(')');
+ }
+ sql.Append(" ON CONFLICT (id) DO UPDATE SET ").Append(updates);
+ await session.ExecuteAsync(sql.ToString(), args, cancellationToken).ConfigureAwait(false);
+ }
+ return await session.ExecuteAsync("DELETE FROM frames WHERE sequence_id = $1 AND idx < 0", [sequenceId], cancellationToken).ConfigureAwait(false);
+ }, cancellationToken);
+
+ /// Number of frames of a sequence.
+ public async Task CountBySequenceAsync(Guid sequenceId, CancellationToken cancellationToken = default) =>
+ (int)await _db.ScalarAsync("SELECT count(*) FROM frames WHERE sequence_id = $1", [sequenceId], cancellationToken).ConfigureAwait(false);
+
+ /// Frames of an archive with this size and capture time (card import duplicate candidates).
+ public Task> FindBySizeAndCaptureAsync(Guid archiveId, long size, DateTime capturedAt, CancellationToken cancellationToken = default) =>
+ _db.ListAsync(
+ $"SELECT {string.Join(", ", ColumnNames.Select(c => "f." + c))} FROM frames f JOIN sequences s ON s.id = f.sequence_id " +
+ "WHERE s.archive_id = $1 AND f.size = $2 AND f.captured_at = $3",
+ [archiveId, size, SequenceRepository.Wall(capturedAt)], Map, cancellationToken);
+
+ /// Every frame of an archive whose SHA-256 occurs more than once, grouped by hash then sequence and index.
+ public Task> ListDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
+ _db.ListAsync(
+ "SELECT f.sha256, f.id, f.sequence_id, f.idx, f.relative_path FROM frames f JOIN sequences s ON s.id = f.sequence_id " +
+ "WHERE s.archive_id = $1 AND f.sha256 IN (SELECT f2.sha256 FROM frames f2 JOIN sequences s2 ON s2.id = f2.sequence_id " +
+ "WHERE s2.archive_id = $1 AND f2.sha256 IS NOT NULL GROUP BY f2.sha256 HAVING count(*) > 1) " +
+ "ORDER BY f.sha256, s.first_capture, f.sequence_id, f.idx",
+ [archiveId], r => new DuplicateFrame(r.GetString(0), r.GetGuid(1), r.GetGuid(2), r.GetInt32(3), r.GetString(4)), cancellationToken);
+
+ /// Frames of an archive that carry every bit of .
+ public Task> ListFlaggedAsync(Guid archiveId, FrameFlags flags, CancellationToken cancellationToken = default) =>
+ _db.ListAsync(
+ "SELECT f.id, f.sequence_id FROM frames f JOIN sequences s ON s.id = f.sequence_id WHERE s.archive_id = $1 AND (f.flags & $2) = $2",
+ [archiveId, (int)flags], r => new FrameRef(r.GetGuid(0), r.GetGuid(1)), cancellationToken);
+
+ /// Rewrites the relative paths of a sequence's frames after its folder moved (mass rename).
+ /// Only paths still under change, so repeating it after a crash is harmless.
+ public Task RebaseFolderAsync(Guid sequenceId, string oldFolder, string newFolder, CancellationToken cancellationToken = default)
+ {
+ // substr counts characters (code points) on both dialects; the tail starts with the '/' after the folder.
+ var prefix = oldFolder + "/";
+ var length = prefix.EnumerateRunes().Count();
+ return _db.ExecuteAsync(
+ "UPDATE frames SET relative_path = $2 || substr(relative_path, $3) WHERE sequence_id = $1 AND substr(relative_path, 1, $4) = $5",
+ [sequenceId, newFolder, length, length, prefix], cancellationToken);
+ }
+
+ /// Writes flags and hashes found by a verify, in one transaction; returns the number of frames updated.
+ public Task UpdateIntegrityAsync(IReadOnlyList frames, CancellationToken cancellationToken = default) =>
+ _db.InTransactionAsync(async session =>
+ {
+ var updated = 0;
+ foreach (var f in frames)
+ {
+ updated += await session.ExecuteAsync("UPDATE frames SET flags = $2, sha256 = COALESCE($3, sha256) WHERE id = $1",
+ [f.FrameId, (int)f.Flags, f.Sha256?.ToLowerInvariant()], cancellationToken).ConfigureAwait(false);
+ }
+ return updated;
+ }, cancellationToken);
+
/// Removes every frame of a sequence; returns how many.
public Task DeleteBySequenceAsync(Guid sequenceId, CancellationToken cancellationToken = default) =>
_db.ExecuteAsync("DELETE FROM frames WHERE sequence_id = $1", [sequenceId], cancellationToken);
diff --git a/src/Titano.Data/Repositories/MediaCardRepository.cs b/src/Titano.Data/Repositories/MediaCardRepository.cs
index ab95cc5f..d42c6971 100644
--- a/src/Titano.Data/Repositories/MediaCardRepository.cs
+++ b/src/Titano.Data/Repositories/MediaCardRepository.cs
@@ -96,6 +96,16 @@ public sealed class MediaCardRepository
await _db.ScalarAsync("SELECT 1 FROM import_history WHERE card_id = $1 AND path = $2 AND size = $3 AND mtime = $4",
[cardId, key.Path, key.Size, key.Modified], cancellationToken).ConfigureAwait(false) is not null;
+ /// Every recorded import from a card that has a hash and a frame (candidates for a verified card erase).
+ public Task> ListVerifiableImportsAsync(Guid cardId, CancellationToken cancellationToken = default) =>
+ _db.ListAsync(
+ "SELECT card_id, path, size, mtime, sha256, frame_id, job_id, imported_at FROM import_history " +
+ "WHERE card_id = $1 AND sha256 IS NOT NULL AND frame_id IS NOT NULL ORDER BY path",
+ [cardId],
+ r => new ImportRecord(r.GetGuid(0), new ImportKey(r.GetString(1), r.GetInt64(2), r.GetDateTimeOffset(3)),
+ r.GetStringOrNull(4), r.GetGuidOrNull(5), r.GetGuidOrNull(6), r.GetDateTimeOffset(7)),
+ cancellationToken);
+
/// Most recent imports from a card.
public Task> ListImportsAsync(Guid cardId, int limit = 1000, CancellationToken cancellationToken = default) =>
_db.ListAsync(
diff --git a/src/Titano.Data/Repositories/SequenceRepository.cs b/src/Titano.Data/Repositories/SequenceRepository.cs
index 4912cec5..eb566ded 100644
--- a/src/Titano.Data/Repositories/SequenceRepository.cs
+++ b/src/Titano.Data/Repositories/SequenceRepository.cs
@@ -79,6 +79,9 @@ public sealed record ArchiveHealth(
long CorruptFrames,
long DuplicateFrames);
+/// Where a sequence lives in its archive.
+public sealed record SequenceLocation(Guid Id, string Folder, string? Title);
+
/// Sequences (projects) with their tags; Library listing and Archive statistics.
public sealed class SequenceRepository
{
@@ -259,6 +262,45 @@ public sealed class SequenceRepository
}, cancellationToken);
}
+ /// Folder and title of every sequence of an archive, by folder.
+ public Task> ListLocationsAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
+ _db.ListAsync("SELECT id, folder, title FROM sequences WHERE archive_id = $1 ORDER BY folder, id", [archiveId],
+ r => new SequenceLocation(r.GetGuid(0), r.GetString(1), r.GetStringOrNull(2)), cancellationToken);
+
+ /// The sequence created for a session key (automatic grouping of uploads and card sessions).
+ public Task FindBySessionKeyAsync(Guid archiveId, string sessionKey, CancellationToken cancellationToken = default) =>
+ _db.ScalarAsync("SELECT sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key = $2",
+ [archiveId, sessionKey], cancellationToken);
+
+ /// Binds a session key to a sequence unless it is already bound; returns the sequence the key resolves to.
+ public Task MapSessionKeyAsync(Guid archiveId, string sessionKey, Guid sequenceId, CancellationToken cancellationToken = default) =>
+ _db.InTransactionAsync(async session =>
+ {
+ await session.ExecuteAsync(
+ "INSERT INTO sequence_sessions (archive_id, session_key, sequence_id) VALUES ($1, $2, $3) ON CONFLICT (archive_id, session_key) DO NOTHING",
+ [archiveId, sessionKey, sequenceId], cancellationToken).ConfigureAwait(false);
+ return (await session.ScalarAsync("SELECT sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key = $2",
+ [archiveId, sessionKey], cancellationToken).ConfigureAwait(false))!.Value;
+ }, cancellationToken);
+
+ /// Moves a sequence to another folder and/or title (mass rename); false when it does not exist.
+ public Task RelocateAsync(Guid id, string folder, string? title, CancellationToken cancellationToken = default) =>
+ _db.InTransactionAsync(async session =>
+ {
+ var current = await session.QuerySingleOrDefaultAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken)
+ .ConfigureAwait(false);
+ if (current is null) return false;
+ var titleChanged = current.Title != title;
+ await UpsertRowAsync(session, current with
+ {
+ Folder = folder,
+ Title = title,
+ SettingsVersion = titleChanged ? current.SettingsVersion + 1 : current.SettingsVersion,
+ UpdatedAt = DateTimeOffset.UtcNow,
+ }, cancellationToken).ConfigureAwait(false);
+ return true;
+ }, cancellationToken);
+
private static List Filter(SequenceQuery query, SqlParameters p)
{
var where = new List();