feat(data): session keys, scan stamps and archive queries
Migration 0002 adds sequence_sessions and archive_folders. New repository methods: folder scan state, session-key mapping, sequence locations and relocation, frame sync from sidecars, duplicate and candidate lookups, folder rebase, integrity updates and verifiable card import history. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
6 files changed
+228
No files matched your search
@@ -0,0 +1,23 @@
|
||||
-- Archive index state (BRIEF §7, §14.4). Mirrors sqlite/0002_archive_index.sql.
|
||||
|
||||
-- Automatic grouping: a session key (upload session, card session) resolves to the sequence created for it.
|
||||
CREATE TABLE sequence_sessions (
|
||||
archive_id uuid NOT NULL REFERENCES archives (id) ON DELETE CASCADE,
|
||||
session_key text NOT NULL,
|
||||
sequence_id uuid NOT NULL REFERENCES sequences (id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (archive_id, session_key)
|
||||
);
|
||||
CREATE INDEX ix_sequence_sessions_sequence ON sequence_sessions (sequence_id);
|
||||
|
||||
-- What the last scan saw in each sequence folder, so incremental scans skip unchanged folders by size and time.
|
||||
-- Times are UTC ticks (100 ns), compared exactly with the file system's.
|
||||
CREATE TABLE archive_folders (
|
||||
archive_id uuid NOT NULL REFERENCES archives (id) ON DELETE CASCADE,
|
||||
folder text COLLATE "C" NOT NULL,
|
||||
sequence_id uuid REFERENCES sequences (id) ON DELETE SET NULL,
|
||||
sidecar_size bigint,
|
||||
sidecar_ticks bigint,
|
||||
photos_ticks bigint,
|
||||
scanned_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (archive_id, folder)
|
||||
);
|
||||
@@ -0,0 +1,23 @@
|
||||
-- Archive index state (BRIEF §7, §14.4). Mirrors postgres/0002_archive_index.sql.
|
||||
|
||||
-- Automatic grouping: a session key (upload session, card session) resolves to the sequence created for it.
|
||||
CREATE TABLE sequence_sessions (
|
||||
archive_id TEXT NOT NULL REFERENCES archives (id) ON DELETE CASCADE,
|
||||
session_key TEXT NOT NULL,
|
||||
sequence_id TEXT NOT NULL REFERENCES sequences (id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (archive_id, session_key)
|
||||
);
|
||||
CREATE INDEX ix_sequence_sessions_sequence ON sequence_sessions (sequence_id);
|
||||
|
||||
-- What the last scan saw in each sequence folder, so incremental scans skip unchanged folders by size and time.
|
||||
-- Times are UTC ticks (100 ns), compared exactly with the file system's.
|
||||
CREATE TABLE archive_folders (
|
||||
archive_id TEXT NOT NULL REFERENCES archives (id) ON DELETE CASCADE,
|
||||
folder TEXT NOT NULL,
|
||||
sequence_id TEXT REFERENCES sequences (id) ON DELETE SET NULL,
|
||||
sidecar_size INTEGER,
|
||||
sidecar_ticks INTEGER,
|
||||
photos_ticks INTEGER,
|
||||
scanned_at TEXT NOT NULL,
|
||||
PRIMARY KEY (archive_id, folder)
|
||||
);
|
||||
@@ -1,7 +1,11 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Model;
|
||||
|
||||
namespace Titano.Data.Repositories;
|
||||
|
||||
/// <summary>What the last scan saw in one sequence folder (sidecar and photo-directory stamps).</summary>
|
||||
public sealed record FolderScanState(string Folder, Guid? SequenceId, FileStamp? Sidecar, FileStamp? Photos);
|
||||
|
||||
/// <summary>Archive roots (BRIEF §5, §7).</summary>
|
||||
public sealed class ArchiveRepository
|
||||
{
|
||||
@@ -10,6 +14,9 @@ public sealed class ArchiveRepository
|
||||
private static readonly string UpsertSql = SqlDialect.Upsert("archives",
|
||||
["id", "name", "root_path", "folder_template", "file_template", "schema_version", "created", "proxy_long_edge"], 1);
|
||||
|
||||
private static readonly string FolderStateUpsertSql = SqlDialect.Upsert("archive_folders",
|
||||
["archive_id", "folder", "sequence_id", "sidecar_size", "sidecar_ticks", "photos_ticks", "scanned_at"], 2);
|
||||
|
||||
private readonly DbAccess _db;
|
||||
|
||||
internal ArchiveRepository(DbAccess db) => _db = db;
|
||||
@@ -29,6 +36,31 @@ public sealed class ArchiveRepository
|
||||
public async Task<bool> DeleteAsync(Guid id, CancellationToken cancellationToken = default) =>
|
||||
await _db.ExecuteAsync("DELETE FROM archives WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) > 0;
|
||||
|
||||
/// <summary>Scan stamps of every folder of an archive, by folder.</summary>
|
||||
public async Task<IReadOnlyDictionary<string, FolderScanState>> GetFolderStatesAsync(Guid archiveId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var rows = await _db.ListAsync(
|
||||
"SELECT folder, sequence_id, sidecar_size, sidecar_ticks, photos_ticks FROM archive_folders WHERE archive_id = $1", [archiveId],
|
||||
r => new FolderScanState(r.GetString(0), r.GetGuidOrNull(1),
|
||||
r.GetInt64OrNull(3) is { } sidecar ? new FileStamp(r.GetInt64(2), new DateTime(sidecar, DateTimeKind.Utc)) : null,
|
||||
r.GetInt64OrNull(4) is { } photos ? new FileStamp(0, new DateTime(photos, DateTimeKind.Utc)) : null),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
return rows.ToDictionary(s => s.Folder, StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
public async Task SetFolderStateAsync(Guid archiveId, FolderScanState state, CancellationToken cancellationToken = default) =>
|
||||
await _db.ExecuteAsync(FolderStateUpsertSql,
|
||||
[archiveId, state.Folder, state.SequenceId, state.Sidecar?.Size, state.Sidecar?.ModifiedUtc.Ticks, state.Photos?.ModifiedUtc.Ticks, DateTimeOffset.UtcNow],
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
public async Task<bool> DeleteFolderStateAsync(Guid archiveId, string folder, CancellationToken cancellationToken = default) =>
|
||||
await _db.ExecuteAsync("DELETE FROM archive_folders WHERE archive_id = $1 AND folder = $2", [archiveId, folder], cancellationToken)
|
||||
.ConfigureAwait(false) > 0;
|
||||
|
||||
/// <summary>Forgets every scan stamp of an archive, so the next scan reads every sidecar (rebuild).</summary>
|
||||
public Task<int> ClearFolderStatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
|
||||
_db.ExecuteAsync("DELETE FROM archive_folders WHERE archive_id = $1", [archiveId], cancellationToken);
|
||||
|
||||
private static Archive Map(DbRow r) => new()
|
||||
{
|
||||
Id = r.GetGuid(0),
|
||||
|
||||
@@ -4,6 +4,15 @@ using Titano.Core.Model;
|
||||
|
||||
namespace Titano.Data.Repositories;
|
||||
|
||||
/// <summary>Integrity fields a verify rewrites: the flags and the hash of a newly hashed file.</summary>
|
||||
public readonly record struct FrameIntegrity(Guid FrameId, FrameFlags Flags, string? Sha256);
|
||||
|
||||
/// <summary>A frame and the sequence it belongs to.</summary>
|
||||
public readonly record struct FrameRef(Guid FrameId, Guid SequenceId);
|
||||
|
||||
/// <summary>A frame whose content (SHA-256) also exists elsewhere in the archive.</summary>
|
||||
public sealed record DuplicateFrame(string Sha256, Guid FrameId, Guid SequenceId, int Index, string RelativePath);
|
||||
|
||||
/// <summary>Light statistics of one frame, written by the analysis job.</summary>
|
||||
public readonly record struct FrameLightStats(Guid FrameId, double? MeanLogLuminance, double? ClipLow, double? ClipHigh);
|
||||
|
||||
@@ -25,6 +34,12 @@ public sealed class FrameRepository
|
||||
/// <summary>Rows per statement of the batched PostgreSQL updates (4 or fewer parameters per row).</summary>
|
||||
private const int Batch = 1000;
|
||||
|
||||
/// <summary>Rows per multi-row upsert: 30 x 27 parameters stays below SQLite's historical 999-variable limit.</summary>
|
||||
private const int UpsertRows = 30;
|
||||
|
||||
/// <summary>Light statistics belong to the analysis job; a sync from the sidecar never clears them.</summary>
|
||||
private static readonly string[] AnalysisColumns = ["mean_log_luminance", "clip_low", "clip_high"];
|
||||
|
||||
private readonly DbAccess _db;
|
||||
|
||||
internal FrameRepository(DbAccess db) => _db = db;
|
||||
@@ -112,6 +127,89 @@ public sealed class FrameRepository
|
||||
return updated;
|
||||
}, cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Makes the frames of a sequence exactly <paramref name="frames"/> (from its sidecar): rows are upserted by id,
|
||||
/// rows not listed are deleted, light statistics of kept rows survive. Indexes may be renumbered freely.
|
||||
/// </summary>
|
||||
public Task<int> SyncSequenceAsync(Guid sequenceId, IReadOnlyList<Frame> frames, CancellationToken cancellationToken = default) =>
|
||||
_db.InTransactionAsync(async session =>
|
||||
{
|
||||
// Park current rows on negative indexes so renumbered rows never collide on (sequence_id, idx).
|
||||
await session.ExecuteAsync("UPDATE frames SET idx = -idx - 1 WHERE sequence_id = $1", [sequenceId], cancellationToken).ConfigureAwait(false);
|
||||
var updates = string.Join(", ", ColumnNames.Skip(1).Where(c => !AnalysisColumns.Contains(c)).Select(c => $"{c} = excluded.{c}"));
|
||||
for (var start = 0; start < frames.Count; start += UpsertRows)
|
||||
{
|
||||
var count = Math.Min(UpsertRows, frames.Count - start);
|
||||
var sql = new StringBuilder("INSERT INTO frames (").Append(Columns).Append(") VALUES ");
|
||||
var args = new DbParam[count * ColumnNames.Length];
|
||||
for (var i = 0; i < count; i++)
|
||||
{
|
||||
var values = Values(frames[start + i]);
|
||||
sql.Append(i > 0 ? ", (" : "(");
|
||||
for (var c = 0; c < values.Length; c++)
|
||||
{
|
||||
args[i * values.Length + c] = new DbParam(values[c]);
|
||||
sql.Append(c > 0 ? ", $" : "$").Append(i * values.Length + c + 1);
|
||||
}
|
||||
sql.Append(')');
|
||||
}
|
||||
sql.Append(" ON CONFLICT (id) DO UPDATE SET ").Append(updates);
|
||||
await session.ExecuteAsync(sql.ToString(), args, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
return await session.ExecuteAsync("DELETE FROM frames WHERE sequence_id = $1 AND idx < 0", [sequenceId], cancellationToken).ConfigureAwait(false);
|
||||
}, cancellationToken);
|
||||
|
||||
/// <summary>Number of frames of a sequence.</summary>
|
||||
public async Task<int> CountBySequenceAsync(Guid sequenceId, CancellationToken cancellationToken = default) =>
|
||||
(int)await _db.ScalarAsync<long>("SELECT count(*) FROM frames WHERE sequence_id = $1", [sequenceId], cancellationToken).ConfigureAwait(false);
|
||||
|
||||
/// <summary>Frames of an archive with this size and capture time (card import duplicate candidates).</summary>
|
||||
public Task<IReadOnlyList<Frame>> FindBySizeAndCaptureAsync(Guid archiveId, long size, DateTime capturedAt, CancellationToken cancellationToken = default) =>
|
||||
_db.ListAsync(
|
||||
$"SELECT {string.Join(", ", ColumnNames.Select(c => "f." + c))} FROM frames f JOIN sequences s ON s.id = f.sequence_id " +
|
||||
"WHERE s.archive_id = $1 AND f.size = $2 AND f.captured_at = $3",
|
||||
[archiveId, size, SequenceRepository.Wall(capturedAt)], Map, cancellationToken);
|
||||
|
||||
/// <summary>Every frame of an archive whose SHA-256 occurs more than once, grouped by hash then sequence and index.</summary>
|
||||
public Task<IReadOnlyList<DuplicateFrame>> ListDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
|
||||
_db.ListAsync(
|
||||
"SELECT f.sha256, f.id, f.sequence_id, f.idx, f.relative_path FROM frames f JOIN sequences s ON s.id = f.sequence_id " +
|
||||
"WHERE s.archive_id = $1 AND f.sha256 IN (SELECT f2.sha256 FROM frames f2 JOIN sequences s2 ON s2.id = f2.sequence_id " +
|
||||
"WHERE s2.archive_id = $1 AND f2.sha256 IS NOT NULL GROUP BY f2.sha256 HAVING count(*) > 1) " +
|
||||
"ORDER BY f.sha256, s.first_capture, f.sequence_id, f.idx",
|
||||
[archiveId], r => new DuplicateFrame(r.GetString(0), r.GetGuid(1), r.GetGuid(2), r.GetInt32(3), r.GetString(4)), cancellationToken);
|
||||
|
||||
/// <summary>Frames of an archive that carry every bit of <paramref name="flags"/>.</summary>
|
||||
public Task<IReadOnlyList<FrameRef>> ListFlaggedAsync(Guid archiveId, FrameFlags flags, CancellationToken cancellationToken = default) =>
|
||||
_db.ListAsync(
|
||||
"SELECT f.id, f.sequence_id FROM frames f JOIN sequences s ON s.id = f.sequence_id WHERE s.archive_id = $1 AND (f.flags & $2) = $2",
|
||||
[archiveId, (int)flags], r => new FrameRef(r.GetGuid(0), r.GetGuid(1)), cancellationToken);
|
||||
|
||||
/// <summary>Rewrites the relative paths of a sequence's frames after its folder moved (mass rename).</summary>
|
||||
/// <remarks>Only paths still under <paramref name="oldFolder"/> change, so repeating it after a crash is harmless.</remarks>
|
||||
public Task<int> RebaseFolderAsync(Guid sequenceId, string oldFolder, string newFolder, CancellationToken cancellationToken = default)
|
||||
{
|
||||
// substr counts characters (code points) on both dialects; the tail starts with the '/' after the folder.
|
||||
var prefix = oldFolder + "/";
|
||||
var length = prefix.EnumerateRunes().Count();
|
||||
return _db.ExecuteAsync(
|
||||
"UPDATE frames SET relative_path = $2 || substr(relative_path, $3) WHERE sequence_id = $1 AND substr(relative_path, 1, $4) = $5",
|
||||
[sequenceId, newFolder, length, length, prefix], cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>Writes flags and hashes found by a verify, in one transaction; returns the number of frames updated.</summary>
|
||||
public Task<int> UpdateIntegrityAsync(IReadOnlyList<FrameIntegrity> frames, CancellationToken cancellationToken = default) =>
|
||||
_db.InTransactionAsync(async session =>
|
||||
{
|
||||
var updated = 0;
|
||||
foreach (var f in frames)
|
||||
{
|
||||
updated += await session.ExecuteAsync("UPDATE frames SET flags = $2, sha256 = COALESCE($3, sha256) WHERE id = $1",
|
||||
[f.FrameId, (int)f.Flags, f.Sha256?.ToLowerInvariant()], cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
return updated;
|
||||
}, cancellationToken);
|
||||
|
||||
/// <summary>Removes every frame of a sequence; returns how many.</summary>
|
||||
public Task<int> DeleteBySequenceAsync(Guid sequenceId, CancellationToken cancellationToken = default) =>
|
||||
_db.ExecuteAsync("DELETE FROM frames WHERE sequence_id = $1", [sequenceId], cancellationToken);
|
||||
|
||||
@@ -96,6 +96,16 @@ public sealed class MediaCardRepository
|
||||
await _db.ScalarAsync<int?>("SELECT 1 FROM import_history WHERE card_id = $1 AND path = $2 AND size = $3 AND mtime = $4",
|
||||
[cardId, key.Path, key.Size, key.Modified], cancellationToken).ConfigureAwait(false) is not null;
|
||||
|
||||
/// <summary>Every recorded import from a card that has a hash and a frame (candidates for a verified card erase).</summary>
|
||||
public Task<IReadOnlyList<ImportRecord>> ListVerifiableImportsAsync(Guid cardId, CancellationToken cancellationToken = default) =>
|
||||
_db.ListAsync(
|
||||
"SELECT card_id, path, size, mtime, sha256, frame_id, job_id, imported_at FROM import_history " +
|
||||
"WHERE card_id = $1 AND sha256 IS NOT NULL AND frame_id IS NOT NULL ORDER BY path",
|
||||
[cardId],
|
||||
r => new ImportRecord(r.GetGuid(0), new ImportKey(r.GetString(1), r.GetInt64(2), r.GetDateTimeOffset(3)),
|
||||
r.GetStringOrNull(4), r.GetGuidOrNull(5), r.GetGuidOrNull(6), r.GetDateTimeOffset(7)),
|
||||
cancellationToken);
|
||||
|
||||
/// <summary>Most recent imports from a card.</summary>
|
||||
public Task<IReadOnlyList<ImportRecord>> ListImportsAsync(Guid cardId, int limit = 1000, CancellationToken cancellationToken = default) =>
|
||||
_db.ListAsync(
|
||||
|
||||
@@ -79,6 +79,9 @@ public sealed record ArchiveHealth(
|
||||
long CorruptFrames,
|
||||
long DuplicateFrames);
|
||||
|
||||
/// <summary>Where a sequence lives in its archive.</summary>
|
||||
public sealed record SequenceLocation(Guid Id, string Folder, string? Title);
|
||||
|
||||
/// <summary>Sequences (projects) with their tags; Library listing and Archive statistics.</summary>
|
||||
public sealed class SequenceRepository
|
||||
{
|
||||
@@ -259,6 +262,45 @@ public sealed class SequenceRepository
|
||||
}, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>Folder and title of every sequence of an archive, by folder.</summary>
|
||||
public Task<IReadOnlyList<SequenceLocation>> ListLocationsAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
|
||||
_db.ListAsync("SELECT id, folder, title FROM sequences WHERE archive_id = $1 ORDER BY folder, id", [archiveId],
|
||||
r => new SequenceLocation(r.GetGuid(0), r.GetString(1), r.GetStringOrNull(2)), cancellationToken);
|
||||
|
||||
/// <summary>The sequence created for a session key (automatic grouping of uploads and card sessions).</summary>
|
||||
public Task<Guid?> FindBySessionKeyAsync(Guid archiveId, string sessionKey, CancellationToken cancellationToken = default) =>
|
||||
_db.ScalarAsync<Guid?>("SELECT sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key = $2",
|
||||
[archiveId, sessionKey], cancellationToken);
|
||||
|
||||
/// <summary>Binds a session key to a sequence unless it is already bound; returns the sequence the key resolves to.</summary>
|
||||
public Task<Guid> MapSessionKeyAsync(Guid archiveId, string sessionKey, Guid sequenceId, CancellationToken cancellationToken = default) =>
|
||||
_db.InTransactionAsync(async session =>
|
||||
{
|
||||
await session.ExecuteAsync(
|
||||
"INSERT INTO sequence_sessions (archive_id, session_key, sequence_id) VALUES ($1, $2, $3) ON CONFLICT (archive_id, session_key) DO NOTHING",
|
||||
[archiveId, sessionKey, sequenceId], cancellationToken).ConfigureAwait(false);
|
||||
return (await session.ScalarAsync<Guid?>("SELECT sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key = $2",
|
||||
[archiveId, sessionKey], cancellationToken).ConfigureAwait(false))!.Value;
|
||||
}, cancellationToken);
|
||||
|
||||
/// <summary>Moves a sequence to another folder and/or title (mass rename); false when it does not exist.</summary>
|
||||
public Task<bool> RelocateAsync(Guid id, string folder, string? title, CancellationToken cancellationToken = default) =>
|
||||
_db.InTransactionAsync(async session =>
|
||||
{
|
||||
var current = await session.QuerySingleOrDefaultAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
if (current is null) return false;
|
||||
var titleChanged = current.Title != title;
|
||||
await UpsertRowAsync(session, current with
|
||||
{
|
||||
Folder = folder,
|
||||
Title = title,
|
||||
SettingsVersion = titleChanged ? current.SettingsVersion + 1 : current.SettingsVersion,
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
return true;
|
||||
}, cancellationToken);
|
||||
|
||||
private static List<string> Filter(SequenceQuery query, SqlParameters p)
|
||||
{
|
||||
var where = new List<string>();
|
||||
|
||||
Reference in new issue
Block a user