fix(archive): partial writers keep the folder changed for the scan; image flags follow queued passes and hashed photos

Writers that change part of a sequence (decode tiers, image-pass flags,
analyses, verify flushes and stamps, duplicate flags, title/tags/rating/
notes edits, mass rename) now ask ArchiveIndexer.InStepAsync before their
own sidecar write and record the folder's scan stamps afterwards only when
the index held the folder as it was: the stored stamps are still the
folder's and the row holds the sidecar's frames. What another process
changed in the sidecar alone (frames, a new title) stays visible to the
next incremental scan, which writes the whole row from the sidecar.

EditSequenceInfoAsync reads sidecar and row under the lease and writes the
row through SequenceRepository.EditAsync at the settings version it read,
applying the edit again on a conflict, then the sidecar from the stored
row; settings and frames committed meanwhile stay. MassRenamer writes only
folder and title (RelocateAsync, whose null title now keeps the current
one). Verify flushes and duplicate flags run under one lease.

A scan that finds frames another writer listed in the sidecar itself
clears AnalyzedLight, ThumbsBuilt and ProxiesBuilt only in its own row and
only on a server (ImagesAfterScan); the shared sidecar keeps the writer's
flags, so a desktop's poll no longer makes the server redo its passes.

An image pass overtaken by an upload keeps its flag while a pass of the
same kind is queued for the sequence. ThumbPack.IsFrom accepts a thumbnail
drawn before a verify hashed its photo, in the pass, in SameFrames and in
the server's thumbnail lookup, which answered 404 for every such frame.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-12 20:16:58 +00:00
1 parent 0475fb9ee4
commit 94ef3c88db
18 files changed
+532 -116

No files matched your search

+9
View File
@@ -223,6 +223,15 @@ public static class ThumbPack
sha256 is { Length: HashLength * 2 }
? Convert.FromHexString(sha256)
: System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes($"{file}|{size}"));
/// <summary>
/// Whether a thumbnail with <paramref name="sourceHash"/> was drawn from this original: its <see cref="Fingerprint"/>,
/// or, for an original hashed since (a verify hashes the photos a scan adopted), the fingerprint of its file name and
/// size it was drawn with before. The hash changes nothing about the photo.
/// </summary>
public static bool IsFrom(ReadOnlySpan<byte> sourceHash, string? sha256, string file, long size) =>
sourceHash.SequenceEqual(Fingerprint(sha256, file, size))
|| (sha256 is { Length: HashLength * 2 } && sourceHash.SequenceEqual(Fingerprint(null, file, size)));
}
/// <summary>Random access to the thumbnails of a pack without loading the file; safe while a writer appends.</summary>
+21 -14
View File
@@ -209,8 +209,7 @@ public sealed class ArchiveImages
for (var i = 0; i < startFrame && i < manifest.Frames.Count; i++)
{
var frame = manifest.Frames[i];
complete &= pack.Entries.GetValueOrDefault(frame.I) is { } entry
&& entry.SourceHash.AsSpan().SequenceEqual(ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size));
complete &= pack.Entries.GetValueOrDefault(frame.I) is { } entry && ThumbPack.IsFrom(entry.SourceHash, frame.Sha256, frame.File, frame.Size);
}
var cadence = new Cadence();
for (var i = startFrame; i < manifest.Frames.Count; i++)
@@ -220,7 +219,7 @@ public sealed class ArchiveImages
var fingerprint = ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size);
var entry = pack.Entries.GetValueOrDefault(frame.I);
var edge = entry is null ? 0 : System.Math.Max(entry.Width, entry.Height);
var current = !force && entry is not null && entry.SourceHash.AsSpan().SequenceEqual(fingerprint);
var current = !force && entry is not null && ThumbPack.IsFrom(entry.SourceHash, frame.Sha256, frame.File, frame.Size);
// A frame this host cannot decode shows its proxy once one arrived; until then its original's embedded
// preview, whose smaller thumbnail the proxy's replaces.
var fromProxy = false;
@@ -255,7 +254,8 @@ public sealed class ArchiveImages
pack.Commit();
}
await _tiers.RecordAsync(archive, sequence, tiers, proxies: false, cancellationToken).ConfigureAwait(false);
await MarkAsync(archive, paths, sequence, manifest, SequenceStatus.ThumbsBuilt, complete && undecodable == 0, cancellationToken).ConfigureAwait(false);
await MarkAsync(archive, paths, sequence, manifest, JobKind.Thumbs, SequenceStatus.ThumbsBuilt, complete && undecodable == 0, cancellationToken)
.ConfigureAwait(false);
return new ImagePassResult(built, reused, undecodable);
}
@@ -327,7 +327,7 @@ public sealed class ArchiveImages
if (cadence.Due()) await checkpoint(i + 1, manifest.Frames.Count).ConfigureAwait(false);
}
await _tiers.RecordAsync(archive, sequence, tiers, proxies: true, cancellationToken).ConfigureAwait(false);
await MarkAsync(archive, paths, sequence, manifest, SequenceStatus.ProxiesBuilt, complete, cancellationToken).ConfigureAwait(false);
await MarkAsync(archive, paths, sequence, manifest, JobKind.Proxies, SequenceStatus.ProxiesBuilt, complete, cancellationToken).ConfigureAwait(false);
return new ImagePassResult(built, reused, undecodable, waiting);
}
@@ -525,17 +525,22 @@ public sealed class ArchiveImages
/// (<paramref name="complete"/>) and the sidecar, as it is under the lease, still lists those frames
/// (<see cref="SameFrames"/>); clears it otherwise. Frames a scan's reconcile or an ingest added, removed or
/// renumbered after the pass read <paramref name="read"/> have nothing of it, and a scan that cleared the flag for
/// them queues the next pass only while the flag stays clear. The index is written under the same lease, so the flag
/// never lands there after a scan cleared it in both.
/// them queues the next pass only while the flag stays clear. A <paramref name="pass"/> of the same kind already
/// queued for the sequence (an upload's, or a scan's) draws them when it runs, so the flag stays set meanwhile, as
/// between an upload and its pass: a camera uploading faster than a pass ends would otherwise keep it clear, and a
/// server would offer no thumbnail or proxy of the sequence at all until uploads paused. The index is written under
/// the same lease, so the flag never lands there after a scan cleared it in both.
/// </summary>
private async Task MarkAsync(Archive archive, ArchivePaths paths, Sequence sequence, SequenceManifest read, SequenceStatus flag, bool complete,
CancellationToken cancellationToken)
private async Task MarkAsync(Archive archive, ArchivePaths paths, Sequence sequence, SequenceManifest read, JobKind pass, SequenceStatus flag,
bool complete, CancellationToken cancellationToken)
{
var folder = paths.Sequence(sequence.Folder);
await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
{
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
var built = complete && SameFrames(manifest.Frames, read.Frames);
var built = complete && (SameFrames(manifest.Frames, read.Frames)
|| await _db.Jobs.IsQueuedAsync(pass, sequence.Id, cancellationToken).ConfigureAwait(false));
var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
var status = built ? manifest.Status | flag : manifest.Status & ~flag;
if (status != manifest.Status)
{
@@ -544,14 +549,15 @@ public sealed class ArchiveImages
}
await _db.Sequences.UpdateStatusAsync(sequence.Id, built ? flag : SequenceStatus.None, built ? SequenceStatus.None : flag, cancellationToken)
.ConfigureAwait(false);
await _indexer.RecordStampsInStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
if (inStep) await _indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
}
}
/// <summary>
/// Whether <paramref name="current"/> lists the photos <paramref name="read"/> did, each at the same index with the
/// same file, size and capture time: thumbnails and proxies are kept by frame index, so photos added, removed,
/// renumbered or replaced since have none of what the pass made.
/// same file, size, capture time and content hash: thumbnails and proxies are kept by frame index, so photos added,
/// removed, renumbered or replaced since have none of what the pass made. A hash learnt since (a verify hashing a
/// photo the scan adopted) changes nothing about the photo, and its thumbnail stays its own (<see cref="ThumbPack.IsFrom"/>).
/// </summary>
private static bool SameFrames(IReadOnlyList<SidecarFrame> current, IReadOnlyList<SidecarFrame> read)
{
@@ -559,7 +565,8 @@ public sealed class ArchiveImages
for (var i = 0; i < current.Count; i++)
{
var (now, then) = (current[i], read[i]);
if (now.Id != then.Id || now.I != then.I || now.File != then.File || now.Size != then.Size || now.CapturedAt != then.CapturedAt) return false;
if (now.Id != then.Id || now.I != then.I || now.File != then.File || now.Size != then.Size || now.CapturedAt != then.CapturedAt
|| (then.Sha256 is not null && !string.Equals(now.Sha256, then.Sha256, StringComparison.OrdinalIgnoreCase))) return false;
}
return true;
}
+58 -33
View File
@@ -135,16 +135,29 @@ public sealed class ArchiveIndexer
FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))), cancellationToken);
/// <summary>
/// <see cref="RecordStampsAsync"/> for a writer that changed, under the sequence lease, only what it owns of the
/// sidecar and the index row (status, decode tiers): recorded only while the index holds the frames of
/// <paramref name="manifest"/>, the sidecar as that writer read and wrote it. Frames another process (a desktop
/// sharing the archive) added, removed or renumbered without this index keep the stamps as they were, so the next
/// scan still indexes them and queues what they need.
/// Whether this index holds the folder of <paramref name="manifest"/> (the sidecar read under the sequence lease) as it
/// is now: the stamps it recorded when it last synced the folder are still the folder's, and its row holds the
/// sidecar's frames. A writer that changes, under the lease, only what it owns of the sidecar and the index row
/// (statuses, decode tiers, frame flags, title, tags, rating and notes, the folder) asks before its own write, and
/// records the folder's new stamps after it only when this held (<see cref="RecordStampsAsync"/>). What another
/// process (a desktop sharing the archive) changed in the sidecar alone, frames or a new title, then keeps the folder
/// changed for the next incremental scan, which writes the whole row from the sidecar.
/// </summary>
internal async Task RecordStampsInStepAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken)
internal Task<bool> InStepAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) =>
InStepAsync(archive, sequence.RelativeFolder, sequence, manifest, cancellationToken);
/// <inheritdoc cref="InStepAsync(Archive, SequencePaths, SequenceManifest, CancellationToken)"/>
/// <param name="recordedFolder">The folder the stamps were recorded at: the one a rename just moved the sequence from,
/// whose files keep their stamps.</param>
internal async Task<bool> InStepAsync(Archive archive, string recordedFolder, SequencePaths sequence, SequenceManifest manifest,
CancellationToken cancellationToken)
{
if (await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { } indexed && indexed.FramesVersion == manifest.Versions.Frames)
await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
if (await _db.Archives.GetFolderStateAsync(archive.Id, recordedFolder, cancellationToken).ConfigureAwait(false) is not { } state
|| state.SequenceId != manifest.Id || state.Sidecar != FileStamp.Of(sequence.ManifestPath)
|| state.Photos != FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))) return false;
// A sidecar rewritten within its file system's time resolution, at the same size, keeps its stamp.
return await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { } indexed
&& indexed.FramesVersion == manifest.Versions.Frames;
}
/// <summary>Indexes unindexed photos and flags missing files of one sequence (the orphans fix action).</summary>
@@ -153,34 +166,45 @@ public sealed class ArchiveIndexer
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var folder = paths.Sequence(sequence.Folder);
return await UpdateAndSyncAsync(archive, paths, folder, sequence.Id, (m, _) => SequenceAdopter.Reconcile(folder, m, cancellationToken), cancellationToken)
return await UpdateAndSyncAsync(archive, paths, folder, sequence.Id, m => SequenceAdopter.Reconcile(folder, m, cancellationToken), cancellationToken)
.ConfigureAwait(false);
}
/// <summary>
/// Applies <paramref name="edit"/> to the sidecar (null writes nothing) and indexes the result, both under the sequence
/// lease; the edit also gets the sequence's index row as it is then (null when it has none). Frames inserted before
/// indexed ones move the project's positions (<see cref="ProjectPositions"/>) without a new settings version; an editor
/// saving its project holds the same lease, so it reads them moved and merges onto them rather than writing its old
/// positions back.
/// lease. Frames inserted before indexed ones move the project's positions (<see cref="ProjectPositions"/>) without a
/// new settings version; an editor saving its project holds the same lease, so it reads them moved and merges onto them
/// rather than writing its old positions back. When the sidecar, as read under the lease, lists other frames than the
/// index row (another process changed them together with their sidecar entries), the row is written without what
/// this host makes for the frames (<see cref="Uncovered"/>).
/// </summary>
private async Task<SequenceManifest> UpdateAndSyncAsync(Archive archive, ArchivePaths paths, SequencePaths sequence, Guid sequenceId,
Func<SequenceManifest, Sequence?, SequenceManifest?> edit, CancellationToken cancellationToken)
Func<SequenceManifest, SequenceManifest?> edit, CancellationToken cancellationToken)
{
await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false))
{
var manifest = SequenceWriter.ReadLocked(sequence, sequenceId);
var indexed = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false);
if (edit(manifest, indexed) is { } updated)
var moved = await FramesMovedAsync(manifest, cancellationToken).ConfigureAwait(false);
if (edit(manifest) is { } updated)
{
ManifestStore.WriteSequence(sequence.ManifestPath, updated);
manifest = updated;
}
await SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
await SyncAsync(archive, sequence, moved ? manifest with { Status = manifest.Status & ~Uncovered } : manifest, cancellationToken).ConfigureAwait(false);
return manifest;
}
}
/// <summary>
/// What a scan clears in this host's own row, never in the shared sidecar, when another process added, removed or
/// renumbered frames together with their sidecar entries: on a server (<see cref="ArchiveServiceOptions.ImagesAfterScan"/>),
/// which analyses and draws what its clients load, the light analysis, thumbnails and proxies, so the scan's follow-ups
/// make them for the frames as they are, and their passes set the flags in both places again. Nothing on a desktop:
/// the flags in the sidecar are the writer's that added the frames (which analyses them, and on a server draws them
/// after every upload), and clearing them there would have the server make everything again.
/// </summary>
private SequenceStatus Uncovered => _writer.Options.ImagesAfterScan ? CoveringStatus : SequenceStatus.None;
/// <summary>Indexes one folder; returns its sequence and the sidecar bytes when they were read and not rewritten since (for the snapshot).</summary>
private async Task<(Guid Id, byte[]? Bytes)> IndexFolderAsync(Archive archive, ArchivePaths paths, SequenceFolder folder, bool reconcile, IndexSnapshot? snapshot,
Locations locations, CancellationToken cancellationToken)
@@ -212,23 +236,24 @@ public sealed class ArchiveIndexer
ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
bytes = null;
}
var reconcilePhotos = reconcile || File.Exists(sequence.IngestJournalPath);
if (reconcilePhotos || await FramesMovedAsync(manifest, cancellationToken).ConfigureAwait(false))
if (reconcile || File.Exists(sequence.IngestJournalPath))
{
// Photos added, gone or back, or frames another process (a desktop sharing the archive) added, removed or
// renumbered together with their sidecar entries, which leaves the statuses its sidecar had: the light analysis,
// thumbnails and proxies no longer cover the frames, so the scan's follow-ups make them again (the images on a
// server, ArchiveServiceOptions.ImagesAfterScan). Decided under the lease, on the sidecar as it is then against
// the index row, under which the project's positions also move.
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, (m, indexed) =>
{
var updated = reconcilePhotos ? SequenceAdopter.Reconcile(sequence, m, cancellationToken) : null;
if (updated is null && (indexed is null || indexed.FramesVersion == m.Versions.Frames)) return null;
updated ??= m;
var status = updated.Status & ~CoveringStatus;
if (ReferenceEquals(updated, m) && status == m.Status) return null;
return updated with { Status = status, UpdatedAt = DateTimeOffset.UtcNow };
}, cancellationToken).ConfigureAwait(false);
// Photos added, gone or back: the light analysis, thumbnails and proxies no longer cover the frames, as the
// sidecar then tells every host, so the scan's follow-ups make them again (the images on a server,
// ArchiveServiceOptions.ImagesAfterScan).
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id,
m => SequenceAdopter.Reconcile(sequence, m, cancellationToken) is { } reconciled
? reconciled with { Status = reconciled.Status & ~CoveringStatus }
: null,
cancellationToken).ConfigureAwait(false);
bytes = null;
}
else if (await FramesMovedAsync(manifest, cancellationToken).ConfigureAwait(false))
{
// Another process (a desktop sharing the archive, or the server it shares it with) added, removed or renumbered
// frames together with their sidecar entries: read again under the lease, under which the project's positions
// move, and indexed without what this host makes for the frames (Uncovered).
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false);
bytes = null;
}
else
@@ -251,12 +251,20 @@ public sealed class ArchiveMaintenance
if (await _db.Sequences.GetAsync(bySequence.Key, cancellationToken).ConfigureAwait(false) is not { } sequence) continue;
var changes = bySequence.ToDictionary(x => x.Key, x => (Func<SidecarFrame, SidecarFrame>)(f =>
f with { Flags = x.Set ? f.Flags | FrameFlags.Duplicate : f.Flags & ~FrameFlags.Duplicate }));
await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m => SequenceFrames.Update(m, changes), cancellationToken).ConfigureAwait(false);
var setIds = bySequence.Where(x => x.Set).Select(x => x.Key).ToList();
var clearIds = bySequence.Where(x => !x.Set).Select(x => x.Key).ToList();
if (setIds.Count > 0) await _db.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false);
if (clearIds.Count > 0) await _db.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false);
await _indexer.RecordStampsAsync(archive, paths.Sequence(sequence.Folder), sequence.Id, cancellationToken).ConfigureAwait(false);
var folder = paths.Sequence(sequence.Folder);
await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
{
// The scan stamps follow only when the index held the folder as it was: frames another process listed in the
// sidecar meanwhile are still indexed by the next scan.
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
ManifestStore.WriteSequence(folder.ManifestPath, SequenceFrames.Update(manifest, changes));
var setIds = bySequence.Where(x => x.Set).Select(x => x.Key).ToList();
var clearIds = bySequence.Where(x => !x.Set).Select(x => x.Key).ToList();
if (setIds.Count > 0) await _db.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false);
if (clearIds.Count > 0) await _db.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false);
if (inStep) await _indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
}
}
return (groups.Count, set.Count + clear.Count);
}
+47 -23
View File
@@ -6,6 +6,7 @@ using Titano.Core.Model;
using Titano.Core.Naming;
using Titano.Data.Archiving.Jobs;
using Titano.Data.Jobs;
using Titano.Data.Repositories;
namespace Titano.Data.Archiving;
@@ -20,6 +21,9 @@ public sealed class ArchiveService : IJobHandlerProvider
/// <summary>Lease serialising read-modify-write of <c>archive.json</c> between the processes sharing an archive.</summary>
private const string SettingsLease = "archive-settings";
/// <summary>Times a title, tags, rating or notes edit is applied at the version it read before it writes regardless.</summary>
private const int InfoEditAttempts = 3;
private readonly TitanoDatabase _db;
private readonly SequenceWriter _writer;
@@ -224,35 +228,54 @@ public sealed class ArchiveService : IJobHandlerProvider
/// Changes some of title, tags, rating and notes from their current values — read under the sequence's lease from
/// the sidecar, or from the index when it is ahead (an edit made there only) — so fields <paramref name="edit"/>
/// leaves alone keep any change made since the caller read the sequence. Sidecar and index change together
/// (settings version + 1).
/// (settings version + 1), the index row in these fields only, at the settings version read under the lease
/// (<see cref="SequenceRepository.EditAsync(Guid, SequenceEdit, int?, CancellationToken)"/>): a settings edit committed
/// since the caller read the sequence (an analysis's Auto values, the server's PATCH, which takes no lease) stays, and
/// so do frames another process listed in the sidecar, which the next scan indexes
/// (<see cref="ArchiveIndexer.InStepAsync(Archive, SequencePaths, SequenceManifest, CancellationToken)"/>).
/// <paramref name="edit"/> runs again on top of an edit the API committed meanwhile.
/// </summary>
/// <exception cref="ArgumentException">The sequence is not in the index (or left it before the edit).</exception>
public async Task<Sequence> EditSequenceInfoAsync(Guid sequenceId, Func<SequenceInfo, SequenceInfo> edit, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(edit);
var (archive, sequence) = await LoadSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false);
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var manifest = await _writer.UpdateAsync(paths, sequenceId, sequence.Folder, m =>
{
var current = sequence.SettingsVersion > m.Versions.Settings
? new SequenceInfo(sequence.Title, sequence.Tags, sequence.Rating, sequence.Notes)
: new SequenceInfo(m.Title, m.Tags, m.Rating, m.Notes);
var info = edit(current);
return m with
{
Title = string.IsNullOrWhiteSpace(info.Title) ? null : info.Title.Trim(),
Tags = info.Tags,
Rating = info.Rating,
Notes = info.Notes,
Versions = m.Versions with { Settings = System.Math.Max(m.Versions.Settings, sequence.SettingsVersion) + 1 },
UpdatedAt = DateTimeOffset.UtcNow,
};
}, cancellationToken).ConfigureAwait(false);
var folder = paths.Sequence(sequence.Folder);
var updated = SequenceMapping.ToSequence(manifest, archive.Id, folder, sequence);
await _db.Sequences.UpsertAsync(updated, cancellationToken).ConfigureAwait(false);
await Indexer.RecordStampsAsync(archive, folder, sequenceId, cancellationToken).ConfigureAwait(false);
return updated;
await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false))
{
var manifest = SequenceWriter.ReadLocked(folder, sequenceId);
var inStep = await Indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
for (var attempt = 1; ; attempt++)
{
var row = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? throw Unknown(sequenceId);
var current = row.SettingsVersion > manifest.Versions.Settings
? new SequenceInfo(row.Title, row.Tags, row.Rating, row.Notes)
: new SequenceInfo(manifest.Title, manifest.Tags, manifest.Rating, manifest.Notes);
var info = edit(current);
var title = string.IsNullOrWhiteSpace(info.Title) ? null : info.Title.Trim();
// The last attempt writes over whatever the index holds then rather than fail the user's edit.
var result = await _db.Sequences.EditAsync(sequenceId,
new SequenceEdit { Title = title ?? string.Empty, Tags = info.Tags, Rating = info.Rating, Notes = info.Notes ?? string.Empty },
attempt < InfoEditAttempts ? row.SettingsVersion : null, cancellationToken).ConfigureAwait(false);
// A conflict means the API changed the sequence after the read: the edit applies again on top of that.
if (result.Outcome == SequenceEditOutcome.Conflict) continue;
if (result.Sequence is not { } updated) throw Unknown(sequenceId);
// The sidecar takes the fields as the index stored them (trimmed, tags without duplicates).
ManifestStore.WriteSequence(folder.ManifestPath, manifest with
{
Title = updated.Title,
Tags = updated.Tags,
Rating = updated.Rating,
Notes = updated.Notes,
Versions = manifest.Versions with { Settings = System.Math.Max(manifest.Versions.Settings + 1, updated.SettingsVersion) },
UpdatedAt = DateTimeOffset.UtcNow,
});
if (inStep) await Indexer.RecordStampsAsync(archive, folder, sequenceId, cancellationToken).ConfigureAwait(false);
return updated;
}
}
}
public Task<IReadOnlyList<DuplicateGroup>> FindDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
@@ -367,10 +390,11 @@ public sealed class ArchiveService : IJobHandlerProvider
private async Task<(Archive, Sequence)> LoadSequenceAsync(Guid sequenceId, CancellationToken cancellationToken)
{
var sequence = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
?? throw new ArgumentException($"Unknown sequence {sequenceId}.", nameof(sequenceId));
var sequence = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? throw Unknown(sequenceId);
return (await LoadArchiveAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false), sequence);
}
private static ArgumentException Unknown(Guid sequenceId) => new($"Unknown sequence {sequenceId}.", nameof(sequenceId));
}
/// <summary>The fields of a sequence the user describes it with (sidecar and index).</summary>
+20 -7
View File
@@ -95,23 +95,35 @@ public sealed class ArchiveVerifier
return targets;
}
/// <summary>Writes the flags and new hashes of checked frames to the sidecar and the index.</summary>
/// <summary>
/// Writes the flags and new hashes of checked frames to the sidecar and the index, under one lease. The folder's scan
/// stamps follow only when the index held the folder as it was
/// (<see cref="ArchiveIndexer.InStepAsync(Archive, SequencePaths, SequenceManifest, CancellationToken)"/>), so frames
/// another process listed in the sidecar meanwhile are still indexed by the next scan.
/// </summary>
private async Task FlushAsync(Archive archive, ArchivePaths paths, Sequence target, List<FrameCheck> pending, CancellationToken cancellationToken)
{
if (pending.Count == 0) return;
var changes = pending.ToDictionary(c => c.Frame.Id, c => (Func<SidecarFrame, SidecarFrame>)(f =>
f with { Flags = c.Flags, Sha256 = c.Health == FrameHealth.Hashed ? c.Sha256 : f.Sha256 }));
await _writer.UpdateAsync(paths, target.Id, target.Folder, m => SequenceFrames.Update(m, changes), cancellationToken).ConfigureAwait(false);
await _db.Frames.UpdateIntegrityAsync(pending.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)),
cancellationToken).ConfigureAwait(false);
await _indexer.RecordStampsAsync(archive, paths.Sequence(target.Folder), target.Id, cancellationToken).ConfigureAwait(false);
var sequence = paths.Sequence(target.Folder);
await using (await _writer.LockAsync(paths, target.Id, cancellationToken).ConfigureAwait(false))
{
var manifest = SequenceWriter.ReadLocked(sequence, target.Id);
var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
ManifestStore.WriteSequence(sequence.ManifestPath, SequenceFrames.Update(manifest, changes));
await _db.Frames.UpdateIntegrityAsync(pending.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)),
cancellationToken).ConfigureAwait(false);
if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false);
}
pending.Clear();
}
/// <summary>
/// Stamps the verify time; the sequence counts as verified only when no frame is missing or corrupt, and has errors
/// while one is. Sidecar and index are written under one lease, the index row with these fields only
/// (<see cref="SequenceRepository.RecordVerifiedAsync"/>), so a settings edit or an ingest committed meanwhile stays.
/// (<see cref="SequenceRepository.RecordVerifiedAsync"/>), so a settings edit or an ingest committed meanwhile stays;
/// the scan stamps as in <see cref="FlushAsync"/>.
/// </summary>
private async Task StampAsync(Archive archive, ArchivePaths paths, Sequence target, CancellationToken cancellationToken)
{
@@ -119,6 +131,7 @@ public sealed class ArchiveVerifier
await using (await _writer.LockAsync(paths, target.Id, cancellationToken).ConfigureAwait(false))
{
var m = SequenceWriter.ReadLocked(sequence, target.Id);
var inStep = await _indexer.InStepAsync(archive, sequence, m, cancellationToken).ConfigureAwait(false);
var clean = m.Frames.All(f => (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0);
var now = DateTimeOffset.UtcNow;
var manifest = m with
@@ -130,7 +143,7 @@ public sealed class ArchiveVerifier
ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
await _db.Sequences.RecordVerifiedAsync(target.Id, clean ? SequenceStatus.Verified : SequenceStatus.HasErrors,
clean ? SequenceStatus.HasErrors : SequenceStatus.Verified, now, cancellationToken).ConfigureAwait(false);
await _indexer.RecordStampsInStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false);
}
}
}
+4 -2
View File
@@ -65,7 +65,8 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi
/// The sidecar and the index are written under one lease, the index from the sidecar read under it, and the index
/// row gets only what tiers change (<see cref="SequenceRepository.RecordTiersAsync"/>): a whole row written back
/// would undo what another writer committed meanwhile, an ingest's frames or a settings edit (the server's PATCH
/// takes no lease).
/// takes no lease). The folder's scan stamps are recorded only when the index held the folder as it was
/// (<see cref="ArchiveIndexer.InStepAsync(Archive, SequencePaths, SequenceManifest, CancellationToken)"/>).
/// </remarks>
/// <exception cref="ArchiveOfflineException">The archive is offline or read-only.</exception>
public async Task<IReadOnlyList<SidecarFrame>> RecordAsync(Archive archive, Sequence sequence, IReadOnlyDictionary<Guid, FileTier> tiers,
@@ -103,6 +104,7 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi
changed.Add(updated);
}
if (changed.Count == 0) return m.Frames;
var inStep = await indexer.InStepAsync(archive, folder, m, cancellationToken).ConfigureAwait(false);
var manifest = m with
{
Frames = frames,
@@ -118,7 +120,7 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi
frames.Count(f => (f.Flags & FrameFlags.PreviewOnly) != 0), cancellationToken).ConfigureAwait(false);
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
}
await indexer.RecordStampsInStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
if (inStep) await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
return manifest.Frames;
}
}
@@ -67,11 +67,11 @@ internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer,
/// <summary>
/// Queues a background <paramref name="pass"/> (thumbnails or proxies) of each sequence this run indexed whose
/// <paramref name="built"/> flag is clear: new and adopted sequences, and those whose photos changed (the scan clears the
/// flag then, also when another writer listed them in the sidecar itself). After a resumed run, which may have indexed
/// sequences before the interruption, of every sequence of the archive without the flag but those in
/// <paramref name="awaiting"/>. One pass per sequence, as after an upload: a folder renamed, deleted or damaged before
/// its pass runs fails that pass alone, and the other sequences still get their images. A sequence whose own pass of
/// that kind is already queued is left out.
/// flag then, in this server's own row when another writer listed them in the sidecar itself). After a resumed run,
/// which may have indexed sequences before the interruption, of every sequence of the archive without the flag but
/// those in <paramref name="awaiting"/>. One pass per sequence, as after an upload: a folder renamed, deleted or
/// damaged before its pass runs fails that pass alone, and the other sequences still get their images. A sequence
/// whose own pass of that kind is already queued is left out.
/// </summary>
private async Task QueueImagesAsync(JobContext context, Archive archive, JobKind pass, SequenceStatus built, IReadOnlyList<Guid> indexed,
bool resumed, IReadOnlySet<Guid> awaiting)
+14 -5
View File
@@ -210,12 +210,22 @@ public sealed class MassRenamer
.FirstOrDefault(l => !taken.Contains(l.Id) && folders.Any(f => RenamePlanner.IsInside(f, l.Folder)));
}
/// <summary>Sidecar title, then index (folder, frames' paths); each part is idempotent so a resumed step can repeat it.</summary>
/// <summary>
/// Sidecar title, then index (folder, title, frames' paths); each part is idempotent so a resumed step can repeat it.
/// The index row changes in its folder and title only (<see cref="SequenceRepository.RelocateAsync"/>): a settings edit
/// committed meanwhile stays, and frames another process listed in the sidecar are left to the next scan, for which
/// the folder keeps no stamps unless the index held it as it was
/// (<see cref="ArchiveIndexer.InStepAsync(Archive, string, SequencePaths, SequenceManifest, CancellationToken)"/>).
/// </summary>
private async Task RecordAsync(Archive archive, ArchivePaths paths, RenameStep step, CancellationToken cancellationToken)
{
var sequence = paths.Sequence(step.ToFolder);
var manifest = SequenceWriter.ReadLocked(sequence, step.SequenceId);
if (manifest.Title != step.ToTitle)
// The folder moved with its files' stamps, which the index recorded at its old place.
var inStep = await _indexer.InStepAsync(archive, step.FromFolder, sequence, manifest, cancellationToken).ConfigureAwait(false);
// A step that only moves the folder keeps the title, which an edit may have changed since the plan.
var retitle = step.FromTitle != step.ToTitle;
if (retitle && manifest.Title != step.ToTitle)
{
manifest = manifest with
{
@@ -227,12 +237,11 @@ public sealed class MassRenamer
}
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
{
var existing = await tx.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false);
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
await tx.Sequences.RelocateAsync(step.SequenceId, step.ToFolder, retitle ? step.ToTitle : null, cancellationToken).ConfigureAwait(false);
if (step.MovesFolder) await tx.Frames.RebaseFolderAsync(step.SequenceId, step.FromFolder, step.ToFolder, cancellationToken).ConfigureAwait(false);
await tx.Archives.DeleteFolderStateAsync(archive.Id, step.FromFolder, cancellationToken).ConfigureAwait(false);
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
}
await _indexer.RecordStampsAsync(archive, sequence, step.SequenceId, cancellationToken).ConfigureAwait(false);
if (inStep) await _indexer.RecordStampsAsync(archive, sequence, step.SequenceId, cancellationToken).ConfigureAwait(false);
}
}
@@ -101,15 +101,12 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti
// Without a current report (none, or made before frames were added) the deep status no longer holds.
var clear = (report is null ? SequenceStatus.AnalyzedDeep : SequenceStatus.None) | (warnings > 0 ? SequenceStatus.None : SequenceStatus.HasWarnings);
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
var inStep = await indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
var status = (manifest.Status | set) & ~clear;
if (status != manifest.Status)
{
manifest = manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow };
ManifestStore.WriteSequence(folder.ManifestPath, manifest);
}
if (status != manifest.Status) ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow });
var summary = deep && report is not null ? AnalysisSummary.From(report).ToJson() : null;
await db.Sequences.RecordAnalysisAsync(sequence.Id, set, clear, warnings, summary, cancellationToken).ConfigureAwait(false);
await indexer.RecordStampsInStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
if (inStep) await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
}
if (changed) await options.NotifyAsync(sequence.Id, ProcessingOptions.SettingsChange, cancellationToken).ConfigureAwait(false);
@@ -10,6 +10,7 @@ public sealed record FolderScanState(string Folder, Guid? SequenceId, FileStamp?
public sealed class ArchiveRepository
{
private const string Columns = "id, name, root_path, folder_template, file_template, schema_version, created, proxy_long_edge";
private const string FolderStateSelect = "SELECT folder, sequence_id, sidecar_size, sidecar_ticks, photos_ticks FROM archive_folders";
private static readonly string UpsertSql = SqlDialect.Upsert("archives",
["id", "name", "root_path", "folder_template", "file_template", "schema_version", "created", "proxy_long_edge"], 1);
@@ -39,15 +40,14 @@ public sealed class ArchiveRepository
/// <summary>Scan stamps of every folder of an archive, by folder.</summary>
public async Task<IReadOnlyDictionary<string, FolderScanState>> GetFolderStatesAsync(Guid archiveId, CancellationToken cancellationToken = default)
{
var rows = await _db.ListAsync(
"SELECT folder, sequence_id, sidecar_size, sidecar_ticks, photos_ticks FROM archive_folders WHERE archive_id = $1", [archiveId],
r => new FolderScanState(r.GetString(0), r.GetGuidOrNull(1),
r.GetInt64OrNull(3) is { } sidecar ? new FileStamp(r.GetInt64(2), new DateTime(sidecar, DateTimeKind.Utc)) : null,
r.GetInt64OrNull(4) is { } photos ? new FileStamp(0, new DateTime(photos, DateTimeKind.Utc)) : null),
cancellationToken).ConfigureAwait(false);
var rows = await _db.ListAsync($"{FolderStateSelect} WHERE archive_id = $1", [archiveId], MapFolderState, cancellationToken).ConfigureAwait(false);
return rows.ToDictionary(s => s.Folder, StringComparer.Ordinal);
}
/// <summary>Scan stamps of one folder of an archive; null when none are recorded.</summary>
public Task<FolderScanState?> GetFolderStateAsync(Guid archiveId, string folder, CancellationToken cancellationToken = default) =>
_db.SingleAsync($"{FolderStateSelect} WHERE archive_id = $1 AND folder = $2", [archiveId, folder], MapFolderState, cancellationToken);
public async Task SetFolderStateAsync(Guid archiveId, FolderScanState state, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync(FolderStateUpsertSql,
[archiveId, state.Folder, state.SequenceId, state.Sidecar?.Size, state.Sidecar?.ModifiedUtc.Ticks, state.Photos?.ModifiedUtc.Ticks, DateTimeOffset.UtcNow],
@@ -61,6 +61,10 @@ public sealed class ArchiveRepository
public Task<int> ClearFolderStatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
_db.ExecuteAsync("DELETE FROM archive_folders WHERE archive_id = $1", [archiveId], cancellationToken);
private static FolderScanState MapFolderState(DbRow r) => new(r.GetString(0), r.GetGuidOrNull(1),
r.GetInt64OrNull(3) is { } sidecar ? new FileStamp(r.GetInt64(2), new DateTime(sidecar, DateTimeKind.Utc)) : null,
r.GetInt64OrNull(4) is { } photos ? new FileStamp(0, new DateTime(photos, DateTimeKind.Utc)) : null);
private static Archive Map(DbRow r) => new()
{
Id = r.GetGuid(0),
@@ -408,19 +408,23 @@ public sealed class SequenceRepository
[archiveId, sessionKey], cancellationToken).ConfigureAwait(false))!.Value;
}, cancellationToken);
/// <summary>Moves a sequence to another folder and/or title (mass rename); false when it does not exist.</summary>
/// <summary>
/// Moves a sequence to another folder and, with <paramref name="title"/>, retitles it (mass rename; settings version
/// + 1 when the title changes); every other field stays as the locked row has it. False when it does not exist.
/// </summary>
/// <param name="title">The new title; null keeps the current one.</param>
public Task<bool> RelocateAsync(Guid id, string folder, string? title, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
var current = await session.QuerySingleOrDefaultAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken)
.ConfigureAwait(false);
if (current is null) return false;
var titleChanged = current.Title != title;
var retitled = title is not null && current.Title != title;
await UpsertRowAsync(session, current with
{
Folder = folder,
Title = title,
SettingsVersion = titleChanged ? current.SettingsVersion + 1 : current.SettingsVersion,
Title = retitled ? title : current.Title,
SettingsVersion = retitled ? current.SettingsVersion + 1 : current.SettingsVersion,
UpdatedAt = DateTimeOffset.UtcNow,
}, cancellationToken).ConfigureAwait(false);
return true;
@@ -7,8 +7,9 @@ namespace Titano.Server.Storage;
/// <summary>
/// Proxies are files under the sequence's <c>proxies/</c> folder; a thumbnail is the JPEG byte range of its entry in
/// <c>thumbs.pack</c>. A thumbnail made from a different original (the frame was renumbered or replaced) counts as not
/// built yet, like a missing one, and so does a proxy whose capture time is another frame's, or one smaller than the
/// archive's policy sent for a frame the server cannot decode (<see cref="ServerProxies.HasProxy"/>).
/// built yet, like a missing one (<see cref="ThumbPack.IsFrom"/>: one drawn before a verify hashed the photo is still its
/// own), and so does a proxy whose capture time is another frame's, or one smaller than the archive's policy sent for a
/// frame the server cannot decode (<see cref="ServerProxies.HasProxy"/>).
/// </summary>
internal sealed class ArchiveAssetLocator : IFrameAssetLocator
{
@@ -34,7 +35,14 @@ internal sealed class ArchiveAssetLocator : IFrameAssetLocator
{
using var pack = ThumbPackReader.Open(paths.ThumbsPackPath);
if (!pack.Entries.TryGetValue(frame.Index, out var entry) || entry.Offset + entry.Length > pack.Length) return null;
if (frame.Sha256 is { Length: ThumbPack.HashLength * 2 } sha && !entry.SourceHash.AsSpan().SequenceEqual(Convert.FromHexString(sha))) return null;
if (frame.Sha256 is { Length: ThumbPack.HashLength * 2 } sha && !ThumbPack.IsFrom(entry.SourceHash, sha, SequenceFile(paths, frame), frame.Size)) return null;
return new AssetLocation(paths.ThumbsPackPath, entry.Offset, entry.Length, Jpeg);
}
/// <summary>The frame's file as its sidecar names it (relative to the sequence folder), which a thumbnail drawn before the photo was hashed is fingerprinted with.</summary>
private static string SequenceFile(SequencePaths paths, Frame frame)
{
var prefix = paths.ArchiveRelative(string.Empty);
return frame.RelativePath.StartsWith(prefix, StringComparison.Ordinal) ? frame.RelativePath[prefix.Length..] : frame.RelativePath;
}
}
@@ -150,6 +150,25 @@ public sealed class ThumbPackTests(TestContext context)
for (var i = 0; i < 20; i++) Assert.BytesEqual(Payload(i + 300), compacted.Read(i));
}
[Test]
public void AThumbnailIsFromItsOriginalAlsoAfterTheOriginalWasHashed()
{
var sha = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData("photo"u8));
var other = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData("another photo"u8));
var drawnUnhashed = ThumbPack.Fingerprint(null, "IMG_0001.JPG", 1000);
var drawnHashed = ThumbPack.Fingerprint(sha, "IMG_0001.JPG", 1000);
Assert.True(ThumbPack.IsFrom(drawnUnhashed, null, "IMG_0001.JPG", 1000), "unhashed");
Assert.True(ThumbPack.IsFrom(drawnHashed, sha, "IMG_0001.JPG", 1000), "hashed");
// Drawn before a verify hashed the photo: the hash changes nothing about it.
Assert.True(ThumbPack.IsFrom(drawnUnhashed, sha, "IMG_0001.JPG", 1000), "hashed since");
// Another photo: another hash, or another file or size at that frame index.
Assert.False(ThumbPack.IsFrom(drawnHashed, other, "IMG_0001.JPG", 1000), "another hash");
Assert.False(ThumbPack.IsFrom(drawnUnhashed, sha, "IMG_0002.JPG", 1000), "another file");
Assert.False(ThumbPack.IsFrom(drawnUnhashed, null, "IMG_0001.JPG", 1001), "another size");
Assert.False(ThumbPack.IsFrom(drawnHashed, null, "IMG_0001.JPG", 1000), "a hash the frame no longer has");
}
[Test]
public void AMissingOrForeignFileReadsAsEmpty()
{
@@ -207,6 +207,7 @@ public sealed class ArchiveJobTests(TestContext context)
var again = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
await fx.WaitForAsync(again.Id, JobState.Done);
Assert.True(Built(ManifestStore.ReadSequence(sidecar).Status), "the server clears what it makes in its own row only");
// The frames changed behind the index: the statuses no longer hold, and the sequence gets a pass of each kind.
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
@@ -270,6 +271,114 @@ public sealed class ArchiveJobTests(TestContext context)
Assert.Count(202, pack.Entries);
}
[Test]
public async Task AnImagePassOvertakenByAnUploadKeepsTheSequenceBuiltWhileThePassAfterTheUploadWaits()
{
await using var fx = await CreateAsync(context);
// The pass commits its pack and checkpoints after 200 frames.
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 201);
await fx.ScanAsync();
var a = (await LocationsAsync(fx)).Single();
var sequence = (await fx.Db.Sequences.GetAsync(a.Id))!;
// While it runs, an upload lands in the sequence and queues the next pass, as the server does after every upload.
await using var engine = ServerEngine(fx);
Job? next = null;
await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, async (_, _) =>
{
if (next is not null) return;
var staged = fx.Stage(Start.AddMinutes(-5), seed: 90);
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, a.Id, null, null, staged.Path, "IMG_0000.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
next = await engine.EnqueueAsync(ArchiveJobs.Thumbnails([a.Id], "a"), cancellationToken: fx.Ct);
}, fx.Ct);
// The pass did not draw the new photo, but the one queued after it does: the sequence stays built meanwhile, so a
// server still offers the thumbnails it has.
Assert.NotNull(next);
Assert.Equal(202, (await fx.Db.Sequences.GetAsync(a.Id))!.FrameCount);
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "row flag");
Assert.True(ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Status.HasFlag(SequenceStatus.ThumbsBuilt), "sidecar flag");
await engine.StartAsync(fx.Ct);
await fx.WaitForAsync(next!.Id, JobState.Done);
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "built");
var folder = fx.Paths.Sequence("a");
using var pack = ThumbPackReader.Open(folder.ThumbsPackPath);
foreach (var frame in ManifestStore.ReadSequence(folder.ManifestPath).Frames)
{
Assert.True(pack.Entries.GetValueOrDefault(frame.I) is { } entry && ThumbPack.IsFrom(entry.SourceHash, frame.Sha256, frame.File, frame.Size),
$"thumbnail of {frame.File}");
}
}
[Test]
public async Task ThumbnailsDrawnBeforeAVerifyHashedThePhotosStayTheirs()
{
await using var fx = await CreateAsync(context);
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3);
await fx.ScanAsync();
var a = (await LocationsAsync(fx)).Single();
var first = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(a.Id))!, false, 0,
(_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(3, first.Built);
// The verify gives the photos the scan adopted their first hash; what was drawn from them is still theirs.
var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [a.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
Assert.Equal(3L, report.Hashed);
var again = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(a.Id))!, false, 0,
(_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((0, 3), (again.Built, again.Reused));
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "built");
Assert.True(ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Frames.All(f => f.Sha256 is not null), "hashed");
}
[Test]
public async Task ADesktopScanAfterTheServersUploadLeavesWhatTheServerMadeForTheFramesAlone()
{
await using var fx = await CreateAsync(context);
Populate(fx);
await using var engine = ServerEngine(fx);
await engine.StartAsync(fx.Ct);
var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
await fx.WaitForAsync(scan.Id, JobState.Done);
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
{
foreach (var pass in (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items) await fx.WaitForAsync(pass.Id, JobState.Done);
}
var a = (await LocationsAsync(fx)).Single(l => l.Folder == "a");
// A desktop sharing the archive, with an index of its own.
await using var desktopIndex = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("desktop"), "titano.db")), fx.Ct);
var desktop = new ArchiveService(desktopIndex, new ArchiveServiceOptions { Owner = "desktop", LeaseWait = TimeSpan.FromSeconds(20) });
var archive = await desktop.OpenAsync(fx.Archive.RootPath, fx.Ct);
await desktop.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct);
// The server takes an upload into the sequence and makes its thumbnail and proxy, as after every upload.
var staged = fx.Stage(Start.AddMinutes(-5), seed: 90);
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, a.Id, null, null, staged.Path, "IMG_0000.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
foreach (var request in new[] { ArchiveJobs.Thumbnails([a.Id], "a"), ArchiveJobs.Proxies([a.Id], "a") })
await fx.WaitForAsync((await engine.EnqueueAsync(request, cancellationToken: fx.Ct)).Id, JobState.Done);
var sidecar = fx.Paths.Sequence("a").ManifestPath;
Assert.True(Built(ManifestStore.ReadSequence(sidecar).Status), "built after the upload");
var stamp = FileStamp.Of(sidecar);
var jobs = (await fx.Db.Jobs.ListAsync(null, null, cancellationToken: fx.Ct)).Items.Count;
// The desktop's index lags behind the upload: its scan indexes the frame and takes the statuses as the sidecar has them.
var summary = await desktop.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct);
Assert.SequenceEqual([a.Id], summary.Sequences);
Assert.Equal(stamp, FileStamp.Of(sidecar));
var row = (await desktopIndex.Sequences.GetAsync(a.Id, fx.Ct))!;
Assert.Equal(7, row.FrameCount);
Assert.True(Built(row.Status), row.Status.ToString());
// So the server's next scan finds nothing changed, and makes nothing again.
var again = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
await fx.WaitForAsync(again.Id, JobState.Done);
Assert.Equal(jobs + 1, (await fx.Db.Jobs.ListAsync(null, null, cancellationToken: fx.Ct)).Items.Count);
Assert.True(Built((await fx.Db.Sequences.GetAsync(a.Id))!.Status), "the server's row");
}
[Test]
public async Task AFolderGoneBeforeItsImagePassesFailsThemAloneAndTheOtherSequencesGetTheirImages()
{
@@ -226,6 +226,10 @@ public sealed class MassRenameTests(TestContext context)
Assert.SequenceEqual(["moved/2026-09-28", "moved/2026-09-29", "moved/2026-09-30"], (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id))
.Select(l => l.Folder).Order(StringComparer.Ordinal));
Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids));
// The folders a nested sequence moved out of changed after their rows were written, as a folder another process
// changed would: the next scan reads those two again, and finds the index as it is.
Assert.SequenceEqual([ids[0], ids[1]], (await fx.ScanAsync()).Sequences);
Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids));
Assert.Equal(0, (await fx.ScanAsync()).Indexed);
}
@@ -10,8 +10,9 @@ using static Titano.Data.Tests.Archiving.ArchiveFixture;
namespace Titano.Data.Tests.Archiving;
/// <summary>
/// Writers that change part of a sequence's sidecar and index row (decode tiers, the verify stamp) while a settings edit
/// or another writer commits, on both dialects: they write only what they own, under the sequence lease.
/// Writers that change part of a sequence's sidecar and index row (decode tiers, the verify, title, tags, rating and
/// notes, a rename) while a settings edit or another writer commits, on both dialects: they write only what they own,
/// under the sequence lease, and leave what another process changed in the sidecar to the next scan.
/// </summary>
[Category("archive")]
public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(context)
@@ -101,6 +102,150 @@ public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(
Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
}
[Test]
public async Task TiersRecordedAfterAnotherWriterRetitledTheSequenceLeaveTheTitleToTheNextScan()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
// A desktop sharing the archive renames and rates the sequence, in the sidecar and its own index.
var (desktop, _) = await fx.OtherWriterAsync();
await desktop.SetSequenceInfoAsync(sequence.Id, "Ridge", ["fog"], 3, null, fx.Ct);
// A tier recorded here meanwhile writes only the tier, and leaves the folder changed for the next scan.
await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(("Ridge", "fog", 3, 1), (row.Title, row.Tags.Single(), row.Rating, row.PreviewOnlyCount));
}
[Test]
public async Task AnInfoEditAfterAnotherWriterAddedFramesLeavesThoseFramesToTheNextScan()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
// The rating goes into the sidecar and this index; the frames this index holds stay as they are.
var rated = await fx.Service.EditSequenceInfoAsync(sequence.Id, info => info with { Rating = 4 }, fx.Ct);
Assert.Equal((3, sequence.FramesVersion, 4), (rated.FrameCount, rated.FramesVersion, rated.Rating));
Assert.Equal(4, ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Rating);
// The next scan still finds the folder changed, and indexes the photo.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal((4, sidecar.Versions.Frames, 4), (row.FrameCount, row.FramesVersion, row.Rating));
Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
}
[Test]
public async Task AnInfoEditKeepsTheSettingsTheApiSavedAfterItReadTheSequence()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
// The server's PATCH, which takes no lease, saves settings after the edit read the sequence: the edit applies again.
var calls = 0;
var rated = await fx.Service.EditSequenceInfoAsync(sequence.Id, info =>
{
if (++calls == 1)
{
var patch = fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":2}""" }, sequence.SettingsVersion, fx.Ct)
.GetAwaiter().GetResult();
Assert.Equal(SequenceEditOutcome.Updated, patch.Outcome);
}
return info with { Rating = 5 };
}, fx.Ct);
Assert.Equal(2, calls);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Contains("heroFrame", row.SettingsJson);
Assert.Equal((5, sequence.SettingsVersion + 2), (row.Rating, row.SettingsVersion));
Assert.Equal(rated.SettingsVersion, row.SettingsVersion);
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
Assert.Equal((5, row.SettingsVersion), (sidecar.Rating, sidecar.Versions.Settings));
}
[Test]
public async Task ARenameAfterAnotherWriterAddedFramesLeavesThoseFramesToTheNextScan()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
// The rename moves the folder and retitles the sequence; the row keeps the frames this index holds.
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [sequence.Id], "b", "Ridge", fx.Ct);
await fx.Service.ApplyRenameAsync(plan, null, fx.Ct);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(("b", "Ridge", 3, sequence.FramesVersion), (row.Folder, row.Title, row.FrameCount, row.FramesVersion));
// The next scan still finds the folder changed, and indexes the photo where it is now.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("b").ManifestPath);
row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(("Ridge", 4, sidecar.Versions.Frames), (row.Title, row.FrameCount, row.FramesVersion));
Assert.True((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).All(f => f.RelativePath.StartsWith("b/", StringComparison.Ordinal)),
"frames in b");
}
[Test]
public async Task AVerifyStoppedAfterAFlushLeavesFramesAnotherWriterAddedToTheNextScan()
{
var fx = await FixtureAsync();
// Enough photos the scan adopted unhashed for the verify to flush their hashes before it reaches the end.
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 513);
await fx.ScanAsync();
var sequence = (await fx.Db.Sequences.GetAsync((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single().Id, fx.Ct))!;
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddHours(3), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0600.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
// The verify writes the hashes of the first 512 photos to the sidecar and this index, and stops before the end.
await Assert.ThrowsAsync<OperationCanceledException>(() => fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default,
(_, _, _) => throw new OperationCanceledException(), null, fx.Ct));
Assert.Equal(512, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Count(f => f.Sha256 is not null));
Assert.Equal(513, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
// The next scan still finds the folder changed, and indexes the photo.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
Assert.Equal(514, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
}
[Test]
public async Task DuplicatesFlaggedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan()
{
var fx = await FixtureAsync();
// Two sequences holding the same photo, hashed by a verify.
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3);
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "b"), Start, 1);
await fx.ScanAsync();
await fx.Service.Verifier.VerifyAsync(fx.Archive, null, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
var copy = (await fx.Service.FindDuplicatesAsync(fx.Archive.Id, fx.Ct)).Single().Frames[1];
var sequence = (await fx.Db.Sequences.GetAsync(copy.SequenceId, fx.Ct))!;
// A desktop sharing the archive imports a photo into the sequence whose copy maintenance flags.
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddHours(3), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0600.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
var report = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, firstStep: 3, _ => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(1, report.DuplicateFlagChanges);
Assert.Equal(sequence.FrameCount, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
// The next scan still finds the folder changed, and indexes the photo.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
Assert.Equal(sequence.FrameCount + 1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
}
[Test]
public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit()
{
@@ -148,6 +148,35 @@ public sealed class ArchiveModuleTests(TestContext context)
await UntilImagesAsync(server, editor, archive.Id, "roof");
}
[Test]
public async Task AThumbnailDrawnBeforeAVerifyHashedThePhotoIsStillServed()
{
await using var server = await TestServer.StartAsync(context, options =>
{
Directory.CreateDirectory(Path.Combine(options.ArchivePath, "roof"));
File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0001.JPG"), Photo());
}, ArchiveModule.Register);
var archive = server.Services.GetRequiredService<ServerArchive>();
var (editor, _, _) = await server.UserAsync("alberto", "editor");
await UntilIndexedAsync(server, archive.Id, "roof");
await UntilScannedAsync(server);
await UntilImagesAsync(server, editor, archive.Id, "roof");
await UntilIdleAsync(server);
var roof = (await server.Database.Sequences.ListLocationsAsync(archive.Id)).Single(l => l.Folder == "roof");
var frame = (await server.Database.Frames.ListBySequenceAsync(roof.Id)).Single();
Assert.Null(frame.Sha256);
// A verify gives the photo the scan adopted its first hash; the thumbnail drawn from it before is still its own.
var verified = await server.Services.GetRequiredService<ArchiveService>().Verifier.VerifyAsync(
Assert.NotNull(await server.Database.Archives.GetAsync(archive.Id)), [roof.Id], default, (_, _, _) => ValueTask.CompletedTask, null,
context.Cancellation);
Assert.Equal(1L, verified.Hashed);
Assert.NotNull(Assert.NotNull(await server.Database.Frames.GetAsync(frame.Id)).Sha256);
using var thumb = await editor.GetAsync($"/api/v1/frames/{frame.Id}/thumb");
Assert.Equal(HttpStatusCode.OK, thumb.StatusCode);
Assert.True(Assert.NotNull(await server.Database.Sequences.GetAsync(roof.Id)).Status.HasFlag(SequenceStatus.ThumbsBuilt), "built");
}
[Test]
public async Task SettingsADesktopChangedInArchiveJsonReachUploadsAndTheAdminPanel()
{