diff --git a/src/Titano.Core/Archiving/ThumbPack.cs b/src/Titano.Core/Archiving/ThumbPack.cs
index c2bf8eaa..b31fd1d8 100644
--- a/src/Titano.Core/Archiving/ThumbPack.cs
+++ b/src/Titano.Core/Archiving/ThumbPack.cs
@@ -223,6 +223,15 @@ public static class ThumbPack
sha256 is { Length: HashLength * 2 }
? Convert.FromHexString(sha256)
: System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes($"{file}|{size}"));
+
+ ///
+ /// Whether a thumbnail with was drawn from this original: its ,
+ /// or, for an original hashed since (a verify hashes the photos a scan adopted), the fingerprint of its file name and
+ /// size it was drawn with before. The hash changes nothing about the photo.
+ ///
+ public static bool IsFrom(ReadOnlySpan sourceHash, string? sha256, string file, long size) =>
+ sourceHash.SequenceEqual(Fingerprint(sha256, file, size))
+ || (sha256 is { Length: HashLength * 2 } && sourceHash.SequenceEqual(Fingerprint(null, file, size)));
}
/// Random access to the thumbnails of a pack without loading the file; safe while a writer appends.
diff --git a/src/Titano.Data/Archiving/ArchiveImages.cs b/src/Titano.Data/Archiving/ArchiveImages.cs
index adee216a..9f9cf69e 100644
--- a/src/Titano.Data/Archiving/ArchiveImages.cs
+++ b/src/Titano.Data/Archiving/ArchiveImages.cs
@@ -209,8 +209,7 @@ public sealed class ArchiveImages
for (var i = 0; i < startFrame && i < manifest.Frames.Count; i++)
{
var frame = manifest.Frames[i];
- complete &= pack.Entries.GetValueOrDefault(frame.I) is { } entry
- && entry.SourceHash.AsSpan().SequenceEqual(ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size));
+ complete &= pack.Entries.GetValueOrDefault(frame.I) is { } entry && ThumbPack.IsFrom(entry.SourceHash, frame.Sha256, frame.File, frame.Size);
}
var cadence = new Cadence();
for (var i = startFrame; i < manifest.Frames.Count; i++)
@@ -220,7 +219,7 @@ public sealed class ArchiveImages
var fingerprint = ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size);
var entry = pack.Entries.GetValueOrDefault(frame.I);
var edge = entry is null ? 0 : System.Math.Max(entry.Width, entry.Height);
- var current = !force && entry is not null && entry.SourceHash.AsSpan().SequenceEqual(fingerprint);
+ var current = !force && entry is not null && ThumbPack.IsFrom(entry.SourceHash, frame.Sha256, frame.File, frame.Size);
// A frame this host cannot decode shows its proxy once one arrived; until then its original's embedded
// preview, whose smaller thumbnail the proxy's replaces.
var fromProxy = false;
@@ -255,7 +254,8 @@ public sealed class ArchiveImages
pack.Commit();
}
await _tiers.RecordAsync(archive, sequence, tiers, proxies: false, cancellationToken).ConfigureAwait(false);
- await MarkAsync(archive, paths, sequence, SequenceStatus.ThumbsBuilt, complete && undecodable == 0, cancellationToken).ConfigureAwait(false);
+ await MarkAsync(archive, paths, sequence, manifest, JobKind.Thumbs, SequenceStatus.ThumbsBuilt, complete && undecodable == 0, cancellationToken)
+ .ConfigureAwait(false);
return new ImagePassResult(built, reused, undecodable);
}
@@ -327,7 +327,7 @@ public sealed class ArchiveImages
if (cadence.Due()) await checkpoint(i + 1, manifest.Frames.Count).ConfigureAwait(false);
}
await _tiers.RecordAsync(archive, sequence, tiers, proxies: true, cancellationToken).ConfigureAwait(false);
- await MarkAsync(archive, paths, sequence, SequenceStatus.ProxiesBuilt, complete, cancellationToken).ConfigureAwait(false);
+ await MarkAsync(archive, paths, sequence, manifest, JobKind.Proxies, SequenceStatus.ProxiesBuilt, complete, cancellationToken).ConfigureAwait(false);
return new ImagePassResult(built, reused, undecodable, waiting);
}
@@ -520,15 +520,54 @@ public sealed class ArchiveImages
}
}
- private async Task MarkAsync(Archive archive, ArchivePaths paths, Sequence sequence, SequenceStatus flag, bool complete, CancellationToken cancellationToken)
+ ///
+ /// Sets in the sidecar and the index when the pass covered every frame it read
+ /// () and the sidecar, as it is under the lease, still lists those frames
+ /// (); clears it otherwise. Frames a scan's reconcile or an ingest added, removed or
+ /// renumbered after the pass read have nothing of it, and a scan that cleared the flag for
+ /// them queues the next pass only while the flag stays clear. A of the same kind already
+ /// queued for the sequence (an upload's, or a scan's) draws them when it runs, so the flag stays set meanwhile, as
+ /// between an upload and its pass: a camera uploading faster than a pass ends would otherwise keep it clear, and a
+ /// server would offer no thumbnail or proxy of the sequence at all until uploads paused. The index is written under
+ /// the same lease, so the flag never lands there after a scan cleared it in both.
+ ///
+ private async Task MarkAsync(Archive archive, ArchivePaths paths, Sequence sequence, SequenceManifest read, JobKind pass, SequenceStatus flag,
+ bool complete, CancellationToken cancellationToken)
{
- await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m =>
+ var folder = paths.Sequence(sequence.Folder);
+ await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
{
- var status = complete ? m.Status | flag : m.Status & ~flag;
- return status == m.Status ? null : m with { Status = status, UpdatedAt = DateTimeOffset.UtcNow };
- }, cancellationToken).ConfigureAwait(false);
- await _db.Sequences.UpdateStatusAsync(sequence.Id, complete ? flag : SequenceStatus.None, complete ? SequenceStatus.None : flag, cancellationToken)
- .ConfigureAwait(false);
- await _indexer.RecordStampsAsync(archive, paths.Sequence(sequence.Folder), sequence.Id, cancellationToken).ConfigureAwait(false);
+ var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
+ var built = complete && (SameFrames(manifest.Frames, read.Frames)
+ || await _db.Jobs.IsQueuedAsync(pass, sequence.Id, cancellationToken).ConfigureAwait(false));
+ var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
+ var status = built ? manifest.Status | flag : manifest.Status & ~flag;
+ if (status != manifest.Status)
+ {
+ manifest = manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow };
+ ManifestStore.WriteSequence(folder.ManifestPath, manifest);
+ }
+ await _db.Sequences.UpdateStatusAsync(sequence.Id, built ? flag : SequenceStatus.None, built ? SequenceStatus.None : flag, cancellationToken)
+ .ConfigureAwait(false);
+ if (inStep) await _indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
+ }
+ }
+
+ ///
+ /// Whether lists the photos did, each at the same index with the
+ /// same file, size, capture time and content hash: thumbnails and proxies are kept by frame index, so photos added,
+ /// removed, renumbered or replaced since have none of what the pass made. A hash learnt since (a verify hashing a
+ /// photo the scan adopted) changes nothing about the photo, and its thumbnail stays its own ().
+ ///
+ private static bool SameFrames(IReadOnlyList current, IReadOnlyList read)
+ {
+ if (current.Count != read.Count) return false;
+ for (var i = 0; i < current.Count; i++)
+ {
+ var (now, then) = (current[i], read[i]);
+ if (now.Id != then.Id || now.I != then.I || now.File != then.File || now.Size != then.Size || now.CapturedAt != then.CapturedAt
+ || (then.Sha256 is not null && !string.Equals(now.Sha256, then.Sha256, StringComparison.OrdinalIgnoreCase))) return false;
+ }
+ return true;
}
}
diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs
index 263376df..09abf740 100644
--- a/src/Titano.Data/Archiving/ArchiveIndexer.cs
+++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs
@@ -31,6 +31,9 @@ public sealed record ScanSummary(int Folders, int Indexed, int Adopted, int Remo
///
public sealed class ArchiveIndexer
{
+ /// What covers the frames as they were: a scan that finds them changed clears it, and its follow-ups make it again.
+ private const SequenceStatus CoveringStatus = SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt;
+
private readonly TitanoDatabase _db;
private readonly SequenceWriter _writer;
@@ -131,6 +134,32 @@ public sealed class ArchiveIndexer
_db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(sequence.RelativeFolder, sequenceId,
FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))), cancellationToken);
+ ///
+ /// Whether this index holds the folder of (the sidecar read under the sequence lease) as it
+ /// is now: the stamps it recorded when it last synced the folder are still the folder's, and its row holds the
+ /// sidecar's frames. A writer that changes, under the lease, only what it owns of the sidecar and the index row
+ /// (statuses, decode tiers, frame flags, title, tags, rating and notes, the folder) asks before its own write, and
+ /// records the folder's new stamps after it only when this held (). What another
+ /// process (a desktop sharing the archive) changed in the sidecar alone, frames or a new title, then keeps the folder
+ /// changed for the next incremental scan, which writes the whole row from the sidecar.
+ ///
+ internal Task InStepAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) =>
+ InStepAsync(archive, sequence.RelativeFolder, sequence, manifest, cancellationToken);
+
+ ///
+ /// The folder the stamps were recorded at: the one a rename just moved the sequence from,
+ /// whose files keep their stamps.
+ internal async Task InStepAsync(Archive archive, string recordedFolder, SequencePaths sequence, SequenceManifest manifest,
+ CancellationToken cancellationToken)
+ {
+ if (await _db.Archives.GetFolderStateAsync(archive.Id, recordedFolder, cancellationToken).ConfigureAwait(false) is not { } state
+ || state.SequenceId != manifest.Id || state.Sidecar != FileStamp.Of(sequence.ManifestPath)
+ || state.Photos != FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))) return false;
+ // A sidecar rewritten within its file system's time resolution, at the same size, keeps its stamp.
+ return await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { } indexed
+ && indexed.FramesVersion == manifest.Versions.Frames;
+ }
+
/// Indexes unindexed photos and flags missing files of one sequence (the orphans fix action).
public async Task ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default)
{
@@ -144,8 +173,10 @@ public sealed class ArchiveIndexer
///
/// Applies to the sidecar (null writes nothing) and indexes the result, both under the sequence
/// lease. Frames inserted before indexed ones move the project's positions () without a
- /// new settings version; an editor saving its project holds the same lease, so it reads them moved and merges onto
- /// them rather than writing its old positions back.
+ /// new settings version; an editor saving its project holds the same lease, so it reads them moved and merges onto them
+ /// rather than writing its old positions back. When the sidecar, as read under the lease, lists other frames than the
+ /// index row (another process changed them together with their sidecar entries), the row is written without what
+ /// this host makes for the frames ().
///
private async Task UpdateAndSyncAsync(Archive archive, ArchivePaths paths, SequencePaths sequence, Guid sequenceId,
Func edit, CancellationToken cancellationToken)
@@ -153,16 +184,27 @@ public sealed class ArchiveIndexer
await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false))
{
var manifest = SequenceWriter.ReadLocked(sequence, sequenceId);
+ var moved = await FramesMovedAsync(manifest, cancellationToken).ConfigureAwait(false);
if (edit(manifest) is { } updated)
{
ManifestStore.WriteSequence(sequence.ManifestPath, updated);
manifest = updated;
}
- await SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
+ await SyncAsync(archive, sequence, moved ? manifest with { Status = manifest.Status & ~Uncovered } : manifest, cancellationToken).ConfigureAwait(false);
return manifest;
}
}
+ ///
+ /// What a scan clears in this host's own row, never in the shared sidecar, when another process added, removed or
+ /// renumbered frames together with their sidecar entries: on a server (),
+ /// which analyses and draws what its clients load, the light analysis, thumbnails and proxies, so the scan's follow-ups
+ /// make them for the frames as they are, and their passes set the flags in both places again. Nothing on a desktop:
+ /// the flags in the sidecar are the writer's that added the frames (which analyses them, and on a server draws them
+ /// after every upload), and clearing them there would have the server make everything again.
+ ///
+ private SequenceStatus Uncovered => _writer.Options.ImagesAfterScan ? CoveringStatus : SequenceStatus.None;
+
/// Indexes one folder; returns its sequence and the sidecar bytes when they were read and not rewritten since (for the snapshot).
private async Task<(Guid Id, byte[]? Bytes)> IndexFolderAsync(Archive archive, ArchivePaths paths, SequenceFolder folder, bool reconcile, IndexSnapshot? snapshot,
Locations locations, CancellationToken cancellationToken)
@@ -196,18 +238,21 @@ public sealed class ArchiveIndexer
}
if (reconcile || File.Exists(sequence.IngestJournalPath))
{
- // Photos added, gone or back: the light analysis, thumbnails and proxies no longer cover the frames, so the scan's
- // follow-ups make them again (the images on a server, ArchiveServiceOptions.ImagesAfterScan).
+ // Photos added, gone or back: the light analysis, thumbnails and proxies no longer cover the frames, as the
+ // sidecar then tells every host, so the scan's follow-ups make them again (the images on a server,
+ // ArchiveServiceOptions.ImagesAfterScan).
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id,
m => SequenceAdopter.Reconcile(sequence, m, cancellationToken) is { } reconciled
- ? reconciled with { Status = reconciled.Status & ~(SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt) }
+ ? reconciled with { Status = reconciled.Status & ~CoveringStatus }
: null,
cancellationToken).ConfigureAwait(false);
bytes = null;
}
- else if (await MayMovePositionsAsync(manifest, cancellationToken).ConfigureAwait(false))
+ else if (await FramesMovedAsync(manifest, cancellationToken).ConfigureAwait(false))
{
- // Another process changed the frames: the sidecar is read again under the lease the project moves under.
+ // Another process (a desktop sharing the archive, or the server it shares it with) added, removed or renumbered
+ // frames together with their sidecar entries: read again under the lease, under which the project's positions
+ // move, and indexed without what this host makes for the frames (Uncovered).
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false);
bytes = null;
}
@@ -257,9 +302,12 @@ public sealed class ArchiveIndexer
}
}
- /// The index holds a project for this sequence and other frames than the sidecar: its positions may move.
- private async Task MayMovePositionsAsync(SequenceManifest manifest, CancellationToken cancellationToken) =>
- await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { SettingsJson: not null } indexed
+ ///
+ /// The index holds this sequence with other frames than the sidecar: another process changed them (its project's
+ /// positions may move, and what covered the frames no longer does).
+ ///
+ private async Task FramesMovedAsync(SequenceManifest manifest, CancellationToken cancellationToken) =>
+ await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { } indexed
&& indexed.FramesVersion != manifest.Versions.Frames;
/// A sidecar for a folder that has none; ids of a sequence the index already knew at this folder are kept.
@@ -309,7 +357,10 @@ public sealed class ArchiveIndexer
///
/// Rebuild only: rows whose folder no longer holds a sidecar leave the index (their files are untouched). A sequence
- /// waiting for its first frame has no sidecar yet and stays.
+ /// waiting for its first frame has no sidecar yet and stays. A row the folders listed when the rebuild began do not
+ /// back is looked at again under its lease, as it and its folder are when it would go ():
+ /// uploads go on during a rebuild, which can take minutes, so a sequence that got its first frame or was created
+ /// meanwhile has a folder the listing does not hold.
///
private async Task RemoveUnbackedAsync(Archive archive, ArchivePaths paths, IReadOnlyList folders, Locations locations,
CancellationToken cancellationToken)
@@ -322,12 +373,46 @@ public sealed class ArchiveIndexer
foreach (var location in rows)
{
if (awaiting.Contains(location.Id) || (present.Contains(location.Folder) && locations.AtFolder(location.Folder) == location.Id)) continue;
- if (await _db.Sequences.DeleteAsync(location.Id, cancellationToken).ConfigureAwait(false)) removed++;
- await _db.Archives.DeleteFolderStateAsync(archive.Id, location.Folder, cancellationToken).ConfigureAwait(false);
+ IAsyncDisposable held;
+ try
+ {
+ held = await _writer.LockAsync(paths, location.Id, cancellationToken).ConfigureAwait(false);
+ }
+ catch (LeaseUnavailableException)
+ {
+ // Another process is writing the sequence, so it is there.
+ continue;
+ }
+ await using (held)
+ {
+ if (await BackedAsync(paths, location.Id, cancellationToken).ConfigureAwait(false)) continue;
+ if (await _db.Sequences.DeleteAsync(location.Id, cancellationToken).ConfigureAwait(false)) removed++;
+ await _db.Archives.DeleteFolderStateAsync(archive.Id, location.Folder, cancellationToken).ConfigureAwait(false);
+ }
}
return removed;
}
+ ///
+ /// Whether the index row of keeps its place as it is now: it waits for its first frame,
+ /// its folder holds a sidecar naming it (one that cannot be read proves nothing), or it is gone already.
+ ///
+ private async Task BackedAsync(ArchivePaths paths, Guid sequenceId, CancellationToken cancellationToken)
+ {
+ if (await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) is not { } row) return true;
+ if ((await AwaitingFolderAsync(paths, [new SequenceLocation(row.Id, row.Folder, row.Title)], cancellationToken).ConfigureAwait(false)).Count > 0)
+ return true;
+ var sidecar = paths.Sequence(row.Folder).ManifestPath;
+ try
+ {
+ return File.Exists(sidecar) && ManifestStore.ReadSequence(sidecar).Id == sequenceId;
+ }
+ catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
+ {
+ return true;
+ }
+ }
+
/// Which folder each indexed sequence id lives in, updated as the scan goes.
private sealed class Locations
{
diff --git a/src/Titano.Data/Archiving/ArchiveMaintenance.cs b/src/Titano.Data/Archiving/ArchiveMaintenance.cs
index e9c921c3..9d137b18 100644
--- a/src/Titano.Data/Archiving/ArchiveMaintenance.cs
+++ b/src/Titano.Data/Archiving/ArchiveMaintenance.cs
@@ -251,12 +251,20 @@ public sealed class ArchiveMaintenance
if (await _db.Sequences.GetAsync(bySequence.Key, cancellationToken).ConfigureAwait(false) is not { } sequence) continue;
var changes = bySequence.ToDictionary(x => x.Key, x => (Func)(f =>
f with { Flags = x.Set ? f.Flags | FrameFlags.Duplicate : f.Flags & ~FrameFlags.Duplicate }));
- await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m => SequenceFrames.Update(m, changes), cancellationToken).ConfigureAwait(false);
- var setIds = bySequence.Where(x => x.Set).Select(x => x.Key).ToList();
- var clearIds = bySequence.Where(x => !x.Set).Select(x => x.Key).ToList();
- if (setIds.Count > 0) await _db.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false);
- if (clearIds.Count > 0) await _db.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false);
- await _indexer.RecordStampsAsync(archive, paths.Sequence(sequence.Folder), sequence.Id, cancellationToken).ConfigureAwait(false);
+ var folder = paths.Sequence(sequence.Folder);
+ await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
+ {
+ // The scan stamps follow only when the index held the folder as it was: frames another process listed in the
+ // sidecar meanwhile are still indexed by the next scan.
+ var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
+ var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
+ ManifestStore.WriteSequence(folder.ManifestPath, SequenceFrames.Update(manifest, changes));
+ var setIds = bySequence.Where(x => x.Set).Select(x => x.Key).ToList();
+ var clearIds = bySequence.Where(x => !x.Set).Select(x => x.Key).ToList();
+ if (setIds.Count > 0) await _db.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false);
+ if (clearIds.Count > 0) await _db.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false);
+ if (inStep) await _indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
+ }
}
return (groups.Count, set.Count + clear.Count);
}
diff --git a/src/Titano.Data/Archiving/ArchiveService.cs b/src/Titano.Data/Archiving/ArchiveService.cs
index 1ee7c6dd..b76809a2 100644
--- a/src/Titano.Data/Archiving/ArchiveService.cs
+++ b/src/Titano.Data/Archiving/ArchiveService.cs
@@ -6,6 +6,7 @@ using Titano.Core.Model;
using Titano.Core.Naming;
using Titano.Data.Archiving.Jobs;
using Titano.Data.Jobs;
+using Titano.Data.Repositories;
namespace Titano.Data.Archiving;
@@ -20,6 +21,9 @@ public sealed class ArchiveService : IJobHandlerProvider
/// Lease serialising read-modify-write of archive.json between the processes sharing an archive.
private const string SettingsLease = "archive-settings";
+ /// Times a title, tags, rating or notes edit is applied at the version it read before it writes regardless.
+ private const int InfoEditAttempts = 3;
+
private readonly TitanoDatabase _db;
private readonly SequenceWriter _writer;
@@ -224,35 +228,54 @@ public sealed class ArchiveService : IJobHandlerProvider
/// Changes some of title, tags, rating and notes from their current values — read under the sequence's lease from
/// the sidecar, or from the index when it is ahead (an edit made there only) — so fields
/// leaves alone keep any change made since the caller read the sequence. Sidecar and index change together
- /// (settings version + 1).
+ /// (settings version + 1), the index row in these fields only, at the settings version read under the lease
+ /// (): a settings edit committed
+ /// since the caller read the sequence (an analysis's Auto values, the server's PATCH, which takes no lease) stays, and
+ /// so do frames another process listed in the sidecar, which the next scan indexes
+ /// ().
+ /// runs again on top of an edit the API committed meanwhile.
///
+ /// The sequence is not in the index (or left it before the edit).
public async Task EditSequenceInfoAsync(Guid sequenceId, Func edit, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(edit);
var (archive, sequence) = await LoadSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false);
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
- var manifest = await _writer.UpdateAsync(paths, sequenceId, sequence.Folder, m =>
- {
- var current = sequence.SettingsVersion > m.Versions.Settings
- ? new SequenceInfo(sequence.Title, sequence.Tags, sequence.Rating, sequence.Notes)
- : new SequenceInfo(m.Title, m.Tags, m.Rating, m.Notes);
- var info = edit(current);
- return m with
- {
- Title = string.IsNullOrWhiteSpace(info.Title) ? null : info.Title.Trim(),
- Tags = info.Tags,
- Rating = info.Rating,
- Notes = info.Notes,
- Versions = m.Versions with { Settings = System.Math.Max(m.Versions.Settings, sequence.SettingsVersion) + 1 },
- UpdatedAt = DateTimeOffset.UtcNow,
- };
- }, cancellationToken).ConfigureAwait(false);
var folder = paths.Sequence(sequence.Folder);
- var updated = SequenceMapping.ToSequence(manifest, archive.Id, folder, sequence);
- await _db.Sequences.UpsertAsync(updated, cancellationToken).ConfigureAwait(false);
- await Indexer.RecordStampsAsync(archive, folder, sequenceId, cancellationToken).ConfigureAwait(false);
- return updated;
+ await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false))
+ {
+ var manifest = SequenceWriter.ReadLocked(folder, sequenceId);
+ var inStep = await Indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
+ for (var attempt = 1; ; attempt++)
+ {
+ var row = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? throw Unknown(sequenceId);
+ var current = row.SettingsVersion > manifest.Versions.Settings
+ ? new SequenceInfo(row.Title, row.Tags, row.Rating, row.Notes)
+ : new SequenceInfo(manifest.Title, manifest.Tags, manifest.Rating, manifest.Notes);
+ var info = edit(current);
+ var title = string.IsNullOrWhiteSpace(info.Title) ? null : info.Title.Trim();
+ // The last attempt writes over whatever the index holds then rather than fail the user's edit.
+ var result = await _db.Sequences.EditAsync(sequenceId,
+ new SequenceEdit { Title = title ?? string.Empty, Tags = info.Tags, Rating = info.Rating, Notes = info.Notes ?? string.Empty },
+ attempt < InfoEditAttempts ? row.SettingsVersion : null, cancellationToken).ConfigureAwait(false);
+ // A conflict means the API changed the sequence after the read: the edit applies again on top of that.
+ if (result.Outcome == SequenceEditOutcome.Conflict) continue;
+ if (result.Sequence is not { } updated) throw Unknown(sequenceId);
+ // The sidecar takes the fields as the index stored them (trimmed, tags without duplicates).
+ ManifestStore.WriteSequence(folder.ManifestPath, manifest with
+ {
+ Title = updated.Title,
+ Tags = updated.Tags,
+ Rating = updated.Rating,
+ Notes = updated.Notes,
+ Versions = manifest.Versions with { Settings = System.Math.Max(manifest.Versions.Settings + 1, updated.SettingsVersion) },
+ UpdatedAt = DateTimeOffset.UtcNow,
+ });
+ if (inStep) await Indexer.RecordStampsAsync(archive, folder, sequenceId, cancellationToken).ConfigureAwait(false);
+ return updated;
+ }
+ }
}
public Task> FindDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
@@ -367,10 +390,11 @@ public sealed class ArchiveService : IJobHandlerProvider
private async Task<(Archive, Sequence)> LoadSequenceAsync(Guid sequenceId, CancellationToken cancellationToken)
{
- var sequence = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
- ?? throw new ArgumentException($"Unknown sequence {sequenceId}.", nameof(sequenceId));
+ var sequence = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? throw Unknown(sequenceId);
return (await LoadArchiveAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false), sequence);
}
+
+ private static ArgumentException Unknown(Guid sequenceId) => new($"Unknown sequence {sequenceId}.", nameof(sequenceId));
}
/// The fields of a sequence the user describes it with (sidecar and index).
diff --git a/src/Titano.Data/Archiving/ArchiveVerifier.cs b/src/Titano.Data/Archiving/ArchiveVerifier.cs
index 3cdf58a7..108149cd 100644
--- a/src/Titano.Data/Archiving/ArchiveVerifier.cs
+++ b/src/Titano.Data/Archiving/ArchiveVerifier.cs
@@ -95,35 +95,55 @@ public sealed class ArchiveVerifier
return targets;
}
- /// Writes the flags and new hashes of checked frames to the sidecar and the index.
+ ///
+ /// Writes the flags and new hashes of checked frames to the sidecar and the index, under one lease. The folder's scan
+ /// stamps follow only when the index held the folder as it was
+ /// (), so frames
+ /// another process listed in the sidecar meanwhile are still indexed by the next scan.
+ ///
private async Task FlushAsync(Archive archive, ArchivePaths paths, Sequence target, List pending, CancellationToken cancellationToken)
{
if (pending.Count == 0) return;
var changes = pending.ToDictionary(c => c.Frame.Id, c => (Func)(f =>
f with { Flags = c.Flags, Sha256 = c.Health == FrameHealth.Hashed ? c.Sha256 : f.Sha256 }));
- await _writer.UpdateAsync(paths, target.Id, target.Folder, m => SequenceFrames.Update(m, changes), cancellationToken).ConfigureAwait(false);
- await _db.Frames.UpdateIntegrityAsync(pending.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)),
- cancellationToken).ConfigureAwait(false);
- await _indexer.RecordStampsAsync(archive, paths.Sequence(target.Folder), target.Id, cancellationToken).ConfigureAwait(false);
+ var sequence = paths.Sequence(target.Folder);
+ await using (await _writer.LockAsync(paths, target.Id, cancellationToken).ConfigureAwait(false))
+ {
+ var manifest = SequenceWriter.ReadLocked(sequence, target.Id);
+ var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
+ ManifestStore.WriteSequence(sequence.ManifestPath, SequenceFrames.Update(manifest, changes));
+ await _db.Frames.UpdateIntegrityAsync(pending.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)),
+ cancellationToken).ConfigureAwait(false);
+ if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false);
+ }
pending.Clear();
}
- /// Stamps the verify time; the sequence counts as verified only when no frame is missing or corrupt.
+ ///
+ /// Stamps the verify time; the sequence counts as verified only when no frame is missing or corrupt, and has errors
+ /// while one is. Sidecar and index are written under one lease, the index row with these fields only
+ /// (), so a settings edit or an ingest committed meanwhile stays;
+ /// the scan stamps as in .
+ ///
private async Task StampAsync(Archive archive, ArchivePaths paths, Sequence target, CancellationToken cancellationToken)
{
var sequence = paths.Sequence(target.Folder);
- var manifest = await _writer.UpdateAsync(paths, target.Id, target.Folder, m =>
+ await using (await _writer.LockAsync(paths, target.Id, cancellationToken).ConfigureAwait(false))
{
+ var m = SequenceWriter.ReadLocked(sequence, target.Id);
+ var inStep = await _indexer.InStepAsync(archive, sequence, m, cancellationToken).ConfigureAwait(false);
var clean = m.Frames.All(f => (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0);
- return m with
+ var now = DateTimeOffset.UtcNow;
+ var manifest = m with
{
Status = clean ? m.Status | SequenceStatus.Verified : m.Status & ~SequenceStatus.Verified,
- VerifiedAt = DateTimeOffset.UtcNow,
- UpdatedAt = DateTimeOffset.UtcNow,
+ VerifiedAt = now,
+ UpdatedAt = now,
};
- }, cancellationToken).ConfigureAwait(false);
- var existing = await _db.Sequences.GetAsync(target.Id, cancellationToken).ConfigureAwait(false);
- await _db.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
- await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false);
+ ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
+ await _db.Sequences.RecordVerifiedAsync(target.Id, clean ? SequenceStatus.Verified : SequenceStatus.HasErrors,
+ clean ? SequenceStatus.HasErrors : SequenceStatus.Verified, now, cancellationToken).ConfigureAwait(false);
+ if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false);
+ }
}
}
diff --git a/src/Titano.Data/Archiving/FrameTiers.cs b/src/Titano.Data/Archiving/FrameTiers.cs
index 6b7d27ed..86110d20 100644
--- a/src/Titano.Data/Archiving/FrameTiers.cs
+++ b/src/Titano.Data/Archiving/FrameTiers.cs
@@ -61,6 +61,13 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi
///
/// Whether every such frame follows its proxy even where no tier is recorded (analyses and the
/// proxy pass); otherwise only the frames in do.
+ ///
+ /// The sidecar and the index are written under one lease, the index from the sidecar read under it, and the index
+ /// row gets only what tiers change (): a whole row written back
+ /// would undo what another writer committed meanwhile, an ingest's frames or a settings edit (the server's PATCH
+ /// takes no lease). The folder's scan stamps are recorded only when the index held the folder as it was
+ /// ().
+ ///
/// The archive is offline or read-only.
public async Task> RecordAsync(Archive archive, Sequence sequence, IReadOnlyDictionary tiers,
bool proxies, CancellationToken cancellationToken)
@@ -69,12 +76,12 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var folder = paths.Sequence(sequence.Folder);
- var changed = new List();
- ProxyPolicy? policy = null;
- var manifest = await writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m =>
+ await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
{
- changed.Clear();
+ var m = SequenceWriter.ReadLocked(folder, sequence.Id);
+ var changed = new List();
var gained = false;
+ ProxyPolicy? policy = null;
var frames = new SidecarFrame[m.Frames.Count];
for (var i = 0; i < frames.Length; i++)
{
@@ -96,21 +103,26 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi
frames[i] = updated;
changed.Add(updated);
}
- if (changed.Count == 0) return null;
- return m with
+ if (changed.Count == 0) return m.Frames;
+ var inStep = await indexer.InStepAsync(archive, folder, m, cancellationToken).ConfigureAwait(false);
+ var manifest = m with
{
Frames = frames,
Status = gained ? m.Status | SequenceStatus.HasWarnings : m.Status,
UpdatedAt = DateTimeOffset.UtcNow,
};
- }, cancellationToken).ConfigureAwait(false);
- if (changed.Count == 0) return manifest.Frames;
- await db.Frames.UpdateTiersAsync(changed.ConvertAll(f => new FrameTierFields(f.Id, f.Flags, f.PreviewW ?? 0, f.PreviewH ?? 0)), cancellationToken)
- .ConfigureAwait(false);
- if (await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) is { } existing)
- await db.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, folder, existing), cancellationToken).ConfigureAwait(false);
- await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
- return manifest.Frames;
+ ManifestStore.WriteSequence(folder.ManifestPath, manifest);
+ await using (var tx = await db.BeginAsync(cancellationToken).ConfigureAwait(false))
+ {
+ await tx.Frames.UpdateTiersAsync(changed.ConvertAll(f => new FrameTierFields(f.Id, f.Flags, f.PreviewW ?? 0, f.PreviewH ?? 0)), cancellationToken)
+ .ConfigureAwait(false);
+ await tx.Sequences.RecordTiersAsync(sequence.Id, gained ? SequenceStatus.HasWarnings : SequenceStatus.None,
+ frames.Count(f => (f.Flags & FrameFlags.PreviewOnly) != 0), cancellationToken).ConfigureAwait(false);
+ await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
+ }
+ if (inStep) await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
+ return manifest.Frames;
+ }
}
///
diff --git a/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs b/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs
index fa1580d7..c498e324 100644
--- a/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs
+++ b/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs
@@ -67,10 +67,11 @@ internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer,
///
/// Queues a background (thumbnails or proxies) of each sequence this run indexed whose
/// flag is clear: new and adopted sequences, and those whose photos changed (the scan clears the
- /// flag then). After a resumed run, which may have indexed sequences before the interruption, of every sequence of the
- /// archive without the flag but those in . One pass per sequence, as after an upload: a folder
- /// renamed, deleted or damaged before its pass runs fails that pass alone, and the other sequences still get their
- /// images. A sequence whose own pass of that kind is already queued is left out.
+ /// flag then, in this server's own row when another writer listed them in the sidecar itself). After a resumed run,
+ /// which may have indexed sequences before the interruption, of every sequence of the archive without the flag but
+ /// those in . One pass per sequence, as after an upload: a folder renamed, deleted or
+ /// damaged before its pass runs fails that pass alone, and the other sequences still get their images. A sequence
+ /// whose own pass of that kind is already queued is left out.
///
private async Task QueueImagesAsync(JobContext context, Archive archive, JobKind pass, SequenceStatus built, IReadOnlyList indexed,
bool resumed, IReadOnlySet awaiting)
diff --git a/src/Titano.Data/Archiving/MassRenamer.cs b/src/Titano.Data/Archiving/MassRenamer.cs
index e6b1c11e..43fe6a25 100644
--- a/src/Titano.Data/Archiving/MassRenamer.cs
+++ b/src/Titano.Data/Archiving/MassRenamer.cs
@@ -210,12 +210,22 @@ public sealed class MassRenamer
.FirstOrDefault(l => !taken.Contains(l.Id) && folders.Any(f => RenamePlanner.IsInside(f, l.Folder)));
}
- /// Sidecar title, then index (folder, frames' paths); each part is idempotent so a resumed step can repeat it.
+ ///
+ /// Sidecar title, then index (folder, title, frames' paths); each part is idempotent so a resumed step can repeat it.
+ /// The index row changes in its folder and title only (): a settings edit
+ /// committed meanwhile stays, and frames another process listed in the sidecar are left to the next scan, for which
+ /// the folder keeps no stamps unless the index held it as it was
+ /// ().
+ ///
private async Task RecordAsync(Archive archive, ArchivePaths paths, RenameStep step, CancellationToken cancellationToken)
{
var sequence = paths.Sequence(step.ToFolder);
var manifest = SequenceWriter.ReadLocked(sequence, step.SequenceId);
- if (manifest.Title != step.ToTitle)
+ // The folder moved with its files' stamps, which the index recorded at its old place.
+ var inStep = await _indexer.InStepAsync(archive, step.FromFolder, sequence, manifest, cancellationToken).ConfigureAwait(false);
+ // A step that only moves the folder keeps the title, which an edit may have changed since the plan.
+ var retitle = step.FromTitle != step.ToTitle;
+ if (retitle && manifest.Title != step.ToTitle)
{
manifest = manifest with
{
@@ -227,12 +237,11 @@ public sealed class MassRenamer
}
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
{
- var existing = await tx.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false);
- await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
+ await tx.Sequences.RelocateAsync(step.SequenceId, step.ToFolder, retitle ? step.ToTitle : null, cancellationToken).ConfigureAwait(false);
if (step.MovesFolder) await tx.Frames.RebaseFolderAsync(step.SequenceId, step.FromFolder, step.ToFolder, cancellationToken).ConfigureAwait(false);
await tx.Archives.DeleteFolderStateAsync(archive.Id, step.FromFolder, cancellationToken).ConfigureAwait(false);
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
}
- await _indexer.RecordStampsAsync(archive, sequence, step.SequenceId, cancellationToken).ConfigureAwait(false);
+ if (inStep) await _indexer.RecordStampsAsync(archive, sequence, step.SequenceId, cancellationToken).ConfigureAwait(false);
}
}
diff --git a/src/Titano.Data/Processing/ProjectStore.cs b/src/Titano.Data/Processing/ProjectStore.cs
index 9204c0f9..4c12d5bf 100644
--- a/src/Titano.Data/Processing/ProjectStore.cs
+++ b/src/Titano.Data/Processing/ProjectStore.cs
@@ -40,25 +40,30 @@ internal sealed class ProjectStore(TitanoDatabase db, SequenceWriter writer)
/// The stored settings cannot be read; nothing is written.
public async Task UpdateAsync(ArchivePaths paths, Sequence sequence, Func edit, CancellationToken cancellationToken)
{
- var folder = paths.Sequence(sequence.Folder);
await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
+ return await UpdateLockedAsync(paths, sequence, edit, cancellationToken).ConfigureAwait(false);
+ }
+
+ /// for a caller that holds the sequence lock already.
+ /// The stored settings cannot be read; nothing is written.
+ public async Task UpdateLockedAsync(ArchivePaths paths, Sequence sequence, Func edit, CancellationToken cancellationToken)
+ {
+ var folder = paths.Sequence(sequence.Folder);
+ for (var attempt = 1; ; attempt++)
{
- for (var attempt = 1; ; attempt++)
- {
- var row = await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) ?? sequence;
- var current = Read(row, folder);
- var before = current.ToJson();
- var edited = edit(current);
- var after = edited.ToJson();
- if (after == before) return false;
- AtomicFile.WriteAllText(folder.ProjectPath, after);
- // The Library shows the project's hero frame (advice fills it, a sync takes another PC's).
- var hero = await HeroFrame.EditAsync(db, sequence.Id, current.HeroFrame, edited.HeroFrame, cancellationToken).ConfigureAwait(false);
- var result = await db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = after, HeroFrameId = hero }, row.SettingsVersion,
- cancellationToken).ConfigureAwait(false);
- // A conflict means the API changed the settings meanwhile: apply the edit again on top of them.
- if (result.Outcome != SequenceEditOutcome.Conflict || attempt == Attempts) return true;
- }
+ var row = await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) ?? sequence;
+ var current = Read(row, folder);
+ var before = current.ToJson();
+ var edited = edit(current);
+ var after = edited.ToJson();
+ if (after == before) return false;
+ AtomicFile.WriteAllText(folder.ProjectPath, after);
+ // The Library shows the project's hero frame (advice fills it, a sync takes another PC's).
+ var hero = await HeroFrame.EditAsync(db, sequence.Id, current.HeroFrame, edited.HeroFrame, cancellationToken).ConfigureAwait(false);
+ var result = await db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = after, HeroFrameId = hero }, row.SettingsVersion,
+ cancellationToken).ConfigureAwait(false);
+ // A conflict means the API changed the settings meanwhile: apply the edit again on top of them.
+ if (result.Outcome != SequenceEditOutcome.Conflict || attempt == Attempts) return true;
}
}
}
diff --git a/src/Titano.Data/Processing/SequenceAnalyser.cs b/src/Titano.Data/Processing/SequenceAnalyser.cs
index 46b73035..8307590d 100644
--- a/src/Titano.Data/Processing/SequenceAnalyser.cs
+++ b/src/Titano.Data/Processing/SequenceAnalyser.cs
@@ -52,56 +52,64 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti
frames = WithTiers(frames, await _tiers.RecordAsync(archive, sequence, probed, proxies: true, cancellationToken).ConfigureAwait(false));
var metadata = frames.LightFrames();
- DeepAnalysisReport? report;
+ DeepAnalysisReport? report = null;
if (deep)
{
- // Proxies may have arrived during the pass: the advice below follows the frames as they are at its end.
+ // Proxies may have arrived during the pass: the flags below follow the frames as they are at its end, and the
+ // report is checked against them under the lock.
(report, var decoded) = await AnalyseDeepAsync(archive, frames, metadata, folder, progress, cancellationToken).ConfigureAwait(false);
frames = WithTiers(frames, await _tiers.RecordAsync(archive, sequence, decoded, proxies: true, cancellationToken).ConfigureAwait(false));
metadata = [.. metadata.Select((m, i) => m with { Flags = frames.Frames[i].Flags })];
}
- else
- {
- report = ReadReport(folder, frames);
- }
var light = LightAnalyzer.Analyze(metadata);
var context = SequenceContext.FromFrames(frames.Frames, options.AdvisorCodecs);
bool changed, unreadable = false;
- try
- {
- // The frame rate follows the user's "fit to N seconds" when one is set (BRIEF §10.3).
- changed = await projects.UpdateAsync(paths, sequence,
- settings => Advisor.Apply(settings, Advisor.Recommend(light, report, context with { TargetDurationSeconds = settings.Timing.FitToSeconds })),
- cancellationToken).ConfigureAwait(false);
- }
- catch (InvalidDataException ex)
- {
- Core.Diagnostics.Log.Warning("processing", $"The project of {sequence.DisplayTitle} cannot be read", ex);
- (changed, unreadable) = (false, true);
- }
-
- if (changed) await options.NotifyAsync(sequence.Id, ProcessingOptions.SettingsChange, cancellationToken).ConfigureAwait(false);
-
int warnings;
// Under the sequence lock, which a deep pass of the same sequence also takes to record its report: a light pass
- // then either sees that report or runs before it is recorded, and never clears what it recorded.
+ // then either sees that report or runs before it is recorded, and never clears what it recorded. The report counts
+ // only while it describes the frames as they are under the lock (ReadReport), so a deep pass loads them again: a
+ // proxy that arrived while it ran (for hours) makes its verdict on that frame's original stale. The light analysis
+ // the proxy queued then either ran before this, and the advice, status and warnings stay without the report as it
+ // left them, or runs after.
await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
{
- report ??= ReadReport(folder, frames);
+ if (deep)
+ {
+ var now = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, options.Archive.Decoder, cancellationToken).ConfigureAwait(false);
+ if (!Describes(report!, now)) report = null;
+ }
+ else
+ {
+ report = ReadReport(folder, frames);
+ }
+ try
+ {
+ // The frame rate follows the user's "fit to N seconds" when one is set (BRIEF §10.3).
+ changed = await projects.UpdateLockedAsync(paths, sequence,
+ settings => Advisor.Apply(settings, Advisor.Recommend(light, report, context with { TargetDurationSeconds = settings.Timing.FitToSeconds })),
+ cancellationToken).ConfigureAwait(false);
+ }
+ catch (InvalidDataException ex)
+ {
+ Core.Diagnostics.Log.Warning("processing", $"The project of {sequence.DisplayTitle} cannot be read", ex);
+ (changed, unreadable) = (false, true);
+ }
warnings = Advisor.Recommend(light, report, context).Findings.Count(f => f.Severity >= FindingSeverity.Warning);
var set = SequenceStatus.AnalyzedLight | (report is null ? SequenceStatus.None : SequenceStatus.AnalyzedDeep)
| (warnings > 0 ? SequenceStatus.HasWarnings : SequenceStatus.None);
// Without a current report (none, or made before frames were added) the deep status no longer holds.
var clear = (report is null ? SequenceStatus.AnalyzedDeep : SequenceStatus.None) | (warnings > 0 ? SequenceStatus.None : SequenceStatus.HasWarnings);
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
+ var inStep = await indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
var status = (manifest.Status | set) & ~clear;
if (status != manifest.Status) ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow });
var summary = deep && report is not null ? AnalysisSummary.From(report).ToJson() : null;
await db.Sequences.RecordAnalysisAsync(sequence.Id, set, clear, warnings, summary, cancellationToken).ConfigureAwait(false);
- await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
+ if (inStep) await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
}
+ if (changed) await options.NotifyAsync(sequence.Id, ProcessingOptions.SettingsChange, cancellationToken).ConfigureAwait(false);
var presets = deep ? SequencePresets.Generate(light, report, context) : [];
return new AnalysisOutcome(sequence, frames.Count, warnings, changed, unreadable, presets);
}
@@ -111,17 +119,21 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti
/// frame it found unreadable has been read from its proxy since (one sent for an original this host cannot decode,
/// BRIEF §2.3), so that verdict and the frame's missing measurements no longer hold.
///
- public static DeepAnalysisReport? ReadReport(SequencePaths folder, SequenceFrames frames)
+ public static DeepAnalysisReport? ReadReport(SequencePaths folder, SequenceFrames frames) =>
+ ReadReport(folder, frames.Count) is { } report && Describes(report, frames) ? report : null;
+
+ /// Whether describes as this host reads them now ().
+ private static bool Describes(DeepAnalysisReport report, SequenceFrames frames)
{
- var report = ReadReport(folder, frames.Count);
- if (report is null || frames.Proxied is not { } proxied) return report;
+ if (report.FrameCount != frames.Count) return false;
+ if (frames.Proxied is not { } proxied) return true;
HashSet measured = report.ProxyFrames is { } read ? [.. read] : [];
foreach (var index in report.Issues.Undecodable)
{
if (index < frames.Count && proxied[index] && !measured.Contains(index) && (frames.Frames[index].Flags & SequenceFrames.NotAnalysed) == 0)
- return null;
+ return false;
}
- return report;
+ return true;
}
/// The saved deep report when it covers frames; null when absent, stale or unreadable.
diff --git a/src/Titano.Data/Repositories/ArchiveRepository.cs b/src/Titano.Data/Repositories/ArchiveRepository.cs
index dec57a0a..d59d8087 100644
--- a/src/Titano.Data/Repositories/ArchiveRepository.cs
+++ b/src/Titano.Data/Repositories/ArchiveRepository.cs
@@ -10,6 +10,7 @@ public sealed record FolderScanState(string Folder, Guid? SequenceId, FileStamp?
public sealed class ArchiveRepository
{
private const string Columns = "id, name, root_path, folder_template, file_template, schema_version, created, proxy_long_edge";
+ private const string FolderStateSelect = "SELECT folder, sequence_id, sidecar_size, sidecar_ticks, photos_ticks FROM archive_folders";
private static readonly string UpsertSql = SqlDialect.Upsert("archives",
["id", "name", "root_path", "folder_template", "file_template", "schema_version", "created", "proxy_long_edge"], 1);
@@ -39,15 +40,14 @@ public sealed class ArchiveRepository
/// Scan stamps of every folder of an archive, by folder.
public async Task> GetFolderStatesAsync(Guid archiveId, CancellationToken cancellationToken = default)
{
- var rows = await _db.ListAsync(
- "SELECT folder, sequence_id, sidecar_size, sidecar_ticks, photos_ticks FROM archive_folders WHERE archive_id = $1", [archiveId],
- r => new FolderScanState(r.GetString(0), r.GetGuidOrNull(1),
- r.GetInt64OrNull(3) is { } sidecar ? new FileStamp(r.GetInt64(2), new DateTime(sidecar, DateTimeKind.Utc)) : null,
- r.GetInt64OrNull(4) is { } photos ? new FileStamp(0, new DateTime(photos, DateTimeKind.Utc)) : null),
- cancellationToken).ConfigureAwait(false);
+ var rows = await _db.ListAsync($"{FolderStateSelect} WHERE archive_id = $1", [archiveId], MapFolderState, cancellationToken).ConfigureAwait(false);
return rows.ToDictionary(s => s.Folder, StringComparer.Ordinal);
}
+ /// Scan stamps of one folder of an archive; null when none are recorded.
+ public Task GetFolderStateAsync(Guid archiveId, string folder, CancellationToken cancellationToken = default) =>
+ _db.SingleAsync($"{FolderStateSelect} WHERE archive_id = $1 AND folder = $2", [archiveId, folder], MapFolderState, cancellationToken);
+
public async Task SetFolderStateAsync(Guid archiveId, FolderScanState state, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync(FolderStateUpsertSql,
[archiveId, state.Folder, state.SequenceId, state.Sidecar?.Size, state.Sidecar?.ModifiedUtc.Ticks, state.Photos?.ModifiedUtc.Ticks, DateTimeOffset.UtcNow],
@@ -61,6 +61,10 @@ public sealed class ArchiveRepository
public Task ClearFolderStatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
_db.ExecuteAsync("DELETE FROM archive_folders WHERE archive_id = $1", [archiveId], cancellationToken);
+ private static FolderScanState MapFolderState(DbRow r) => new(r.GetString(0), r.GetGuidOrNull(1),
+ r.GetInt64OrNull(3) is { } sidecar ? new FileStamp(r.GetInt64(2), new DateTime(sidecar, DateTimeKind.Utc)) : null,
+ r.GetInt64OrNull(4) is { } photos ? new FileStamp(0, new DateTime(photos, DateTimeKind.Utc)) : null);
+
private static Archive Map(DbRow r) => new()
{
Id = r.GetGuid(0),
diff --git a/src/Titano.Data/Repositories/SequenceRepository.cs b/src/Titano.Data/Repositories/SequenceRepository.cs
index 3ee1a531..9a17e6f3 100644
--- a/src/Titano.Data/Repositories/SequenceRepository.cs
+++ b/src/Titano.Data/Repositories/SequenceRepository.cs
@@ -195,6 +195,24 @@ public sealed class SequenceRepository
await _db.ExecuteAsync("UPDATE sequences SET status = (status | $2) & ~$3, updated_at = $4 WHERE id = $1",
[id, (int)set, (int)clear, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0;
+ ///
+ /// Records decode tiers written to the sequence's frames (): the status
+ /// flags in and the preview-only count, in one statement. The frames, their versions and the
+ /// settings stay as they are, so an ingest or a settings edit committed meanwhile is never written back over.
+ ///
+ public async Task RecordTiersAsync(Guid id, SequenceStatus set, int previewOnlyCount, CancellationToken cancellationToken = default) =>
+ await _db.ExecuteAsync("UPDATE sequences SET status = status | $2, preview_only_count = $3, updated_at = $4 WHERE id = $1",
+ [id, (int)set, previewOnlyCount, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0;
+
+ ///
+ /// Records a verify: status flags set and cleared and the verify time, in one statement; every other field stays as
+ /// it is ().
+ ///
+ public async Task RecordVerifiedAsync(Guid id, SequenceStatus set, SequenceStatus clear, DateTimeOffset verifiedAt,
+ CancellationToken cancellationToken = default) =>
+ await _db.ExecuteAsync("UPDATE sequences SET status = (status | $2) & ~$3, verified_at = $4, updated_at = $5 WHERE id = $1",
+ [id, (int)set, (int)clear, verifiedAt, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0;
+
///
/// Records an analysis: status flags set and cleared, the warning count and, for a deep analysis, the compact
/// summary ( null keeps the stored one).
@@ -390,19 +408,23 @@ public sealed class SequenceRepository
[archiveId, sessionKey], cancellationToken).ConfigureAwait(false))!.Value;
}, cancellationToken);
- /// Moves a sequence to another folder and/or title (mass rename); false when it does not exist.
+ ///
+ /// Moves a sequence to another folder and, with , retitles it (mass rename; settings version
+ /// + 1 when the title changes); every other field stays as the locked row has it. False when it does not exist.
+ ///
+ /// The new title; null keeps the current one.
public Task RelocateAsync(Guid id, string folder, string? title, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
var current = await session.QuerySingleOrDefaultAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken)
.ConfigureAwait(false);
if (current is null) return false;
- var titleChanged = current.Title != title;
+ var retitled = title is not null && current.Title != title;
await UpsertRowAsync(session, current with
{
Folder = folder,
- Title = title,
- SettingsVersion = titleChanged ? current.SettingsVersion + 1 : current.SettingsVersion,
+ Title = retitled ? title : current.Title,
+ SettingsVersion = retitled ? current.SettingsVersion + 1 : current.SettingsVersion,
UpdatedAt = DateTimeOffset.UtcNow,
}, cancellationToken).ConfigureAwait(false);
return true;
diff --git a/src/Titano.Server/Storage/ArchiveAssetLocator.cs b/src/Titano.Server/Storage/ArchiveAssetLocator.cs
index d9d3d1c6..a36480d6 100644
--- a/src/Titano.Server/Storage/ArchiveAssetLocator.cs
+++ b/src/Titano.Server/Storage/ArchiveAssetLocator.cs
@@ -7,8 +7,9 @@ namespace Titano.Server.Storage;
///
/// Proxies are files under the sequence's proxies/ folder; a thumbnail is the JPEG byte range of its entry in
/// thumbs.pack. A thumbnail made from a different original (the frame was renumbered or replaced) counts as not
-/// built yet, like a missing one, and so does a proxy whose capture time is another frame's, or one smaller than the
-/// archive's policy sent for a frame the server cannot decode ().
+/// built yet, like a missing one (: one drawn before a verify hashed the photo is still its
+/// own), and so does a proxy whose capture time is another frame's, or one smaller than the archive's policy sent for a
+/// frame the server cannot decode ().
///
internal sealed class ArchiveAssetLocator : IFrameAssetLocator
{
@@ -34,7 +35,14 @@ internal sealed class ArchiveAssetLocator : IFrameAssetLocator
{
using var pack = ThumbPackReader.Open(paths.ThumbsPackPath);
if (!pack.Entries.TryGetValue(frame.Index, out var entry) || entry.Offset + entry.Length > pack.Length) return null;
- if (frame.Sha256 is { Length: ThumbPack.HashLength * 2 } sha && !entry.SourceHash.AsSpan().SequenceEqual(Convert.FromHexString(sha))) return null;
+ if (frame.Sha256 is { Length: ThumbPack.HashLength * 2 } sha && !ThumbPack.IsFrom(entry.SourceHash, sha, SequenceFile(paths, frame), frame.Size)) return null;
return new AssetLocation(paths.ThumbsPackPath, entry.Offset, entry.Length, Jpeg);
}
+
+ /// The frame's file as its sidecar names it (relative to the sequence folder), which a thumbnail drawn before the photo was hashed is fingerprinted with.
+ private static string SequenceFile(SequencePaths paths, Frame frame)
+ {
+ var prefix = paths.ArchiveRelative(string.Empty);
+ return frame.RelativePath.StartsWith(prefix, StringComparison.Ordinal) ? frame.RelativePath[prefix.Length..] : frame.RelativePath;
+ }
}
diff --git a/tests/Titano.Core.Tests/Archiving/ThumbPackTests.cs b/tests/Titano.Core.Tests/Archiving/ThumbPackTests.cs
index b360aafd..38dd3abf 100644
--- a/tests/Titano.Core.Tests/Archiving/ThumbPackTests.cs
+++ b/tests/Titano.Core.Tests/Archiving/ThumbPackTests.cs
@@ -150,6 +150,25 @@ public sealed class ThumbPackTests(TestContext context)
for (var i = 0; i < 20; i++) Assert.BytesEqual(Payload(i + 300), compacted.Read(i));
}
+ [Test]
+ public void AThumbnailIsFromItsOriginalAlsoAfterTheOriginalWasHashed()
+ {
+ var sha = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData("photo"u8));
+ var other = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData("another photo"u8));
+ var drawnUnhashed = ThumbPack.Fingerprint(null, "IMG_0001.JPG", 1000);
+ var drawnHashed = ThumbPack.Fingerprint(sha, "IMG_0001.JPG", 1000);
+
+ Assert.True(ThumbPack.IsFrom(drawnUnhashed, null, "IMG_0001.JPG", 1000), "unhashed");
+ Assert.True(ThumbPack.IsFrom(drawnHashed, sha, "IMG_0001.JPG", 1000), "hashed");
+ // Drawn before a verify hashed the photo: the hash changes nothing about it.
+ Assert.True(ThumbPack.IsFrom(drawnUnhashed, sha, "IMG_0001.JPG", 1000), "hashed since");
+ // Another photo: another hash, or another file or size at that frame index.
+ Assert.False(ThumbPack.IsFrom(drawnHashed, other, "IMG_0001.JPG", 1000), "another hash");
+ Assert.False(ThumbPack.IsFrom(drawnUnhashed, sha, "IMG_0002.JPG", 1000), "another file");
+ Assert.False(ThumbPack.IsFrom(drawnUnhashed, null, "IMG_0001.JPG", 1001), "another size");
+ Assert.False(ThumbPack.IsFrom(drawnHashed, null, "IMG_0001.JPG", 1000), "a hash the frame no longer has");
+ }
+
[Test]
public void AMissingOrForeignFileReadsAsEmpty()
{
diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveFixture.cs b/tests/Titano.Data.Tests/Archiving/ArchiveFixture.cs
index 1d1aea42..5cbf7510 100644
--- a/tests/Titano.Data.Tests/Archiving/ArchiveFixture.cs
+++ b/tests/Titano.Data.Tests/Archiving/ArchiveFixture.cs
@@ -55,6 +55,20 @@ internal sealed class ArchiveFixture : IAsyncDisposable
public Task ScanAsync(ScanMode mode = ScanMode.Incremental) =>
Service.Indexer.ScanAsync(Archive, mode, null, _ => ValueTask.CompletedTask, Ct);
+ ///
+ /// Another process writing the same archive, such as a desktop sharing a server's archive folder: its own index
+ /// (scanned) and archive service; disposed with the fixture.
+ ///
+ public async Task<(ArchiveService Service, Archive Archive)> OtherWriterAsync()
+ {
+ var db = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(_context.CreateTempDirectory("other"), "titano.db")), Ct);
+ _owned.Add(db);
+ var service = new ArchiveService(db, new ArchiveServiceOptions { Owner = "other", LeaseWait = TimeSpan.FromSeconds(20) });
+ var archive = await service.OpenAsync(Archive.RootPath, Ct);
+ await service.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, Ct);
+ return (service, archive);
+ }
+
/// A job engine with the archive handlers, started; disposed with the fixture.
public async Task StartEngineAsync()
{
diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs
index 04b6bdae..ddde498f 100644
--- a/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs
+++ b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs
@@ -365,6 +365,38 @@ public sealed class ArchiveIngestTests(TestContext context)
Assert.Equal("2026/2026-09-29 Canon EOS R5", (await fx.Db.Sequences.GetAsync(waiting.Id))!.Folder);
}
+ [Test]
+ public async Task ARebuildKeepsSequencesThatGetTheirFolderWhileItRuns()
+ {
+ await using var fx = await CreateAsync(context);
+ Photos.WriteSeries(Path.Combine(fx.Archive.RootPath, "Trips", "Comet"), Start, 2);
+ await fx.ScanAsync();
+ var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset");
+
+ // Uploads go on while the rebuild runs: after it listed the folders, the sequence waiting for its first frame gets
+ // it, and a new upload session creates a sequence.
+ IngestResult? first = null, session = null;
+ var rebuilt = await fx.Service.Indexer.ScanAsync(fx.Archive, ScanMode.Rebuild, null, async progress =>
+ {
+ if (progress.FoldersDone < progress.FoldersTotal || first is not null) return;
+ first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddDays(1), 1), sequenceId: waiting.Id), fx.Ct);
+ session = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddDays(2), 2), sessionKey: "upload-2"), fx.Ct);
+ }, fx.Ct);
+
+ // Neither is dropped, though the folders listed at the start hold neither.
+ Assert.Equal(0, rebuilt.Removed);
+ Assert.Equal(waiting.Id, Assert.NotNull(first).SequenceId);
+ var placed = Assert.NotNull(await fx.Db.Sequences.GetAsync(waiting.Id));
+ Assert.Equal(("2026/2026-09-29 Canon EOS R5", 1, "Harbour sunset"), (placed.Folder, placed.FrameCount, placed.Title));
+ var created = Assert.NotNull(await fx.Db.Sequences.GetAsync(Assert.NotNull(session).SequenceId));
+ Assert.Equal(1, created.FrameCount);
+ // The session's next upload lands in the same sequence, not in a second one beside it.
+ Assert.Equal(created.Id, await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, "upload-2", fx.Ct));
+ var next = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddDays(2).AddSeconds(10), 3), sessionKey: "upload-2"), fx.Ct);
+ Assert.Equal(created.Id, next.SequenceId);
+ Assert.Equal(3, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count);
+ }
+
[Test]
public async Task ADamagedRootSidecarIsRebuiltUnderTheRootSequenceNotOneWaitingForItsFirstFrame()
{
diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs
index 8412d43f..90a9aa89 100644
--- a/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs
+++ b/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs
@@ -175,6 +175,210 @@ public sealed class ArchiveJobTests(TestContext context)
Assert.Count(7, Directory.GetFiles(fx.Paths.Sequence("a").Proxies));
}
+ [Test]
+ public async Task AServerScanQueuesTheImagesOfFramesAnotherWriterListedInTheSidecarItself()
+ {
+ await using var fx = await CreateAsync(context);
+ Populate(fx);
+ await using var engine = ServerEngine(fx);
+ await engine.StartAsync(fx.Ct);
+ var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
+ await fx.WaitForAsync(scan.Id, JobState.Done);
+ var first = new HashSet();
+ foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
+ {
+ foreach (var pass in (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items)
+ {
+ await fx.WaitForAsync(pass.Id, JobState.Done);
+ first.Add(pass.Id);
+ }
+ }
+ var a = (await LocationsAsync(fx)).Single(l => l.Folder == "a");
+ Assert.True(Built((await fx.Db.Sequences.GetAsync(a.Id))!.Status), "built");
+
+ // A desktop sharing the archive imports a photo taken before the others into the sequence: it files the photo and
+ // lists it in the sidecar itself, whose statuses it keeps, so the scan finds no photo to reconcile.
+ var (desktop, archive) = await fx.OtherWriterAsync();
+ var staged = fx.Stage(Start.AddMinutes(-5), seed: 90);
+ await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, a.Id, null, null, staged.Path, "IMG_0000.JPG", staged.Size, staged.Sha256, null, null),
+ fx.Ct);
+ var sidecar = fx.Paths.Sequence("a").ManifestPath;
+ Assert.True(Built(ManifestStore.ReadSequence(sidecar).Status), "the other writer keeps the statuses");
+
+ var again = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
+ await fx.WaitForAsync(again.Id, JobState.Done);
+ Assert.True(Built(ManifestStore.ReadSequence(sidecar).Status), "the server clears what it makes in its own row only");
+
+ // The frames changed behind the index: the statuses no longer hold, and the sequence gets a pass of each kind.
+ foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
+ {
+ var pass = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items.Single(j => !first.Contains(j.Id));
+ Assert.SequenceEqual([a.Id], pass.TargetIds);
+ await fx.WaitForAsync(pass.Id, JobState.Done);
+ }
+ var row = (await fx.Db.Sequences.GetAsync(a.Id))!;
+ Assert.Equal(7, row.FrameCount);
+ Assert.True(Built(row.Status), "built again");
+ // Every frame has its own thumbnail and proxy: the new first one, and the others it renumbered.
+ var folder = fx.Paths.Sequence("a");
+ using var pack = ThumbPackReader.Open(folder.ThumbsPackPath);
+ foreach (var frame in ManifestStore.ReadSequence(sidecar).Frames)
+ {
+ Assert.True(pack.Entries.GetValueOrDefault(frame.I) is { } entry
+ && entry.SourceHash.AsSpan().SequenceEqual(ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size)), $"thumbnail of {frame.File}");
+ Assert.True(ArchiveImages.IsProxyOf(folder.Proxy(frame.I), frame.CapturedAt, frame.W, frame.H, null), $"proxy of {frame.File}");
+ }
+ }
+
+ [Test]
+ public async Task AnImagePassMarksTheSequenceBuiltOnlyWhenItsSidecarStillListsTheFramesThePassRead()
+ {
+ await using var fx = await CreateAsync(context);
+ // The pass commits its pack and checkpoints after 200 frames.
+ Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 201);
+ await fx.ScanAsync();
+ var a = (await LocationsAsync(fx)).Single();
+ var sequence = (await fx.Db.Sequences.GetAsync(a.Id))!;
+
+ // While it runs, another writer copies in a photo taken before the others, and a scan reconciles it.
+ var scanned = false;
+ var result = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, async (_, _) =>
+ {
+ if (scanned) return;
+ scanned = true;
+ Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start.AddMinutes(-5), 1, firstSeed: 900, firstNumber: 900);
+ Assert.SequenceEqual([a.Id], (await fx.ScanAsync()).Sequences);
+ }, fx.Ct);
+
+ // The pass drew every frame it read, but not the new photo: the sequence is not built.
+ Assert.True(scanned, "scanned during the pass");
+ Assert.Equal((201, 0), (result.Built, result.Undecodable));
+ var row = (await fx.Db.Sequences.GetAsync(a.Id))!;
+ Assert.Equal(202, row.FrameCount);
+ Assert.False(row.Status.HasFlag(SequenceStatus.ThumbsBuilt), row.Status.ToString());
+ Assert.False(ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Status.HasFlag(SequenceStatus.ThumbsBuilt), "sidecar flag");
+
+ // So a scan's follow-up (here a resumed one, which lists the whole archive) queues the next pass, which makes them.
+ await using var engine = ServerEngine(fx);
+ var resumed = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Scan(fx.Archive), "after:a");
+ await engine.StartAsync(fx.Ct);
+ await fx.WaitForAsync(resumed.Id, JobState.Done);
+ var pass = (await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items.Single();
+ Assert.SequenceEqual([a.Id], pass.TargetIds);
+ await fx.WaitForAsync(pass.Id, JobState.Done);
+ Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "built");
+ using var pack = ThumbPackReader.Open(fx.Paths.Sequence("a").ThumbsPackPath);
+ Assert.Count(202, pack.Entries);
+ }
+
+ [Test]
+ public async Task AnImagePassOvertakenByAnUploadKeepsTheSequenceBuiltWhileThePassAfterTheUploadWaits()
+ {
+ await using var fx = await CreateAsync(context);
+ // The pass commits its pack and checkpoints after 200 frames.
+ Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 201);
+ await fx.ScanAsync();
+ var a = (await LocationsAsync(fx)).Single();
+ var sequence = (await fx.Db.Sequences.GetAsync(a.Id))!;
+
+ // While it runs, an upload lands in the sequence and queues the next pass, as the server does after every upload.
+ await using var engine = ServerEngine(fx);
+ Job? next = null;
+ await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, async (_, _) =>
+ {
+ if (next is not null) return;
+ var staged = fx.Stage(Start.AddMinutes(-5), seed: 90);
+ await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, a.Id, null, null, staged.Path, "IMG_0000.JPG", staged.Size, staged.Sha256, null, null),
+ fx.Ct);
+ next = await engine.EnqueueAsync(ArchiveJobs.Thumbnails([a.Id], "a"), cancellationToken: fx.Ct);
+ }, fx.Ct);
+
+ // The pass did not draw the new photo, but the one queued after it does: the sequence stays built meanwhile, so a
+ // server still offers the thumbnails it has.
+ Assert.NotNull(next);
+ Assert.Equal(202, (await fx.Db.Sequences.GetAsync(a.Id))!.FrameCount);
+ Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "row flag");
+ Assert.True(ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Status.HasFlag(SequenceStatus.ThumbsBuilt), "sidecar flag");
+
+ await engine.StartAsync(fx.Ct);
+ await fx.WaitForAsync(next!.Id, JobState.Done);
+ Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "built");
+ var folder = fx.Paths.Sequence("a");
+ using var pack = ThumbPackReader.Open(folder.ThumbsPackPath);
+ foreach (var frame in ManifestStore.ReadSequence(folder.ManifestPath).Frames)
+ {
+ Assert.True(pack.Entries.GetValueOrDefault(frame.I) is { } entry && ThumbPack.IsFrom(entry.SourceHash, frame.Sha256, frame.File, frame.Size),
+ $"thumbnail of {frame.File}");
+ }
+ }
+
+ [Test]
+ public async Task ThumbnailsDrawnBeforeAVerifyHashedThePhotosStayTheirs()
+ {
+ await using var fx = await CreateAsync(context);
+ Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3);
+ await fx.ScanAsync();
+ var a = (await LocationsAsync(fx)).Single();
+ var first = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(a.Id))!, false, 0,
+ (_, _) => ValueTask.CompletedTask, fx.Ct);
+ Assert.Equal(3, first.Built);
+
+ // The verify gives the photos the scan adopted their first hash; what was drawn from them is still theirs.
+ var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [a.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
+ Assert.Equal(3L, report.Hashed);
+ var again = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(a.Id))!, false, 0,
+ (_, _) => ValueTask.CompletedTask, fx.Ct);
+ Assert.Equal((0, 3), (again.Built, again.Reused));
+ Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "built");
+ Assert.True(ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Frames.All(f => f.Sha256 is not null), "hashed");
+ }
+
+ [Test]
+ public async Task ADesktopScanAfterTheServersUploadLeavesWhatTheServerMadeForTheFramesAlone()
+ {
+ await using var fx = await CreateAsync(context);
+ Populate(fx);
+ await using var engine = ServerEngine(fx);
+ await engine.StartAsync(fx.Ct);
+ var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
+ await fx.WaitForAsync(scan.Id, JobState.Done);
+ foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
+ {
+ foreach (var pass in (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items) await fx.WaitForAsync(pass.Id, JobState.Done);
+ }
+ var a = (await LocationsAsync(fx)).Single(l => l.Folder == "a");
+ // A desktop sharing the archive, with an index of its own.
+ await using var desktopIndex = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("desktop"), "titano.db")), fx.Ct);
+ var desktop = new ArchiveService(desktopIndex, new ArchiveServiceOptions { Owner = "desktop", LeaseWait = TimeSpan.FromSeconds(20) });
+ var archive = await desktop.OpenAsync(fx.Archive.RootPath, fx.Ct);
+ await desktop.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct);
+
+ // The server takes an upload into the sequence and makes its thumbnail and proxy, as after every upload.
+ var staged = fx.Stage(Start.AddMinutes(-5), seed: 90);
+ await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, a.Id, null, null, staged.Path, "IMG_0000.JPG", staged.Size, staged.Sha256, null, null),
+ fx.Ct);
+ foreach (var request in new[] { ArchiveJobs.Thumbnails([a.Id], "a"), ArchiveJobs.Proxies([a.Id], "a") })
+ await fx.WaitForAsync((await engine.EnqueueAsync(request, cancellationToken: fx.Ct)).Id, JobState.Done);
+ var sidecar = fx.Paths.Sequence("a").ManifestPath;
+ Assert.True(Built(ManifestStore.ReadSequence(sidecar).Status), "built after the upload");
+ var stamp = FileStamp.Of(sidecar);
+ var jobs = (await fx.Db.Jobs.ListAsync(null, null, cancellationToken: fx.Ct)).Items.Count;
+
+ // The desktop's index lags behind the upload: its scan indexes the frame and takes the statuses as the sidecar has them.
+ var summary = await desktop.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct);
+ Assert.SequenceEqual([a.Id], summary.Sequences);
+ Assert.Equal(stamp, FileStamp.Of(sidecar));
+ var row = (await desktopIndex.Sequences.GetAsync(a.Id, fx.Ct))!;
+ Assert.Equal(7, row.FrameCount);
+ Assert.True(Built(row.Status), row.Status.ToString());
+
+ // So the server's next scan finds nothing changed, and makes nothing again.
+ var again = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
+ await fx.WaitForAsync(again.Id, JobState.Done);
+ Assert.Equal(jobs + 1, (await fx.Db.Jobs.ListAsync(null, null, cancellationToken: fx.Ct)).Items.Count);
+ Assert.True(Built((await fx.Db.Sequences.GetAsync(a.Id))!.Status), "the server's row");
+ }
+
[Test]
public async Task AFolderGoneBeforeItsImagePassesFailsThemAloneAndTheOtherSequencesGetTheirImages()
{
diff --git a/tests/Titano.Data.Tests/Archiving/HeifFile.cs b/tests/Titano.Data.Tests/Archiving/HeifFile.cs
index 39c79aab..8e807c8c 100644
--- a/tests/Titano.Data.Tests/Archiving/HeifFile.cs
+++ b/tests/Titano.Data.Tests/Archiving/HeifFile.cs
@@ -14,13 +14,17 @@ namespace Titano.Data.Tests.Archiving;
internal static class HeifFile
{
/// A × HEIF captured at , its Exif
- /// thumbnail × .
+ /// thumbnail × (none when the width is 0: such a
+ /// host finds no pixels in it at all).
public static byte[] Create(DateTime captured, int width, int height, int thumbnailWidth = 160, int thumbnailHeight = 120)
{
- using var small = new ImageBuffer(thumbnailWidth, thumbnailHeight);
- for (var c = 0; c < 3; c++) small.Plane(c).Fill(0.2f + c * 0.2f);
- var thumbnail = JpegEncoder.Encode(small, new JpegEncodeOptions { Quality = 80 });
- var tiff = WithThumbnail(ExifWriter.BuildTiff(new FrameMetadata { Make = "TestCam", Model = "H-1", CapturedAt = captured }), thumbnail);
+ var tiff = ExifWriter.BuildTiff(new FrameMetadata { Make = "TestCam", Model = "H-1", CapturedAt = captured });
+ if (thumbnailWidth > 0)
+ {
+ using var small = new ImageBuffer(thumbnailWidth, thumbnailHeight);
+ for (var c = 0; c < 3; c++) small.Plane(c).Fill(0.2f + c * 0.2f);
+ tiff = WithThumbnail(tiff, JpegEncoder.Encode(small, new JpegEncodeOptions { Quality = 80 }));
+ }
byte[] exif = [.. U32(6), .. "Exif\0\0"u8, .. tiff];
// The HEVC-coded primary image, which no managed decoder reads.
var coded = new byte[64];
diff --git a/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs b/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs
index cdcee194..a394c82d 100644
--- a/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs
+++ b/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs
@@ -226,6 +226,10 @@ public sealed class MassRenameTests(TestContext context)
Assert.SequenceEqual(["moved/2026-09-28", "moved/2026-09-29", "moved/2026-09-30"], (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id))
.Select(l => l.Folder).Order(StringComparer.Ordinal));
Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids));
+ // The folders a nested sequence moved out of changed after their rows were written, as a folder another process
+ // changed would: the next scan reads those two again, and finds the index as it is.
+ Assert.SequenceEqual([ids[0], ids[1]], (await fx.ScanAsync()).Sequences);
+ Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids));
Assert.Equal(0, (await fx.ScanAsync()).Indexed);
}
diff --git a/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs
new file mode 100644
index 00000000..2ec4c3b7
--- /dev/null
+++ b/tests/Titano.Data.Tests/Archiving/PartialRowWriterTests.cs
@@ -0,0 +1,286 @@
+using Titano.Core.Archiving;
+using Titano.Core.Imaging;
+using Titano.Core.Model;
+using Titano.Data.Archiving;
+using Titano.Data.Repositories;
+using Titano.Data.Tests.Infrastructure;
+using Titano.Testing;
+using static Titano.Data.Tests.Archiving.ArchiveFixture;
+
+namespace Titano.Data.Tests.Archiving;
+
+///
+/// Writers that change part of a sequence's sidecar and index row (decode tiers, the verify, title, tags, rating and
+/// notes, a rename) while a settings edit or another writer commits, on both dialects: they write only what they own,
+/// under the sequence lease, and leave what another process changed in the sidecar to the next scan.
+///
+[Category("archive")]
+public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(context)
+{
+ private async Task FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true)));
+
+ /// One indexed sequence of three photos in a.
+ private static async Task IndexedAsync(ArchiveFixture fx)
+ {
+ Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3);
+ await fx.ScanAsync();
+ var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single();
+ return (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!;
+ }
+
+ /// Records decode tiers as analyses, image passes and renders do.
+ private static FrameTiers Tiers(ArchiveFixture fx)
+ {
+ var writer = new SequenceWriter(new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) });
+ return new FrameTiers(fx.Db, writer, new ArchiveIndexer(fx.Db, writer));
+ }
+
+ /// The frame decodes only through its 48 × 32 embedded preview.
+ private static Dictionary Preview(Guid frameId) =>
+ new() { [frameId] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32) };
+
+ /// Commits once the writer under test had time to read the row and wait for it.
+ private static async Task CommitLaterAsync(TitanoTransaction tx, CancellationToken cancellationToken)
+ {
+ await using (tx)
+ {
+ await Task.Delay(500, cancellationToken);
+ await tx.CommitAsync(cancellationToken);
+ }
+ }
+
+ [Test]
+ public async Task TiersRecordedWhileASettingsEditCommitsLeaveTheEdit()
+ {
+ var fx = await FixtureAsync();
+ var sequence = await IndexedAsync(fx);
+ var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
+
+ // The server's PATCH holds the row (and SQLite's write lock) while an analysis records a tier.
+ Task record;
+ await using (var tx = await fx.Db.BeginAsync(fx.Ct))
+ {
+ var edit = await tx.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":2}""" }, sequence.SettingsVersion, fx.Ct);
+ Assert.Equal(SequenceEditOutcome.Updated, edit.Outcome);
+ record = Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
+ await Task.WhenAny(record, Task.Delay(500, fx.Ct));
+ Assert.False(record.IsCompleted, "the record waits for the row");
+ await tx.CommitAsync(fx.Ct);
+ }
+ await record;
+
+ // The tier and what follows from it are recorded, and the edit committed meanwhile stays.
+ var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.Contains("heroFrame", row.SettingsJson);
+ Assert.Equal((sequence.SettingsVersion + 1, 1), (row.SettingsVersion, row.PreviewOnlyCount));
+ Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags);
+ }
+
+ [Test]
+ public async Task TiersRecordedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan()
+ {
+ var fx = await FixtureAsync();
+ var sequence = await IndexedAsync(fx);
+ var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
+
+ // A desktop sharing the archive imports a photo into the sequence: its own index follows the sidecar, this one not yet.
+ var (desktop, archive) = await fx.OtherWriterAsync();
+ var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
+ await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
+ fx.Ct);
+
+ // A tier recorded meanwhile changes the tiers here, not the frames this index holds.
+ await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
+ var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.Equal((3, sequence.FramesVersion, 1), (row.FrameCount, row.FramesVersion, row.PreviewOnlyCount));
+
+ // The next scan still finds the folder changed, and indexes the photo.
+ Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
+ var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
+ row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.Equal((4, sidecar.Versions.Frames), (row.FrameCount, row.FramesVersion));
+ Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
+ }
+
+ [Test]
+ public async Task TiersRecordedAfterAnotherWriterRetitledTheSequenceLeaveTheTitleToTheNextScan()
+ {
+ var fx = await FixtureAsync();
+ var sequence = await IndexedAsync(fx);
+ var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
+
+ // A desktop sharing the archive renames and rates the sequence, in the sidecar and its own index.
+ var (desktop, _) = await fx.OtherWriterAsync();
+ await desktop.SetSequenceInfoAsync(sequence.Id, "Ridge", ["fog"], 3, null, fx.Ct);
+
+ // A tier recorded here meanwhile writes only the tier, and leaves the folder changed for the next scan.
+ await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
+ Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
+ var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.Equal(("Ridge", "fog", 3, 1), (row.Title, row.Tags.Single(), row.Rating, row.PreviewOnlyCount));
+ }
+
+ [Test]
+ public async Task AnInfoEditAfterAnotherWriterAddedFramesLeavesThoseFramesToTheNextScan()
+ {
+ var fx = await FixtureAsync();
+ var sequence = await IndexedAsync(fx);
+ var (desktop, archive) = await fx.OtherWriterAsync();
+ var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
+ await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
+ fx.Ct);
+
+ // The rating goes into the sidecar and this index; the frames this index holds stay as they are.
+ var rated = await fx.Service.EditSequenceInfoAsync(sequence.Id, info => info with { Rating = 4 }, fx.Ct);
+ Assert.Equal((3, sequence.FramesVersion, 4), (rated.FrameCount, rated.FramesVersion, rated.Rating));
+ Assert.Equal(4, ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Rating);
+
+ // The next scan still finds the folder changed, and indexes the photo.
+ Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
+ var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
+ var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.Equal((4, sidecar.Versions.Frames, 4), (row.FrameCount, row.FramesVersion, row.Rating));
+ Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
+ }
+
+ [Test]
+ public async Task AnInfoEditKeepsTheSettingsTheApiSavedAfterItReadTheSequence()
+ {
+ var fx = await FixtureAsync();
+ var sequence = await IndexedAsync(fx);
+
+ // The server's PATCH, which takes no lease, saves settings after the edit read the sequence: the edit applies again.
+ var calls = 0;
+ var rated = await fx.Service.EditSequenceInfoAsync(sequence.Id, info =>
+ {
+ if (++calls == 1)
+ {
+ var patch = fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":2}""" }, sequence.SettingsVersion, fx.Ct)
+ .GetAwaiter().GetResult();
+ Assert.Equal(SequenceEditOutcome.Updated, patch.Outcome);
+ }
+ return info with { Rating = 5 };
+ }, fx.Ct);
+
+ Assert.Equal(2, calls);
+ var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.Contains("heroFrame", row.SettingsJson);
+ Assert.Equal((5, sequence.SettingsVersion + 2), (row.Rating, row.SettingsVersion));
+ Assert.Equal(rated.SettingsVersion, row.SettingsVersion);
+ var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
+ Assert.Equal((5, row.SettingsVersion), (sidecar.Rating, sidecar.Versions.Settings));
+ }
+
+ [Test]
+ public async Task ARenameAfterAnotherWriterAddedFramesLeavesThoseFramesToTheNextScan()
+ {
+ var fx = await FixtureAsync();
+ var sequence = await IndexedAsync(fx);
+ var (desktop, archive) = await fx.OtherWriterAsync();
+ var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
+ await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
+ fx.Ct);
+
+ // The rename moves the folder and retitles the sequence; the row keeps the frames this index holds.
+ var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [sequence.Id], "b", "Ridge", fx.Ct);
+ await fx.Service.ApplyRenameAsync(plan, null, fx.Ct);
+ var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.Equal(("b", "Ridge", 3, sequence.FramesVersion), (row.Folder, row.Title, row.FrameCount, row.FramesVersion));
+
+ // The next scan still finds the folder changed, and indexes the photo where it is now.
+ Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
+ var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("b").ManifestPath);
+ row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.Equal(("Ridge", 4, sidecar.Versions.Frames), (row.Title, row.FrameCount, row.FramesVersion));
+ Assert.True((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).All(f => f.RelativePath.StartsWith("b/", StringComparison.Ordinal)),
+ "frames in b");
+ }
+
+ [Test]
+ public async Task AVerifyStoppedAfterAFlushLeavesFramesAnotherWriterAddedToTheNextScan()
+ {
+ var fx = await FixtureAsync();
+ // Enough photos the scan adopted unhashed for the verify to flush their hashes before it reaches the end.
+ Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 513);
+ await fx.ScanAsync();
+ var sequence = (await fx.Db.Sequences.GetAsync((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single().Id, fx.Ct))!;
+ var (desktop, archive) = await fx.OtherWriterAsync();
+ var staged = fx.Stage(Start.AddHours(3), seed: 90);
+ await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0600.JPG", staged.Size, staged.Sha256, null, null),
+ fx.Ct);
+
+ // The verify writes the hashes of the first 512 photos to the sidecar and this index, and stops before the end.
+ await Assert.ThrowsAsync(() => fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default,
+ (_, _, _) => throw new OperationCanceledException(), null, fx.Ct));
+ Assert.Equal(512, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Count(f => f.Sha256 is not null));
+ Assert.Equal(513, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
+
+ // The next scan still finds the folder changed, and indexes the photo.
+ Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
+ Assert.Equal(514, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
+ }
+
+ [Test]
+ public async Task DuplicatesFlaggedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan()
+ {
+ var fx = await FixtureAsync();
+ // Two sequences holding the same photo, hashed by a verify.
+ Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3);
+ Photos.WriteSeries(Path.Combine(fx.Paths.Root, "b"), Start, 1);
+ await fx.ScanAsync();
+ await fx.Service.Verifier.VerifyAsync(fx.Archive, null, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
+ var copy = (await fx.Service.FindDuplicatesAsync(fx.Archive.Id, fx.Ct)).Single().Frames[1];
+ var sequence = (await fx.Db.Sequences.GetAsync(copy.SequenceId, fx.Ct))!;
+
+ // A desktop sharing the archive imports a photo into the sequence whose copy maintenance flags.
+ var (desktop, archive) = await fx.OtherWriterAsync();
+ var staged = fx.Stage(Start.AddHours(3), seed: 90);
+ await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0600.JPG", staged.Size, staged.Sha256, null, null),
+ fx.Ct);
+ var report = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, firstStep: 3, _ => ValueTask.CompletedTask, fx.Ct);
+ Assert.Equal(1, report.DuplicateFlagChanges);
+ Assert.Equal(sequence.FrameCount, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
+
+ // The next scan still finds the folder changed, and indexes the photo.
+ Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
+ Assert.Equal(sequence.FrameCount + 1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
+ }
+
+ [Test]
+ public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit()
+ {
+ var fx = await FixtureAsync();
+ var sequence = await IndexedAsync(fx);
+ // Hashed by a first verify, so the next one changes no frame and only stamps the sequence.
+ await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
+ sequence = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+
+ // The server's PATCH takes the row as the last photo is checked, and commits while the verify stamps the sequence.
+ Task? commit = null;
+ await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (done, total) =>
+ {
+ if (done < total || commit is not null) return;
+ var tx = await fx.Db.BeginAsync(fx.Ct);
+ await tx.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":1}""" }, sequence.SettingsVersion, fx.Ct);
+ commit = CommitLaterAsync(tx, fx.Ct);
+ }, fx.Ct);
+ await Assert.NotNull(commit);
+
+ var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.Contains("heroFrame", row.SettingsJson);
+ Assert.Equal(sequence.SettingsVersion + 1, row.SettingsVersion);
+ Assert.True(row.Status.HasFlag(SequenceStatus.Verified), row.Status.ToString());
+ Assert.True(row.VerifiedAt > sequence.VerifiedAt, "stamped again");
+ }
+}
+
+public sealed class SqlitePartialRowWriterTests(TestContext context) : PartialRowWriterTests(context)
+{
+ protected override Backend Backend => Backend.SqliteFile;
+}
+
+[RequiresEnvironment(PgTestServer.Variable)]
+public sealed class PostgresPartialRowWriterTests(TestContext context) : PartialRowWriterTests(context)
+{
+ protected override Backend Backend => Backend.Postgres;
+}
diff --git a/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs b/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs
index b6947cca..3805a27f 100644
--- a/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs
+++ b/tests/Titano.Data.Tests/Processing/AnalysisJobTests.cs
@@ -206,6 +206,54 @@ public sealed class AnalysisJobTests(TestContext context)
Assert.Equal("4k-hevc", await PresetAsync());
}
+ [Test]
+ public async Task ProxiesThatArriveDuringADeepAnalysisOfHeifsWithoutThumbnailsMakeItsUnreadableVerdictStale()
+ {
+ // A server, which finds no pixels at all in these HEIF files (they carry no Exif thumbnail) until proxies come (BRIEF §2.3).
+ await using var fx = await ProcessingFixture.CreateAsync(context, o => o with { Host = EncoderHost.Server });
+ var directory = Path.Combine(fx.Archive.Paths.Root, "Heif");
+ Directory.CreateDirectory(directory);
+ var start = new DateTime(2026, 9, 28, 18, 0, 0);
+ for (var i = 0; i < 4; i++)
+ {
+ await File.WriteAllBytesAsync(Path.Combine(directory, $"IMG_{i + 1:0000}.HEIC"), HeifFile.Create(start.AddSeconds(10 * i), 4032, 3024, thumbnailWidth: 0),
+ fx.Ct);
+ }
+ await fx.Archive.ScanAsync(ScanMode.Full);
+ var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Archive.Id, fx.Ct)).Single(l => l.Folder == "Heif");
+ var sequence = (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!;
+ var folder = fx.Folder(sequence);
+ await fx.StartEngineAsync(workers: 2);
+ async Task AutoValuesAsync() =>
+ File.Exists(folder.ProjectPath) ? ProjectSettings.FromJson(await File.ReadAllTextAsync(folder.ProjectPath, fx.Ct)).ToJson() : string.Empty;
+
+ // A deep analysis starts on the originals; while it runs, the proxies arrive and a light analysis on the other
+ // worker reads the frames from them.
+ fx.Decoder.Hold = "IMG_0001.HEIC";
+ var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
+ await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
+ Directory.CreateDirectory(folder.Proxies);
+ foreach (var frame in await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))
+ await File.WriteAllBytesAsync(folder.Proxy(frame.Index), Proxy(frame.CapturedAt!.Value, 1920, 1440), fx.Ct);
+ await LightAnalysisAsync(fx, sequence);
+ var light = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ var advice = await AutoValuesAsync();
+ fx.Decoder.Release();
+ await fx.WaitForAsync(deep.Id, JobState.Done);
+
+ // The deep pass found no pixels in any original, a verdict the proxies made stale before it ended, as every reader
+ // of its report finds: it records no deep status, and neither warnings nor advice of its own.
+ Assert.Count(4, SequenceAnalyser.ReadReport(folder, sequence.FrameCount)!.Issues.Undecodable);
+ var now = await Titano.Data.Processing.SequenceFrames.LoadAsync(fx.Db, fx.Archive.Archive, sequence, useProxies: false,
+ new ImageDecoder([fx.Decoder]), fx.Ct);
+ Assert.Null(SequenceAnalyser.ReadReport(folder, now));
+ var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
+ Assert.False(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), row.Status.ToString());
+ Assert.False(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedDeep), "sidecar status");
+ Assert.Equal((light.WarningCount, light.Status.HasFlag(SequenceStatus.HasWarnings)), (row.WarningCount, row.Status.HasFlag(SequenceStatus.HasWarnings)));
+ Assert.Equal(advice, await AutoValuesAsync());
+ }
+
[Test]
public async Task AFrameTheDeepAnalysisCouldNotReadIsReadAgainOnceItsProxyArrives()
{
diff --git a/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs b/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs
index df1ac4ff..7b6fe256 100644
--- a/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs
+++ b/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs
@@ -1,8 +1,11 @@
using System.Net;
+using System.Security.Cryptography;
using Microsoft.Extensions.DependencyInjection;
using Titano.Core.Archiving;
using Titano.Core.Codecs.Jpeg;
using Titano.Core.Model;
+using Titano.Data;
+using Titano.Data.Archiving;
using Titano.Protocol;
using Titano.Protocol.Client;
using Titano.Server.Jobs;
@@ -106,6 +109,74 @@ public sealed class ArchiveModuleTests(TestContext context)
await UntilImagesAsync(server, editor, archive.Id, "garden");
}
+ [Test]
+ public async Task FramesADesktopSharingTheArchiveAddsToASequenceGetThumbnailsAndProxiesAfterThePoll()
+ {
+ await using var server = await TestServer.StartAsync(context, options =>
+ {
+ Directory.CreateDirectory(Path.Combine(options.ArchivePath, "roof"));
+ File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0002.JPG"), Photo());
+ }, ArchiveModule.Register);
+ var archive = server.Services.GetRequiredService();
+ var (editor, _, _) = await server.UserAsync("alberto", "editor");
+ await UntilIndexedAsync(server, archive.Id, "roof");
+ await UntilScannedAsync(server);
+ await UntilImagesAsync(server, editor, archive.Id, "roof");
+ await UntilIdleAsync(server);
+ var roof = (await server.Database.Sequences.ListLocationsAsync(archive.Id)).Single(l => l.Folder == "roof");
+
+ // A desktop sharing the archive, with an index of its own, imports a photo taken earlier into the sequence: it files
+ // the photo and lists it in the sidecar itself, keeping the sidecar's ThumbsBuilt and ProxiesBuilt.
+ var desktopFolder = context.CreateTempDirectory("desktop");
+ await using (var desktopIndex = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(desktopFolder, "titano.db")), context.Cancellation))
+ {
+ var desktop = new ArchiveService(desktopIndex, new ArchiveServiceOptions { Owner = "desktop" });
+ var shared = await desktop.OpenAsync(server.Options.ArchivePath, context.Cancellation);
+ await desktop.Indexer.ScanAsync(shared, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, context.Cancellation);
+ var photo = Photo(seed: 5);
+ var staged = Path.Combine(desktopFolder, "IMG_0001.JPG");
+ await File.WriteAllBytesAsync(staged, photo);
+ await desktop.Ingest.CommitAsync(new IngestRequest(shared.Id, roof.Id, null, null, staged, "IMG_0001.JPG", photo.Length,
+ Convert.ToHexStringLower(SHA256.HashData(photo)), new DateTime(2026, 9, 28, 18, 0, 0), null), context.Cancellation);
+ }
+ Assert.Equal(1, Assert.NotNull(await server.Database.Sequences.GetAsync(roof.Id)).FrameCount);
+
+ // The next poll indexes it and queues the images: the new frame and the one it moved get theirs.
+ Assert.NotNull(await server.Services.GetRequiredService().PollAsync(context.Cancellation));
+ await UntilScannedAsync(server);
+ Assert.Equal(2, Assert.NotNull(await server.Database.Sequences.GetAsync(roof.Id)).FrameCount);
+ await UntilImagesAsync(server, editor, archive.Id, "roof");
+ }
+
+ [Test]
+ public async Task AThumbnailDrawnBeforeAVerifyHashedThePhotoIsStillServed()
+ {
+ await using var server = await TestServer.StartAsync(context, options =>
+ {
+ Directory.CreateDirectory(Path.Combine(options.ArchivePath, "roof"));
+ File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0001.JPG"), Photo());
+ }, ArchiveModule.Register);
+ var archive = server.Services.GetRequiredService();
+ var (editor, _, _) = await server.UserAsync("alberto", "editor");
+ await UntilIndexedAsync(server, archive.Id, "roof");
+ await UntilScannedAsync(server);
+ await UntilImagesAsync(server, editor, archive.Id, "roof");
+ await UntilIdleAsync(server);
+ var roof = (await server.Database.Sequences.ListLocationsAsync(archive.Id)).Single(l => l.Folder == "roof");
+ var frame = (await server.Database.Frames.ListBySequenceAsync(roof.Id)).Single();
+ Assert.Null(frame.Sha256);
+
+ // A verify gives the photo the scan adopted its first hash; the thumbnail drawn from it before is still its own.
+ var verified = await server.Services.GetRequiredService().Verifier.VerifyAsync(
+ Assert.NotNull(await server.Database.Archives.GetAsync(archive.Id)), [roof.Id], default, (_, _, _) => ValueTask.CompletedTask, null,
+ context.Cancellation);
+ Assert.Equal(1L, verified.Hashed);
+ Assert.NotNull(Assert.NotNull(await server.Database.Frames.GetAsync(frame.Id)).Sha256);
+ using var thumb = await editor.GetAsync($"/api/v1/frames/{frame.Id}/thumb");
+ Assert.Equal(HttpStatusCode.OK, thumb.StatusCode);
+ Assert.True(Assert.NotNull(await server.Database.Sequences.GetAsync(roof.Id)).Status.HasFlag(SequenceStatus.ThumbsBuilt), "built");
+ }
+
[Test]
public async Task SettingsADesktopChangedInArchiveJsonReachUploadsAndTheAdminPanel()
{
@@ -157,28 +228,51 @@ public sealed class ArchiveModuleTests(TestContext context)
}
}
- /// Waits until the first frame of the sequence in has its thumbnail and proxy served.
+ ///
+ /// Waits until every frame of the sequence in has its thumbnail and proxy served and the
+ /// passes that made them have marked the sequence (a pass writes its files before it marks).
+ ///
private static async Task UntilImagesAsync(TestServer server, HttpClient client, Guid archiveId, string folder)
{
var sequence = (await server.Database.Sequences.ListLocationsAsync(archiveId)).Single(l => l.Folder == folder);
- var frame = (await server.Database.Frames.ListBySequenceAsync(sequence.Id)).First();
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
- foreach (var asset in new[] { "thumb", "proxy" })
+ foreach (var frame in await server.Database.Frames.ListBySequenceAsync(sequence.Id))
{
- while (true)
+ foreach (var asset in new[] { "thumb", "proxy" })
{
- using var response = await client.GetAsync($"/api/v1/frames/{frame.Id}/{asset}");
- if (response.StatusCode == HttpStatusCode.OK)
+ while (true)
{
- Assert.True(await response.Content.ReadAsByteArrayAsync() is [0xFF, 0xD8, ..], $"a JPEG {asset} of {folder}");
- break;
+ using var response = await client.GetAsync($"/api/v1/frames/{frame.Id}/{asset}");
+ if (response.StatusCode == HttpStatusCode.OK)
+ {
+ Assert.True(await response.Content.ReadAsByteArrayAsync() is [0xFF, 0xD8, ..], $"a JPEG {asset} of {frame.FileName}");
+ break;
+ }
+ Assert.True(DateTime.UtcNow < until, $"no {asset} of {frame.FileName} in {folder}: {(int)response.StatusCode}");
+ await Task.Delay(100);
}
- Assert.True(DateTime.UtcNow < until, $"no {asset} of {folder}: {(int)response.StatusCode}");
- await Task.Delay(100);
}
}
- var built = Assert.NotNull(await server.Database.Sequences.GetAsync(sequence.Id)).Status;
- Assert.True(built.HasFlag(SequenceStatus.ThumbsBuilt) && built.HasFlag(SequenceStatus.ProxiesBuilt), $"{folder}: {built}");
+ while (Assert.NotNull(await server.Database.Sequences.GetAsync(sequence.Id)).Status is var built
+ && !(built.HasFlag(SequenceStatus.ThumbsBuilt) && built.HasFlag(SequenceStatus.ProxiesBuilt)))
+ {
+ Assert.True(DateTime.UtcNow < until, $"{folder}: {built}");
+ await Task.Delay(50);
+ }
+ }
+
+ /// Waits until no job is waiting or running.
+ private static async Task UntilIdleAsync(TestServer server)
+ {
+ var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
+ foreach (var state in new[] { JobState.Queued, JobState.Running, JobState.Queued })
+ {
+ while ((await server.Database.Jobs.ListAsync(state)).Items.Count > 0)
+ {
+ Assert.True(DateTime.UtcNow < until, $"jobs still {state}");
+ await Task.Delay(50);
+ }
+ }
}
/// Waits for the scan queued at start to finish: its sequences are indexed before it ends.