merge: third-pass archive, image and analysis fixes (W-1, W-5, W-7, W-13, W-14)

Merge branch fix3/data-images into the verification branch.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-12 20:18:40 +00:00
commit f1720e7779
23 files changed
+1132 -169

No files matched your search

+9
View File
@@ -223,6 +223,15 @@ public static class ThumbPack
sha256 is { Length: HashLength * 2 }
? Convert.FromHexString(sha256)
: System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes($"{file}|{size}"));
/// <summary>
/// Whether a thumbnail with <paramref name="sourceHash"/> was drawn from this original: its <see cref="Fingerprint"/>,
/// or, for an original hashed since (a verify hashes the photos a scan adopted), the fingerprint of its file name and
/// size it was drawn with before. The hash changes nothing about the photo.
/// </summary>
public static bool IsFrom(ReadOnlySpan<byte> sourceHash, string? sha256, string file, long size) =>
sourceHash.SequenceEqual(Fingerprint(sha256, file, size))
|| (sha256 is { Length: HashLength * 2 } && sourceHash.SequenceEqual(Fingerprint(null, file, size)));
}
/// <summary>Random access to the thumbnails of a pack without loading the file; safe while a writer appends.</summary>
+52 -13
View File
@@ -209,8 +209,7 @@ public sealed class ArchiveImages
for (var i = 0; i < startFrame && i < manifest.Frames.Count; i++)
{
var frame = manifest.Frames[i];
complete &= pack.Entries.GetValueOrDefault(frame.I) is { } entry
&& entry.SourceHash.AsSpan().SequenceEqual(ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size));
complete &= pack.Entries.GetValueOrDefault(frame.I) is { } entry && ThumbPack.IsFrom(entry.SourceHash, frame.Sha256, frame.File, frame.Size);
}
var cadence = new Cadence();
for (var i = startFrame; i < manifest.Frames.Count; i++)
@@ -220,7 +219,7 @@ public sealed class ArchiveImages
var fingerprint = ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size);
var entry = pack.Entries.GetValueOrDefault(frame.I);
var edge = entry is null ? 0 : System.Math.Max(entry.Width, entry.Height);
var current = !force && entry is not null && entry.SourceHash.AsSpan().SequenceEqual(fingerprint);
var current = !force && entry is not null && ThumbPack.IsFrom(entry.SourceHash, frame.Sha256, frame.File, frame.Size);
// A frame this host cannot decode shows its proxy once one arrived; until then its original's embedded
// preview, whose smaller thumbnail the proxy's replaces.
var fromProxy = false;
@@ -255,7 +254,8 @@ public sealed class ArchiveImages
pack.Commit();
}
await _tiers.RecordAsync(archive, sequence, tiers, proxies: false, cancellationToken).ConfigureAwait(false);
await MarkAsync(archive, paths, sequence, SequenceStatus.ThumbsBuilt, complete && undecodable == 0, cancellationToken).ConfigureAwait(false);
await MarkAsync(archive, paths, sequence, manifest, JobKind.Thumbs, SequenceStatus.ThumbsBuilt, complete && undecodable == 0, cancellationToken)
.ConfigureAwait(false);
return new ImagePassResult(built, reused, undecodable);
}
@@ -327,7 +327,7 @@ public sealed class ArchiveImages
if (cadence.Due()) await checkpoint(i + 1, manifest.Frames.Count).ConfigureAwait(false);
}
await _tiers.RecordAsync(archive, sequence, tiers, proxies: true, cancellationToken).ConfigureAwait(false);
await MarkAsync(archive, paths, sequence, SequenceStatus.ProxiesBuilt, complete, cancellationToken).ConfigureAwait(false);
await MarkAsync(archive, paths, sequence, manifest, JobKind.Proxies, SequenceStatus.ProxiesBuilt, complete, cancellationToken).ConfigureAwait(false);
return new ImagePassResult(built, reused, undecodable, waiting);
}
@@ -520,15 +520,54 @@ public sealed class ArchiveImages
}
}
private async Task MarkAsync(Archive archive, ArchivePaths paths, Sequence sequence, SequenceStatus flag, bool complete, CancellationToken cancellationToken)
/// <summary>
/// Sets <paramref name="flag"/> in the sidecar and the index when the pass covered every frame it read
/// (<paramref name="complete"/>) and the sidecar, as it is under the lease, still lists those frames
/// (<see cref="SameFrames"/>); clears it otherwise. Frames a scan's reconcile or an ingest added, removed or
/// renumbered after the pass read <paramref name="read"/> have nothing of it, and a scan that cleared the flag for
/// them queues the next pass only while the flag stays clear. A <paramref name="pass"/> of the same kind already
/// queued for the sequence (an upload's, or a scan's) draws them when it runs, so the flag stays set meanwhile, as
/// between an upload and its pass: a camera uploading faster than a pass ends would otherwise keep it clear, and a
/// server would offer no thumbnail or proxy of the sequence at all until uploads paused. The index is written under
/// the same lease, so the flag never lands there after a scan cleared it in both.
/// </summary>
private async Task MarkAsync(Archive archive, ArchivePaths paths, Sequence sequence, SequenceManifest read, JobKind pass, SequenceStatus flag,
bool complete, CancellationToken cancellationToken)
{
await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m =>
var folder = paths.Sequence(sequence.Folder);
await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
{
var status = complete ? m.Status | flag : m.Status & ~flag;
return status == m.Status ? null : m with { Status = status, UpdatedAt = DateTimeOffset.UtcNow };
}, cancellationToken).ConfigureAwait(false);
await _db.Sequences.UpdateStatusAsync(sequence.Id, complete ? flag : SequenceStatus.None, complete ? SequenceStatus.None : flag, cancellationToken)
.ConfigureAwait(false);
await _indexer.RecordStampsAsync(archive, paths.Sequence(sequence.Folder), sequence.Id, cancellationToken).ConfigureAwait(false);
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
var built = complete && (SameFrames(manifest.Frames, read.Frames)
|| await _db.Jobs.IsQueuedAsync(pass, sequence.Id, cancellationToken).ConfigureAwait(false));
var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
var status = built ? manifest.Status | flag : manifest.Status & ~flag;
if (status != manifest.Status)
{
manifest = manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow };
ManifestStore.WriteSequence(folder.ManifestPath, manifest);
}
await _db.Sequences.UpdateStatusAsync(sequence.Id, built ? flag : SequenceStatus.None, built ? SequenceStatus.None : flag, cancellationToken)
.ConfigureAwait(false);
if (inStep) await _indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
}
}
/// <summary>
/// Whether <paramref name="current"/> lists the photos <paramref name="read"/> did, each at the same index with the
/// same file, size, capture time and content hash: thumbnails and proxies are kept by frame index, so photos added,
/// removed, renumbered or replaced since have none of what the pass made. A hash learnt since (a verify hashing a
/// photo the scan adopted) changes nothing about the photo, and its thumbnail stays its own (<see cref="ThumbPack.IsFrom"/>).
/// </summary>
private static bool SameFrames(IReadOnlyList<SidecarFrame> current, IReadOnlyList<SidecarFrame> read)
{
if (current.Count != read.Count) return false;
for (var i = 0; i < current.Count; i++)
{
var (now, then) = (current[i], read[i]);
if (now.Id != then.Id || now.I != then.I || now.File != then.File || now.Size != then.Size || now.CapturedAt != then.CapturedAt
|| (then.Sha256 is not null && !string.Equals(now.Sha256, then.Sha256, StringComparison.OrdinalIgnoreCase))) return false;
}
return true;
}
}
+99 -14
View File
@@ -31,6 +31,9 @@ public sealed record ScanSummary(int Folders, int Indexed, int Adopted, int Remo
/// </summary>
public sealed class ArchiveIndexer
{
/// <summary>What covers the frames as they were: a scan that finds them changed clears it, and its follow-ups make it again.</summary>
private const SequenceStatus CoveringStatus = SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt;
private readonly TitanoDatabase _db;
private readonly SequenceWriter _writer;
@@ -131,6 +134,32 @@ public sealed class ArchiveIndexer
_db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(sequence.RelativeFolder, sequenceId,
FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))), cancellationToken);
/// <summary>
/// Whether this index holds the folder of <paramref name="manifest"/> (the sidecar read under the sequence lease) as it
/// is now: the stamps it recorded when it last synced the folder are still the folder's, and its row holds the
/// sidecar's frames. A writer that changes, under the lease, only what it owns of the sidecar and the index row
/// (statuses, decode tiers, frame flags, title, tags, rating and notes, the folder) asks before its own write, and
/// records the folder's new stamps after it only when this held (<see cref="RecordStampsAsync"/>). What another
/// process (a desktop sharing the archive) changed in the sidecar alone, frames or a new title, then keeps the folder
/// changed for the next incremental scan, which writes the whole row from the sidecar.
/// </summary>
internal Task<bool> InStepAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) =>
InStepAsync(archive, sequence.RelativeFolder, sequence, manifest, cancellationToken);
/// <inheritdoc cref="InStepAsync(Archive, SequencePaths, SequenceManifest, CancellationToken)"/>
/// <param name="recordedFolder">The folder the stamps were recorded at: the one a rename just moved the sequence from,
/// whose files keep their stamps.</param>
internal async Task<bool> InStepAsync(Archive archive, string recordedFolder, SequencePaths sequence, SequenceManifest manifest,
CancellationToken cancellationToken)
{
if (await _db.Archives.GetFolderStateAsync(archive.Id, recordedFolder, cancellationToken).ConfigureAwait(false) is not { } state
|| state.SequenceId != manifest.Id || state.Sidecar != FileStamp.Of(sequence.ManifestPath)
|| state.Photos != FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))) return false;
// A sidecar rewritten within its file system's time resolution, at the same size, keeps its stamp.
return await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { } indexed
&& indexed.FramesVersion == manifest.Versions.Frames;
}
/// <summary>Indexes unindexed photos and flags missing files of one sequence (the orphans fix action).</summary>
public async Task<SequenceManifest> ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default)
{
@@ -144,8 +173,10 @@ public sealed class ArchiveIndexer
/// <summary>
/// Applies <paramref name="edit"/> to the sidecar (null writes nothing) and indexes the result, both under the sequence
/// lease. Frames inserted before indexed ones move the project's positions (<see cref="ProjectPositions"/>) without a
/// new settings version; an editor saving its project holds the same lease, so it reads them moved and merges onto
/// them rather than writing its old positions back.
/// new settings version; an editor saving its project holds the same lease, so it reads them moved and merges onto them
/// rather than writing its old positions back. When the sidecar, as read under the lease, lists other frames than the
/// index row (another process changed them together with their sidecar entries), the row is written without what
/// this host makes for the frames (<see cref="Uncovered"/>).
/// </summary>
private async Task<SequenceManifest> UpdateAndSyncAsync(Archive archive, ArchivePaths paths, SequencePaths sequence, Guid sequenceId,
Func<SequenceManifest, SequenceManifest?> edit, CancellationToken cancellationToken)
@@ -153,16 +184,27 @@ public sealed class ArchiveIndexer
await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false))
{
var manifest = SequenceWriter.ReadLocked(sequence, sequenceId);
var moved = await FramesMovedAsync(manifest, cancellationToken).ConfigureAwait(false);
if (edit(manifest) is { } updated)
{
ManifestStore.WriteSequence(sequence.ManifestPath, updated);
manifest = updated;
}
await SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
await SyncAsync(archive, sequence, moved ? manifest with { Status = manifest.Status & ~Uncovered } : manifest, cancellationToken).ConfigureAwait(false);
return manifest;
}
}
/// <summary>
/// What a scan clears in this host's own row, never in the shared sidecar, when another process added, removed or
/// renumbered frames together with their sidecar entries: on a server (<see cref="ArchiveServiceOptions.ImagesAfterScan"/>),
/// which analyses and draws what its clients load, the light analysis, thumbnails and proxies, so the scan's follow-ups
/// make them for the frames as they are, and their passes set the flags in both places again. Nothing on a desktop:
/// the flags in the sidecar are the writer's that added the frames (which analyses them, and on a server draws them
/// after every upload), and clearing them there would have the server make everything again.
/// </summary>
private SequenceStatus Uncovered => _writer.Options.ImagesAfterScan ? CoveringStatus : SequenceStatus.None;
/// <summary>Indexes one folder; returns its sequence and the sidecar bytes when they were read and not rewritten since (for the snapshot).</summary>
private async Task<(Guid Id, byte[]? Bytes)> IndexFolderAsync(Archive archive, ArchivePaths paths, SequenceFolder folder, bool reconcile, IndexSnapshot? snapshot,
Locations locations, CancellationToken cancellationToken)
@@ -196,18 +238,21 @@ public sealed class ArchiveIndexer
}
if (reconcile || File.Exists(sequence.IngestJournalPath))
{
// Photos added, gone or back: the light analysis, thumbnails and proxies no longer cover the frames, so the scan's
// follow-ups make them again (the images on a server, ArchiveServiceOptions.ImagesAfterScan).
// Photos added, gone or back: the light analysis, thumbnails and proxies no longer cover the frames, as the
// sidecar then tells every host, so the scan's follow-ups make them again (the images on a server,
// ArchiveServiceOptions.ImagesAfterScan).
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id,
m => SequenceAdopter.Reconcile(sequence, m, cancellationToken) is { } reconciled
? reconciled with { Status = reconciled.Status & ~(SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt) }
? reconciled with { Status = reconciled.Status & ~CoveringStatus }
: null,
cancellationToken).ConfigureAwait(false);
bytes = null;
}
else if (await MayMovePositionsAsync(manifest, cancellationToken).ConfigureAwait(false))
else if (await FramesMovedAsync(manifest, cancellationToken).ConfigureAwait(false))
{
// Another process changed the frames: the sidecar is read again under the lease the project moves under.
// Another process (a desktop sharing the archive, or the server it shares it with) added, removed or renumbered
// frames together with their sidecar entries: read again under the lease, under which the project's positions
// move, and indexed without what this host makes for the frames (Uncovered).
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false);
bytes = null;
}
@@ -257,9 +302,12 @@ public sealed class ArchiveIndexer
}
}
/// <summary>The index holds a project for this sequence and other frames than the sidecar: its positions may move.</summary>
private async Task<bool> MayMovePositionsAsync(SequenceManifest manifest, CancellationToken cancellationToken) =>
await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { SettingsJson: not null } indexed
/// <summary>
/// The index holds this sequence with other frames than the sidecar: another process changed them (its project's
/// positions may move, and what covered the frames no longer does).
/// </summary>
private async Task<bool> FramesMovedAsync(SequenceManifest manifest, CancellationToken cancellationToken) =>
await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { } indexed
&& indexed.FramesVersion != manifest.Versions.Frames;
/// <summary>A sidecar for a folder that has none; ids of a sequence the index already knew at this folder are kept.</summary>
@@ -309,7 +357,10 @@ public sealed class ArchiveIndexer
/// <summary>
/// Rebuild only: rows whose folder no longer holds a sidecar leave the index (their files are untouched). A sequence
/// waiting for its first frame has no sidecar yet and stays.
/// waiting for its first frame has no sidecar yet and stays. A row the folders listed when the rebuild began do not
/// back is looked at again under its lease, as it and its folder are when it would go (<see cref="BackedAsync"/>):
/// uploads go on during a rebuild, which can take minutes, so a sequence that got its first frame or was created
/// meanwhile has a folder the listing does not hold.
/// </summary>
private async Task<int> RemoveUnbackedAsync(Archive archive, ArchivePaths paths, IReadOnlyList<SequenceFolder> folders, Locations locations,
CancellationToken cancellationToken)
@@ -322,12 +373,46 @@ public sealed class ArchiveIndexer
foreach (var location in rows)
{
if (awaiting.Contains(location.Id) || (present.Contains(location.Folder) && locations.AtFolder(location.Folder) == location.Id)) continue;
if (await _db.Sequences.DeleteAsync(location.Id, cancellationToken).ConfigureAwait(false)) removed++;
await _db.Archives.DeleteFolderStateAsync(archive.Id, location.Folder, cancellationToken).ConfigureAwait(false);
IAsyncDisposable held;
try
{
held = await _writer.LockAsync(paths, location.Id, cancellationToken).ConfigureAwait(false);
}
catch (LeaseUnavailableException)
{
// Another process is writing the sequence, so it is there.
continue;
}
await using (held)
{
if (await BackedAsync(paths, location.Id, cancellationToken).ConfigureAwait(false)) continue;
if (await _db.Sequences.DeleteAsync(location.Id, cancellationToken).ConfigureAwait(false)) removed++;
await _db.Archives.DeleteFolderStateAsync(archive.Id, location.Folder, cancellationToken).ConfigureAwait(false);
}
}
return removed;
}
/// <summary>
/// Whether the index row of <paramref name="sequenceId"/> keeps its place as it is now: it waits for its first frame,
/// its folder holds a sidecar naming it (one that cannot be read proves nothing), or it is gone already.
/// </summary>
private async Task<bool> BackedAsync(ArchivePaths paths, Guid sequenceId, CancellationToken cancellationToken)
{
if (await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) is not { } row) return true;
if ((await AwaitingFolderAsync(paths, [new SequenceLocation(row.Id, row.Folder, row.Title)], cancellationToken).ConfigureAwait(false)).Count > 0)
return true;
var sidecar = paths.Sequence(row.Folder).ManifestPath;
try
{
return File.Exists(sidecar) && ManifestStore.ReadSequence(sidecar).Id == sequenceId;
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
return true;
}
}
/// <summary>Which folder each indexed sequence id lives in, updated as the scan goes.</summary>
private sealed class Locations
{
@@ -251,12 +251,20 @@ public sealed class ArchiveMaintenance
if (await _db.Sequences.GetAsync(bySequence.Key, cancellationToken).ConfigureAwait(false) is not { } sequence) continue;
var changes = bySequence.ToDictionary(x => x.Key, x => (Func<SidecarFrame, SidecarFrame>)(f =>
f with { Flags = x.Set ? f.Flags | FrameFlags.Duplicate : f.Flags & ~FrameFlags.Duplicate }));
await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m => SequenceFrames.Update(m, changes), cancellationToken).ConfigureAwait(false);
var setIds = bySequence.Where(x => x.Set).Select(x => x.Key).ToList();
var clearIds = bySequence.Where(x => !x.Set).Select(x => x.Key).ToList();
if (setIds.Count > 0) await _db.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false);
if (clearIds.Count > 0) await _db.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false);
await _indexer.RecordStampsAsync(archive, paths.Sequence(sequence.Folder), sequence.Id, cancellationToken).ConfigureAwait(false);
var folder = paths.Sequence(sequence.Folder);
await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
{
// The scan stamps follow only when the index held the folder as it was: frames another process listed in the
// sidecar meanwhile are still indexed by the next scan.
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
ManifestStore.WriteSequence(folder.ManifestPath, SequenceFrames.Update(manifest, changes));
var setIds = bySequence.Where(x => x.Set).Select(x => x.Key).ToList();
var clearIds = bySequence.Where(x => !x.Set).Select(x => x.Key).ToList();
if (setIds.Count > 0) await _db.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false);
if (clearIds.Count > 0) await _db.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false);
if (inStep) await _indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
}
}
return (groups.Count, set.Count + clear.Count);
}
+47 -23
View File
@@ -6,6 +6,7 @@ using Titano.Core.Model;
using Titano.Core.Naming;
using Titano.Data.Archiving.Jobs;
using Titano.Data.Jobs;
using Titano.Data.Repositories;
namespace Titano.Data.Archiving;
@@ -20,6 +21,9 @@ public sealed class ArchiveService : IJobHandlerProvider
/// <summary>Lease serialising read-modify-write of <c>archive.json</c> between the processes sharing an archive.</summary>
private const string SettingsLease = "archive-settings";
/// <summary>Times a title, tags, rating or notes edit is applied at the version it read before it writes regardless.</summary>
private const int InfoEditAttempts = 3;
private readonly TitanoDatabase _db;
private readonly SequenceWriter _writer;
@@ -224,35 +228,54 @@ public sealed class ArchiveService : IJobHandlerProvider
/// Changes some of title, tags, rating and notes from their current values — read under the sequence's lease from
/// the sidecar, or from the index when it is ahead (an edit made there only) — so fields <paramref name="edit"/>
/// leaves alone keep any change made since the caller read the sequence. Sidecar and index change together
/// (settings version + 1).
/// (settings version + 1), the index row in these fields only, at the settings version read under the lease
/// (<see cref="SequenceRepository.EditAsync(Guid, SequenceEdit, int?, CancellationToken)"/>): a settings edit committed
/// since the caller read the sequence (an analysis's Auto values, the server's PATCH, which takes no lease) stays, and
/// so do frames another process listed in the sidecar, which the next scan indexes
/// (<see cref="ArchiveIndexer.InStepAsync(Archive, SequencePaths, SequenceManifest, CancellationToken)"/>).
/// <paramref name="edit"/> runs again on top of an edit the API committed meanwhile.
/// </summary>
/// <exception cref="ArgumentException">The sequence is not in the index (or left it before the edit).</exception>
public async Task<Sequence> EditSequenceInfoAsync(Guid sequenceId, Func<SequenceInfo, SequenceInfo> edit, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(edit);
var (archive, sequence) = await LoadSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false);
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var manifest = await _writer.UpdateAsync(paths, sequenceId, sequence.Folder, m =>
{
var current = sequence.SettingsVersion > m.Versions.Settings
? new SequenceInfo(sequence.Title, sequence.Tags, sequence.Rating, sequence.Notes)
: new SequenceInfo(m.Title, m.Tags, m.Rating, m.Notes);
var info = edit(current);
return m with
{
Title = string.IsNullOrWhiteSpace(info.Title) ? null : info.Title.Trim(),
Tags = info.Tags,
Rating = info.Rating,
Notes = info.Notes,
Versions = m.Versions with { Settings = System.Math.Max(m.Versions.Settings, sequence.SettingsVersion) + 1 },
UpdatedAt = DateTimeOffset.UtcNow,
};
}, cancellationToken).ConfigureAwait(false);
var folder = paths.Sequence(sequence.Folder);
var updated = SequenceMapping.ToSequence(manifest, archive.Id, folder, sequence);
await _db.Sequences.UpsertAsync(updated, cancellationToken).ConfigureAwait(false);
await Indexer.RecordStampsAsync(archive, folder, sequenceId, cancellationToken).ConfigureAwait(false);
return updated;
await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false))
{
var manifest = SequenceWriter.ReadLocked(folder, sequenceId);
var inStep = await Indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
for (var attempt = 1; ; attempt++)
{
var row = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? throw Unknown(sequenceId);
var current = row.SettingsVersion > manifest.Versions.Settings
? new SequenceInfo(row.Title, row.Tags, row.Rating, row.Notes)
: new SequenceInfo(manifest.Title, manifest.Tags, manifest.Rating, manifest.Notes);
var info = edit(current);
var title = string.IsNullOrWhiteSpace(info.Title) ? null : info.Title.Trim();
// The last attempt writes over whatever the index holds then rather than fail the user's edit.
var result = await _db.Sequences.EditAsync(sequenceId,
new SequenceEdit { Title = title ?? string.Empty, Tags = info.Tags, Rating = info.Rating, Notes = info.Notes ?? string.Empty },
attempt < InfoEditAttempts ? row.SettingsVersion : null, cancellationToken).ConfigureAwait(false);
// A conflict means the API changed the sequence after the read: the edit applies again on top of that.
if (result.Outcome == SequenceEditOutcome.Conflict) continue;
if (result.Sequence is not { } updated) throw Unknown(sequenceId);
// The sidecar takes the fields as the index stored them (trimmed, tags without duplicates).
ManifestStore.WriteSequence(folder.ManifestPath, manifest with
{
Title = updated.Title,
Tags = updated.Tags,
Rating = updated.Rating,
Notes = updated.Notes,
Versions = manifest.Versions with { Settings = System.Math.Max(manifest.Versions.Settings + 1, updated.SettingsVersion) },
UpdatedAt = DateTimeOffset.UtcNow,
});
if (inStep) await Indexer.RecordStampsAsync(archive, folder, sequenceId, cancellationToken).ConfigureAwait(false);
return updated;
}
}
}
public Task<IReadOnlyList<DuplicateGroup>> FindDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
@@ -367,10 +390,11 @@ public sealed class ArchiveService : IJobHandlerProvider
private async Task<(Archive, Sequence)> LoadSequenceAsync(Guid sequenceId, CancellationToken cancellationToken)
{
var sequence = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
?? throw new ArgumentException($"Unknown sequence {sequenceId}.", nameof(sequenceId));
var sequence = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? throw Unknown(sequenceId);
return (await LoadArchiveAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false), sequence);
}
private static ArgumentException Unknown(Guid sequenceId) => new($"Unknown sequence {sequenceId}.", nameof(sequenceId));
}
/// <summary>The fields of a sequence the user describes it with (sidecar and index).</summary>
+34 -14
View File
@@ -95,35 +95,55 @@ public sealed class ArchiveVerifier
return targets;
}
/// <summary>Writes the flags and new hashes of checked frames to the sidecar and the index.</summary>
/// <summary>
/// Writes the flags and new hashes of checked frames to the sidecar and the index, under one lease. The folder's scan
/// stamps follow only when the index held the folder as it was
/// (<see cref="ArchiveIndexer.InStepAsync(Archive, SequencePaths, SequenceManifest, CancellationToken)"/>), so frames
/// another process listed in the sidecar meanwhile are still indexed by the next scan.
/// </summary>
private async Task FlushAsync(Archive archive, ArchivePaths paths, Sequence target, List<FrameCheck> pending, CancellationToken cancellationToken)
{
if (pending.Count == 0) return;
var changes = pending.ToDictionary(c => c.Frame.Id, c => (Func<SidecarFrame, SidecarFrame>)(f =>
f with { Flags = c.Flags, Sha256 = c.Health == FrameHealth.Hashed ? c.Sha256 : f.Sha256 }));
await _writer.UpdateAsync(paths, target.Id, target.Folder, m => SequenceFrames.Update(m, changes), cancellationToken).ConfigureAwait(false);
await _db.Frames.UpdateIntegrityAsync(pending.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)),
cancellationToken).ConfigureAwait(false);
await _indexer.RecordStampsAsync(archive, paths.Sequence(target.Folder), target.Id, cancellationToken).ConfigureAwait(false);
var sequence = paths.Sequence(target.Folder);
await using (await _writer.LockAsync(paths, target.Id, cancellationToken).ConfigureAwait(false))
{
var manifest = SequenceWriter.ReadLocked(sequence, target.Id);
var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
ManifestStore.WriteSequence(sequence.ManifestPath, SequenceFrames.Update(manifest, changes));
await _db.Frames.UpdateIntegrityAsync(pending.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)),
cancellationToken).ConfigureAwait(false);
if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false);
}
pending.Clear();
}
/// <summary>Stamps the verify time; the sequence counts as verified only when no frame is missing or corrupt.</summary>
/// <summary>
/// Stamps the verify time; the sequence counts as verified only when no frame is missing or corrupt, and has errors
/// while one is. Sidecar and index are written under one lease, the index row with these fields only
/// (<see cref="SequenceRepository.RecordVerifiedAsync"/>), so a settings edit or an ingest committed meanwhile stays;
/// the scan stamps as in <see cref="FlushAsync"/>.
/// </summary>
private async Task StampAsync(Archive archive, ArchivePaths paths, Sequence target, CancellationToken cancellationToken)
{
var sequence = paths.Sequence(target.Folder);
var manifest = await _writer.UpdateAsync(paths, target.Id, target.Folder, m =>
await using (await _writer.LockAsync(paths, target.Id, cancellationToken).ConfigureAwait(false))
{
var m = SequenceWriter.ReadLocked(sequence, target.Id);
var inStep = await _indexer.InStepAsync(archive, sequence, m, cancellationToken).ConfigureAwait(false);
var clean = m.Frames.All(f => (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0);
return m with
var now = DateTimeOffset.UtcNow;
var manifest = m with
{
Status = clean ? m.Status | SequenceStatus.Verified : m.Status & ~SequenceStatus.Verified,
VerifiedAt = DateTimeOffset.UtcNow,
UpdatedAt = DateTimeOffset.UtcNow,
VerifiedAt = now,
UpdatedAt = now,
};
}, cancellationToken).ConfigureAwait(false);
var existing = await _db.Sequences.GetAsync(target.Id, cancellationToken).ConfigureAwait(false);
await _db.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false);
ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
await _db.Sequences.RecordVerifiedAsync(target.Id, clean ? SequenceStatus.Verified : SequenceStatus.HasErrors,
clean ? SequenceStatus.HasErrors : SequenceStatus.Verified, now, cancellationToken).ConfigureAwait(false);
if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false);
}
}
}
+26 -14
View File
@@ -61,6 +61,13 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi
/// </summary>
/// <param name="proxies">Whether every such frame follows its proxy even where no tier is recorded (analyses and the
/// proxy pass); otherwise only the frames in <paramref name="tiers"/> do.</param>
/// <remarks>
/// The sidecar and the index are written under one lease, the index from the sidecar read under it, and the index
/// row gets only what tiers change (<see cref="SequenceRepository.RecordTiersAsync"/>): a whole row written back
/// would undo what another writer committed meanwhile, an ingest's frames or a settings edit (the server's PATCH
/// takes no lease). The folder's scan stamps are recorded only when the index held the folder as it was
/// (<see cref="ArchiveIndexer.InStepAsync(Archive, SequencePaths, SequenceManifest, CancellationToken)"/>).
/// </remarks>
/// <exception cref="ArchiveOfflineException">The archive is offline or read-only.</exception>
public async Task<IReadOnlyList<SidecarFrame>> RecordAsync(Archive archive, Sequence sequence, IReadOnlyDictionary<Guid, FileTier> tiers,
bool proxies, CancellationToken cancellationToken)
@@ -69,12 +76,12 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var folder = paths.Sequence(sequence.Folder);
var changed = new List<SidecarFrame>();
ProxyPolicy? policy = null;
var manifest = await writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m =>
await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
{
changed.Clear();
var m = SequenceWriter.ReadLocked(folder, sequence.Id);
var changed = new List<SidecarFrame>();
var gained = false;
ProxyPolicy? policy = null;
var frames = new SidecarFrame[m.Frames.Count];
for (var i = 0; i < frames.Length; i++)
{
@@ -96,21 +103,26 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi
frames[i] = updated;
changed.Add(updated);
}
if (changed.Count == 0) return null;
return m with
if (changed.Count == 0) return m.Frames;
var inStep = await indexer.InStepAsync(archive, folder, m, cancellationToken).ConfigureAwait(false);
var manifest = m with
{
Frames = frames,
Status = gained ? m.Status | SequenceStatus.HasWarnings : m.Status,
UpdatedAt = DateTimeOffset.UtcNow,
};
}, cancellationToken).ConfigureAwait(false);
if (changed.Count == 0) return manifest.Frames;
await db.Frames.UpdateTiersAsync(changed.ConvertAll(f => new FrameTierFields(f.Id, f.Flags, f.PreviewW ?? 0, f.PreviewH ?? 0)), cancellationToken)
.ConfigureAwait(false);
if (await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) is { } existing)
await db.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, folder, existing), cancellationToken).ConfigureAwait(false);
await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
return manifest.Frames;
ManifestStore.WriteSequence(folder.ManifestPath, manifest);
await using (var tx = await db.BeginAsync(cancellationToken).ConfigureAwait(false))
{
await tx.Frames.UpdateTiersAsync(changed.ConvertAll(f => new FrameTierFields(f.Id, f.Flags, f.PreviewW ?? 0, f.PreviewH ?? 0)), cancellationToken)
.ConfigureAwait(false);
await tx.Sequences.RecordTiersAsync(sequence.Id, gained ? SequenceStatus.HasWarnings : SequenceStatus.None,
frames.Count(f => (f.Flags & FrameFlags.PreviewOnly) != 0), cancellationToken).ConfigureAwait(false);
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
}
if (inStep) await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
return manifest.Frames;
}
}
/// <summary>
@@ -67,10 +67,11 @@ internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer,
/// <summary>
/// Queues a background <paramref name="pass"/> (thumbnails or proxies) of each sequence this run indexed whose
/// <paramref name="built"/> flag is clear: new and adopted sequences, and those whose photos changed (the scan clears the
/// flag then). After a resumed run, which may have indexed sequences before the interruption, of every sequence of the
/// archive without the flag but those in <paramref name="awaiting"/>. One pass per sequence, as after an upload: a folder
/// renamed, deleted or damaged before its pass runs fails that pass alone, and the other sequences still get their
/// images. A sequence whose own pass of that kind is already queued is left out.
/// flag then, in this server's own row when another writer listed them in the sidecar itself). After a resumed run,
/// which may have indexed sequences before the interruption, of every sequence of the archive without the flag but
/// those in <paramref name="awaiting"/>. One pass per sequence, as after an upload: a folder renamed, deleted or
/// damaged before its pass runs fails that pass alone, and the other sequences still get their images. A sequence
/// whose own pass of that kind is already queued is left out.
/// </summary>
private async Task QueueImagesAsync(JobContext context, Archive archive, JobKind pass, SequenceStatus built, IReadOnlyList<Guid> indexed,
bool resumed, IReadOnlySet<Guid> awaiting)
+14 -5
View File
@@ -210,12 +210,22 @@ public sealed class MassRenamer
.FirstOrDefault(l => !taken.Contains(l.Id) && folders.Any(f => RenamePlanner.IsInside(f, l.Folder)));
}
/// <summary>Sidecar title, then index (folder, frames' paths); each part is idempotent so a resumed step can repeat it.</summary>
/// <summary>
/// Sidecar title, then index (folder, title, frames' paths); each part is idempotent so a resumed step can repeat it.
/// The index row changes in its folder and title only (<see cref="SequenceRepository.RelocateAsync"/>): a settings edit
/// committed meanwhile stays, and frames another process listed in the sidecar are left to the next scan, for which
/// the folder keeps no stamps unless the index held it as it was
/// (<see cref="ArchiveIndexer.InStepAsync(Archive, string, SequencePaths, SequenceManifest, CancellationToken)"/>).
/// </summary>
private async Task RecordAsync(Archive archive, ArchivePaths paths, RenameStep step, CancellationToken cancellationToken)
{
var sequence = paths.Sequence(step.ToFolder);
var manifest = SequenceWriter.ReadLocked(sequence, step.SequenceId);
if (manifest.Title != step.ToTitle)
// The folder moved with its files' stamps, which the index recorded at its old place.
var inStep = await _indexer.InStepAsync(archive, step.FromFolder, sequence, manifest, cancellationToken).ConfigureAwait(false);
// A step that only moves the folder keeps the title, which an edit may have changed since the plan.
var retitle = step.FromTitle != step.ToTitle;
if (retitle && manifest.Title != step.ToTitle)
{
manifest = manifest with
{
@@ -227,12 +237,11 @@ public sealed class MassRenamer
}
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
{
var existing = await tx.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false);
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
await tx.Sequences.RelocateAsync(step.SequenceId, step.ToFolder, retitle ? step.ToTitle : null, cancellationToken).ConfigureAwait(false);
if (step.MovesFolder) await tx.Frames.RebaseFolderAsync(step.SequenceId, step.FromFolder, step.ToFolder, cancellationToken).ConfigureAwait(false);
await tx.Archives.DeleteFolderStateAsync(archive.Id, step.FromFolder, cancellationToken).ConfigureAwait(false);
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
}
await _indexer.RecordStampsAsync(archive, sequence, step.SequenceId, cancellationToken).ConfigureAwait(false);
if (inStep) await _indexer.RecordStampsAsync(archive, sequence, step.SequenceId, cancellationToken).ConfigureAwait(false);
}
}
+22 -17
View File
@@ -40,25 +40,30 @@ internal sealed class ProjectStore(TitanoDatabase db, SequenceWriter writer)
/// <exception cref="InvalidDataException">The stored settings cannot be read; nothing is written.</exception>
public async Task<bool> UpdateAsync(ArchivePaths paths, Sequence sequence, Func<ProjectSettings, ProjectSettings> edit, CancellationToken cancellationToken)
{
var folder = paths.Sequence(sequence.Folder);
await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
return await UpdateLockedAsync(paths, sequence, edit, cancellationToken).ConfigureAwait(false);
}
/// <summary><see cref="UpdateAsync"/> for a caller that holds the sequence lock already.</summary>
/// <exception cref="InvalidDataException">The stored settings cannot be read; nothing is written.</exception>
public async Task<bool> UpdateLockedAsync(ArchivePaths paths, Sequence sequence, Func<ProjectSettings, ProjectSettings> edit, CancellationToken cancellationToken)
{
var folder = paths.Sequence(sequence.Folder);
for (var attempt = 1; ; attempt++)
{
for (var attempt = 1; ; attempt++)
{
var row = await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) ?? sequence;
var current = Read(row, folder);
var before = current.ToJson();
var edited = edit(current);
var after = edited.ToJson();
if (after == before) return false;
AtomicFile.WriteAllText(folder.ProjectPath, after);
// The Library shows the project's hero frame (advice fills it, a sync takes another PC's).
var hero = await HeroFrame.EditAsync(db, sequence.Id, current.HeroFrame, edited.HeroFrame, cancellationToken).ConfigureAwait(false);
var result = await db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = after, HeroFrameId = hero }, row.SettingsVersion,
cancellationToken).ConfigureAwait(false);
// A conflict means the API changed the settings meanwhile: apply the edit again on top of them.
if (result.Outcome != SequenceEditOutcome.Conflict || attempt == Attempts) return true;
}
var row = await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) ?? sequence;
var current = Read(row, folder);
var before = current.ToJson();
var edited = edit(current);
var after = edited.ToJson();
if (after == before) return false;
AtomicFile.WriteAllText(folder.ProjectPath, after);
// The Library shows the project's hero frame (advice fills it, a sync takes another PC's).
var hero = await HeroFrame.EditAsync(db, sequence.Id, current.HeroFrame, edited.HeroFrame, cancellationToken).ConfigureAwait(false);
var result = await db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = after, HeroFrameId = hero }, row.SettingsVersion,
cancellationToken).ConfigureAwait(false);
// A conflict means the API changed the settings meanwhile: apply the edit again on top of them.
if (result.Outcome != SequenceEditOutcome.Conflict || attempt == Attempts) return true;
}
}
}
+41 -29
View File
@@ -52,56 +52,64 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti
frames = WithTiers(frames, await _tiers.RecordAsync(archive, sequence, probed, proxies: true, cancellationToken).ConfigureAwait(false));
var metadata = frames.LightFrames();
DeepAnalysisReport? report;
DeepAnalysisReport? report = null;
if (deep)
{
// Proxies may have arrived during the pass: the advice below follows the frames as they are at its end.
// Proxies may have arrived during the pass: the flags below follow the frames as they are at its end, and the
// report is checked against them under the lock.
(report, var decoded) = await AnalyseDeepAsync(archive, frames, metadata, folder, progress, cancellationToken).ConfigureAwait(false);
frames = WithTiers(frames, await _tiers.RecordAsync(archive, sequence, decoded, proxies: true, cancellationToken).ConfigureAwait(false));
metadata = [.. metadata.Select((m, i) => m with { Flags = frames.Frames[i].Flags })];
}
else
{
report = ReadReport(folder, frames);
}
var light = LightAnalyzer.Analyze(metadata);
var context = SequenceContext.FromFrames(frames.Frames, options.AdvisorCodecs);
bool changed, unreadable = false;
try
{
// The frame rate follows the user's "fit to N seconds" when one is set (BRIEF §10.3).
changed = await projects.UpdateAsync(paths, sequence,
settings => Advisor.Apply(settings, Advisor.Recommend(light, report, context with { TargetDurationSeconds = settings.Timing.FitToSeconds })),
cancellationToken).ConfigureAwait(false);
}
catch (InvalidDataException ex)
{
Core.Diagnostics.Log.Warning("processing", $"The project of {sequence.DisplayTitle} cannot be read", ex);
(changed, unreadable) = (false, true);
}
if (changed) await options.NotifyAsync(sequence.Id, ProcessingOptions.SettingsChange, cancellationToken).ConfigureAwait(false);
int warnings;
// Under the sequence lock, which a deep pass of the same sequence also takes to record its report: a light pass
// then either sees that report or runs before it is recorded, and never clears what it recorded.
// then either sees that report or runs before it is recorded, and never clears what it recorded. The report counts
// only while it describes the frames as they are under the lock (ReadReport), so a deep pass loads them again: a
// proxy that arrived while it ran (for hours) makes its verdict on that frame's original stale. The light analysis
// the proxy queued then either ran before this, and the advice, status and warnings stay without the report as it
// left them, or runs after.
await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
{
report ??= ReadReport(folder, frames);
if (deep)
{
var now = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, options.Archive.Decoder, cancellationToken).ConfigureAwait(false);
if (!Describes(report!, now)) report = null;
}
else
{
report = ReadReport(folder, frames);
}
try
{
// The frame rate follows the user's "fit to N seconds" when one is set (BRIEF §10.3).
changed = await projects.UpdateLockedAsync(paths, sequence,
settings => Advisor.Apply(settings, Advisor.Recommend(light, report, context with { TargetDurationSeconds = settings.Timing.FitToSeconds })),
cancellationToken).ConfigureAwait(false);
}
catch (InvalidDataException ex)
{
Core.Diagnostics.Log.Warning("processing", $"The project of {sequence.DisplayTitle} cannot be read", ex);
(changed, unreadable) = (false, true);
}
warnings = Advisor.Recommend(light, report, context).Findings.Count(f => f.Severity >= FindingSeverity.Warning);
var set = SequenceStatus.AnalyzedLight | (report is null ? SequenceStatus.None : SequenceStatus.AnalyzedDeep)
| (warnings > 0 ? SequenceStatus.HasWarnings : SequenceStatus.None);
// Without a current report (none, or made before frames were added) the deep status no longer holds.
var clear = (report is null ? SequenceStatus.AnalyzedDeep : SequenceStatus.None) | (warnings > 0 ? SequenceStatus.None : SequenceStatus.HasWarnings);
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
var inStep = await indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
var status = (manifest.Status | set) & ~clear;
if (status != manifest.Status) ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow });
var summary = deep && report is not null ? AnalysisSummary.From(report).ToJson() : null;
await db.Sequences.RecordAnalysisAsync(sequence.Id, set, clear, warnings, summary, cancellationToken).ConfigureAwait(false);
await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
if (inStep) await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
}
if (changed) await options.NotifyAsync(sequence.Id, ProcessingOptions.SettingsChange, cancellationToken).ConfigureAwait(false);
var presets = deep ? SequencePresets.Generate(light, report, context) : [];
return new AnalysisOutcome(sequence, frames.Count, warnings, changed, unreadable, presets);
}
@@ -111,17 +119,21 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti
/// frame it found unreadable has been read from its proxy since (one sent for an original this host cannot decode,
/// BRIEF §2.3), so that verdict and the frame's missing measurements no longer hold.
/// </summary>
public static DeepAnalysisReport? ReadReport(SequencePaths folder, SequenceFrames frames)
public static DeepAnalysisReport? ReadReport(SequencePaths folder, SequenceFrames frames) =>
ReadReport(folder, frames.Count) is { } report && Describes(report, frames) ? report : null;
/// <summary>Whether <paramref name="report"/> describes <paramref name="frames"/> as this host reads them now (<see cref="ReadReport(SequencePaths, SequenceFrames)"/>).</summary>
private static bool Describes(DeepAnalysisReport report, SequenceFrames frames)
{
var report = ReadReport(folder, frames.Count);
if (report is null || frames.Proxied is not { } proxied) return report;
if (report.FrameCount != frames.Count) return false;
if (frames.Proxied is not { } proxied) return true;
HashSet<int> measured = report.ProxyFrames is { } read ? [.. read] : [];
foreach (var index in report.Issues.Undecodable)
{
if (index < frames.Count && proxied[index] && !measured.Contains(index) && (frames.Frames[index].Flags & SequenceFrames.NotAnalysed) == 0)
return null;
return false;
}
return report;
return true;
}
/// <summary>The saved deep report when it covers <paramref name="frameCount"/> frames; null when absent, stale or unreadable.</summary>
@@ -10,6 +10,7 @@ public sealed record FolderScanState(string Folder, Guid? SequenceId, FileStamp?
public sealed class ArchiveRepository
{
private const string Columns = "id, name, root_path, folder_template, file_template, schema_version, created, proxy_long_edge";
private const string FolderStateSelect = "SELECT folder, sequence_id, sidecar_size, sidecar_ticks, photos_ticks FROM archive_folders";
private static readonly string UpsertSql = SqlDialect.Upsert("archives",
["id", "name", "root_path", "folder_template", "file_template", "schema_version", "created", "proxy_long_edge"], 1);
@@ -39,15 +40,14 @@ public sealed class ArchiveRepository
/// <summary>Scan stamps of every folder of an archive, by folder.</summary>
public async Task<IReadOnlyDictionary<string, FolderScanState>> GetFolderStatesAsync(Guid archiveId, CancellationToken cancellationToken = default)
{
var rows = await _db.ListAsync(
"SELECT folder, sequence_id, sidecar_size, sidecar_ticks, photos_ticks FROM archive_folders WHERE archive_id = $1", [archiveId],
r => new FolderScanState(r.GetString(0), r.GetGuidOrNull(1),
r.GetInt64OrNull(3) is { } sidecar ? new FileStamp(r.GetInt64(2), new DateTime(sidecar, DateTimeKind.Utc)) : null,
r.GetInt64OrNull(4) is { } photos ? new FileStamp(0, new DateTime(photos, DateTimeKind.Utc)) : null),
cancellationToken).ConfigureAwait(false);
var rows = await _db.ListAsync($"{FolderStateSelect} WHERE archive_id = $1", [archiveId], MapFolderState, cancellationToken).ConfigureAwait(false);
return rows.ToDictionary(s => s.Folder, StringComparer.Ordinal);
}
/// <summary>Scan stamps of one folder of an archive; null when none are recorded.</summary>
public Task<FolderScanState?> GetFolderStateAsync(Guid archiveId, string folder, CancellationToken cancellationToken = default) =>
_db.SingleAsync($"{FolderStateSelect} WHERE archive_id = $1 AND folder = $2", [archiveId, folder], MapFolderState, cancellationToken);
public async Task SetFolderStateAsync(Guid archiveId, FolderScanState state, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync(FolderStateUpsertSql,
[archiveId, state.Folder, state.SequenceId, state.Sidecar?.Size, state.Sidecar?.ModifiedUtc.Ticks, state.Photos?.ModifiedUtc.Ticks, DateTimeOffset.UtcNow],
@@ -61,6 +61,10 @@ public sealed class ArchiveRepository
public Task<int> ClearFolderStatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
_db.ExecuteAsync("DELETE FROM archive_folders WHERE archive_id = $1", [archiveId], cancellationToken);
private static FolderScanState MapFolderState(DbRow r) => new(r.GetString(0), r.GetGuidOrNull(1),
r.GetInt64OrNull(3) is { } sidecar ? new FileStamp(r.GetInt64(2), new DateTime(sidecar, DateTimeKind.Utc)) : null,
r.GetInt64OrNull(4) is { } photos ? new FileStamp(0, new DateTime(photos, DateTimeKind.Utc)) : null);
private static Archive Map(DbRow r) => new()
{
Id = r.GetGuid(0),
@@ -195,6 +195,24 @@ public sealed class SequenceRepository
await _db.ExecuteAsync("UPDATE sequences SET status = (status | $2) & ~$3, updated_at = $4 WHERE id = $1",
[id, (int)set, (int)clear, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>
/// Records decode tiers written to the sequence's frames (<see cref="FrameRepository.UpdateTiersAsync"/>): the status
/// flags in <paramref name="set"/> and the preview-only count, in one statement. The frames, their versions and the
/// settings stay as they are, so an ingest or a settings edit committed meanwhile is never written back over.
/// </summary>
public async Task<bool> RecordTiersAsync(Guid id, SequenceStatus set, int previewOnlyCount, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("UPDATE sequences SET status = status | $2, preview_only_count = $3, updated_at = $4 WHERE id = $1",
[id, (int)set, previewOnlyCount, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>
/// Records a verify: status flags set and cleared and the verify time, in one statement; every other field stays as
/// it is (<see cref="RecordTiersAsync"/>).
/// </summary>
public async Task<bool> RecordVerifiedAsync(Guid id, SequenceStatus set, SequenceStatus clear, DateTimeOffset verifiedAt,
CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("UPDATE sequences SET status = (status | $2) & ~$3, verified_at = $4, updated_at = $5 WHERE id = $1",
[id, (int)set, (int)clear, verifiedAt, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>
/// Records an analysis: status flags set and cleared, the warning count and, for a deep analysis, the compact
/// summary (<paramref name="analysisJson"/> null keeps the stored one).
@@ -390,19 +408,23 @@ public sealed class SequenceRepository
[archiveId, sessionKey], cancellationToken).ConfigureAwait(false))!.Value;
}, cancellationToken);
/// <summary>Moves a sequence to another folder and/or title (mass rename); false when it does not exist.</summary>
/// <summary>
/// Moves a sequence to another folder and, with <paramref name="title"/>, retitles it (mass rename; settings version
/// + 1 when the title changes); every other field stays as the locked row has it. False when it does not exist.
/// </summary>
/// <param name="title">The new title; null keeps the current one.</param>
public Task<bool> RelocateAsync(Guid id, string folder, string? title, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
var current = await session.QuerySingleOrDefaultAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken)
.ConfigureAwait(false);
if (current is null) return false;
var titleChanged = current.Title != title;
var retitled = title is not null && current.Title != title;
await UpsertRowAsync(session, current with
{
Folder = folder,
Title = title,
SettingsVersion = titleChanged ? current.SettingsVersion + 1 : current.SettingsVersion,
Title = retitled ? title : current.Title,
SettingsVersion = retitled ? current.SettingsVersion + 1 : current.SettingsVersion,
UpdatedAt = DateTimeOffset.UtcNow,
}, cancellationToken).ConfigureAwait(false);
return true;
@@ -7,8 +7,9 @@ namespace Titano.Server.Storage;
/// <summary>
/// Proxies are files under the sequence's <c>proxies/</c> folder; a thumbnail is the JPEG byte range of its entry in
/// <c>thumbs.pack</c>. A thumbnail made from a different original (the frame was renumbered or replaced) counts as not
/// built yet, like a missing one, and so does a proxy whose capture time is another frame's, or one smaller than the
/// archive's policy sent for a frame the server cannot decode (<see cref="ServerProxies.HasProxy"/>).
/// built yet, like a missing one (<see cref="ThumbPack.IsFrom"/>: one drawn before a verify hashed the photo is still its
/// own), and so does a proxy whose capture time is another frame's, or one smaller than the archive's policy sent for a
/// frame the server cannot decode (<see cref="ServerProxies.HasProxy"/>).
/// </summary>
internal sealed class ArchiveAssetLocator : IFrameAssetLocator
{
@@ -34,7 +35,14 @@ internal sealed class ArchiveAssetLocator : IFrameAssetLocator
{
using var pack = ThumbPackReader.Open(paths.ThumbsPackPath);
if (!pack.Entries.TryGetValue(frame.Index, out var entry) || entry.Offset + entry.Length > pack.Length) return null;
if (frame.Sha256 is { Length: ThumbPack.HashLength * 2 } sha && !entry.SourceHash.AsSpan().SequenceEqual(Convert.FromHexString(sha))) return null;
if (frame.Sha256 is { Length: ThumbPack.HashLength * 2 } sha && !ThumbPack.IsFrom(entry.SourceHash, sha, SequenceFile(paths, frame), frame.Size)) return null;
return new AssetLocation(paths.ThumbsPackPath, entry.Offset, entry.Length, Jpeg);
}
/// <summary>The frame's file as its sidecar names it (relative to the sequence folder), which a thumbnail drawn before the photo was hashed is fingerprinted with.</summary>
private static string SequenceFile(SequencePaths paths, Frame frame)
{
var prefix = paths.ArchiveRelative(string.Empty);
return frame.RelativePath.StartsWith(prefix, StringComparison.Ordinal) ? frame.RelativePath[prefix.Length..] : frame.RelativePath;
}
}
@@ -150,6 +150,25 @@ public sealed class ThumbPackTests(TestContext context)
for (var i = 0; i < 20; i++) Assert.BytesEqual(Payload(i + 300), compacted.Read(i));
}
[Test]
public void AThumbnailIsFromItsOriginalAlsoAfterTheOriginalWasHashed()
{
var sha = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData("photo"u8));
var other = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData("another photo"u8));
var drawnUnhashed = ThumbPack.Fingerprint(null, "IMG_0001.JPG", 1000);
var drawnHashed = ThumbPack.Fingerprint(sha, "IMG_0001.JPG", 1000);
Assert.True(ThumbPack.IsFrom(drawnUnhashed, null, "IMG_0001.JPG", 1000), "unhashed");
Assert.True(ThumbPack.IsFrom(drawnHashed, sha, "IMG_0001.JPG", 1000), "hashed");
// Drawn before a verify hashed the photo: the hash changes nothing about it.
Assert.True(ThumbPack.IsFrom(drawnUnhashed, sha, "IMG_0001.JPG", 1000), "hashed since");
// Another photo: another hash, or another file or size at that frame index.
Assert.False(ThumbPack.IsFrom(drawnHashed, other, "IMG_0001.JPG", 1000), "another hash");
Assert.False(ThumbPack.IsFrom(drawnUnhashed, sha, "IMG_0002.JPG", 1000), "another file");
Assert.False(ThumbPack.IsFrom(drawnUnhashed, null, "IMG_0001.JPG", 1001), "another size");
Assert.False(ThumbPack.IsFrom(drawnHashed, null, "IMG_0001.JPG", 1000), "a hash the frame no longer has");
}
[Test]
public void AMissingOrForeignFileReadsAsEmpty()
{
@@ -55,6 +55,20 @@ internal sealed class ArchiveFixture : IAsyncDisposable
public Task<ScanSummary> ScanAsync(ScanMode mode = ScanMode.Incremental) =>
Service.Indexer.ScanAsync(Archive, mode, null, _ => ValueTask.CompletedTask, Ct);
/// <summary>
/// Another process writing the same archive, such as a desktop sharing a server's archive folder: its own index
/// (scanned) and archive service; disposed with the fixture.
/// </summary>
public async Task<(ArchiveService Service, Archive Archive)> OtherWriterAsync()
{
var db = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(_context.CreateTempDirectory("other"), "titano.db")), Ct);
_owned.Add(db);
var service = new ArchiveService(db, new ArchiveServiceOptions { Owner = "other", LeaseWait = TimeSpan.FromSeconds(20) });
var archive = await service.OpenAsync(Archive.RootPath, Ct);
await service.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, Ct);
return (service, archive);
}
/// <summary>A job engine with the archive handlers, started; disposed with the fixture.</summary>
public async Task<JobEngine> StartEngineAsync()
{
@@ -365,6 +365,38 @@ public sealed class ArchiveIngestTests(TestContext context)
Assert.Equal("2026/2026-09-29 Canon EOS R5", (await fx.Db.Sequences.GetAsync(waiting.Id))!.Folder);
}
[Test]
public async Task ARebuildKeepsSequencesThatGetTheirFolderWhileItRuns()
{
await using var fx = await CreateAsync(context);
Photos.WriteSeries(Path.Combine(fx.Archive.RootPath, "Trips", "Comet"), Start, 2);
await fx.ScanAsync();
var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset");
// Uploads go on while the rebuild runs: after it listed the folders, the sequence waiting for its first frame gets
// it, and a new upload session creates a sequence.
IngestResult? first = null, session = null;
var rebuilt = await fx.Service.Indexer.ScanAsync(fx.Archive, ScanMode.Rebuild, null, async progress =>
{
if (progress.FoldersDone < progress.FoldersTotal || first is not null) return;
first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddDays(1), 1), sequenceId: waiting.Id), fx.Ct);
session = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddDays(2), 2), sessionKey: "upload-2"), fx.Ct);
}, fx.Ct);
// Neither is dropped, though the folders listed at the start hold neither.
Assert.Equal(0, rebuilt.Removed);
Assert.Equal(waiting.Id, Assert.NotNull(first).SequenceId);
var placed = Assert.NotNull(await fx.Db.Sequences.GetAsync(waiting.Id));
Assert.Equal(("2026/2026-09-29 Canon EOS R5", 1, "Harbour sunset"), (placed.Folder, placed.FrameCount, placed.Title));
var created = Assert.NotNull(await fx.Db.Sequences.GetAsync(Assert.NotNull(session).SequenceId));
Assert.Equal(1, created.FrameCount);
// The session's next upload lands in the same sequence, not in a second one beside it.
Assert.Equal(created.Id, await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, "upload-2", fx.Ct));
var next = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddDays(2).AddSeconds(10), 3), sessionKey: "upload-2"), fx.Ct);
Assert.Equal(created.Id, next.SequenceId);
Assert.Equal(3, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count);
}
[Test]
public async Task ADamagedRootSidecarIsRebuiltUnderTheRootSequenceNotOneWaitingForItsFirstFrame()
{
@@ -175,6 +175,210 @@ public sealed class ArchiveJobTests(TestContext context)
Assert.Count(7, Directory.GetFiles(fx.Paths.Sequence("a").Proxies));
}
[Test]
public async Task AServerScanQueuesTheImagesOfFramesAnotherWriterListedInTheSidecarItself()
{
await using var fx = await CreateAsync(context);
Populate(fx);
await using var engine = ServerEngine(fx);
await engine.StartAsync(fx.Ct);
var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
await fx.WaitForAsync(scan.Id, JobState.Done);
var first = new HashSet<Guid>();
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
{
foreach (var pass in (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items)
{
await fx.WaitForAsync(pass.Id, JobState.Done);
first.Add(pass.Id);
}
}
var a = (await LocationsAsync(fx)).Single(l => l.Folder == "a");
Assert.True(Built((await fx.Db.Sequences.GetAsync(a.Id))!.Status), "built");
// A desktop sharing the archive imports a photo taken before the others into the sequence: it files the photo and
// lists it in the sidecar itself, whose statuses it keeps, so the scan finds no photo to reconcile.
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddMinutes(-5), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, a.Id, null, null, staged.Path, "IMG_0000.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
var sidecar = fx.Paths.Sequence("a").ManifestPath;
Assert.True(Built(ManifestStore.ReadSequence(sidecar).Status), "the other writer keeps the statuses");
var again = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
await fx.WaitForAsync(again.Id, JobState.Done);
Assert.True(Built(ManifestStore.ReadSequence(sidecar).Status), "the server clears what it makes in its own row only");
// The frames changed behind the index: the statuses no longer hold, and the sequence gets a pass of each kind.
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
{
var pass = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items.Single(j => !first.Contains(j.Id));
Assert.SequenceEqual([a.Id], pass.TargetIds);
await fx.WaitForAsync(pass.Id, JobState.Done);
}
var row = (await fx.Db.Sequences.GetAsync(a.Id))!;
Assert.Equal(7, row.FrameCount);
Assert.True(Built(row.Status), "built again");
// Every frame has its own thumbnail and proxy: the new first one, and the others it renumbered.
var folder = fx.Paths.Sequence("a");
using var pack = ThumbPackReader.Open(folder.ThumbsPackPath);
foreach (var frame in ManifestStore.ReadSequence(sidecar).Frames)
{
Assert.True(pack.Entries.GetValueOrDefault(frame.I) is { } entry
&& entry.SourceHash.AsSpan().SequenceEqual(ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size)), $"thumbnail of {frame.File}");
Assert.True(ArchiveImages.IsProxyOf(folder.Proxy(frame.I), frame.CapturedAt, frame.W, frame.H, null), $"proxy of {frame.File}");
}
}
[Test]
public async Task AnImagePassMarksTheSequenceBuiltOnlyWhenItsSidecarStillListsTheFramesThePassRead()
{
await using var fx = await CreateAsync(context);
// The pass commits its pack and checkpoints after 200 frames.
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 201);
await fx.ScanAsync();
var a = (await LocationsAsync(fx)).Single();
var sequence = (await fx.Db.Sequences.GetAsync(a.Id))!;
// While it runs, another writer copies in a photo taken before the others, and a scan reconciles it.
var scanned = false;
var result = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, async (_, _) =>
{
if (scanned) return;
scanned = true;
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start.AddMinutes(-5), 1, firstSeed: 900, firstNumber: 900);
Assert.SequenceEqual([a.Id], (await fx.ScanAsync()).Sequences);
}, fx.Ct);
// The pass drew every frame it read, but not the new photo: the sequence is not built.
Assert.True(scanned, "scanned during the pass");
Assert.Equal((201, 0), (result.Built, result.Undecodable));
var row = (await fx.Db.Sequences.GetAsync(a.Id))!;
Assert.Equal(202, row.FrameCount);
Assert.False(row.Status.HasFlag(SequenceStatus.ThumbsBuilt), row.Status.ToString());
Assert.False(ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Status.HasFlag(SequenceStatus.ThumbsBuilt), "sidecar flag");
// So a scan's follow-up (here a resumed one, which lists the whole archive) queues the next pass, which makes them.
await using var engine = ServerEngine(fx);
var resumed = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Scan(fx.Archive), "after:a");
await engine.StartAsync(fx.Ct);
await fx.WaitForAsync(resumed.Id, JobState.Done);
var pass = (await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items.Single();
Assert.SequenceEqual([a.Id], pass.TargetIds);
await fx.WaitForAsync(pass.Id, JobState.Done);
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "built");
using var pack = ThumbPackReader.Open(fx.Paths.Sequence("a").ThumbsPackPath);
Assert.Count(202, pack.Entries);
}
[Test]
public async Task AnImagePassOvertakenByAnUploadKeepsTheSequenceBuiltWhileThePassAfterTheUploadWaits()
{
await using var fx = await CreateAsync(context);
// The pass commits its pack and checkpoints after 200 frames.
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 201);
await fx.ScanAsync();
var a = (await LocationsAsync(fx)).Single();
var sequence = (await fx.Db.Sequences.GetAsync(a.Id))!;
// While it runs, an upload lands in the sequence and queues the next pass, as the server does after every upload.
await using var engine = ServerEngine(fx);
Job? next = null;
await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, async (_, _) =>
{
if (next is not null) return;
var staged = fx.Stage(Start.AddMinutes(-5), seed: 90);
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, a.Id, null, null, staged.Path, "IMG_0000.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
next = await engine.EnqueueAsync(ArchiveJobs.Thumbnails([a.Id], "a"), cancellationToken: fx.Ct);
}, fx.Ct);
// The pass did not draw the new photo, but the one queued after it does: the sequence stays built meanwhile, so a
// server still offers the thumbnails it has.
Assert.NotNull(next);
Assert.Equal(202, (await fx.Db.Sequences.GetAsync(a.Id))!.FrameCount);
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "row flag");
Assert.True(ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Status.HasFlag(SequenceStatus.ThumbsBuilt), "sidecar flag");
await engine.StartAsync(fx.Ct);
await fx.WaitForAsync(next!.Id, JobState.Done);
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "built");
var folder = fx.Paths.Sequence("a");
using var pack = ThumbPackReader.Open(folder.ThumbsPackPath);
foreach (var frame in ManifestStore.ReadSequence(folder.ManifestPath).Frames)
{
Assert.True(pack.Entries.GetValueOrDefault(frame.I) is { } entry && ThumbPack.IsFrom(entry.SourceHash, frame.Sha256, frame.File, frame.Size),
$"thumbnail of {frame.File}");
}
}
[Test]
public async Task ThumbnailsDrawnBeforeAVerifyHashedThePhotosStayTheirs()
{
await using var fx = await CreateAsync(context);
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3);
await fx.ScanAsync();
var a = (await LocationsAsync(fx)).Single();
var first = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(a.Id))!, false, 0,
(_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(3, first.Built);
// The verify gives the photos the scan adopted their first hash; what was drawn from them is still theirs.
var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [a.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
Assert.Equal(3L, report.Hashed);
var again = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(a.Id))!, false, 0,
(_, _) => ValueTask.CompletedTask, fx.Ct);
Assert.Equal((0, 3), (again.Built, again.Reused));
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "built");
Assert.True(ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Frames.All(f => f.Sha256 is not null), "hashed");
}
[Test]
public async Task ADesktopScanAfterTheServersUploadLeavesWhatTheServerMadeForTheFramesAlone()
{
await using var fx = await CreateAsync(context);
Populate(fx);
await using var engine = ServerEngine(fx);
await engine.StartAsync(fx.Ct);
var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
await fx.WaitForAsync(scan.Id, JobState.Done);
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
{
foreach (var pass in (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items) await fx.WaitForAsync(pass.Id, JobState.Done);
}
var a = (await LocationsAsync(fx)).Single(l => l.Folder == "a");
// A desktop sharing the archive, with an index of its own.
await using var desktopIndex = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("desktop"), "titano.db")), fx.Ct);
var desktop = new ArchiveService(desktopIndex, new ArchiveServiceOptions { Owner = "desktop", LeaseWait = TimeSpan.FromSeconds(20) });
var archive = await desktop.OpenAsync(fx.Archive.RootPath, fx.Ct);
await desktop.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct);
// The server takes an upload into the sequence and makes its thumbnail and proxy, as after every upload.
var staged = fx.Stage(Start.AddMinutes(-5), seed: 90);
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, a.Id, null, null, staged.Path, "IMG_0000.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
foreach (var request in new[] { ArchiveJobs.Thumbnails([a.Id], "a"), ArchiveJobs.Proxies([a.Id], "a") })
await fx.WaitForAsync((await engine.EnqueueAsync(request, cancellationToken: fx.Ct)).Id, JobState.Done);
var sidecar = fx.Paths.Sequence("a").ManifestPath;
Assert.True(Built(ManifestStore.ReadSequence(sidecar).Status), "built after the upload");
var stamp = FileStamp.Of(sidecar);
var jobs = (await fx.Db.Jobs.ListAsync(null, null, cancellationToken: fx.Ct)).Items.Count;
// The desktop's index lags behind the upload: its scan indexes the frame and takes the statuses as the sidecar has them.
var summary = await desktop.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct);
Assert.SequenceEqual([a.Id], summary.Sequences);
Assert.Equal(stamp, FileStamp.Of(sidecar));
var row = (await desktopIndex.Sequences.GetAsync(a.Id, fx.Ct))!;
Assert.Equal(7, row.FrameCount);
Assert.True(Built(row.Status), row.Status.ToString());
// So the server's next scan finds nothing changed, and makes nothing again.
var again = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
await fx.WaitForAsync(again.Id, JobState.Done);
Assert.Equal(jobs + 1, (await fx.Db.Jobs.ListAsync(null, null, cancellationToken: fx.Ct)).Items.Count);
Assert.True(Built((await fx.Db.Sequences.GetAsync(a.Id))!.Status), "the server's row");
}
[Test]
public async Task AFolderGoneBeforeItsImagePassesFailsThemAloneAndTheOtherSequencesGetTheirImages()
{
@@ -14,13 +14,17 @@ namespace Titano.Data.Tests.Archiving;
internal static class HeifFile
{
/// <summary>A <paramref name="width"/> × <paramref name="height"/> HEIF captured at <paramref name="captured"/>, its Exif
/// thumbnail <paramref name="thumbnailWidth"/> × <paramref name="thumbnailHeight"/>.</summary>
/// thumbnail <paramref name="thumbnailWidth"/> × <paramref name="thumbnailHeight"/> (none when the width is 0: such a
/// host finds no pixels in it at all).</summary>
public static byte[] Create(DateTime captured, int width, int height, int thumbnailWidth = 160, int thumbnailHeight = 120)
{
using var small = new ImageBuffer(thumbnailWidth, thumbnailHeight);
for (var c = 0; c < 3; c++) small.Plane(c).Fill(0.2f + c * 0.2f);
var thumbnail = JpegEncoder.Encode(small, new JpegEncodeOptions { Quality = 80 });
var tiff = WithThumbnail(ExifWriter.BuildTiff(new FrameMetadata { Make = "TestCam", Model = "H-1", CapturedAt = captured }), thumbnail);
var tiff = ExifWriter.BuildTiff(new FrameMetadata { Make = "TestCam", Model = "H-1", CapturedAt = captured });
if (thumbnailWidth > 0)
{
using var small = new ImageBuffer(thumbnailWidth, thumbnailHeight);
for (var c = 0; c < 3; c++) small.Plane(c).Fill(0.2f + c * 0.2f);
tiff = WithThumbnail(tiff, JpegEncoder.Encode(small, new JpegEncodeOptions { Quality = 80 }));
}
byte[] exif = [.. U32(6), .. "Exif\0\0"u8, .. tiff];
// The HEVC-coded primary image, which no managed decoder reads.
var coded = new byte[64];
@@ -226,6 +226,10 @@ public sealed class MassRenameTests(TestContext context)
Assert.SequenceEqual(["moved/2026-09-28", "moved/2026-09-29", "moved/2026-09-30"], (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id))
.Select(l => l.Folder).Order(StringComparer.Ordinal));
Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids));
// The folders a nested sequence moved out of changed after their rows were written, as a folder another process
// changed would: the next scan reads those two again, and finds the index as it is.
Assert.SequenceEqual([ids[0], ids[1]], (await fx.ScanAsync()).Sequences);
Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids));
Assert.Equal(0, (await fx.ScanAsync()).Indexed);
}
@@ -0,0 +1,286 @@
using Titano.Core.Archiving;
using Titano.Core.Imaging;
using Titano.Core.Model;
using Titano.Data.Archiving;
using Titano.Data.Repositories;
using Titano.Data.Tests.Infrastructure;
using Titano.Testing;
using static Titano.Data.Tests.Archiving.ArchiveFixture;
namespace Titano.Data.Tests.Archiving;
/// <summary>
/// Writers that change part of a sequence's sidecar and index row (decode tiers, the verify, title, tags, rating and
/// notes, a rename) while a settings edit or another writer commits, on both dialects: they write only what they own,
/// under the sequence lease, and leave what another process changed in the sidecar to the next scan.
/// </summary>
[Category("archive")]
public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(context)
{
private async Task<ArchiveFixture> FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true)));
/// <summary>One indexed sequence of three photos in <c>a</c>.</summary>
private static async Task<Sequence> IndexedAsync(ArchiveFixture fx)
{
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3);
await fx.ScanAsync();
var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single();
return (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!;
}
/// <summary>Records decode tiers as analyses, image passes and renders do.</summary>
private static FrameTiers Tiers(ArchiveFixture fx)
{
var writer = new SequenceWriter(new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) });
return new FrameTiers(fx.Db, writer, new ArchiveIndexer(fx.Db, writer));
}
/// <summary>The frame decodes only through its 48 × 32 embedded preview.</summary>
private static Dictionary<Guid, FileTier> Preview(Guid frameId) =>
new() { [frameId] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32) };
/// <summary>Commits <paramref name="tx"/> once the writer under test had time to read the row and wait for it.</summary>
private static async Task CommitLaterAsync(TitanoTransaction tx, CancellationToken cancellationToken)
{
await using (tx)
{
await Task.Delay(500, cancellationToken);
await tx.CommitAsync(cancellationToken);
}
}
[Test]
public async Task TiersRecordedWhileASettingsEditCommitsLeaveTheEdit()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
// The server's PATCH holds the row (and SQLite's write lock) while an analysis records a tier.
Task record;
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
{
var edit = await tx.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":2}""" }, sequence.SettingsVersion, fx.Ct);
Assert.Equal(SequenceEditOutcome.Updated, edit.Outcome);
record = Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
await Task.WhenAny(record, Task.Delay(500, fx.Ct));
Assert.False(record.IsCompleted, "the record waits for the row");
await tx.CommitAsync(fx.Ct);
}
await record;
// The tier and what follows from it are recorded, and the edit committed meanwhile stays.
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Contains("heroFrame", row.SettingsJson);
Assert.Equal((sequence.SettingsVersion + 1, 1), (row.SettingsVersion, row.PreviewOnlyCount));
Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags);
}
[Test]
public async Task TiersRecordedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
// A desktop sharing the archive imports a photo into the sequence: its own index follows the sidecar, this one not yet.
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
// A tier recorded meanwhile changes the tiers here, not the frames this index holds.
await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal((3, sequence.FramesVersion, 1), (row.FrameCount, row.FramesVersion, row.PreviewOnlyCount));
// The next scan still finds the folder changed, and indexes the photo.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal((4, sidecar.Versions.Frames), (row.FrameCount, row.FramesVersion));
Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
}
[Test]
public async Task TiersRecordedAfterAnotherWriterRetitledTheSequenceLeaveTheTitleToTheNextScan()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
// A desktop sharing the archive renames and rates the sequence, in the sidecar and its own index.
var (desktop, _) = await fx.OtherWriterAsync();
await desktop.SetSequenceInfoAsync(sequence.Id, "Ridge", ["fog"], 3, null, fx.Ct);
// A tier recorded here meanwhile writes only the tier, and leaves the folder changed for the next scan.
await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(("Ridge", "fog", 3, 1), (row.Title, row.Tags.Single(), row.Rating, row.PreviewOnlyCount));
}
[Test]
public async Task AnInfoEditAfterAnotherWriterAddedFramesLeavesThoseFramesToTheNextScan()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
// The rating goes into the sidecar and this index; the frames this index holds stay as they are.
var rated = await fx.Service.EditSequenceInfoAsync(sequence.Id, info => info with { Rating = 4 }, fx.Ct);
Assert.Equal((3, sequence.FramesVersion, 4), (rated.FrameCount, rated.FramesVersion, rated.Rating));
Assert.Equal(4, ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Rating);
// The next scan still finds the folder changed, and indexes the photo.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal((4, sidecar.Versions.Frames, 4), (row.FrameCount, row.FramesVersion, row.Rating));
Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
}
[Test]
public async Task AnInfoEditKeepsTheSettingsTheApiSavedAfterItReadTheSequence()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
// The server's PATCH, which takes no lease, saves settings after the edit read the sequence: the edit applies again.
var calls = 0;
var rated = await fx.Service.EditSequenceInfoAsync(sequence.Id, info =>
{
if (++calls == 1)
{
var patch = fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":2}""" }, sequence.SettingsVersion, fx.Ct)
.GetAwaiter().GetResult();
Assert.Equal(SequenceEditOutcome.Updated, patch.Outcome);
}
return info with { Rating = 5 };
}, fx.Ct);
Assert.Equal(2, calls);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Contains("heroFrame", row.SettingsJson);
Assert.Equal((5, sequence.SettingsVersion + 2), (row.Rating, row.SettingsVersion));
Assert.Equal(rated.SettingsVersion, row.SettingsVersion);
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
Assert.Equal((5, row.SettingsVersion), (sidecar.Rating, sidecar.Versions.Settings));
}
[Test]
public async Task ARenameAfterAnotherWriterAddedFramesLeavesThoseFramesToTheNextScan()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
// The rename moves the folder and retitles the sequence; the row keeps the frames this index holds.
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [sequence.Id], "b", "Ridge", fx.Ct);
await fx.Service.ApplyRenameAsync(plan, null, fx.Ct);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(("b", "Ridge", 3, sequence.FramesVersion), (row.Folder, row.Title, row.FrameCount, row.FramesVersion));
// The next scan still finds the folder changed, and indexes the photo where it is now.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("b").ManifestPath);
row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Equal(("Ridge", 4, sidecar.Versions.Frames), (row.Title, row.FrameCount, row.FramesVersion));
Assert.True((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).All(f => f.RelativePath.StartsWith("b/", StringComparison.Ordinal)),
"frames in b");
}
[Test]
public async Task AVerifyStoppedAfterAFlushLeavesFramesAnotherWriterAddedToTheNextScan()
{
var fx = await FixtureAsync();
// Enough photos the scan adopted unhashed for the verify to flush their hashes before it reaches the end.
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 513);
await fx.ScanAsync();
var sequence = (await fx.Db.Sequences.GetAsync((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single().Id, fx.Ct))!;
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddHours(3), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0600.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
// The verify writes the hashes of the first 512 photos to the sidecar and this index, and stops before the end.
await Assert.ThrowsAsync<OperationCanceledException>(() => fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default,
(_, _, _) => throw new OperationCanceledException(), null, fx.Ct));
Assert.Equal(512, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Count(f => f.Sha256 is not null));
Assert.Equal(513, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
// The next scan still finds the folder changed, and indexes the photo.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
Assert.Equal(514, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
}
[Test]
public async Task DuplicatesFlaggedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan()
{
var fx = await FixtureAsync();
// Two sequences holding the same photo, hashed by a verify.
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3);
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "b"), Start, 1);
await fx.ScanAsync();
await fx.Service.Verifier.VerifyAsync(fx.Archive, null, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
var copy = (await fx.Service.FindDuplicatesAsync(fx.Archive.Id, fx.Ct)).Single().Frames[1];
var sequence = (await fx.Db.Sequences.GetAsync(copy.SequenceId, fx.Ct))!;
// A desktop sharing the archive imports a photo into the sequence whose copy maintenance flags.
var (desktop, archive) = await fx.OtherWriterAsync();
var staged = fx.Stage(Start.AddHours(3), seed: 90);
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0600.JPG", staged.Size, staged.Sha256, null, null),
fx.Ct);
var report = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, firstStep: 3, _ => ValueTask.CompletedTask, fx.Ct);
Assert.Equal(1, report.DuplicateFlagChanges);
Assert.Equal(sequence.FrameCount, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
// The next scan still finds the folder changed, and indexes the photo.
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
Assert.Equal(sequence.FrameCount + 1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
}
[Test]
public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit()
{
var fx = await FixtureAsync();
var sequence = await IndexedAsync(fx);
// Hashed by a first verify, so the next one changes no frame and only stamps the sequence.
await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
sequence = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
// The server's PATCH takes the row as the last photo is checked, and commits while the verify stamps the sequence.
Task? commit = null;
await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (done, total) =>
{
if (done < total || commit is not null) return;
var tx = await fx.Db.BeginAsync(fx.Ct);
await tx.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":1}""" }, sequence.SettingsVersion, fx.Ct);
commit = CommitLaterAsync(tx, fx.Ct);
}, fx.Ct);
await Assert.NotNull(commit);
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.Contains("heroFrame", row.SettingsJson);
Assert.Equal(sequence.SettingsVersion + 1, row.SettingsVersion);
Assert.True(row.Status.HasFlag(SequenceStatus.Verified), row.Status.ToString());
Assert.True(row.VerifiedAt > sequence.VerifiedAt, "stamped again");
}
}
public sealed class SqlitePartialRowWriterTests(TestContext context) : PartialRowWriterTests(context)
{
protected override Backend Backend => Backend.SqliteFile;
}
[RequiresEnvironment(PgTestServer.Variable)]
public sealed class PostgresPartialRowWriterTests(TestContext context) : PartialRowWriterTests(context)
{
protected override Backend Backend => Backend.Postgres;
}
@@ -206,6 +206,54 @@ public sealed class AnalysisJobTests(TestContext context)
Assert.Equal("4k-hevc", await PresetAsync());
}
[Test]
public async Task ProxiesThatArriveDuringADeepAnalysisOfHeifsWithoutThumbnailsMakeItsUnreadableVerdictStale()
{
// A server, which finds no pixels at all in these HEIF files (they carry no Exif thumbnail) until proxies come (BRIEF §2.3).
await using var fx = await ProcessingFixture.CreateAsync(context, o => o with { Host = EncoderHost.Server });
var directory = Path.Combine(fx.Archive.Paths.Root, "Heif");
Directory.CreateDirectory(directory);
var start = new DateTime(2026, 9, 28, 18, 0, 0);
for (var i = 0; i < 4; i++)
{
await File.WriteAllBytesAsync(Path.Combine(directory, $"IMG_{i + 1:0000}.HEIC"), HeifFile.Create(start.AddSeconds(10 * i), 4032, 3024, thumbnailWidth: 0),
fx.Ct);
}
await fx.Archive.ScanAsync(ScanMode.Full);
var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Archive.Id, fx.Ct)).Single(l => l.Folder == "Heif");
var sequence = (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!;
var folder = fx.Folder(sequence);
await fx.StartEngineAsync(workers: 2);
async Task<string> AutoValuesAsync() =>
File.Exists(folder.ProjectPath) ? ProjectSettings.FromJson(await File.ReadAllTextAsync(folder.ProjectPath, fx.Ct)).ToJson() : string.Empty;
// A deep analysis starts on the originals; while it runs, the proxies arrive and a light analysis on the other
// worker reads the frames from them.
fx.Decoder.Hold = "IMG_0001.HEIC";
var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
Directory.CreateDirectory(folder.Proxies);
foreach (var frame in await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))
await File.WriteAllBytesAsync(folder.Proxy(frame.Index), Proxy(frame.CapturedAt!.Value, 1920, 1440), fx.Ct);
await LightAnalysisAsync(fx, sequence);
var light = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
var advice = await AutoValuesAsync();
fx.Decoder.Release();
await fx.WaitForAsync(deep.Id, JobState.Done);
// The deep pass found no pixels in any original, a verdict the proxies made stale before it ended, as every reader
// of its report finds: it records no deep status, and neither warnings nor advice of its own.
Assert.Count(4, SequenceAnalyser.ReadReport(folder, sequence.FrameCount)!.Issues.Undecodable);
var now = await Titano.Data.Processing.SequenceFrames.LoadAsync(fx.Db, fx.Archive.Archive, sequence, useProxies: false,
new ImageDecoder([fx.Decoder]), fx.Ct);
Assert.Null(SequenceAnalyser.ReadReport(folder, now));
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
Assert.False(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), row.Status.ToString());
Assert.False(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedDeep), "sidecar status");
Assert.Equal((light.WarningCount, light.Status.HasFlag(SequenceStatus.HasWarnings)), (row.WarningCount, row.Status.HasFlag(SequenceStatus.HasWarnings)));
Assert.Equal(advice, await AutoValuesAsync());
}
[Test]
public async Task AFrameTheDeepAnalysisCouldNotReadIsReadAgainOnceItsProxyArrives()
{
@@ -1,8 +1,11 @@
using System.Net;
using System.Security.Cryptography;
using Microsoft.Extensions.DependencyInjection;
using Titano.Core.Archiving;
using Titano.Core.Codecs.Jpeg;
using Titano.Core.Model;
using Titano.Data;
using Titano.Data.Archiving;
using Titano.Protocol;
using Titano.Protocol.Client;
using Titano.Server.Jobs;
@@ -106,6 +109,74 @@ public sealed class ArchiveModuleTests(TestContext context)
await UntilImagesAsync(server, editor, archive.Id, "garden");
}
[Test]
public async Task FramesADesktopSharingTheArchiveAddsToASequenceGetThumbnailsAndProxiesAfterThePoll()
{
await using var server = await TestServer.StartAsync(context, options =>
{
Directory.CreateDirectory(Path.Combine(options.ArchivePath, "roof"));
File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0002.JPG"), Photo());
}, ArchiveModule.Register);
var archive = server.Services.GetRequiredService<ServerArchive>();
var (editor, _, _) = await server.UserAsync("alberto", "editor");
await UntilIndexedAsync(server, archive.Id, "roof");
await UntilScannedAsync(server);
await UntilImagesAsync(server, editor, archive.Id, "roof");
await UntilIdleAsync(server);
var roof = (await server.Database.Sequences.ListLocationsAsync(archive.Id)).Single(l => l.Folder == "roof");
// A desktop sharing the archive, with an index of its own, imports a photo taken earlier into the sequence: it files
// the photo and lists it in the sidecar itself, keeping the sidecar's ThumbsBuilt and ProxiesBuilt.
var desktopFolder = context.CreateTempDirectory("desktop");
await using (var desktopIndex = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(desktopFolder, "titano.db")), context.Cancellation))
{
var desktop = new ArchiveService(desktopIndex, new ArchiveServiceOptions { Owner = "desktop" });
var shared = await desktop.OpenAsync(server.Options.ArchivePath, context.Cancellation);
await desktop.Indexer.ScanAsync(shared, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, context.Cancellation);
var photo = Photo(seed: 5);
var staged = Path.Combine(desktopFolder, "IMG_0001.JPG");
await File.WriteAllBytesAsync(staged, photo);
await desktop.Ingest.CommitAsync(new IngestRequest(shared.Id, roof.Id, null, null, staged, "IMG_0001.JPG", photo.Length,
Convert.ToHexStringLower(SHA256.HashData(photo)), new DateTime(2026, 9, 28, 18, 0, 0), null), context.Cancellation);
}
Assert.Equal(1, Assert.NotNull(await server.Database.Sequences.GetAsync(roof.Id)).FrameCount);
// The next poll indexes it and queues the images: the new frame and the one it moved get theirs.
Assert.NotNull(await server.Services.GetRequiredService<ArchivePolling>().PollAsync(context.Cancellation));
await UntilScannedAsync(server);
Assert.Equal(2, Assert.NotNull(await server.Database.Sequences.GetAsync(roof.Id)).FrameCount);
await UntilImagesAsync(server, editor, archive.Id, "roof");
}
[Test]
public async Task AThumbnailDrawnBeforeAVerifyHashedThePhotoIsStillServed()
{
await using var server = await TestServer.StartAsync(context, options =>
{
Directory.CreateDirectory(Path.Combine(options.ArchivePath, "roof"));
File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0001.JPG"), Photo());
}, ArchiveModule.Register);
var archive = server.Services.GetRequiredService<ServerArchive>();
var (editor, _, _) = await server.UserAsync("alberto", "editor");
await UntilIndexedAsync(server, archive.Id, "roof");
await UntilScannedAsync(server);
await UntilImagesAsync(server, editor, archive.Id, "roof");
await UntilIdleAsync(server);
var roof = (await server.Database.Sequences.ListLocationsAsync(archive.Id)).Single(l => l.Folder == "roof");
var frame = (await server.Database.Frames.ListBySequenceAsync(roof.Id)).Single();
Assert.Null(frame.Sha256);
// A verify gives the photo the scan adopted its first hash; the thumbnail drawn from it before is still its own.
var verified = await server.Services.GetRequiredService<ArchiveService>().Verifier.VerifyAsync(
Assert.NotNull(await server.Database.Archives.GetAsync(archive.Id)), [roof.Id], default, (_, _, _) => ValueTask.CompletedTask, null,
context.Cancellation);
Assert.Equal(1L, verified.Hashed);
Assert.NotNull(Assert.NotNull(await server.Database.Frames.GetAsync(frame.Id)).Sha256);
using var thumb = await editor.GetAsync($"/api/v1/frames/{frame.Id}/thumb");
Assert.Equal(HttpStatusCode.OK, thumb.StatusCode);
Assert.True(Assert.NotNull(await server.Database.Sequences.GetAsync(roof.Id)).Status.HasFlag(SequenceStatus.ThumbsBuilt), "built");
}
[Test]
public async Task SettingsADesktopChangedInArchiveJsonReachUploadsAndTheAdminPanel()
{
@@ -157,28 +228,51 @@ public sealed class ArchiveModuleTests(TestContext context)
}
}
/// <summary>Waits until the first frame of the sequence in <paramref name="folder"/> has its thumbnail and proxy served.</summary>
/// <summary>
/// Waits until every frame of the sequence in <paramref name="folder"/> has its thumbnail and proxy served and the
/// passes that made them have marked the sequence (a pass writes its files before it marks).
/// </summary>
private static async Task UntilImagesAsync(TestServer server, HttpClient client, Guid archiveId, string folder)
{
var sequence = (await server.Database.Sequences.ListLocationsAsync(archiveId)).Single(l => l.Folder == folder);
var frame = (await server.Database.Frames.ListBySequenceAsync(sequence.Id)).First();
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
foreach (var asset in new[] { "thumb", "proxy" })
foreach (var frame in await server.Database.Frames.ListBySequenceAsync(sequence.Id))
{
while (true)
foreach (var asset in new[] { "thumb", "proxy" })
{
using var response = await client.GetAsync($"/api/v1/frames/{frame.Id}/{asset}");
if (response.StatusCode == HttpStatusCode.OK)
while (true)
{
Assert.True(await response.Content.ReadAsByteArrayAsync() is [0xFF, 0xD8, ..], $"a JPEG {asset} of {folder}");
break;
using var response = await client.GetAsync($"/api/v1/frames/{frame.Id}/{asset}");
if (response.StatusCode == HttpStatusCode.OK)
{
Assert.True(await response.Content.ReadAsByteArrayAsync() is [0xFF, 0xD8, ..], $"a JPEG {asset} of {frame.FileName}");
break;
}
Assert.True(DateTime.UtcNow < until, $"no {asset} of {frame.FileName} in {folder}: {(int)response.StatusCode}");
await Task.Delay(100);
}
Assert.True(DateTime.UtcNow < until, $"no {asset} of {folder}: {(int)response.StatusCode}");
await Task.Delay(100);
}
}
var built = Assert.NotNull(await server.Database.Sequences.GetAsync(sequence.Id)).Status;
Assert.True(built.HasFlag(SequenceStatus.ThumbsBuilt) && built.HasFlag(SequenceStatus.ProxiesBuilt), $"{folder}: {built}");
while (Assert.NotNull(await server.Database.Sequences.GetAsync(sequence.Id)).Status is var built
&& !(built.HasFlag(SequenceStatus.ThumbsBuilt) && built.HasFlag(SequenceStatus.ProxiesBuilt)))
{
Assert.True(DateTime.UtcNow < until, $"{folder}: {built}");
await Task.Delay(50);
}
}
/// <summary>Waits until no job is waiting or running.</summary>
private static async Task UntilIdleAsync(TestServer server)
{
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
foreach (var state in new[] { JobState.Queued, JobState.Running, JobState.Queued })
{
while ((await server.Database.Jobs.ListAsync(state)).Items.Count > 0)
{
Assert.True(DateTime.UtcNow < until, $"jobs still {state}");
await Task.Delay(50);
}
}
}
/// <summary>Waits for the scan queued at start to finish: its sequences are indexed before it ends.</summary>