merge: third-pass archive, image and analysis fixes (W-1, W-5, W-7, W-13, W-14)
Merge branch fix3/data-images into the verification branch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
23 files changed
+1132
-169
No files matched your search
@@ -223,6 +223,15 @@ public static class ThumbPack
|
||||
sha256 is { Length: HashLength * 2 }
|
||||
? Convert.FromHexString(sha256)
|
||||
: System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes($"{file}|{size}"));
|
||||
|
||||
/// <summary>
|
||||
/// Whether a thumbnail with <paramref name="sourceHash"/> was drawn from this original: its <see cref="Fingerprint"/>,
|
||||
/// or, for an original hashed since (a verify hashes the photos a scan adopted), the fingerprint of its file name and
|
||||
/// size it was drawn with before. The hash changes nothing about the photo.
|
||||
/// </summary>
|
||||
public static bool IsFrom(ReadOnlySpan<byte> sourceHash, string? sha256, string file, long size) =>
|
||||
sourceHash.SequenceEqual(Fingerprint(sha256, file, size))
|
||||
|| (sha256 is { Length: HashLength * 2 } && sourceHash.SequenceEqual(Fingerprint(null, file, size)));
|
||||
}
|
||||
|
||||
/// <summary>Random access to the thumbnails of a pack without loading the file; safe while a writer appends.</summary>
|
||||
|
||||
@@ -209,8 +209,7 @@ public sealed class ArchiveImages
|
||||
for (var i = 0; i < startFrame && i < manifest.Frames.Count; i++)
|
||||
{
|
||||
var frame = manifest.Frames[i];
|
||||
complete &= pack.Entries.GetValueOrDefault(frame.I) is { } entry
|
||||
&& entry.SourceHash.AsSpan().SequenceEqual(ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size));
|
||||
complete &= pack.Entries.GetValueOrDefault(frame.I) is { } entry && ThumbPack.IsFrom(entry.SourceHash, frame.Sha256, frame.File, frame.Size);
|
||||
}
|
||||
var cadence = new Cadence();
|
||||
for (var i = startFrame; i < manifest.Frames.Count; i++)
|
||||
@@ -220,7 +219,7 @@ public sealed class ArchiveImages
|
||||
var fingerprint = ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size);
|
||||
var entry = pack.Entries.GetValueOrDefault(frame.I);
|
||||
var edge = entry is null ? 0 : System.Math.Max(entry.Width, entry.Height);
|
||||
var current = !force && entry is not null && entry.SourceHash.AsSpan().SequenceEqual(fingerprint);
|
||||
var current = !force && entry is not null && ThumbPack.IsFrom(entry.SourceHash, frame.Sha256, frame.File, frame.Size);
|
||||
// A frame this host cannot decode shows its proxy once one arrived; until then its original's embedded
|
||||
// preview, whose smaller thumbnail the proxy's replaces.
|
||||
var fromProxy = false;
|
||||
@@ -255,7 +254,8 @@ public sealed class ArchiveImages
|
||||
pack.Commit();
|
||||
}
|
||||
await _tiers.RecordAsync(archive, sequence, tiers, proxies: false, cancellationToken).ConfigureAwait(false);
|
||||
await MarkAsync(archive, paths, sequence, SequenceStatus.ThumbsBuilt, complete && undecodable == 0, cancellationToken).ConfigureAwait(false);
|
||||
await MarkAsync(archive, paths, sequence, manifest, JobKind.Thumbs, SequenceStatus.ThumbsBuilt, complete && undecodable == 0, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return new ImagePassResult(built, reused, undecodable);
|
||||
}
|
||||
|
||||
@@ -327,7 +327,7 @@ public sealed class ArchiveImages
|
||||
if (cadence.Due()) await checkpoint(i + 1, manifest.Frames.Count).ConfigureAwait(false);
|
||||
}
|
||||
await _tiers.RecordAsync(archive, sequence, tiers, proxies: true, cancellationToken).ConfigureAwait(false);
|
||||
await MarkAsync(archive, paths, sequence, SequenceStatus.ProxiesBuilt, complete, cancellationToken).ConfigureAwait(false);
|
||||
await MarkAsync(archive, paths, sequence, manifest, JobKind.Proxies, SequenceStatus.ProxiesBuilt, complete, cancellationToken).ConfigureAwait(false);
|
||||
return new ImagePassResult(built, reused, undecodable, waiting);
|
||||
}
|
||||
|
||||
@@ -520,15 +520,54 @@ public sealed class ArchiveImages
|
||||
}
|
||||
}
|
||||
|
||||
private async Task MarkAsync(Archive archive, ArchivePaths paths, Sequence sequence, SequenceStatus flag, bool complete, CancellationToken cancellationToken)
|
||||
/// <summary>
|
||||
/// Sets <paramref name="flag"/> in the sidecar and the index when the pass covered every frame it read
|
||||
/// (<paramref name="complete"/>) and the sidecar, as it is under the lease, still lists those frames
|
||||
/// (<see cref="SameFrames"/>); clears it otherwise. Frames a scan's reconcile or an ingest added, removed or
|
||||
/// renumbered after the pass read <paramref name="read"/> have nothing of it, and a scan that cleared the flag for
|
||||
/// them queues the next pass only while the flag stays clear. A <paramref name="pass"/> of the same kind already
|
||||
/// queued for the sequence (an upload's, or a scan's) draws them when it runs, so the flag stays set meanwhile, as
|
||||
/// between an upload and its pass: a camera uploading faster than a pass ends would otherwise keep it clear, and a
|
||||
/// server would offer no thumbnail or proxy of the sequence at all until uploads paused. The index is written under
|
||||
/// the same lease, so the flag never lands there after a scan cleared it in both.
|
||||
/// </summary>
|
||||
private async Task MarkAsync(Archive archive, ArchivePaths paths, Sequence sequence, SequenceManifest read, JobKind pass, SequenceStatus flag,
|
||||
bool complete, CancellationToken cancellationToken)
|
||||
{
|
||||
await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m =>
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var status = complete ? m.Status | flag : m.Status & ~flag;
|
||||
return status == m.Status ? null : m with { Status = status, UpdatedAt = DateTimeOffset.UtcNow };
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
await _db.Sequences.UpdateStatusAsync(sequence.Id, complete ? flag : SequenceStatus.None, complete ? SequenceStatus.None : flag, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
await _indexer.RecordStampsAsync(archive, paths.Sequence(sequence.Folder), sequence.Id, cancellationToken).ConfigureAwait(false);
|
||||
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
|
||||
var built = complete && (SameFrames(manifest.Frames, read.Frames)
|
||||
|| await _db.Jobs.IsQueuedAsync(pass, sequence.Id, cancellationToken).ConfigureAwait(false));
|
||||
var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
|
||||
var status = built ? manifest.Status | flag : manifest.Status & ~flag;
|
||||
if (status != manifest.Status)
|
||||
{
|
||||
manifest = manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow };
|
||||
ManifestStore.WriteSequence(folder.ManifestPath, manifest);
|
||||
}
|
||||
await _db.Sequences.UpdateStatusAsync(sequence.Id, built ? flag : SequenceStatus.None, built ? SequenceStatus.None : flag, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
if (inStep) await _indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether <paramref name="current"/> lists the photos <paramref name="read"/> did, each at the same index with the
|
||||
/// same file, size, capture time and content hash: thumbnails and proxies are kept by frame index, so photos added,
|
||||
/// removed, renumbered or replaced since have none of what the pass made. A hash learnt since (a verify hashing a
|
||||
/// photo the scan adopted) changes nothing about the photo, and its thumbnail stays its own (<see cref="ThumbPack.IsFrom"/>).
|
||||
/// </summary>
|
||||
private static bool SameFrames(IReadOnlyList<SidecarFrame> current, IReadOnlyList<SidecarFrame> read)
|
||||
{
|
||||
if (current.Count != read.Count) return false;
|
||||
for (var i = 0; i < current.Count; i++)
|
||||
{
|
||||
var (now, then) = (current[i], read[i]);
|
||||
if (now.Id != then.Id || now.I != then.I || now.File != then.File || now.Size != then.Size || now.CapturedAt != then.CapturedAt
|
||||
|| (then.Sha256 is not null && !string.Equals(now.Sha256, then.Sha256, StringComparison.OrdinalIgnoreCase))) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -31,6 +31,9 @@ public sealed record ScanSummary(int Folders, int Indexed, int Adopted, int Remo
|
||||
/// </summary>
|
||||
public sealed class ArchiveIndexer
|
||||
{
|
||||
/// <summary>What covers the frames as they were: a scan that finds them changed clears it, and its follow-ups make it again.</summary>
|
||||
private const SequenceStatus CoveringStatus = SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt;
|
||||
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly SequenceWriter _writer;
|
||||
|
||||
@@ -131,6 +134,32 @@ public sealed class ArchiveIndexer
|
||||
_db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(sequence.RelativeFolder, sequenceId,
|
||||
FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))), cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Whether this index holds the folder of <paramref name="manifest"/> (the sidecar read under the sequence lease) as it
|
||||
/// is now: the stamps it recorded when it last synced the folder are still the folder's, and its row holds the
|
||||
/// sidecar's frames. A writer that changes, under the lease, only what it owns of the sidecar and the index row
|
||||
/// (statuses, decode tiers, frame flags, title, tags, rating and notes, the folder) asks before its own write, and
|
||||
/// records the folder's new stamps after it only when this held (<see cref="RecordStampsAsync"/>). What another
|
||||
/// process (a desktop sharing the archive) changed in the sidecar alone, frames or a new title, then keeps the folder
|
||||
/// changed for the next incremental scan, which writes the whole row from the sidecar.
|
||||
/// </summary>
|
||||
internal Task<bool> InStepAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) =>
|
||||
InStepAsync(archive, sequence.RelativeFolder, sequence, manifest, cancellationToken);
|
||||
|
||||
/// <inheritdoc cref="InStepAsync(Archive, SequencePaths, SequenceManifest, CancellationToken)"/>
|
||||
/// <param name="recordedFolder">The folder the stamps were recorded at: the one a rename just moved the sequence from,
|
||||
/// whose files keep their stamps.</param>
|
||||
internal async Task<bool> InStepAsync(Archive archive, string recordedFolder, SequencePaths sequence, SequenceManifest manifest,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (await _db.Archives.GetFolderStateAsync(archive.Id, recordedFolder, cancellationToken).ConfigureAwait(false) is not { } state
|
||||
|| state.SequenceId != manifest.Id || state.Sidecar != FileStamp.Of(sequence.ManifestPath)
|
||||
|| state.Photos != FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))) return false;
|
||||
// A sidecar rewritten within its file system's time resolution, at the same size, keeps its stamp.
|
||||
return await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { } indexed
|
||||
&& indexed.FramesVersion == manifest.Versions.Frames;
|
||||
}
|
||||
|
||||
/// <summary>Indexes unindexed photos and flags missing files of one sequence (the orphans fix action).</summary>
|
||||
public async Task<SequenceManifest> ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default)
|
||||
{
|
||||
@@ -144,8 +173,10 @@ public sealed class ArchiveIndexer
|
||||
/// <summary>
|
||||
/// Applies <paramref name="edit"/> to the sidecar (null writes nothing) and indexes the result, both under the sequence
|
||||
/// lease. Frames inserted before indexed ones move the project's positions (<see cref="ProjectPositions"/>) without a
|
||||
/// new settings version; an editor saving its project holds the same lease, so it reads them moved and merges onto
|
||||
/// them rather than writing its old positions back.
|
||||
/// new settings version; an editor saving its project holds the same lease, so it reads them moved and merges onto them
|
||||
/// rather than writing its old positions back. When the sidecar, as read under the lease, lists other frames than the
|
||||
/// index row (another process changed them together with their sidecar entries), the row is written without what
|
||||
/// this host makes for the frames (<see cref="Uncovered"/>).
|
||||
/// </summary>
|
||||
private async Task<SequenceManifest> UpdateAndSyncAsync(Archive archive, ArchivePaths paths, SequencePaths sequence, Guid sequenceId,
|
||||
Func<SequenceManifest, SequenceManifest?> edit, CancellationToken cancellationToken)
|
||||
@@ -153,16 +184,27 @@ public sealed class ArchiveIndexer
|
||||
await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var manifest = SequenceWriter.ReadLocked(sequence, sequenceId);
|
||||
var moved = await FramesMovedAsync(manifest, cancellationToken).ConfigureAwait(false);
|
||||
if (edit(manifest) is { } updated)
|
||||
{
|
||||
ManifestStore.WriteSequence(sequence.ManifestPath, updated);
|
||||
manifest = updated;
|
||||
}
|
||||
await SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
|
||||
await SyncAsync(archive, sequence, moved ? manifest with { Status = manifest.Status & ~Uncovered } : manifest, cancellationToken).ConfigureAwait(false);
|
||||
return manifest;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// What a scan clears in this host's own row, never in the shared sidecar, when another process added, removed or
|
||||
/// renumbered frames together with their sidecar entries: on a server (<see cref="ArchiveServiceOptions.ImagesAfterScan"/>),
|
||||
/// which analyses and draws what its clients load, the light analysis, thumbnails and proxies, so the scan's follow-ups
|
||||
/// make them for the frames as they are, and their passes set the flags in both places again. Nothing on a desktop:
|
||||
/// the flags in the sidecar are the writer's that added the frames (which analyses them, and on a server draws them
|
||||
/// after every upload), and clearing them there would have the server make everything again.
|
||||
/// </summary>
|
||||
private SequenceStatus Uncovered => _writer.Options.ImagesAfterScan ? CoveringStatus : SequenceStatus.None;
|
||||
|
||||
/// <summary>Indexes one folder; returns its sequence and the sidecar bytes when they were read and not rewritten since (for the snapshot).</summary>
|
||||
private async Task<(Guid Id, byte[]? Bytes)> IndexFolderAsync(Archive archive, ArchivePaths paths, SequenceFolder folder, bool reconcile, IndexSnapshot? snapshot,
|
||||
Locations locations, CancellationToken cancellationToken)
|
||||
@@ -196,18 +238,21 @@ public sealed class ArchiveIndexer
|
||||
}
|
||||
if (reconcile || File.Exists(sequence.IngestJournalPath))
|
||||
{
|
||||
// Photos added, gone or back: the light analysis, thumbnails and proxies no longer cover the frames, so the scan's
|
||||
// follow-ups make them again (the images on a server, ArchiveServiceOptions.ImagesAfterScan).
|
||||
// Photos added, gone or back: the light analysis, thumbnails and proxies no longer cover the frames, as the
|
||||
// sidecar then tells every host, so the scan's follow-ups make them again (the images on a server,
|
||||
// ArchiveServiceOptions.ImagesAfterScan).
|
||||
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id,
|
||||
m => SequenceAdopter.Reconcile(sequence, m, cancellationToken) is { } reconciled
|
||||
? reconciled with { Status = reconciled.Status & ~(SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt) }
|
||||
? reconciled with { Status = reconciled.Status & ~CoveringStatus }
|
||||
: null,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
bytes = null;
|
||||
}
|
||||
else if (await MayMovePositionsAsync(manifest, cancellationToken).ConfigureAwait(false))
|
||||
else if (await FramesMovedAsync(manifest, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
// Another process changed the frames: the sidecar is read again under the lease the project moves under.
|
||||
// Another process (a desktop sharing the archive, or the server it shares it with) added, removed or renumbered
|
||||
// frames together with their sidecar entries: read again under the lease, under which the project's positions
|
||||
// move, and indexed without what this host makes for the frames (Uncovered).
|
||||
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false);
|
||||
bytes = null;
|
||||
}
|
||||
@@ -257,9 +302,12 @@ public sealed class ArchiveIndexer
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The index holds a project for this sequence and other frames than the sidecar: its positions may move.</summary>
|
||||
private async Task<bool> MayMovePositionsAsync(SequenceManifest manifest, CancellationToken cancellationToken) =>
|
||||
await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { SettingsJson: not null } indexed
|
||||
/// <summary>
|
||||
/// The index holds this sequence with other frames than the sidecar: another process changed them (its project's
|
||||
/// positions may move, and what covered the frames no longer does).
|
||||
/// </summary>
|
||||
private async Task<bool> FramesMovedAsync(SequenceManifest manifest, CancellationToken cancellationToken) =>
|
||||
await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { } indexed
|
||||
&& indexed.FramesVersion != manifest.Versions.Frames;
|
||||
|
||||
/// <summary>A sidecar for a folder that has none; ids of a sequence the index already knew at this folder are kept.</summary>
|
||||
@@ -309,7 +357,10 @@ public sealed class ArchiveIndexer
|
||||
|
||||
/// <summary>
|
||||
/// Rebuild only: rows whose folder no longer holds a sidecar leave the index (their files are untouched). A sequence
|
||||
/// waiting for its first frame has no sidecar yet and stays.
|
||||
/// waiting for its first frame has no sidecar yet and stays. A row the folders listed when the rebuild began do not
|
||||
/// back is looked at again under its lease, as it and its folder are when it would go (<see cref="BackedAsync"/>):
|
||||
/// uploads go on during a rebuild, which can take minutes, so a sequence that got its first frame or was created
|
||||
/// meanwhile has a folder the listing does not hold.
|
||||
/// </summary>
|
||||
private async Task<int> RemoveUnbackedAsync(Archive archive, ArchivePaths paths, IReadOnlyList<SequenceFolder> folders, Locations locations,
|
||||
CancellationToken cancellationToken)
|
||||
@@ -322,12 +373,46 @@ public sealed class ArchiveIndexer
|
||||
foreach (var location in rows)
|
||||
{
|
||||
if (awaiting.Contains(location.Id) || (present.Contains(location.Folder) && locations.AtFolder(location.Folder) == location.Id)) continue;
|
||||
if (await _db.Sequences.DeleteAsync(location.Id, cancellationToken).ConfigureAwait(false)) removed++;
|
||||
await _db.Archives.DeleteFolderStateAsync(archive.Id, location.Folder, cancellationToken).ConfigureAwait(false);
|
||||
IAsyncDisposable held;
|
||||
try
|
||||
{
|
||||
held = await _writer.LockAsync(paths, location.Id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (LeaseUnavailableException)
|
||||
{
|
||||
// Another process is writing the sequence, so it is there.
|
||||
continue;
|
||||
}
|
||||
await using (held)
|
||||
{
|
||||
if (await BackedAsync(paths, location.Id, cancellationToken).ConfigureAwait(false)) continue;
|
||||
if (await _db.Sequences.DeleteAsync(location.Id, cancellationToken).ConfigureAwait(false)) removed++;
|
||||
await _db.Archives.DeleteFolderStateAsync(archive.Id, location.Folder, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether the index row of <paramref name="sequenceId"/> keeps its place as it is now: it waits for its first frame,
|
||||
/// its folder holds a sidecar naming it (one that cannot be read proves nothing), or it is gone already.
|
||||
/// </summary>
|
||||
private async Task<bool> BackedAsync(ArchivePaths paths, Guid sequenceId, CancellationToken cancellationToken)
|
||||
{
|
||||
if (await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) is not { } row) return true;
|
||||
if ((await AwaitingFolderAsync(paths, [new SequenceLocation(row.Id, row.Folder, row.Title)], cancellationToken).ConfigureAwait(false)).Count > 0)
|
||||
return true;
|
||||
var sidecar = paths.Sequence(row.Folder).ManifestPath;
|
||||
try
|
||||
{
|
||||
return File.Exists(sidecar) && ManifestStore.ReadSequence(sidecar).Id == sequenceId;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Which folder each indexed sequence id lives in, updated as the scan goes.</summary>
|
||||
private sealed class Locations
|
||||
{
|
||||
|
||||
@@ -251,12 +251,20 @@ public sealed class ArchiveMaintenance
|
||||
if (await _db.Sequences.GetAsync(bySequence.Key, cancellationToken).ConfigureAwait(false) is not { } sequence) continue;
|
||||
var changes = bySequence.ToDictionary(x => x.Key, x => (Func<SidecarFrame, SidecarFrame>)(f =>
|
||||
f with { Flags = x.Set ? f.Flags | FrameFlags.Duplicate : f.Flags & ~FrameFlags.Duplicate }));
|
||||
await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m => SequenceFrames.Update(m, changes), cancellationToken).ConfigureAwait(false);
|
||||
var setIds = bySequence.Where(x => x.Set).Select(x => x.Key).ToList();
|
||||
var clearIds = bySequence.Where(x => !x.Set).Select(x => x.Key).ToList();
|
||||
if (setIds.Count > 0) await _db.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false);
|
||||
if (clearIds.Count > 0) await _db.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false);
|
||||
await _indexer.RecordStampsAsync(archive, paths.Sequence(sequence.Folder), sequence.Id, cancellationToken).ConfigureAwait(false);
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
// The scan stamps follow only when the index held the folder as it was: frames another process listed in the
|
||||
// sidecar meanwhile are still indexed by the next scan.
|
||||
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
|
||||
var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
|
||||
ManifestStore.WriteSequence(folder.ManifestPath, SequenceFrames.Update(manifest, changes));
|
||||
var setIds = bySequence.Where(x => x.Set).Select(x => x.Key).ToList();
|
||||
var clearIds = bySequence.Where(x => !x.Set).Select(x => x.Key).ToList();
|
||||
if (setIds.Count > 0) await _db.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false);
|
||||
if (clearIds.Count > 0) await _db.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false);
|
||||
if (inStep) await _indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
return (groups.Count, set.Count + clear.Count);
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ using Titano.Core.Model;
|
||||
using Titano.Core.Naming;
|
||||
using Titano.Data.Archiving.Jobs;
|
||||
using Titano.Data.Jobs;
|
||||
using Titano.Data.Repositories;
|
||||
|
||||
namespace Titano.Data.Archiving;
|
||||
|
||||
@@ -20,6 +21,9 @@ public sealed class ArchiveService : IJobHandlerProvider
|
||||
/// <summary>Lease serialising read-modify-write of <c>archive.json</c> between the processes sharing an archive.</summary>
|
||||
private const string SettingsLease = "archive-settings";
|
||||
|
||||
/// <summary>Times a title, tags, rating or notes edit is applied at the version it read before it writes regardless.</summary>
|
||||
private const int InfoEditAttempts = 3;
|
||||
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly SequenceWriter _writer;
|
||||
|
||||
@@ -224,35 +228,54 @@ public sealed class ArchiveService : IJobHandlerProvider
|
||||
/// Changes some of title, tags, rating and notes from their current values — read under the sequence's lease from
|
||||
/// the sidecar, or from the index when it is ahead (an edit made there only) — so fields <paramref name="edit"/>
|
||||
/// leaves alone keep any change made since the caller read the sequence. Sidecar and index change together
|
||||
/// (settings version + 1).
|
||||
/// (settings version + 1), the index row in these fields only, at the settings version read under the lease
|
||||
/// (<see cref="SequenceRepository.EditAsync(Guid, SequenceEdit, int?, CancellationToken)"/>): a settings edit committed
|
||||
/// since the caller read the sequence (an analysis's Auto values, the server's PATCH, which takes no lease) stays, and
|
||||
/// so do frames another process listed in the sidecar, which the next scan indexes
|
||||
/// (<see cref="ArchiveIndexer.InStepAsync(Archive, SequencePaths, SequenceManifest, CancellationToken)"/>).
|
||||
/// <paramref name="edit"/> runs again on top of an edit the API committed meanwhile.
|
||||
/// </summary>
|
||||
/// <exception cref="ArgumentException">The sequence is not in the index (or left it before the edit).</exception>
|
||||
public async Task<Sequence> EditSequenceInfoAsync(Guid sequenceId, Func<SequenceInfo, SequenceInfo> edit, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(edit);
|
||||
var (archive, sequence) = await LoadSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false);
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var manifest = await _writer.UpdateAsync(paths, sequenceId, sequence.Folder, m =>
|
||||
{
|
||||
var current = sequence.SettingsVersion > m.Versions.Settings
|
||||
? new SequenceInfo(sequence.Title, sequence.Tags, sequence.Rating, sequence.Notes)
|
||||
: new SequenceInfo(m.Title, m.Tags, m.Rating, m.Notes);
|
||||
var info = edit(current);
|
||||
return m with
|
||||
{
|
||||
Title = string.IsNullOrWhiteSpace(info.Title) ? null : info.Title.Trim(),
|
||||
Tags = info.Tags,
|
||||
Rating = info.Rating,
|
||||
Notes = info.Notes,
|
||||
Versions = m.Versions with { Settings = System.Math.Max(m.Versions.Settings, sequence.SettingsVersion) + 1 },
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
};
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
var updated = SequenceMapping.ToSequence(manifest, archive.Id, folder, sequence);
|
||||
await _db.Sequences.UpsertAsync(updated, cancellationToken).ConfigureAwait(false);
|
||||
await Indexer.RecordStampsAsync(archive, folder, sequenceId, cancellationToken).ConfigureAwait(false);
|
||||
return updated;
|
||||
await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var manifest = SequenceWriter.ReadLocked(folder, sequenceId);
|
||||
var inStep = await Indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
|
||||
for (var attempt = 1; ; attempt++)
|
||||
{
|
||||
var row = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? throw Unknown(sequenceId);
|
||||
var current = row.SettingsVersion > manifest.Versions.Settings
|
||||
? new SequenceInfo(row.Title, row.Tags, row.Rating, row.Notes)
|
||||
: new SequenceInfo(manifest.Title, manifest.Tags, manifest.Rating, manifest.Notes);
|
||||
var info = edit(current);
|
||||
var title = string.IsNullOrWhiteSpace(info.Title) ? null : info.Title.Trim();
|
||||
// The last attempt writes over whatever the index holds then rather than fail the user's edit.
|
||||
var result = await _db.Sequences.EditAsync(sequenceId,
|
||||
new SequenceEdit { Title = title ?? string.Empty, Tags = info.Tags, Rating = info.Rating, Notes = info.Notes ?? string.Empty },
|
||||
attempt < InfoEditAttempts ? row.SettingsVersion : null, cancellationToken).ConfigureAwait(false);
|
||||
// A conflict means the API changed the sequence after the read: the edit applies again on top of that.
|
||||
if (result.Outcome == SequenceEditOutcome.Conflict) continue;
|
||||
if (result.Sequence is not { } updated) throw Unknown(sequenceId);
|
||||
// The sidecar takes the fields as the index stored them (trimmed, tags without duplicates).
|
||||
ManifestStore.WriteSequence(folder.ManifestPath, manifest with
|
||||
{
|
||||
Title = updated.Title,
|
||||
Tags = updated.Tags,
|
||||
Rating = updated.Rating,
|
||||
Notes = updated.Notes,
|
||||
Versions = manifest.Versions with { Settings = System.Math.Max(manifest.Versions.Settings + 1, updated.SettingsVersion) },
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
});
|
||||
if (inStep) await Indexer.RecordStampsAsync(archive, folder, sequenceId, cancellationToken).ConfigureAwait(false);
|
||||
return updated;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<DuplicateGroup>> FindDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
|
||||
@@ -367,10 +390,11 @@ public sealed class ArchiveService : IJobHandlerProvider
|
||||
|
||||
private async Task<(Archive, Sequence)> LoadSequenceAsync(Guid sequenceId, CancellationToken cancellationToken)
|
||||
{
|
||||
var sequence = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
|
||||
?? throw new ArgumentException($"Unknown sequence {sequenceId}.", nameof(sequenceId));
|
||||
var sequence = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? throw Unknown(sequenceId);
|
||||
return (await LoadArchiveAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false), sequence);
|
||||
}
|
||||
|
||||
private static ArgumentException Unknown(Guid sequenceId) => new($"Unknown sequence {sequenceId}.", nameof(sequenceId));
|
||||
}
|
||||
|
||||
/// <summary>The fields of a sequence the user describes it with (sidecar and index).</summary>
|
||||
|
||||
@@ -95,35 +95,55 @@ public sealed class ArchiveVerifier
|
||||
return targets;
|
||||
}
|
||||
|
||||
/// <summary>Writes the flags and new hashes of checked frames to the sidecar and the index.</summary>
|
||||
/// <summary>
|
||||
/// Writes the flags and new hashes of checked frames to the sidecar and the index, under one lease. The folder's scan
|
||||
/// stamps follow only when the index held the folder as it was
|
||||
/// (<see cref="ArchiveIndexer.InStepAsync(Archive, SequencePaths, SequenceManifest, CancellationToken)"/>), so frames
|
||||
/// another process listed in the sidecar meanwhile are still indexed by the next scan.
|
||||
/// </summary>
|
||||
private async Task FlushAsync(Archive archive, ArchivePaths paths, Sequence target, List<FrameCheck> pending, CancellationToken cancellationToken)
|
||||
{
|
||||
if (pending.Count == 0) return;
|
||||
var changes = pending.ToDictionary(c => c.Frame.Id, c => (Func<SidecarFrame, SidecarFrame>)(f =>
|
||||
f with { Flags = c.Flags, Sha256 = c.Health == FrameHealth.Hashed ? c.Sha256 : f.Sha256 }));
|
||||
await _writer.UpdateAsync(paths, target.Id, target.Folder, m => SequenceFrames.Update(m, changes), cancellationToken).ConfigureAwait(false);
|
||||
await _db.Frames.UpdateIntegrityAsync(pending.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
await _indexer.RecordStampsAsync(archive, paths.Sequence(target.Folder), target.Id, cancellationToken).ConfigureAwait(false);
|
||||
var sequence = paths.Sequence(target.Folder);
|
||||
await using (await _writer.LockAsync(paths, target.Id, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var manifest = SequenceWriter.ReadLocked(sequence, target.Id);
|
||||
var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
|
||||
ManifestStore.WriteSequence(sequence.ManifestPath, SequenceFrames.Update(manifest, changes));
|
||||
await _db.Frames.UpdateIntegrityAsync(pending.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
pending.Clear();
|
||||
}
|
||||
|
||||
/// <summary>Stamps the verify time; the sequence counts as verified only when no frame is missing or corrupt.</summary>
|
||||
/// <summary>
|
||||
/// Stamps the verify time; the sequence counts as verified only when no frame is missing or corrupt, and has errors
|
||||
/// while one is. Sidecar and index are written under one lease, the index row with these fields only
|
||||
/// (<see cref="SequenceRepository.RecordVerifiedAsync"/>), so a settings edit or an ingest committed meanwhile stays;
|
||||
/// the scan stamps as in <see cref="FlushAsync"/>.
|
||||
/// </summary>
|
||||
private async Task StampAsync(Archive archive, ArchivePaths paths, Sequence target, CancellationToken cancellationToken)
|
||||
{
|
||||
var sequence = paths.Sequence(target.Folder);
|
||||
var manifest = await _writer.UpdateAsync(paths, target.Id, target.Folder, m =>
|
||||
await using (await _writer.LockAsync(paths, target.Id, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var m = SequenceWriter.ReadLocked(sequence, target.Id);
|
||||
var inStep = await _indexer.InStepAsync(archive, sequence, m, cancellationToken).ConfigureAwait(false);
|
||||
var clean = m.Frames.All(f => (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0);
|
||||
return m with
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var manifest = m with
|
||||
{
|
||||
Status = clean ? m.Status | SequenceStatus.Verified : m.Status & ~SequenceStatus.Verified,
|
||||
VerifiedAt = DateTimeOffset.UtcNow,
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
VerifiedAt = now,
|
||||
UpdatedAt = now,
|
||||
};
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
var existing = await _db.Sequences.GetAsync(target.Id, cancellationToken).ConfigureAwait(false);
|
||||
await _db.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
|
||||
await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false);
|
||||
ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
|
||||
await _db.Sequences.RecordVerifiedAsync(target.Id, clean ? SequenceStatus.Verified : SequenceStatus.HasErrors,
|
||||
clean ? SequenceStatus.HasErrors : SequenceStatus.Verified, now, cancellationToken).ConfigureAwait(false);
|
||||
if (inStep) await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -61,6 +61,13 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi
|
||||
/// </summary>
|
||||
/// <param name="proxies">Whether every such frame follows its proxy even where no tier is recorded (analyses and the
|
||||
/// proxy pass); otherwise only the frames in <paramref name="tiers"/> do.</param>
|
||||
/// <remarks>
|
||||
/// The sidecar and the index are written under one lease, the index from the sidecar read under it, and the index
|
||||
/// row gets only what tiers change (<see cref="SequenceRepository.RecordTiersAsync"/>): a whole row written back
|
||||
/// would undo what another writer committed meanwhile, an ingest's frames or a settings edit (the server's PATCH
|
||||
/// takes no lease). The folder's scan stamps are recorded only when the index held the folder as it was
|
||||
/// (<see cref="ArchiveIndexer.InStepAsync(Archive, SequencePaths, SequenceManifest, CancellationToken)"/>).
|
||||
/// </remarks>
|
||||
/// <exception cref="ArchiveOfflineException">The archive is offline or read-only.</exception>
|
||||
public async Task<IReadOnlyList<SidecarFrame>> RecordAsync(Archive archive, Sequence sequence, IReadOnlyDictionary<Guid, FileTier> tiers,
|
||||
bool proxies, CancellationToken cancellationToken)
|
||||
@@ -69,12 +76,12 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
var changed = new List<SidecarFrame>();
|
||||
ProxyPolicy? policy = null;
|
||||
var manifest = await writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m =>
|
||||
await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
changed.Clear();
|
||||
var m = SequenceWriter.ReadLocked(folder, sequence.Id);
|
||||
var changed = new List<SidecarFrame>();
|
||||
var gained = false;
|
||||
ProxyPolicy? policy = null;
|
||||
var frames = new SidecarFrame[m.Frames.Count];
|
||||
for (var i = 0; i < frames.Length; i++)
|
||||
{
|
||||
@@ -96,21 +103,26 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi
|
||||
frames[i] = updated;
|
||||
changed.Add(updated);
|
||||
}
|
||||
if (changed.Count == 0) return null;
|
||||
return m with
|
||||
if (changed.Count == 0) return m.Frames;
|
||||
var inStep = await indexer.InStepAsync(archive, folder, m, cancellationToken).ConfigureAwait(false);
|
||||
var manifest = m with
|
||||
{
|
||||
Frames = frames,
|
||||
Status = gained ? m.Status | SequenceStatus.HasWarnings : m.Status,
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
};
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
if (changed.Count == 0) return manifest.Frames;
|
||||
await db.Frames.UpdateTiersAsync(changed.ConvertAll(f => new FrameTierFields(f.Id, f.Flags, f.PreviewW ?? 0, f.PreviewH ?? 0)), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
if (await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) is { } existing)
|
||||
await db.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, folder, existing), cancellationToken).ConfigureAwait(false);
|
||||
await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
|
||||
return manifest.Frames;
|
||||
ManifestStore.WriteSequence(folder.ManifestPath, manifest);
|
||||
await using (var tx = await db.BeginAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
await tx.Frames.UpdateTiersAsync(changed.ConvertAll(f => new FrameTierFields(f.Id, f.Flags, f.PreviewW ?? 0, f.PreviewH ?? 0)), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
await tx.Sequences.RecordTiersAsync(sequence.Id, gained ? SequenceStatus.HasWarnings : SequenceStatus.None,
|
||||
frames.Count(f => (f.Flags & FrameFlags.PreviewOnly) != 0), cancellationToken).ConfigureAwait(false);
|
||||
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
if (inStep) await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
|
||||
return manifest.Frames;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -67,10 +67,11 @@ internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer,
|
||||
/// <summary>
|
||||
/// Queues a background <paramref name="pass"/> (thumbnails or proxies) of each sequence this run indexed whose
|
||||
/// <paramref name="built"/> flag is clear: new and adopted sequences, and those whose photos changed (the scan clears the
|
||||
/// flag then). After a resumed run, which may have indexed sequences before the interruption, of every sequence of the
|
||||
/// archive without the flag but those in <paramref name="awaiting"/>. One pass per sequence, as after an upload: a folder
|
||||
/// renamed, deleted or damaged before its pass runs fails that pass alone, and the other sequences still get their
|
||||
/// images. A sequence whose own pass of that kind is already queued is left out.
|
||||
/// flag then, in this server's own row when another writer listed them in the sidecar itself). After a resumed run,
|
||||
/// which may have indexed sequences before the interruption, of every sequence of the archive without the flag but
|
||||
/// those in <paramref name="awaiting"/>. One pass per sequence, as after an upload: a folder renamed, deleted or
|
||||
/// damaged before its pass runs fails that pass alone, and the other sequences still get their images. A sequence
|
||||
/// whose own pass of that kind is already queued is left out.
|
||||
/// </summary>
|
||||
private async Task QueueImagesAsync(JobContext context, Archive archive, JobKind pass, SequenceStatus built, IReadOnlyList<Guid> indexed,
|
||||
bool resumed, IReadOnlySet<Guid> awaiting)
|
||||
|
||||
@@ -210,12 +210,22 @@ public sealed class MassRenamer
|
||||
.FirstOrDefault(l => !taken.Contains(l.Id) && folders.Any(f => RenamePlanner.IsInside(f, l.Folder)));
|
||||
}
|
||||
|
||||
/// <summary>Sidecar title, then index (folder, frames' paths); each part is idempotent so a resumed step can repeat it.</summary>
|
||||
/// <summary>
|
||||
/// Sidecar title, then index (folder, title, frames' paths); each part is idempotent so a resumed step can repeat it.
|
||||
/// The index row changes in its folder and title only (<see cref="SequenceRepository.RelocateAsync"/>): a settings edit
|
||||
/// committed meanwhile stays, and frames another process listed in the sidecar are left to the next scan, for which
|
||||
/// the folder keeps no stamps unless the index held it as it was
|
||||
/// (<see cref="ArchiveIndexer.InStepAsync(Archive, string, SequencePaths, SequenceManifest, CancellationToken)"/>).
|
||||
/// </summary>
|
||||
private async Task RecordAsync(Archive archive, ArchivePaths paths, RenameStep step, CancellationToken cancellationToken)
|
||||
{
|
||||
var sequence = paths.Sequence(step.ToFolder);
|
||||
var manifest = SequenceWriter.ReadLocked(sequence, step.SequenceId);
|
||||
if (manifest.Title != step.ToTitle)
|
||||
// The folder moved with its files' stamps, which the index recorded at its old place.
|
||||
var inStep = await _indexer.InStepAsync(archive, step.FromFolder, sequence, manifest, cancellationToken).ConfigureAwait(false);
|
||||
// A step that only moves the folder keeps the title, which an edit may have changed since the plan.
|
||||
var retitle = step.FromTitle != step.ToTitle;
|
||||
if (retitle && manifest.Title != step.ToTitle)
|
||||
{
|
||||
manifest = manifest with
|
||||
{
|
||||
@@ -227,12 +237,11 @@ public sealed class MassRenamer
|
||||
}
|
||||
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var existing = await tx.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false);
|
||||
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
|
||||
await tx.Sequences.RelocateAsync(step.SequenceId, step.ToFolder, retitle ? step.ToTitle : null, cancellationToken).ConfigureAwait(false);
|
||||
if (step.MovesFolder) await tx.Frames.RebaseFolderAsync(step.SequenceId, step.FromFolder, step.ToFolder, cancellationToken).ConfigureAwait(false);
|
||||
await tx.Archives.DeleteFolderStateAsync(archive.Id, step.FromFolder, cancellationToken).ConfigureAwait(false);
|
||||
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
await _indexer.RecordStampsAsync(archive, sequence, step.SequenceId, cancellationToken).ConfigureAwait(false);
|
||||
if (inStep) await _indexer.RecordStampsAsync(archive, sequence, step.SequenceId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -40,25 +40,30 @@ internal sealed class ProjectStore(TitanoDatabase db, SequenceWriter writer)
|
||||
/// <exception cref="InvalidDataException">The stored settings cannot be read; nothing is written.</exception>
|
||||
public async Task<bool> UpdateAsync(ArchivePaths paths, Sequence sequence, Func<ProjectSettings, ProjectSettings> edit, CancellationToken cancellationToken)
|
||||
{
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
|
||||
return await UpdateLockedAsync(paths, sequence, edit, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary><see cref="UpdateAsync"/> for a caller that holds the sequence lock already.</summary>
|
||||
/// <exception cref="InvalidDataException">The stored settings cannot be read; nothing is written.</exception>
|
||||
public async Task<bool> UpdateLockedAsync(ArchivePaths paths, Sequence sequence, Func<ProjectSettings, ProjectSettings> edit, CancellationToken cancellationToken)
|
||||
{
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
for (var attempt = 1; ; attempt++)
|
||||
{
|
||||
for (var attempt = 1; ; attempt++)
|
||||
{
|
||||
var row = await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) ?? sequence;
|
||||
var current = Read(row, folder);
|
||||
var before = current.ToJson();
|
||||
var edited = edit(current);
|
||||
var after = edited.ToJson();
|
||||
if (after == before) return false;
|
||||
AtomicFile.WriteAllText(folder.ProjectPath, after);
|
||||
// The Library shows the project's hero frame (advice fills it, a sync takes another PC's).
|
||||
var hero = await HeroFrame.EditAsync(db, sequence.Id, current.HeroFrame, edited.HeroFrame, cancellationToken).ConfigureAwait(false);
|
||||
var result = await db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = after, HeroFrameId = hero }, row.SettingsVersion,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
// A conflict means the API changed the settings meanwhile: apply the edit again on top of them.
|
||||
if (result.Outcome != SequenceEditOutcome.Conflict || attempt == Attempts) return true;
|
||||
}
|
||||
var row = await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) ?? sequence;
|
||||
var current = Read(row, folder);
|
||||
var before = current.ToJson();
|
||||
var edited = edit(current);
|
||||
var after = edited.ToJson();
|
||||
if (after == before) return false;
|
||||
AtomicFile.WriteAllText(folder.ProjectPath, after);
|
||||
// The Library shows the project's hero frame (advice fills it, a sync takes another PC's).
|
||||
var hero = await HeroFrame.EditAsync(db, sequence.Id, current.HeroFrame, edited.HeroFrame, cancellationToken).ConfigureAwait(false);
|
||||
var result = await db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = after, HeroFrameId = hero }, row.SettingsVersion,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
// A conflict means the API changed the settings meanwhile: apply the edit again on top of them.
|
||||
if (result.Outcome != SequenceEditOutcome.Conflict || attempt == Attempts) return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -52,56 +52,64 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti
|
||||
frames = WithTiers(frames, await _tiers.RecordAsync(archive, sequence, probed, proxies: true, cancellationToken).ConfigureAwait(false));
|
||||
var metadata = frames.LightFrames();
|
||||
|
||||
DeepAnalysisReport? report;
|
||||
DeepAnalysisReport? report = null;
|
||||
if (deep)
|
||||
{
|
||||
// Proxies may have arrived during the pass: the advice below follows the frames as they are at its end.
|
||||
// Proxies may have arrived during the pass: the flags below follow the frames as they are at its end, and the
|
||||
// report is checked against them under the lock.
|
||||
(report, var decoded) = await AnalyseDeepAsync(archive, frames, metadata, folder, progress, cancellationToken).ConfigureAwait(false);
|
||||
frames = WithTiers(frames, await _tiers.RecordAsync(archive, sequence, decoded, proxies: true, cancellationToken).ConfigureAwait(false));
|
||||
metadata = [.. metadata.Select((m, i) => m with { Flags = frames.Frames[i].Flags })];
|
||||
}
|
||||
else
|
||||
{
|
||||
report = ReadReport(folder, frames);
|
||||
}
|
||||
var light = LightAnalyzer.Analyze(metadata);
|
||||
var context = SequenceContext.FromFrames(frames.Frames, options.AdvisorCodecs);
|
||||
|
||||
bool changed, unreadable = false;
|
||||
try
|
||||
{
|
||||
// The frame rate follows the user's "fit to N seconds" when one is set (BRIEF §10.3).
|
||||
changed = await projects.UpdateAsync(paths, sequence,
|
||||
settings => Advisor.Apply(settings, Advisor.Recommend(light, report, context with { TargetDurationSeconds = settings.Timing.FitToSeconds })),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (InvalidDataException ex)
|
||||
{
|
||||
Core.Diagnostics.Log.Warning("processing", $"The project of {sequence.DisplayTitle} cannot be read", ex);
|
||||
(changed, unreadable) = (false, true);
|
||||
}
|
||||
|
||||
if (changed) await options.NotifyAsync(sequence.Id, ProcessingOptions.SettingsChange, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
int warnings;
|
||||
// Under the sequence lock, which a deep pass of the same sequence also takes to record its report: a light pass
|
||||
// then either sees that report or runs before it is recorded, and never clears what it recorded.
|
||||
// then either sees that report or runs before it is recorded, and never clears what it recorded. The report counts
|
||||
// only while it describes the frames as they are under the lock (ReadReport), so a deep pass loads them again: a
|
||||
// proxy that arrived while it ran (for hours) makes its verdict on that frame's original stale. The light analysis
|
||||
// the proxy queued then either ran before this, and the advice, status and warnings stay without the report as it
|
||||
// left them, or runs after.
|
||||
await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
report ??= ReadReport(folder, frames);
|
||||
if (deep)
|
||||
{
|
||||
var now = await SequenceFrames.LoadAsync(db, archive, sequence, useProxies: false, options.Archive.Decoder, cancellationToken).ConfigureAwait(false);
|
||||
if (!Describes(report!, now)) report = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
report = ReadReport(folder, frames);
|
||||
}
|
||||
try
|
||||
{
|
||||
// The frame rate follows the user's "fit to N seconds" when one is set (BRIEF §10.3).
|
||||
changed = await projects.UpdateLockedAsync(paths, sequence,
|
||||
settings => Advisor.Apply(settings, Advisor.Recommend(light, report, context with { TargetDurationSeconds = settings.Timing.FitToSeconds })),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (InvalidDataException ex)
|
||||
{
|
||||
Core.Diagnostics.Log.Warning("processing", $"The project of {sequence.DisplayTitle} cannot be read", ex);
|
||||
(changed, unreadable) = (false, true);
|
||||
}
|
||||
warnings = Advisor.Recommend(light, report, context).Findings.Count(f => f.Severity >= FindingSeverity.Warning);
|
||||
var set = SequenceStatus.AnalyzedLight | (report is null ? SequenceStatus.None : SequenceStatus.AnalyzedDeep)
|
||||
| (warnings > 0 ? SequenceStatus.HasWarnings : SequenceStatus.None);
|
||||
// Without a current report (none, or made before frames were added) the deep status no longer holds.
|
||||
var clear = (report is null ? SequenceStatus.AnalyzedDeep : SequenceStatus.None) | (warnings > 0 ? SequenceStatus.None : SequenceStatus.HasWarnings);
|
||||
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
|
||||
var inStep = await indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
|
||||
var status = (manifest.Status | set) & ~clear;
|
||||
if (status != manifest.Status) ManifestStore.WriteSequence(folder.ManifestPath, manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow });
|
||||
var summary = deep && report is not null ? AnalysisSummary.From(report).ToJson() : null;
|
||||
await db.Sequences.RecordAnalysisAsync(sequence.Id, set, clear, warnings, summary, cancellationToken).ConfigureAwait(false);
|
||||
await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
|
||||
if (inStep) await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (changed) await options.NotifyAsync(sequence.Id, ProcessingOptions.SettingsChange, cancellationToken).ConfigureAwait(false);
|
||||
var presets = deep ? SequencePresets.Generate(light, report, context) : [];
|
||||
return new AnalysisOutcome(sequence, frames.Count, warnings, changed, unreadable, presets);
|
||||
}
|
||||
@@ -111,17 +119,21 @@ internal sealed class SequenceAnalyser(TitanoDatabase db, ProcessingOptions opti
|
||||
/// frame it found unreadable has been read from its proxy since (one sent for an original this host cannot decode,
|
||||
/// BRIEF §2.3), so that verdict and the frame's missing measurements no longer hold.
|
||||
/// </summary>
|
||||
public static DeepAnalysisReport? ReadReport(SequencePaths folder, SequenceFrames frames)
|
||||
public static DeepAnalysisReport? ReadReport(SequencePaths folder, SequenceFrames frames) =>
|
||||
ReadReport(folder, frames.Count) is { } report && Describes(report, frames) ? report : null;
|
||||
|
||||
/// <summary>Whether <paramref name="report"/> describes <paramref name="frames"/> as this host reads them now (<see cref="ReadReport(SequencePaths, SequenceFrames)"/>).</summary>
|
||||
private static bool Describes(DeepAnalysisReport report, SequenceFrames frames)
|
||||
{
|
||||
var report = ReadReport(folder, frames.Count);
|
||||
if (report is null || frames.Proxied is not { } proxied) return report;
|
||||
if (report.FrameCount != frames.Count) return false;
|
||||
if (frames.Proxied is not { } proxied) return true;
|
||||
HashSet<int> measured = report.ProxyFrames is { } read ? [.. read] : [];
|
||||
foreach (var index in report.Issues.Undecodable)
|
||||
{
|
||||
if (index < frames.Count && proxied[index] && !measured.Contains(index) && (frames.Frames[index].Flags & SequenceFrames.NotAnalysed) == 0)
|
||||
return null;
|
||||
return false;
|
||||
}
|
||||
return report;
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>The saved deep report when it covers <paramref name="frameCount"/> frames; null when absent, stale or unreadable.</summary>
|
||||
|
||||
@@ -10,6 +10,7 @@ public sealed record FolderScanState(string Folder, Guid? SequenceId, FileStamp?
|
||||
public sealed class ArchiveRepository
|
||||
{
|
||||
private const string Columns = "id, name, root_path, folder_template, file_template, schema_version, created, proxy_long_edge";
|
||||
private const string FolderStateSelect = "SELECT folder, sequence_id, sidecar_size, sidecar_ticks, photos_ticks FROM archive_folders";
|
||||
|
||||
private static readonly string UpsertSql = SqlDialect.Upsert("archives",
|
||||
["id", "name", "root_path", "folder_template", "file_template", "schema_version", "created", "proxy_long_edge"], 1);
|
||||
@@ -39,15 +40,14 @@ public sealed class ArchiveRepository
|
||||
/// <summary>Scan stamps of every folder of an archive, by folder.</summary>
|
||||
public async Task<IReadOnlyDictionary<string, FolderScanState>> GetFolderStatesAsync(Guid archiveId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var rows = await _db.ListAsync(
|
||||
"SELECT folder, sequence_id, sidecar_size, sidecar_ticks, photos_ticks FROM archive_folders WHERE archive_id = $1", [archiveId],
|
||||
r => new FolderScanState(r.GetString(0), r.GetGuidOrNull(1),
|
||||
r.GetInt64OrNull(3) is { } sidecar ? new FileStamp(r.GetInt64(2), new DateTime(sidecar, DateTimeKind.Utc)) : null,
|
||||
r.GetInt64OrNull(4) is { } photos ? new FileStamp(0, new DateTime(photos, DateTimeKind.Utc)) : null),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
var rows = await _db.ListAsync($"{FolderStateSelect} WHERE archive_id = $1", [archiveId], MapFolderState, cancellationToken).ConfigureAwait(false);
|
||||
return rows.ToDictionary(s => s.Folder, StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
/// <summary>Scan stamps of one folder of an archive; null when none are recorded.</summary>
|
||||
public Task<FolderScanState?> GetFolderStateAsync(Guid archiveId, string folder, CancellationToken cancellationToken = default) =>
|
||||
_db.SingleAsync($"{FolderStateSelect} WHERE archive_id = $1 AND folder = $2", [archiveId, folder], MapFolderState, cancellationToken);
|
||||
|
||||
public async Task SetFolderStateAsync(Guid archiveId, FolderScanState state, CancellationToken cancellationToken = default) =>
|
||||
await _db.ExecuteAsync(FolderStateUpsertSql,
|
||||
[archiveId, state.Folder, state.SequenceId, state.Sidecar?.Size, state.Sidecar?.ModifiedUtc.Ticks, state.Photos?.ModifiedUtc.Ticks, DateTimeOffset.UtcNow],
|
||||
@@ -61,6 +61,10 @@ public sealed class ArchiveRepository
|
||||
public Task<int> ClearFolderStatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
|
||||
_db.ExecuteAsync("DELETE FROM archive_folders WHERE archive_id = $1", [archiveId], cancellationToken);
|
||||
|
||||
private static FolderScanState MapFolderState(DbRow r) => new(r.GetString(0), r.GetGuidOrNull(1),
|
||||
r.GetInt64OrNull(3) is { } sidecar ? new FileStamp(r.GetInt64(2), new DateTime(sidecar, DateTimeKind.Utc)) : null,
|
||||
r.GetInt64OrNull(4) is { } photos ? new FileStamp(0, new DateTime(photos, DateTimeKind.Utc)) : null);
|
||||
|
||||
private static Archive Map(DbRow r) => new()
|
||||
{
|
||||
Id = r.GetGuid(0),
|
||||
|
||||
@@ -195,6 +195,24 @@ public sealed class SequenceRepository
|
||||
await _db.ExecuteAsync("UPDATE sequences SET status = (status | $2) & ~$3, updated_at = $4 WHERE id = $1",
|
||||
[id, (int)set, (int)clear, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0;
|
||||
|
||||
/// <summary>
|
||||
/// Records decode tiers written to the sequence's frames (<see cref="FrameRepository.UpdateTiersAsync"/>): the status
|
||||
/// flags in <paramref name="set"/> and the preview-only count, in one statement. The frames, their versions and the
|
||||
/// settings stay as they are, so an ingest or a settings edit committed meanwhile is never written back over.
|
||||
/// </summary>
|
||||
public async Task<bool> RecordTiersAsync(Guid id, SequenceStatus set, int previewOnlyCount, CancellationToken cancellationToken = default) =>
|
||||
await _db.ExecuteAsync("UPDATE sequences SET status = status | $2, preview_only_count = $3, updated_at = $4 WHERE id = $1",
|
||||
[id, (int)set, previewOnlyCount, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0;
|
||||
|
||||
/// <summary>
|
||||
/// Records a verify: status flags set and cleared and the verify time, in one statement; every other field stays as
|
||||
/// it is (<see cref="RecordTiersAsync"/>).
|
||||
/// </summary>
|
||||
public async Task<bool> RecordVerifiedAsync(Guid id, SequenceStatus set, SequenceStatus clear, DateTimeOffset verifiedAt,
|
||||
CancellationToken cancellationToken = default) =>
|
||||
await _db.ExecuteAsync("UPDATE sequences SET status = (status | $2) & ~$3, verified_at = $4, updated_at = $5 WHERE id = $1",
|
||||
[id, (int)set, (int)clear, verifiedAt, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0;
|
||||
|
||||
/// <summary>
|
||||
/// Records an analysis: status flags set and cleared, the warning count and, for a deep analysis, the compact
|
||||
/// summary (<paramref name="analysisJson"/> null keeps the stored one).
|
||||
@@ -390,19 +408,23 @@ public sealed class SequenceRepository
|
||||
[archiveId, sessionKey], cancellationToken).ConfigureAwait(false))!.Value;
|
||||
}, cancellationToken);
|
||||
|
||||
/// <summary>Moves a sequence to another folder and/or title (mass rename); false when it does not exist.</summary>
|
||||
/// <summary>
|
||||
/// Moves a sequence to another folder and, with <paramref name="title"/>, retitles it (mass rename; settings version
|
||||
/// + 1 when the title changes); every other field stays as the locked row has it. False when it does not exist.
|
||||
/// </summary>
|
||||
/// <param name="title">The new title; null keeps the current one.</param>
|
||||
public Task<bool> RelocateAsync(Guid id, string folder, string? title, CancellationToken cancellationToken = default) =>
|
||||
_db.InTransactionAsync(async session =>
|
||||
{
|
||||
var current = await session.QuerySingleOrDefaultAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
if (current is null) return false;
|
||||
var titleChanged = current.Title != title;
|
||||
var retitled = title is not null && current.Title != title;
|
||||
await UpsertRowAsync(session, current with
|
||||
{
|
||||
Folder = folder,
|
||||
Title = title,
|
||||
SettingsVersion = titleChanged ? current.SettingsVersion + 1 : current.SettingsVersion,
|
||||
Title = retitled ? title : current.Title,
|
||||
SettingsVersion = retitled ? current.SettingsVersion + 1 : current.SettingsVersion,
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
return true;
|
||||
|
||||
@@ -7,8 +7,9 @@ namespace Titano.Server.Storage;
|
||||
/// <summary>
|
||||
/// Proxies are files under the sequence's <c>proxies/</c> folder; a thumbnail is the JPEG byte range of its entry in
|
||||
/// <c>thumbs.pack</c>. A thumbnail made from a different original (the frame was renumbered or replaced) counts as not
|
||||
/// built yet, like a missing one, and so does a proxy whose capture time is another frame's, or one smaller than the
|
||||
/// archive's policy sent for a frame the server cannot decode (<see cref="ServerProxies.HasProxy"/>).
|
||||
/// built yet, like a missing one (<see cref="ThumbPack.IsFrom"/>: one drawn before a verify hashed the photo is still its
|
||||
/// own), and so does a proxy whose capture time is another frame's, or one smaller than the archive's policy sent for a
|
||||
/// frame the server cannot decode (<see cref="ServerProxies.HasProxy"/>).
|
||||
/// </summary>
|
||||
internal sealed class ArchiveAssetLocator : IFrameAssetLocator
|
||||
{
|
||||
@@ -34,7 +35,14 @@ internal sealed class ArchiveAssetLocator : IFrameAssetLocator
|
||||
{
|
||||
using var pack = ThumbPackReader.Open(paths.ThumbsPackPath);
|
||||
if (!pack.Entries.TryGetValue(frame.Index, out var entry) || entry.Offset + entry.Length > pack.Length) return null;
|
||||
if (frame.Sha256 is { Length: ThumbPack.HashLength * 2 } sha && !entry.SourceHash.AsSpan().SequenceEqual(Convert.FromHexString(sha))) return null;
|
||||
if (frame.Sha256 is { Length: ThumbPack.HashLength * 2 } sha && !ThumbPack.IsFrom(entry.SourceHash, sha, SequenceFile(paths, frame), frame.Size)) return null;
|
||||
return new AssetLocation(paths.ThumbsPackPath, entry.Offset, entry.Length, Jpeg);
|
||||
}
|
||||
|
||||
/// <summary>The frame's file as its sidecar names it (relative to the sequence folder), which a thumbnail drawn before the photo was hashed is fingerprinted with.</summary>
|
||||
private static string SequenceFile(SequencePaths paths, Frame frame)
|
||||
{
|
||||
var prefix = paths.ArchiveRelative(string.Empty);
|
||||
return frame.RelativePath.StartsWith(prefix, StringComparison.Ordinal) ? frame.RelativePath[prefix.Length..] : frame.RelativePath;
|
||||
}
|
||||
}
|
||||
@@ -150,6 +150,25 @@ public sealed class ThumbPackTests(TestContext context)
|
||||
for (var i = 0; i < 20; i++) Assert.BytesEqual(Payload(i + 300), compacted.Read(i));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AThumbnailIsFromItsOriginalAlsoAfterTheOriginalWasHashed()
|
||||
{
|
||||
var sha = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData("photo"u8));
|
||||
var other = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData("another photo"u8));
|
||||
var drawnUnhashed = ThumbPack.Fingerprint(null, "IMG_0001.JPG", 1000);
|
||||
var drawnHashed = ThumbPack.Fingerprint(sha, "IMG_0001.JPG", 1000);
|
||||
|
||||
Assert.True(ThumbPack.IsFrom(drawnUnhashed, null, "IMG_0001.JPG", 1000), "unhashed");
|
||||
Assert.True(ThumbPack.IsFrom(drawnHashed, sha, "IMG_0001.JPG", 1000), "hashed");
|
||||
// Drawn before a verify hashed the photo: the hash changes nothing about it.
|
||||
Assert.True(ThumbPack.IsFrom(drawnUnhashed, sha, "IMG_0001.JPG", 1000), "hashed since");
|
||||
// Another photo: another hash, or another file or size at that frame index.
|
||||
Assert.False(ThumbPack.IsFrom(drawnHashed, other, "IMG_0001.JPG", 1000), "another hash");
|
||||
Assert.False(ThumbPack.IsFrom(drawnUnhashed, sha, "IMG_0002.JPG", 1000), "another file");
|
||||
Assert.False(ThumbPack.IsFrom(drawnUnhashed, null, "IMG_0001.JPG", 1001), "another size");
|
||||
Assert.False(ThumbPack.IsFrom(drawnHashed, null, "IMG_0001.JPG", 1000), "a hash the frame no longer has");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AMissingOrForeignFileReadsAsEmpty()
|
||||
{
|
||||
|
||||
@@ -55,6 +55,20 @@ internal sealed class ArchiveFixture : IAsyncDisposable
|
||||
public Task<ScanSummary> ScanAsync(ScanMode mode = ScanMode.Incremental) =>
|
||||
Service.Indexer.ScanAsync(Archive, mode, null, _ => ValueTask.CompletedTask, Ct);
|
||||
|
||||
/// <summary>
|
||||
/// Another process writing the same archive, such as a desktop sharing a server's archive folder: its own index
|
||||
/// (scanned) and archive service; disposed with the fixture.
|
||||
/// </summary>
|
||||
public async Task<(ArchiveService Service, Archive Archive)> OtherWriterAsync()
|
||||
{
|
||||
var db = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(_context.CreateTempDirectory("other"), "titano.db")), Ct);
|
||||
_owned.Add(db);
|
||||
var service = new ArchiveService(db, new ArchiveServiceOptions { Owner = "other", LeaseWait = TimeSpan.FromSeconds(20) });
|
||||
var archive = await service.OpenAsync(Archive.RootPath, Ct);
|
||||
await service.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, Ct);
|
||||
return (service, archive);
|
||||
}
|
||||
|
||||
/// <summary>A job engine with the archive handlers, started; disposed with the fixture.</summary>
|
||||
public async Task<JobEngine> StartEngineAsync()
|
||||
{
|
||||
|
||||
@@ -365,6 +365,38 @@ public sealed class ArchiveIngestTests(TestContext context)
|
||||
Assert.Equal("2026/2026-09-29 Canon EOS R5", (await fx.Db.Sequences.GetAsync(waiting.Id))!.Folder);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ARebuildKeepsSequencesThatGetTheirFolderWhileItRuns()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
Photos.WriteSeries(Path.Combine(fx.Archive.RootPath, "Trips", "Comet"), Start, 2);
|
||||
await fx.ScanAsync();
|
||||
var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset");
|
||||
|
||||
// Uploads go on while the rebuild runs: after it listed the folders, the sequence waiting for its first frame gets
|
||||
// it, and a new upload session creates a sequence.
|
||||
IngestResult? first = null, session = null;
|
||||
var rebuilt = await fx.Service.Indexer.ScanAsync(fx.Archive, ScanMode.Rebuild, null, async progress =>
|
||||
{
|
||||
if (progress.FoldersDone < progress.FoldersTotal || first is not null) return;
|
||||
first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddDays(1), 1), sequenceId: waiting.Id), fx.Ct);
|
||||
session = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddDays(2), 2), sessionKey: "upload-2"), fx.Ct);
|
||||
}, fx.Ct);
|
||||
|
||||
// Neither is dropped, though the folders listed at the start hold neither.
|
||||
Assert.Equal(0, rebuilt.Removed);
|
||||
Assert.Equal(waiting.Id, Assert.NotNull(first).SequenceId);
|
||||
var placed = Assert.NotNull(await fx.Db.Sequences.GetAsync(waiting.Id));
|
||||
Assert.Equal(("2026/2026-09-29 Canon EOS R5", 1, "Harbour sunset"), (placed.Folder, placed.FrameCount, placed.Title));
|
||||
var created = Assert.NotNull(await fx.Db.Sequences.GetAsync(Assert.NotNull(session).SequenceId));
|
||||
Assert.Equal(1, created.FrameCount);
|
||||
// The session's next upload lands in the same sequence, not in a second one beside it.
|
||||
Assert.Equal(created.Id, await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, "upload-2", fx.Ct));
|
||||
var next = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddDays(2).AddSeconds(10), 3), sessionKey: "upload-2"), fx.Ct);
|
||||
Assert.Equal(created.Id, next.SequenceId);
|
||||
Assert.Equal(3, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ADamagedRootSidecarIsRebuiltUnderTheRootSequenceNotOneWaitingForItsFirstFrame()
|
||||
{
|
||||
|
||||
@@ -175,6 +175,210 @@ public sealed class ArchiveJobTests(TestContext context)
|
||||
Assert.Count(7, Directory.GetFiles(fx.Paths.Sequence("a").Proxies));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AServerScanQueuesTheImagesOfFramesAnotherWriterListedInTheSidecarItself()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
Populate(fx);
|
||||
await using var engine = ServerEngine(fx);
|
||||
await engine.StartAsync(fx.Ct);
|
||||
var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
|
||||
await fx.WaitForAsync(scan.Id, JobState.Done);
|
||||
var first = new HashSet<Guid>();
|
||||
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
|
||||
{
|
||||
foreach (var pass in (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items)
|
||||
{
|
||||
await fx.WaitForAsync(pass.Id, JobState.Done);
|
||||
first.Add(pass.Id);
|
||||
}
|
||||
}
|
||||
var a = (await LocationsAsync(fx)).Single(l => l.Folder == "a");
|
||||
Assert.True(Built((await fx.Db.Sequences.GetAsync(a.Id))!.Status), "built");
|
||||
|
||||
// A desktop sharing the archive imports a photo taken before the others into the sequence: it files the photo and
|
||||
// lists it in the sidecar itself, whose statuses it keeps, so the scan finds no photo to reconcile.
|
||||
var (desktop, archive) = await fx.OtherWriterAsync();
|
||||
var staged = fx.Stage(Start.AddMinutes(-5), seed: 90);
|
||||
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, a.Id, null, null, staged.Path, "IMG_0000.JPG", staged.Size, staged.Sha256, null, null),
|
||||
fx.Ct);
|
||||
var sidecar = fx.Paths.Sequence("a").ManifestPath;
|
||||
Assert.True(Built(ManifestStore.ReadSequence(sidecar).Status), "the other writer keeps the statuses");
|
||||
|
||||
var again = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
|
||||
await fx.WaitForAsync(again.Id, JobState.Done);
|
||||
Assert.True(Built(ManifestStore.ReadSequence(sidecar).Status), "the server clears what it makes in its own row only");
|
||||
|
||||
// The frames changed behind the index: the statuses no longer hold, and the sequence gets a pass of each kind.
|
||||
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
|
||||
{
|
||||
var pass = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items.Single(j => !first.Contains(j.Id));
|
||||
Assert.SequenceEqual([a.Id], pass.TargetIds);
|
||||
await fx.WaitForAsync(pass.Id, JobState.Done);
|
||||
}
|
||||
var row = (await fx.Db.Sequences.GetAsync(a.Id))!;
|
||||
Assert.Equal(7, row.FrameCount);
|
||||
Assert.True(Built(row.Status), "built again");
|
||||
// Every frame has its own thumbnail and proxy: the new first one, and the others it renumbered.
|
||||
var folder = fx.Paths.Sequence("a");
|
||||
using var pack = ThumbPackReader.Open(folder.ThumbsPackPath);
|
||||
foreach (var frame in ManifestStore.ReadSequence(sidecar).Frames)
|
||||
{
|
||||
Assert.True(pack.Entries.GetValueOrDefault(frame.I) is { } entry
|
||||
&& entry.SourceHash.AsSpan().SequenceEqual(ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size)), $"thumbnail of {frame.File}");
|
||||
Assert.True(ArchiveImages.IsProxyOf(folder.Proxy(frame.I), frame.CapturedAt, frame.W, frame.H, null), $"proxy of {frame.File}");
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnImagePassMarksTheSequenceBuiltOnlyWhenItsSidecarStillListsTheFramesThePassRead()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
// The pass commits its pack and checkpoints after 200 frames.
|
||||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 201);
|
||||
await fx.ScanAsync();
|
||||
var a = (await LocationsAsync(fx)).Single();
|
||||
var sequence = (await fx.Db.Sequences.GetAsync(a.Id))!;
|
||||
|
||||
// While it runs, another writer copies in a photo taken before the others, and a scan reconciles it.
|
||||
var scanned = false;
|
||||
var result = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, async (_, _) =>
|
||||
{
|
||||
if (scanned) return;
|
||||
scanned = true;
|
||||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start.AddMinutes(-5), 1, firstSeed: 900, firstNumber: 900);
|
||||
Assert.SequenceEqual([a.Id], (await fx.ScanAsync()).Sequences);
|
||||
}, fx.Ct);
|
||||
|
||||
// The pass drew every frame it read, but not the new photo: the sequence is not built.
|
||||
Assert.True(scanned, "scanned during the pass");
|
||||
Assert.Equal((201, 0), (result.Built, result.Undecodable));
|
||||
var row = (await fx.Db.Sequences.GetAsync(a.Id))!;
|
||||
Assert.Equal(202, row.FrameCount);
|
||||
Assert.False(row.Status.HasFlag(SequenceStatus.ThumbsBuilt), row.Status.ToString());
|
||||
Assert.False(ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Status.HasFlag(SequenceStatus.ThumbsBuilt), "sidecar flag");
|
||||
|
||||
// So a scan's follow-up (here a resumed one, which lists the whole archive) queues the next pass, which makes them.
|
||||
await using var engine = ServerEngine(fx);
|
||||
var resumed = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Scan(fx.Archive), "after:a");
|
||||
await engine.StartAsync(fx.Ct);
|
||||
await fx.WaitForAsync(resumed.Id, JobState.Done);
|
||||
var pass = (await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items.Single();
|
||||
Assert.SequenceEqual([a.Id], pass.TargetIds);
|
||||
await fx.WaitForAsync(pass.Id, JobState.Done);
|
||||
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "built");
|
||||
using var pack = ThumbPackReader.Open(fx.Paths.Sequence("a").ThumbsPackPath);
|
||||
Assert.Count(202, pack.Entries);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnImagePassOvertakenByAnUploadKeepsTheSequenceBuiltWhileThePassAfterTheUploadWaits()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
// The pass commits its pack and checkpoints after 200 frames.
|
||||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 201);
|
||||
await fx.ScanAsync();
|
||||
var a = (await LocationsAsync(fx)).Single();
|
||||
var sequence = (await fx.Db.Sequences.GetAsync(a.Id))!;
|
||||
|
||||
// While it runs, an upload lands in the sequence and queues the next pass, as the server does after every upload.
|
||||
await using var engine = ServerEngine(fx);
|
||||
Job? next = null;
|
||||
await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, async (_, _) =>
|
||||
{
|
||||
if (next is not null) return;
|
||||
var staged = fx.Stage(Start.AddMinutes(-5), seed: 90);
|
||||
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, a.Id, null, null, staged.Path, "IMG_0000.JPG", staged.Size, staged.Sha256, null, null),
|
||||
fx.Ct);
|
||||
next = await engine.EnqueueAsync(ArchiveJobs.Thumbnails([a.Id], "a"), cancellationToken: fx.Ct);
|
||||
}, fx.Ct);
|
||||
|
||||
// The pass did not draw the new photo, but the one queued after it does: the sequence stays built meanwhile, so a
|
||||
// server still offers the thumbnails it has.
|
||||
Assert.NotNull(next);
|
||||
Assert.Equal(202, (await fx.Db.Sequences.GetAsync(a.Id))!.FrameCount);
|
||||
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "row flag");
|
||||
Assert.True(ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Status.HasFlag(SequenceStatus.ThumbsBuilt), "sidecar flag");
|
||||
|
||||
await engine.StartAsync(fx.Ct);
|
||||
await fx.WaitForAsync(next!.Id, JobState.Done);
|
||||
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "built");
|
||||
var folder = fx.Paths.Sequence("a");
|
||||
using var pack = ThumbPackReader.Open(folder.ThumbsPackPath);
|
||||
foreach (var frame in ManifestStore.ReadSequence(folder.ManifestPath).Frames)
|
||||
{
|
||||
Assert.True(pack.Entries.GetValueOrDefault(frame.I) is { } entry && ThumbPack.IsFrom(entry.SourceHash, frame.Sha256, frame.File, frame.Size),
|
||||
$"thumbnail of {frame.File}");
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ThumbnailsDrawnBeforeAVerifyHashedThePhotosStayTheirs()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3);
|
||||
await fx.ScanAsync();
|
||||
var a = (await LocationsAsync(fx)).Single();
|
||||
var first = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(a.Id))!, false, 0,
|
||||
(_, _) => ValueTask.CompletedTask, fx.Ct);
|
||||
Assert.Equal(3, first.Built);
|
||||
|
||||
// The verify gives the photos the scan adopted their first hash; what was drawn from them is still theirs.
|
||||
var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, [a.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
|
||||
Assert.Equal(3L, report.Hashed);
|
||||
var again = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, (await fx.Db.Sequences.GetAsync(a.Id))!, false, 0,
|
||||
(_, _) => ValueTask.CompletedTask, fx.Ct);
|
||||
Assert.Equal((0, 3), (again.Built, again.Reused));
|
||||
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "built");
|
||||
Assert.True(ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Frames.All(f => f.Sha256 is not null), "hashed");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ADesktopScanAfterTheServersUploadLeavesWhatTheServerMadeForTheFramesAlone()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
Populate(fx);
|
||||
await using var engine = ServerEngine(fx);
|
||||
await engine.StartAsync(fx.Ct);
|
||||
var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
|
||||
await fx.WaitForAsync(scan.Id, JobState.Done);
|
||||
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
|
||||
{
|
||||
foreach (var pass in (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items) await fx.WaitForAsync(pass.Id, JobState.Done);
|
||||
}
|
||||
var a = (await LocationsAsync(fx)).Single(l => l.Folder == "a");
|
||||
// A desktop sharing the archive, with an index of its own.
|
||||
await using var desktopIndex = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("desktop"), "titano.db")), fx.Ct);
|
||||
var desktop = new ArchiveService(desktopIndex, new ArchiveServiceOptions { Owner = "desktop", LeaseWait = TimeSpan.FromSeconds(20) });
|
||||
var archive = await desktop.OpenAsync(fx.Archive.RootPath, fx.Ct);
|
||||
await desktop.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct);
|
||||
|
||||
// The server takes an upload into the sequence and makes its thumbnail and proxy, as after every upload.
|
||||
var staged = fx.Stage(Start.AddMinutes(-5), seed: 90);
|
||||
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, a.Id, null, null, staged.Path, "IMG_0000.JPG", staged.Size, staged.Sha256, null, null),
|
||||
fx.Ct);
|
||||
foreach (var request in new[] { ArchiveJobs.Thumbnails([a.Id], "a"), ArchiveJobs.Proxies([a.Id], "a") })
|
||||
await fx.WaitForAsync((await engine.EnqueueAsync(request, cancellationToken: fx.Ct)).Id, JobState.Done);
|
||||
var sidecar = fx.Paths.Sequence("a").ManifestPath;
|
||||
Assert.True(Built(ManifestStore.ReadSequence(sidecar).Status), "built after the upload");
|
||||
var stamp = FileStamp.Of(sidecar);
|
||||
var jobs = (await fx.Db.Jobs.ListAsync(null, null, cancellationToken: fx.Ct)).Items.Count;
|
||||
|
||||
// The desktop's index lags behind the upload: its scan indexes the frame and takes the statuses as the sidecar has them.
|
||||
var summary = await desktop.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, fx.Ct);
|
||||
Assert.SequenceEqual([a.Id], summary.Sequences);
|
||||
Assert.Equal(stamp, FileStamp.Of(sidecar));
|
||||
var row = (await desktopIndex.Sequences.GetAsync(a.Id, fx.Ct))!;
|
||||
Assert.Equal(7, row.FrameCount);
|
||||
Assert.True(Built(row.Status), row.Status.ToString());
|
||||
|
||||
// So the server's next scan finds nothing changed, and makes nothing again.
|
||||
var again = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
|
||||
await fx.WaitForAsync(again.Id, JobState.Done);
|
||||
Assert.Equal(jobs + 1, (await fx.Db.Jobs.ListAsync(null, null, cancellationToken: fx.Ct)).Items.Count);
|
||||
Assert.True(Built((await fx.Db.Sequences.GetAsync(a.Id))!.Status), "the server's row");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AFolderGoneBeforeItsImagePassesFailsThemAloneAndTheOtherSequencesGetTheirImages()
|
||||
{
|
||||
|
||||
@@ -14,13 +14,17 @@ namespace Titano.Data.Tests.Archiving;
|
||||
internal static class HeifFile
|
||||
{
|
||||
/// <summary>A <paramref name="width"/> × <paramref name="height"/> HEIF captured at <paramref name="captured"/>, its Exif
|
||||
/// thumbnail <paramref name="thumbnailWidth"/> × <paramref name="thumbnailHeight"/>.</summary>
|
||||
/// thumbnail <paramref name="thumbnailWidth"/> × <paramref name="thumbnailHeight"/> (none when the width is 0: such a
|
||||
/// host finds no pixels in it at all).</summary>
|
||||
public static byte[] Create(DateTime captured, int width, int height, int thumbnailWidth = 160, int thumbnailHeight = 120)
|
||||
{
|
||||
using var small = new ImageBuffer(thumbnailWidth, thumbnailHeight);
|
||||
for (var c = 0; c < 3; c++) small.Plane(c).Fill(0.2f + c * 0.2f);
|
||||
var thumbnail = JpegEncoder.Encode(small, new JpegEncodeOptions { Quality = 80 });
|
||||
var tiff = WithThumbnail(ExifWriter.BuildTiff(new FrameMetadata { Make = "TestCam", Model = "H-1", CapturedAt = captured }), thumbnail);
|
||||
var tiff = ExifWriter.BuildTiff(new FrameMetadata { Make = "TestCam", Model = "H-1", CapturedAt = captured });
|
||||
if (thumbnailWidth > 0)
|
||||
{
|
||||
using var small = new ImageBuffer(thumbnailWidth, thumbnailHeight);
|
||||
for (var c = 0; c < 3; c++) small.Plane(c).Fill(0.2f + c * 0.2f);
|
||||
tiff = WithThumbnail(tiff, JpegEncoder.Encode(small, new JpegEncodeOptions { Quality = 80 }));
|
||||
}
|
||||
byte[] exif = [.. U32(6), .. "Exif\0\0"u8, .. tiff];
|
||||
// The HEVC-coded primary image, which no managed decoder reads.
|
||||
var coded = new byte[64];
|
||||
|
||||
@@ -226,6 +226,10 @@ public sealed class MassRenameTests(TestContext context)
|
||||
Assert.SequenceEqual(["moved/2026-09-28", "moved/2026-09-29", "moved/2026-09-30"], (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id))
|
||||
.Select(l => l.Folder).Order(StringComparer.Ordinal));
|
||||
Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids));
|
||||
// The folders a nested sequence moved out of changed after their rows were written, as a folder another process
|
||||
// changed would: the next scan reads those two again, and finds the index as it is.
|
||||
Assert.SequenceEqual([ids[0], ids[1]], (await fx.ScanAsync()).Sequences);
|
||||
Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids));
|
||||
Assert.Equal(0, (await fx.ScanAsync()).Indexed);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,286 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Data.Repositories;
|
||||
using Titano.Data.Tests.Infrastructure;
|
||||
using Titano.Testing;
|
||||
using static Titano.Data.Tests.Archiving.ArchiveFixture;
|
||||
|
||||
namespace Titano.Data.Tests.Archiving;
|
||||
|
||||
/// <summary>
|
||||
/// Writers that change part of a sequence's sidecar and index row (decode tiers, the verify, title, tags, rating and
|
||||
/// notes, a rename) while a settings edit or another writer commits, on both dialects: they write only what they own,
|
||||
/// under the sequence lease, and leave what another process changed in the sidecar to the next scan.
|
||||
/// </summary>
|
||||
[Category("archive")]
|
||||
public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(context)
|
||||
{
|
||||
private async Task<ArchiveFixture> FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true)));
|
||||
|
||||
/// <summary>One indexed sequence of three photos in <c>a</c>.</summary>
|
||||
private static async Task<Sequence> IndexedAsync(ArchiveFixture fx)
|
||||
{
|
||||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3);
|
||||
await fx.ScanAsync();
|
||||
var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single();
|
||||
return (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!;
|
||||
}
|
||||
|
||||
/// <summary>Records decode tiers as analyses, image passes and renders do.</summary>
|
||||
private static FrameTiers Tiers(ArchiveFixture fx)
|
||||
{
|
||||
var writer = new SequenceWriter(new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) });
|
||||
return new FrameTiers(fx.Db, writer, new ArchiveIndexer(fx.Db, writer));
|
||||
}
|
||||
|
||||
/// <summary>The frame decodes only through its 48 × 32 embedded preview.</summary>
|
||||
private static Dictionary<Guid, FileTier> Preview(Guid frameId) =>
|
||||
new() { [frameId] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32) };
|
||||
|
||||
/// <summary>Commits <paramref name="tx"/> once the writer under test had time to read the row and wait for it.</summary>
|
||||
private static async Task CommitLaterAsync(TitanoTransaction tx, CancellationToken cancellationToken)
|
||||
{
|
||||
await using (tx)
|
||||
{
|
||||
await Task.Delay(500, cancellationToken);
|
||||
await tx.CommitAsync(cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TiersRecordedWhileASettingsEditCommitsLeaveTheEdit()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var sequence = await IndexedAsync(fx);
|
||||
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||||
|
||||
// The server's PATCH holds the row (and SQLite's write lock) while an analysis records a tier.
|
||||
Task record;
|
||||
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
|
||||
{
|
||||
var edit = await tx.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":2}""" }, sequence.SettingsVersion, fx.Ct);
|
||||
Assert.Equal(SequenceEditOutcome.Updated, edit.Outcome);
|
||||
record = Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
|
||||
await Task.WhenAny(record, Task.Delay(500, fx.Ct));
|
||||
Assert.False(record.IsCompleted, "the record waits for the row");
|
||||
await tx.CommitAsync(fx.Ct);
|
||||
}
|
||||
await record;
|
||||
|
||||
// The tier and what follows from it are recorded, and the edit committed meanwhile stays.
|
||||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
Assert.Contains("heroFrame", row.SettingsJson);
|
||||
Assert.Equal((sequence.SettingsVersion + 1, 1), (row.SettingsVersion, row.PreviewOnlyCount));
|
||||
Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TiersRecordedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var sequence = await IndexedAsync(fx);
|
||||
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||||
|
||||
// A desktop sharing the archive imports a photo into the sequence: its own index follows the sidecar, this one not yet.
|
||||
var (desktop, archive) = await fx.OtherWriterAsync();
|
||||
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
|
||||
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
|
||||
fx.Ct);
|
||||
|
||||
// A tier recorded meanwhile changes the tiers here, not the frames this index holds.
|
||||
await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
|
||||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
Assert.Equal((3, sequence.FramesVersion, 1), (row.FrameCount, row.FramesVersion, row.PreviewOnlyCount));
|
||||
|
||||
// The next scan still finds the folder changed, and indexes the photo.
|
||||
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
|
||||
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
|
||||
row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
Assert.Equal((4, sidecar.Versions.Frames), (row.FrameCount, row.FramesVersion));
|
||||
Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TiersRecordedAfterAnotherWriterRetitledTheSequenceLeaveTheTitleToTheNextScan()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var sequence = await IndexedAsync(fx);
|
||||
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||||
|
||||
// A desktop sharing the archive renames and rates the sequence, in the sidecar and its own index.
|
||||
var (desktop, _) = await fx.OtherWriterAsync();
|
||||
await desktop.SetSequenceInfoAsync(sequence.Id, "Ridge", ["fog"], 3, null, fx.Ct);
|
||||
|
||||
// A tier recorded here meanwhile writes only the tier, and leaves the folder changed for the next scan.
|
||||
await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
|
||||
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
|
||||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
Assert.Equal(("Ridge", "fog", 3, 1), (row.Title, row.Tags.Single(), row.Rating, row.PreviewOnlyCount));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnInfoEditAfterAnotherWriterAddedFramesLeavesThoseFramesToTheNextScan()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var sequence = await IndexedAsync(fx);
|
||||
var (desktop, archive) = await fx.OtherWriterAsync();
|
||||
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
|
||||
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
|
||||
fx.Ct);
|
||||
|
||||
// The rating goes into the sidecar and this index; the frames this index holds stay as they are.
|
||||
var rated = await fx.Service.EditSequenceInfoAsync(sequence.Id, info => info with { Rating = 4 }, fx.Ct);
|
||||
Assert.Equal((3, sequence.FramesVersion, 4), (rated.FrameCount, rated.FramesVersion, rated.Rating));
|
||||
Assert.Equal(4, ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Rating);
|
||||
|
||||
// The next scan still finds the folder changed, and indexes the photo.
|
||||
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
|
||||
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
|
||||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
Assert.Equal((4, sidecar.Versions.Frames, 4), (row.FrameCount, row.FramesVersion, row.Rating));
|
||||
Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnInfoEditKeepsTheSettingsTheApiSavedAfterItReadTheSequence()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var sequence = await IndexedAsync(fx);
|
||||
|
||||
// The server's PATCH, which takes no lease, saves settings after the edit read the sequence: the edit applies again.
|
||||
var calls = 0;
|
||||
var rated = await fx.Service.EditSequenceInfoAsync(sequence.Id, info =>
|
||||
{
|
||||
if (++calls == 1)
|
||||
{
|
||||
var patch = fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":2}""" }, sequence.SettingsVersion, fx.Ct)
|
||||
.GetAwaiter().GetResult();
|
||||
Assert.Equal(SequenceEditOutcome.Updated, patch.Outcome);
|
||||
}
|
||||
return info with { Rating = 5 };
|
||||
}, fx.Ct);
|
||||
|
||||
Assert.Equal(2, calls);
|
||||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
Assert.Contains("heroFrame", row.SettingsJson);
|
||||
Assert.Equal((5, sequence.SettingsVersion + 2), (row.Rating, row.SettingsVersion));
|
||||
Assert.Equal(rated.SettingsVersion, row.SettingsVersion);
|
||||
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
|
||||
Assert.Equal((5, row.SettingsVersion), (sidecar.Rating, sidecar.Versions.Settings));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ARenameAfterAnotherWriterAddedFramesLeavesThoseFramesToTheNextScan()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var sequence = await IndexedAsync(fx);
|
||||
var (desktop, archive) = await fx.OtherWriterAsync();
|
||||
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
|
||||
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
|
||||
fx.Ct);
|
||||
|
||||
// The rename moves the folder and retitles the sequence; the row keeps the frames this index holds.
|
||||
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [sequence.Id], "b", "Ridge", fx.Ct);
|
||||
await fx.Service.ApplyRenameAsync(plan, null, fx.Ct);
|
||||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
Assert.Equal(("b", "Ridge", 3, sequence.FramesVersion), (row.Folder, row.Title, row.FrameCount, row.FramesVersion));
|
||||
|
||||
// The next scan still finds the folder changed, and indexes the photo where it is now.
|
||||
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
|
||||
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("b").ManifestPath);
|
||||
row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
Assert.Equal(("Ridge", 4, sidecar.Versions.Frames), (row.Title, row.FrameCount, row.FramesVersion));
|
||||
Assert.True((await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).All(f => f.RelativePath.StartsWith("b/", StringComparison.Ordinal)),
|
||||
"frames in b");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AVerifyStoppedAfterAFlushLeavesFramesAnotherWriterAddedToTheNextScan()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
// Enough photos the scan adopted unhashed for the verify to flush their hashes before it reaches the end.
|
||||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 513);
|
||||
await fx.ScanAsync();
|
||||
var sequence = (await fx.Db.Sequences.GetAsync((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single().Id, fx.Ct))!;
|
||||
var (desktop, archive) = await fx.OtherWriterAsync();
|
||||
var staged = fx.Stage(Start.AddHours(3), seed: 90);
|
||||
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0600.JPG", staged.Size, staged.Sha256, null, null),
|
||||
fx.Ct);
|
||||
|
||||
// The verify writes the hashes of the first 512 photos to the sidecar and this index, and stops before the end.
|
||||
await Assert.ThrowsAsync<OperationCanceledException>(() => fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default,
|
||||
(_, _, _) => throw new OperationCanceledException(), null, fx.Ct));
|
||||
Assert.Equal(512, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Count(f => f.Sha256 is not null));
|
||||
Assert.Equal(513, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
|
||||
|
||||
// The next scan still finds the folder changed, and indexes the photo.
|
||||
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
|
||||
Assert.Equal(514, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task DuplicatesFlaggedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
// Two sequences holding the same photo, hashed by a verify.
|
||||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3);
|
||||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "b"), Start, 1);
|
||||
await fx.ScanAsync();
|
||||
await fx.Service.Verifier.VerifyAsync(fx.Archive, null, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
|
||||
var copy = (await fx.Service.FindDuplicatesAsync(fx.Archive.Id, fx.Ct)).Single().Frames[1];
|
||||
var sequence = (await fx.Db.Sequences.GetAsync(copy.SequenceId, fx.Ct))!;
|
||||
|
||||
// A desktop sharing the archive imports a photo into the sequence whose copy maintenance flags.
|
||||
var (desktop, archive) = await fx.OtherWriterAsync();
|
||||
var staged = fx.Stage(Start.AddHours(3), seed: 90);
|
||||
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0600.JPG", staged.Size, staged.Sha256, null, null),
|
||||
fx.Ct);
|
||||
var report = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, firstStep: 3, _ => ValueTask.CompletedTask, fx.Ct);
|
||||
Assert.Equal(1, report.DuplicateFlagChanges);
|
||||
Assert.Equal(sequence.FrameCount, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
|
||||
|
||||
// The next scan still finds the folder changed, and indexes the photo.
|
||||
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
|
||||
Assert.Equal(sequence.FrameCount + 1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.FrameCount);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var sequence = await IndexedAsync(fx);
|
||||
// Hashed by a first verify, so the next one changes no frame and only stamps the sequence.
|
||||
await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
|
||||
sequence = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
|
||||
// The server's PATCH takes the row as the last photo is checked, and commits while the verify stamps the sequence.
|
||||
Task? commit = null;
|
||||
await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (done, total) =>
|
||||
{
|
||||
if (done < total || commit is not null) return;
|
||||
var tx = await fx.Db.BeginAsync(fx.Ct);
|
||||
await tx.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":1}""" }, sequence.SettingsVersion, fx.Ct);
|
||||
commit = CommitLaterAsync(tx, fx.Ct);
|
||||
}, fx.Ct);
|
||||
await Assert.NotNull(commit);
|
||||
|
||||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
Assert.Contains("heroFrame", row.SettingsJson);
|
||||
Assert.Equal(sequence.SettingsVersion + 1, row.SettingsVersion);
|
||||
Assert.True(row.Status.HasFlag(SequenceStatus.Verified), row.Status.ToString());
|
||||
Assert.True(row.VerifiedAt > sequence.VerifiedAt, "stamped again");
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class SqlitePartialRowWriterTests(TestContext context) : PartialRowWriterTests(context)
|
||||
{
|
||||
protected override Backend Backend => Backend.SqliteFile;
|
||||
}
|
||||
|
||||
[RequiresEnvironment(PgTestServer.Variable)]
|
||||
public sealed class PostgresPartialRowWriterTests(TestContext context) : PartialRowWriterTests(context)
|
||||
{
|
||||
protected override Backend Backend => Backend.Postgres;
|
||||
}
|
||||
@@ -206,6 +206,54 @@ public sealed class AnalysisJobTests(TestContext context)
|
||||
Assert.Equal("4k-hevc", await PresetAsync());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ProxiesThatArriveDuringADeepAnalysisOfHeifsWithoutThumbnailsMakeItsUnreadableVerdictStale()
|
||||
{
|
||||
// A server, which finds no pixels at all in these HEIF files (they carry no Exif thumbnail) until proxies come (BRIEF §2.3).
|
||||
await using var fx = await ProcessingFixture.CreateAsync(context, o => o with { Host = EncoderHost.Server });
|
||||
var directory = Path.Combine(fx.Archive.Paths.Root, "Heif");
|
||||
Directory.CreateDirectory(directory);
|
||||
var start = new DateTime(2026, 9, 28, 18, 0, 0);
|
||||
for (var i = 0; i < 4; i++)
|
||||
{
|
||||
await File.WriteAllBytesAsync(Path.Combine(directory, $"IMG_{i + 1:0000}.HEIC"), HeifFile.Create(start.AddSeconds(10 * i), 4032, 3024, thumbnailWidth: 0),
|
||||
fx.Ct);
|
||||
}
|
||||
await fx.Archive.ScanAsync(ScanMode.Full);
|
||||
var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Archive.Id, fx.Ct)).Single(l => l.Folder == "Heif");
|
||||
var sequence = (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!;
|
||||
var folder = fx.Folder(sequence);
|
||||
await fx.StartEngineAsync(workers: 2);
|
||||
async Task<string> AutoValuesAsync() =>
|
||||
File.Exists(folder.ProjectPath) ? ProjectSettings.FromJson(await File.ReadAllTextAsync(folder.ProjectPath, fx.Ct)).ToJson() : string.Empty;
|
||||
|
||||
// A deep analysis starts on the originals; while it runs, the proxies arrive and a light analysis on the other
|
||||
// worker reads the frames from them.
|
||||
fx.Decoder.Hold = "IMG_0001.HEIC";
|
||||
var deep = await fx.EnqueueAsync(ProcessingJobs.DeepAnalysis([sequence.Id]));
|
||||
await fx.Decoder.Holding.Task.WaitAsync(TimeSpan.FromSeconds(60), fx.Ct);
|
||||
Directory.CreateDirectory(folder.Proxies);
|
||||
foreach (var frame in await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))
|
||||
await File.WriteAllBytesAsync(folder.Proxy(frame.Index), Proxy(frame.CapturedAt!.Value, 1920, 1440), fx.Ct);
|
||||
await LightAnalysisAsync(fx, sequence);
|
||||
var light = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
var advice = await AutoValuesAsync();
|
||||
fx.Decoder.Release();
|
||||
await fx.WaitForAsync(deep.Id, JobState.Done);
|
||||
|
||||
// The deep pass found no pixels in any original, a verdict the proxies made stale before it ended, as every reader
|
||||
// of its report finds: it records no deep status, and neither warnings nor advice of its own.
|
||||
Assert.Count(4, SequenceAnalyser.ReadReport(folder, sequence.FrameCount)!.Issues.Undecodable);
|
||||
var now = await Titano.Data.Processing.SequenceFrames.LoadAsync(fx.Db, fx.Archive.Archive, sequence, useProxies: false,
|
||||
new ImageDecoder([fx.Decoder]), fx.Ct);
|
||||
Assert.Null(SequenceAnalyser.ReadReport(folder, now));
|
||||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
Assert.False(row.Status.HasFlag(SequenceStatus.AnalyzedDeep), row.Status.ToString());
|
||||
Assert.False(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedDeep), "sidecar status");
|
||||
Assert.Equal((light.WarningCount, light.Status.HasFlag(SequenceStatus.HasWarnings)), (row.WarningCount, row.Status.HasFlag(SequenceStatus.HasWarnings)));
|
||||
Assert.Equal(advice, await AutoValuesAsync());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AFrameTheDeepAnalysisCouldNotReadIsReadAgainOnceItsProxyArrives()
|
||||
{
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
using System.Net;
|
||||
using System.Security.Cryptography;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Codecs.Jpeg;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Protocol;
|
||||
using Titano.Protocol.Client;
|
||||
using Titano.Server.Jobs;
|
||||
@@ -106,6 +109,74 @@ public sealed class ArchiveModuleTests(TestContext context)
|
||||
await UntilImagesAsync(server, editor, archive.Id, "garden");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task FramesADesktopSharingTheArchiveAddsToASequenceGetThumbnailsAndProxiesAfterThePoll()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context, options =>
|
||||
{
|
||||
Directory.CreateDirectory(Path.Combine(options.ArchivePath, "roof"));
|
||||
File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0002.JPG"), Photo());
|
||||
}, ArchiveModule.Register);
|
||||
var archive = server.Services.GetRequiredService<ServerArchive>();
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
await UntilIndexedAsync(server, archive.Id, "roof");
|
||||
await UntilScannedAsync(server);
|
||||
await UntilImagesAsync(server, editor, archive.Id, "roof");
|
||||
await UntilIdleAsync(server);
|
||||
var roof = (await server.Database.Sequences.ListLocationsAsync(archive.Id)).Single(l => l.Folder == "roof");
|
||||
|
||||
// A desktop sharing the archive, with an index of its own, imports a photo taken earlier into the sequence: it files
|
||||
// the photo and lists it in the sidecar itself, keeping the sidecar's ThumbsBuilt and ProxiesBuilt.
|
||||
var desktopFolder = context.CreateTempDirectory("desktop");
|
||||
await using (var desktopIndex = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(desktopFolder, "titano.db")), context.Cancellation))
|
||||
{
|
||||
var desktop = new ArchiveService(desktopIndex, new ArchiveServiceOptions { Owner = "desktop" });
|
||||
var shared = await desktop.OpenAsync(server.Options.ArchivePath, context.Cancellation);
|
||||
await desktop.Indexer.ScanAsync(shared, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, context.Cancellation);
|
||||
var photo = Photo(seed: 5);
|
||||
var staged = Path.Combine(desktopFolder, "IMG_0001.JPG");
|
||||
await File.WriteAllBytesAsync(staged, photo);
|
||||
await desktop.Ingest.CommitAsync(new IngestRequest(shared.Id, roof.Id, null, null, staged, "IMG_0001.JPG", photo.Length,
|
||||
Convert.ToHexStringLower(SHA256.HashData(photo)), new DateTime(2026, 9, 28, 18, 0, 0), null), context.Cancellation);
|
||||
}
|
||||
Assert.Equal(1, Assert.NotNull(await server.Database.Sequences.GetAsync(roof.Id)).FrameCount);
|
||||
|
||||
// The next poll indexes it and queues the images: the new frame and the one it moved get theirs.
|
||||
Assert.NotNull(await server.Services.GetRequiredService<ArchivePolling>().PollAsync(context.Cancellation));
|
||||
await UntilScannedAsync(server);
|
||||
Assert.Equal(2, Assert.NotNull(await server.Database.Sequences.GetAsync(roof.Id)).FrameCount);
|
||||
await UntilImagesAsync(server, editor, archive.Id, "roof");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AThumbnailDrawnBeforeAVerifyHashedThePhotoIsStillServed()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context, options =>
|
||||
{
|
||||
Directory.CreateDirectory(Path.Combine(options.ArchivePath, "roof"));
|
||||
File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0001.JPG"), Photo());
|
||||
}, ArchiveModule.Register);
|
||||
var archive = server.Services.GetRequiredService<ServerArchive>();
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
await UntilIndexedAsync(server, archive.Id, "roof");
|
||||
await UntilScannedAsync(server);
|
||||
await UntilImagesAsync(server, editor, archive.Id, "roof");
|
||||
await UntilIdleAsync(server);
|
||||
var roof = (await server.Database.Sequences.ListLocationsAsync(archive.Id)).Single(l => l.Folder == "roof");
|
||||
var frame = (await server.Database.Frames.ListBySequenceAsync(roof.Id)).Single();
|
||||
Assert.Null(frame.Sha256);
|
||||
|
||||
// A verify gives the photo the scan adopted its first hash; the thumbnail drawn from it before is still its own.
|
||||
var verified = await server.Services.GetRequiredService<ArchiveService>().Verifier.VerifyAsync(
|
||||
Assert.NotNull(await server.Database.Archives.GetAsync(archive.Id)), [roof.Id], default, (_, _, _) => ValueTask.CompletedTask, null,
|
||||
context.Cancellation);
|
||||
Assert.Equal(1L, verified.Hashed);
|
||||
Assert.NotNull(Assert.NotNull(await server.Database.Frames.GetAsync(frame.Id)).Sha256);
|
||||
using var thumb = await editor.GetAsync($"/api/v1/frames/{frame.Id}/thumb");
|
||||
Assert.Equal(HttpStatusCode.OK, thumb.StatusCode);
|
||||
Assert.True(Assert.NotNull(await server.Database.Sequences.GetAsync(roof.Id)).Status.HasFlag(SequenceStatus.ThumbsBuilt), "built");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task SettingsADesktopChangedInArchiveJsonReachUploadsAndTheAdminPanel()
|
||||
{
|
||||
@@ -157,28 +228,51 @@ public sealed class ArchiveModuleTests(TestContext context)
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Waits until the first frame of the sequence in <paramref name="folder"/> has its thumbnail and proxy served.</summary>
|
||||
/// <summary>
|
||||
/// Waits until every frame of the sequence in <paramref name="folder"/> has its thumbnail and proxy served and the
|
||||
/// passes that made them have marked the sequence (a pass writes its files before it marks).
|
||||
/// </summary>
|
||||
private static async Task UntilImagesAsync(TestServer server, HttpClient client, Guid archiveId, string folder)
|
||||
{
|
||||
var sequence = (await server.Database.Sequences.ListLocationsAsync(archiveId)).Single(l => l.Folder == folder);
|
||||
var frame = (await server.Database.Frames.ListBySequenceAsync(sequence.Id)).First();
|
||||
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
||||
foreach (var asset in new[] { "thumb", "proxy" })
|
||||
foreach (var frame in await server.Database.Frames.ListBySequenceAsync(sequence.Id))
|
||||
{
|
||||
while (true)
|
||||
foreach (var asset in new[] { "thumb", "proxy" })
|
||||
{
|
||||
using var response = await client.GetAsync($"/api/v1/frames/{frame.Id}/{asset}");
|
||||
if (response.StatusCode == HttpStatusCode.OK)
|
||||
while (true)
|
||||
{
|
||||
Assert.True(await response.Content.ReadAsByteArrayAsync() is [0xFF, 0xD8, ..], $"a JPEG {asset} of {folder}");
|
||||
break;
|
||||
using var response = await client.GetAsync($"/api/v1/frames/{frame.Id}/{asset}");
|
||||
if (response.StatusCode == HttpStatusCode.OK)
|
||||
{
|
||||
Assert.True(await response.Content.ReadAsByteArrayAsync() is [0xFF, 0xD8, ..], $"a JPEG {asset} of {frame.FileName}");
|
||||
break;
|
||||
}
|
||||
Assert.True(DateTime.UtcNow < until, $"no {asset} of {frame.FileName} in {folder}: {(int)response.StatusCode}");
|
||||
await Task.Delay(100);
|
||||
}
|
||||
Assert.True(DateTime.UtcNow < until, $"no {asset} of {folder}: {(int)response.StatusCode}");
|
||||
await Task.Delay(100);
|
||||
}
|
||||
}
|
||||
var built = Assert.NotNull(await server.Database.Sequences.GetAsync(sequence.Id)).Status;
|
||||
Assert.True(built.HasFlag(SequenceStatus.ThumbsBuilt) && built.HasFlag(SequenceStatus.ProxiesBuilt), $"{folder}: {built}");
|
||||
while (Assert.NotNull(await server.Database.Sequences.GetAsync(sequence.Id)).Status is var built
|
||||
&& !(built.HasFlag(SequenceStatus.ThumbsBuilt) && built.HasFlag(SequenceStatus.ProxiesBuilt)))
|
||||
{
|
||||
Assert.True(DateTime.UtcNow < until, $"{folder}: {built}");
|
||||
await Task.Delay(50);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Waits until no job is waiting or running.</summary>
|
||||
private static async Task UntilIdleAsync(TestServer server)
|
||||
{
|
||||
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
||||
foreach (var state in new[] { JobState.Queued, JobState.Running, JobState.Queued })
|
||||
{
|
||||
while ((await server.Database.Jobs.ListAsync(state)).Items.Count > 0)
|
||||
{
|
||||
Assert.True(DateTime.UtcNow < until, $"jobs still {state}");
|
||||
await Task.Delay(50);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Waits for the scan queued at start to finish: its sequences are indexed before it ends.</summary>
|
||||
|
||||
Reference in new issue
Block a user