fix(archive): scans see frames another writer changed; partial writers keep the row and the lease
A desktop sharing the archive that imports into an existing sequence lists the new frames in the sidecar itself and keeps its ThumbsBuilt and ProxiesBuilt. The server scan cleared the flags only when SequenceAdopter.Reconcile found unindexed photos, so it queued no image pass: the new frames, and every frame they renumbered, answered 404 on /thumb and /proxy until a pass was forced. A scan now also clears AnalyzedLight, ThumbsBuilt and ProxiesBuilt when the sidecar's frames version differs from the index row's, decided under the sequence lease, and its follow-ups queue the light analysis and both passes. Writers that change only part of a sequence wrote the whole index row back from a sidecar read under a lease they had already released (decode tiers) or without any (verify stamp): an ingest or a PATCH committed in between was undone, and a row written from a sidecar the index had not synced claimed its frames version, hiding the change from the scan. FrameTiers.RecordAsync and ArchiveVerifier now write the sidecar and only their own columns (SequenceRepository.RecordTiersAsync, RecordVerifiedAsync) under one lease. They and the image passes record the folder's scan stamps only while the index holds the sidecar's frames version (ArchiveIndexer.RecordStampsInStepAsync), so the next scan still sees frames another process added. An image pass decided ThumbsBuilt/ProxiesBuilt from the sidecar it read at its start and set the flag again after a scan had cleared it for photos added meanwhile, so no pass was queued for them. MarkAsync now holds the lease for the sidecar and the index, and sets the flag only while the sidecar lists the photos the pass read at the same indexes; otherwise it clears it. A rebuild deleted every row whose folder was not among those listed when it began, so a sequence that got its first frame or was created by an upload session while it ran lost its row (404 for the Uploader, a second '(2)' sequence for the session). A row is now checked again under its lease, as it and its folder are then, before it goes. Tests: a desktop's ArchiveIngest commit followed by a server scan (Data and Server, /thumb and /proxy answer 200 for every frame); a thumbnail pass with a scan's reconcile in between; tier records and a verify stamp interleaved with a PATCH and with another writer's frames, on SQLite and PostgreSQL; a rebuild during which uploads place and create sequences. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
11 files changed
+550
-75
No files matched your search
@@ -255,7 +255,7 @@ public sealed class ArchiveImages
|
||||
pack.Commit();
|
||||
}
|
||||
await _tiers.RecordAsync(archive, sequence, tiers, proxies: false, cancellationToken).ConfigureAwait(false);
|
||||
await MarkAsync(archive, paths, sequence, SequenceStatus.ThumbsBuilt, complete && undecodable == 0, cancellationToken).ConfigureAwait(false);
|
||||
await MarkAsync(archive, paths, sequence, manifest, SequenceStatus.ThumbsBuilt, complete && undecodable == 0, cancellationToken).ConfigureAwait(false);
|
||||
return new ImagePassResult(built, reused, undecodable);
|
||||
}
|
||||
|
||||
@@ -327,7 +327,7 @@ public sealed class ArchiveImages
|
||||
if (cadence.Due()) await checkpoint(i + 1, manifest.Frames.Count).ConfigureAwait(false);
|
||||
}
|
||||
await _tiers.RecordAsync(archive, sequence, tiers, proxies: true, cancellationToken).ConfigureAwait(false);
|
||||
await MarkAsync(archive, paths, sequence, SequenceStatus.ProxiesBuilt, complete, cancellationToken).ConfigureAwait(false);
|
||||
await MarkAsync(archive, paths, sequence, manifest, SequenceStatus.ProxiesBuilt, complete, cancellationToken).ConfigureAwait(false);
|
||||
return new ImagePassResult(built, reused, undecodable, waiting);
|
||||
}
|
||||
|
||||
@@ -520,15 +520,47 @@ public sealed class ArchiveImages
|
||||
}
|
||||
}
|
||||
|
||||
private async Task MarkAsync(Archive archive, ArchivePaths paths, Sequence sequence, SequenceStatus flag, bool complete, CancellationToken cancellationToken)
|
||||
/// <summary>
|
||||
/// Sets <paramref name="flag"/> in the sidecar and the index when the pass covered every frame it read
|
||||
/// (<paramref name="complete"/>) and the sidecar, as it is under the lease, still lists those frames
|
||||
/// (<see cref="SameFrames"/>); clears it otherwise. Frames a scan's reconcile or an ingest added, removed or
|
||||
/// renumbered after the pass read <paramref name="read"/> have nothing of it, and a scan that cleared the flag for
|
||||
/// them queues the next pass only while the flag stays clear. The index is written under the same lease, so the flag
|
||||
/// never lands there after a scan cleared it in both.
|
||||
/// </summary>
|
||||
private async Task MarkAsync(Archive archive, ArchivePaths paths, Sequence sequence, SequenceManifest read, SequenceStatus flag, bool complete,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m =>
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var status = complete ? m.Status | flag : m.Status & ~flag;
|
||||
return status == m.Status ? null : m with { Status = status, UpdatedAt = DateTimeOffset.UtcNow };
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
await _db.Sequences.UpdateStatusAsync(sequence.Id, complete ? flag : SequenceStatus.None, complete ? SequenceStatus.None : flag, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
await _indexer.RecordStampsAsync(archive, paths.Sequence(sequence.Folder), sequence.Id, cancellationToken).ConfigureAwait(false);
|
||||
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
|
||||
var built = complete && SameFrames(manifest.Frames, read.Frames);
|
||||
var status = built ? manifest.Status | flag : manifest.Status & ~flag;
|
||||
if (status != manifest.Status)
|
||||
{
|
||||
manifest = manifest with { Status = status, UpdatedAt = DateTimeOffset.UtcNow };
|
||||
ManifestStore.WriteSequence(folder.ManifestPath, manifest);
|
||||
}
|
||||
await _db.Sequences.UpdateStatusAsync(sequence.Id, built ? flag : SequenceStatus.None, built ? SequenceStatus.None : flag, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
await _indexer.RecordStampsInStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether <paramref name="current"/> lists the photos <paramref name="read"/> did, each at the same index with the
|
||||
/// same file, size and capture time: thumbnails and proxies are kept by frame index, so photos added, removed,
|
||||
/// renumbered or replaced since have none of what the pass made.
|
||||
/// </summary>
|
||||
private static bool SameFrames(IReadOnlyList<SidecarFrame> current, IReadOnlyList<SidecarFrame> read)
|
||||
{
|
||||
if (current.Count != read.Count) return false;
|
||||
for (var i = 0; i < current.Count; i++)
|
||||
{
|
||||
var (now, then) = (current[i], read[i]);
|
||||
if (now.Id != then.Id || now.I != then.I || now.File != then.File || now.Size != then.Size || now.CapturedAt != then.CapturedAt) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -31,6 +31,9 @@ public sealed record ScanSummary(int Folders, int Indexed, int Adopted, int Remo
|
||||
/// </summary>
|
||||
public sealed class ArchiveIndexer
|
||||
{
|
||||
/// <summary>What covers the frames as they were: a scan that finds them changed clears it, and its follow-ups make it again.</summary>
|
||||
private const SequenceStatus CoveringStatus = SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt;
|
||||
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly SequenceWriter _writer;
|
||||
|
||||
@@ -131,29 +134,44 @@ public sealed class ArchiveIndexer
|
||||
_db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(sequence.RelativeFolder, sequenceId,
|
||||
FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))), cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="RecordStampsAsync"/> for a writer that changed, under the sequence lease, only what it owns of the
|
||||
/// sidecar and the index row (status, decode tiers): recorded only while the index holds the frames of
|
||||
/// <paramref name="manifest"/>, the sidecar as that writer read and wrote it. Frames another process (a desktop
|
||||
/// sharing the archive) added, removed or renumbered without this index keep the stamps as they were, so the next
|
||||
/// scan still indexes them and queues what they need.
|
||||
/// </summary>
|
||||
internal async Task RecordStampsInStepAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken)
|
||||
{
|
||||
if (await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { } indexed && indexed.FramesVersion == manifest.Versions.Frames)
|
||||
await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>Indexes unindexed photos and flags missing files of one sequence (the orphans fix action).</summary>
|
||||
public async Task<SequenceManifest> ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
return await UpdateAndSyncAsync(archive, paths, folder, sequence.Id, m => SequenceAdopter.Reconcile(folder, m, cancellationToken), cancellationToken)
|
||||
return await UpdateAndSyncAsync(archive, paths, folder, sequence.Id, (m, _) => SequenceAdopter.Reconcile(folder, m, cancellationToken), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Applies <paramref name="edit"/> to the sidecar (null writes nothing) and indexes the result, both under the sequence
|
||||
/// lease. Frames inserted before indexed ones move the project's positions (<see cref="ProjectPositions"/>) without a
|
||||
/// new settings version; an editor saving its project holds the same lease, so it reads them moved and merges onto
|
||||
/// them rather than writing its old positions back.
|
||||
/// lease; the edit also gets the sequence's index row as it is then (null when it has none). Frames inserted before
|
||||
/// indexed ones move the project's positions (<see cref="ProjectPositions"/>) without a new settings version; an editor
|
||||
/// saving its project holds the same lease, so it reads them moved and merges onto them rather than writing its old
|
||||
/// positions back.
|
||||
/// </summary>
|
||||
private async Task<SequenceManifest> UpdateAndSyncAsync(Archive archive, ArchivePaths paths, SequencePaths sequence, Guid sequenceId,
|
||||
Func<SequenceManifest, SequenceManifest?> edit, CancellationToken cancellationToken)
|
||||
Func<SequenceManifest, Sequence?, SequenceManifest?> edit, CancellationToken cancellationToken)
|
||||
{
|
||||
await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var manifest = SequenceWriter.ReadLocked(sequence, sequenceId);
|
||||
if (edit(manifest) is { } updated)
|
||||
var indexed = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false);
|
||||
if (edit(manifest, indexed) is { } updated)
|
||||
{
|
||||
ManifestStore.WriteSequence(sequence.ManifestPath, updated);
|
||||
manifest = updated;
|
||||
@@ -194,21 +212,23 @@ public sealed class ArchiveIndexer
|
||||
ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
|
||||
bytes = null;
|
||||
}
|
||||
if (reconcile || File.Exists(sequence.IngestJournalPath))
|
||||
var reconcilePhotos = reconcile || File.Exists(sequence.IngestJournalPath);
|
||||
if (reconcilePhotos || await FramesMovedAsync(manifest, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
// Photos added, gone or back: the light analysis, thumbnails and proxies no longer cover the frames, so the scan's
|
||||
// follow-ups make them again (the images on a server, ArchiveServiceOptions.ImagesAfterScan).
|
||||
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id,
|
||||
m => SequenceAdopter.Reconcile(sequence, m, cancellationToken) is { } reconciled
|
||||
? reconciled with { Status = reconciled.Status & ~(SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt) }
|
||||
: null,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
bytes = null;
|
||||
}
|
||||
else if (await MayMovePositionsAsync(manifest, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
// Another process changed the frames: the sidecar is read again under the lease the project moves under.
|
||||
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false);
|
||||
// Photos added, gone or back, or frames another process (a desktop sharing the archive) added, removed or
|
||||
// renumbered together with their sidecar entries, which leaves the statuses its sidecar had: the light analysis,
|
||||
// thumbnails and proxies no longer cover the frames, so the scan's follow-ups make them again (the images on a
|
||||
// server, ArchiveServiceOptions.ImagesAfterScan). Decided under the lease, on the sidecar as it is then against
|
||||
// the index row, under which the project's positions also move.
|
||||
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, (m, indexed) =>
|
||||
{
|
||||
var updated = reconcilePhotos ? SequenceAdopter.Reconcile(sequence, m, cancellationToken) : null;
|
||||
if (updated is null && (indexed is null || indexed.FramesVersion == m.Versions.Frames)) return null;
|
||||
updated ??= m;
|
||||
var status = updated.Status & ~CoveringStatus;
|
||||
if (ReferenceEquals(updated, m) && status == m.Status) return null;
|
||||
return updated with { Status = status, UpdatedAt = DateTimeOffset.UtcNow };
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
bytes = null;
|
||||
}
|
||||
else
|
||||
@@ -257,9 +277,12 @@ public sealed class ArchiveIndexer
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The index holds a project for this sequence and other frames than the sidecar: its positions may move.</summary>
|
||||
private async Task<bool> MayMovePositionsAsync(SequenceManifest manifest, CancellationToken cancellationToken) =>
|
||||
await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { SettingsJson: not null } indexed
|
||||
/// <summary>
|
||||
/// The index holds this sequence with other frames than the sidecar: another process changed them (its project's
|
||||
/// positions may move, and what covered the frames no longer does).
|
||||
/// </summary>
|
||||
private async Task<bool> FramesMovedAsync(SequenceManifest manifest, CancellationToken cancellationToken) =>
|
||||
await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { } indexed
|
||||
&& indexed.FramesVersion != manifest.Versions.Frames;
|
||||
|
||||
/// <summary>A sidecar for a folder that has none; ids of a sequence the index already knew at this folder are kept.</summary>
|
||||
@@ -309,7 +332,10 @@ public sealed class ArchiveIndexer
|
||||
|
||||
/// <summary>
|
||||
/// Rebuild only: rows whose folder no longer holds a sidecar leave the index (their files are untouched). A sequence
|
||||
/// waiting for its first frame has no sidecar yet and stays.
|
||||
/// waiting for its first frame has no sidecar yet and stays. A row the folders listed when the rebuild began do not
|
||||
/// back is looked at again under its lease, as it and its folder are when it would go (<see cref="BackedAsync"/>):
|
||||
/// uploads go on during a rebuild, which can take minutes, so a sequence that got its first frame or was created
|
||||
/// meanwhile has a folder the listing does not hold.
|
||||
/// </summary>
|
||||
private async Task<int> RemoveUnbackedAsync(Archive archive, ArchivePaths paths, IReadOnlyList<SequenceFolder> folders, Locations locations,
|
||||
CancellationToken cancellationToken)
|
||||
@@ -322,12 +348,46 @@ public sealed class ArchiveIndexer
|
||||
foreach (var location in rows)
|
||||
{
|
||||
if (awaiting.Contains(location.Id) || (present.Contains(location.Folder) && locations.AtFolder(location.Folder) == location.Id)) continue;
|
||||
if (await _db.Sequences.DeleteAsync(location.Id, cancellationToken).ConfigureAwait(false)) removed++;
|
||||
await _db.Archives.DeleteFolderStateAsync(archive.Id, location.Folder, cancellationToken).ConfigureAwait(false);
|
||||
IAsyncDisposable held;
|
||||
try
|
||||
{
|
||||
held = await _writer.LockAsync(paths, location.Id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (LeaseUnavailableException)
|
||||
{
|
||||
// Another process is writing the sequence, so it is there.
|
||||
continue;
|
||||
}
|
||||
await using (held)
|
||||
{
|
||||
if (await BackedAsync(paths, location.Id, cancellationToken).ConfigureAwait(false)) continue;
|
||||
if (await _db.Sequences.DeleteAsync(location.Id, cancellationToken).ConfigureAwait(false)) removed++;
|
||||
await _db.Archives.DeleteFolderStateAsync(archive.Id, location.Folder, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether the index row of <paramref name="sequenceId"/> keeps its place as it is now: it waits for its first frame,
|
||||
/// its folder holds a sidecar naming it (one that cannot be read proves nothing), or it is gone already.
|
||||
/// </summary>
|
||||
private async Task<bool> BackedAsync(ArchivePaths paths, Guid sequenceId, CancellationToken cancellationToken)
|
||||
{
|
||||
if (await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) is not { } row) return true;
|
||||
if ((await AwaitingFolderAsync(paths, [new SequenceLocation(row.Id, row.Folder, row.Title)], cancellationToken).ConfigureAwait(false)).Count > 0)
|
||||
return true;
|
||||
var sidecar = paths.Sequence(row.Folder).ManifestPath;
|
||||
try
|
||||
{
|
||||
return File.Exists(sidecar) && ManifestStore.ReadSequence(sidecar).Id == sequenceId;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Which folder each indexed sequence id lives in, updated as the scan goes.</summary>
|
||||
private sealed class Locations
|
||||
{
|
||||
|
||||
@@ -108,22 +108,29 @@ public sealed class ArchiveVerifier
|
||||
pending.Clear();
|
||||
}
|
||||
|
||||
/// <summary>Stamps the verify time; the sequence counts as verified only when no frame is missing or corrupt.</summary>
|
||||
/// <summary>
|
||||
/// Stamps the verify time; the sequence counts as verified only when no frame is missing or corrupt, and has errors
|
||||
/// while one is. Sidecar and index are written under one lease, the index row with these fields only
|
||||
/// (<see cref="SequenceRepository.RecordVerifiedAsync"/>), so a settings edit or an ingest committed meanwhile stays.
|
||||
/// </summary>
|
||||
private async Task StampAsync(Archive archive, ArchivePaths paths, Sequence target, CancellationToken cancellationToken)
|
||||
{
|
||||
var sequence = paths.Sequence(target.Folder);
|
||||
var manifest = await _writer.UpdateAsync(paths, target.Id, target.Folder, m =>
|
||||
await using (await _writer.LockAsync(paths, target.Id, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var m = SequenceWriter.ReadLocked(sequence, target.Id);
|
||||
var clean = m.Frames.All(f => (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0);
|
||||
return m with
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var manifest = m with
|
||||
{
|
||||
Status = clean ? m.Status | SequenceStatus.Verified : m.Status & ~SequenceStatus.Verified,
|
||||
VerifiedAt = DateTimeOffset.UtcNow,
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
VerifiedAt = now,
|
||||
UpdatedAt = now,
|
||||
};
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
var existing = await _db.Sequences.GetAsync(target.Id, cancellationToken).ConfigureAwait(false);
|
||||
await _db.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
|
||||
await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false);
|
||||
ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
|
||||
await _db.Sequences.RecordVerifiedAsync(target.Id, clean ? SequenceStatus.Verified : SequenceStatus.HasErrors,
|
||||
clean ? SequenceStatus.HasErrors : SequenceStatus.Verified, now, cancellationToken).ConfigureAwait(false);
|
||||
await _indexer.RecordStampsInStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -61,6 +61,12 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi
|
||||
/// </summary>
|
||||
/// <param name="proxies">Whether every such frame follows its proxy even where no tier is recorded (analyses and the
|
||||
/// proxy pass); otherwise only the frames in <paramref name="tiers"/> do.</param>
|
||||
/// <remarks>
|
||||
/// The sidecar and the index are written under one lease, the index from the sidecar read under it, and the index
|
||||
/// row gets only what tiers change (<see cref="SequenceRepository.RecordTiersAsync"/>): a whole row written back
|
||||
/// would undo what another writer committed meanwhile, an ingest's frames or a settings edit (the server's PATCH
|
||||
/// takes no lease).
|
||||
/// </remarks>
|
||||
/// <exception cref="ArchiveOfflineException">The archive is offline or read-only.</exception>
|
||||
public async Task<IReadOnlyList<SidecarFrame>> RecordAsync(Archive archive, Sequence sequence, IReadOnlyDictionary<Guid, FileTier> tiers,
|
||||
bool proxies, CancellationToken cancellationToken)
|
||||
@@ -69,12 +75,12 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
var changed = new List<SidecarFrame>();
|
||||
ProxyPolicy? policy = null;
|
||||
var manifest = await writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m =>
|
||||
await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
changed.Clear();
|
||||
var m = SequenceWriter.ReadLocked(folder, sequence.Id);
|
||||
var changed = new List<SidecarFrame>();
|
||||
var gained = false;
|
||||
ProxyPolicy? policy = null;
|
||||
var frames = new SidecarFrame[m.Frames.Count];
|
||||
for (var i = 0; i < frames.Length; i++)
|
||||
{
|
||||
@@ -96,21 +102,25 @@ internal sealed class FrameTiers(TitanoDatabase db, SequenceWriter writer, Archi
|
||||
frames[i] = updated;
|
||||
changed.Add(updated);
|
||||
}
|
||||
if (changed.Count == 0) return null;
|
||||
return m with
|
||||
if (changed.Count == 0) return m.Frames;
|
||||
var manifest = m with
|
||||
{
|
||||
Frames = frames,
|
||||
Status = gained ? m.Status | SequenceStatus.HasWarnings : m.Status,
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
};
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
if (changed.Count == 0) return manifest.Frames;
|
||||
await db.Frames.UpdateTiersAsync(changed.ConvertAll(f => new FrameTierFields(f.Id, f.Flags, f.PreviewW ?? 0, f.PreviewH ?? 0)), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
if (await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) is { } existing)
|
||||
await db.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, folder, existing), cancellationToken).ConfigureAwait(false);
|
||||
await indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
|
||||
return manifest.Frames;
|
||||
ManifestStore.WriteSequence(folder.ManifestPath, manifest);
|
||||
await using (var tx = await db.BeginAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
await tx.Frames.UpdateTiersAsync(changed.ConvertAll(f => new FrameTierFields(f.Id, f.Flags, f.PreviewW ?? 0, f.PreviewH ?? 0)), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
await tx.Sequences.RecordTiersAsync(sequence.Id, gained ? SequenceStatus.HasWarnings : SequenceStatus.None,
|
||||
frames.Count(f => (f.Flags & FrameFlags.PreviewOnly) != 0), cancellationToken).ConfigureAwait(false);
|
||||
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
await indexer.RecordStampsInStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
|
||||
return manifest.Frames;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -67,10 +67,11 @@ internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer,
|
||||
/// <summary>
|
||||
/// Queues a background <paramref name="pass"/> (thumbnails or proxies) of each sequence this run indexed whose
|
||||
/// <paramref name="built"/> flag is clear: new and adopted sequences, and those whose photos changed (the scan clears the
|
||||
/// flag then). After a resumed run, which may have indexed sequences before the interruption, of every sequence of the
|
||||
/// archive without the flag but those in <paramref name="awaiting"/>. One pass per sequence, as after an upload: a folder
|
||||
/// renamed, deleted or damaged before its pass runs fails that pass alone, and the other sequences still get their
|
||||
/// images. A sequence whose own pass of that kind is already queued is left out.
|
||||
/// flag then, also when another writer listed them in the sidecar itself). After a resumed run, which may have indexed
|
||||
/// sequences before the interruption, of every sequence of the archive without the flag but those in
|
||||
/// <paramref name="awaiting"/>. One pass per sequence, as after an upload: a folder renamed, deleted or damaged before
|
||||
/// its pass runs fails that pass alone, and the other sequences still get their images. A sequence whose own pass of
|
||||
/// that kind is already queued is left out.
|
||||
/// </summary>
|
||||
private async Task QueueImagesAsync(JobContext context, Archive archive, JobKind pass, SequenceStatus built, IReadOnlyList<Guid> indexed,
|
||||
bool resumed, IReadOnlySet<Guid> awaiting)
|
||||
|
||||
@@ -195,6 +195,24 @@ public sealed class SequenceRepository
|
||||
await _db.ExecuteAsync("UPDATE sequences SET status = (status | $2) & ~$3, updated_at = $4 WHERE id = $1",
|
||||
[id, (int)set, (int)clear, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0;
|
||||
|
||||
/// <summary>
|
||||
/// Records decode tiers written to the sequence's frames (<see cref="FrameRepository.UpdateTiersAsync"/>): the status
|
||||
/// flags in <paramref name="set"/> and the preview-only count, in one statement. The frames, their versions and the
|
||||
/// settings stay as they are, so an ingest or a settings edit committed meanwhile is never written back over.
|
||||
/// </summary>
|
||||
public async Task<bool> RecordTiersAsync(Guid id, SequenceStatus set, int previewOnlyCount, CancellationToken cancellationToken = default) =>
|
||||
await _db.ExecuteAsync("UPDATE sequences SET status = status | $2, preview_only_count = $3, updated_at = $4 WHERE id = $1",
|
||||
[id, (int)set, previewOnlyCount, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0;
|
||||
|
||||
/// <summary>
|
||||
/// Records a verify: status flags set and cleared and the verify time, in one statement; every other field stays as
|
||||
/// it is (<see cref="RecordTiersAsync"/>).
|
||||
/// </summary>
|
||||
public async Task<bool> RecordVerifiedAsync(Guid id, SequenceStatus set, SequenceStatus clear, DateTimeOffset verifiedAt,
|
||||
CancellationToken cancellationToken = default) =>
|
||||
await _db.ExecuteAsync("UPDATE sequences SET status = (status | $2) & ~$3, verified_at = $4, updated_at = $5 WHERE id = $1",
|
||||
[id, (int)set, (int)clear, verifiedAt, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false) > 0;
|
||||
|
||||
/// <summary>
|
||||
/// Records an analysis: status flags set and cleared, the warning count and, for a deep analysis, the compact
|
||||
/// summary (<paramref name="analysisJson"/> null keeps the stored one).
|
||||
|
||||
@@ -55,6 +55,20 @@ internal sealed class ArchiveFixture : IAsyncDisposable
|
||||
public Task<ScanSummary> ScanAsync(ScanMode mode = ScanMode.Incremental) =>
|
||||
Service.Indexer.ScanAsync(Archive, mode, null, _ => ValueTask.CompletedTask, Ct);
|
||||
|
||||
/// <summary>
|
||||
/// Another process writing the same archive, such as a desktop sharing a server's archive folder: its own index
|
||||
/// (scanned) and archive service; disposed with the fixture.
|
||||
/// </summary>
|
||||
public async Task<(ArchiveService Service, Archive Archive)> OtherWriterAsync()
|
||||
{
|
||||
var db = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(_context.CreateTempDirectory("other"), "titano.db")), Ct);
|
||||
_owned.Add(db);
|
||||
var service = new ArchiveService(db, new ArchiveServiceOptions { Owner = "other", LeaseWait = TimeSpan.FromSeconds(20) });
|
||||
var archive = await service.OpenAsync(Archive.RootPath, Ct);
|
||||
await service.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, Ct);
|
||||
return (service, archive);
|
||||
}
|
||||
|
||||
/// <summary>A job engine with the archive handlers, started; disposed with the fixture.</summary>
|
||||
public async Task<JobEngine> StartEngineAsync()
|
||||
{
|
||||
|
||||
@@ -365,6 +365,38 @@ public sealed class ArchiveIngestTests(TestContext context)
|
||||
Assert.Equal("2026/2026-09-29 Canon EOS R5", (await fx.Db.Sequences.GetAsync(waiting.Id))!.Folder);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ARebuildKeepsSequencesThatGetTheirFolderWhileItRuns()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
Photos.WriteSeries(Path.Combine(fx.Archive.RootPath, "Trips", "Comet"), Start, 2);
|
||||
await fx.ScanAsync();
|
||||
var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset");
|
||||
|
||||
// Uploads go on while the rebuild runs: after it listed the folders, the sequence waiting for its first frame gets
|
||||
// it, and a new upload session creates a sequence.
|
||||
IngestResult? first = null, session = null;
|
||||
var rebuilt = await fx.Service.Indexer.ScanAsync(fx.Archive, ScanMode.Rebuild, null, async progress =>
|
||||
{
|
||||
if (progress.FoldersDone < progress.FoldersTotal || first is not null) return;
|
||||
first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddDays(1), 1), sequenceId: waiting.Id), fx.Ct);
|
||||
session = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddDays(2), 2), sessionKey: "upload-2"), fx.Ct);
|
||||
}, fx.Ct);
|
||||
|
||||
// Neither is dropped, though the folders listed at the start hold neither.
|
||||
Assert.Equal(0, rebuilt.Removed);
|
||||
Assert.Equal(waiting.Id, Assert.NotNull(first).SequenceId);
|
||||
var placed = Assert.NotNull(await fx.Db.Sequences.GetAsync(waiting.Id));
|
||||
Assert.Equal(("2026/2026-09-29 Canon EOS R5", 1, "Harbour sunset"), (placed.Folder, placed.FrameCount, placed.Title));
|
||||
var created = Assert.NotNull(await fx.Db.Sequences.GetAsync(Assert.NotNull(session).SequenceId));
|
||||
Assert.Equal(1, created.FrameCount);
|
||||
// The session's next upload lands in the same sequence, not in a second one beside it.
|
||||
Assert.Equal(created.Id, await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, "upload-2", fx.Ct));
|
||||
var next = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddDays(2).AddSeconds(10), 3), sessionKey: "upload-2"), fx.Ct);
|
||||
Assert.Equal(created.Id, next.SequenceId);
|
||||
Assert.Equal(3, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Count);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ADamagedRootSidecarIsRebuiltUnderTheRootSequenceNotOneWaitingForItsFirstFrame()
|
||||
{
|
||||
|
||||
@@ -175,6 +175,101 @@ public sealed class ArchiveJobTests(TestContext context)
|
||||
Assert.Count(7, Directory.GetFiles(fx.Paths.Sequence("a").Proxies));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AServerScanQueuesTheImagesOfFramesAnotherWriterListedInTheSidecarItself()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
Populate(fx);
|
||||
await using var engine = ServerEngine(fx);
|
||||
await engine.StartAsync(fx.Ct);
|
||||
var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
|
||||
await fx.WaitForAsync(scan.Id, JobState.Done);
|
||||
var first = new HashSet<Guid>();
|
||||
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
|
||||
{
|
||||
foreach (var pass in (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items)
|
||||
{
|
||||
await fx.WaitForAsync(pass.Id, JobState.Done);
|
||||
first.Add(pass.Id);
|
||||
}
|
||||
}
|
||||
var a = (await LocationsAsync(fx)).Single(l => l.Folder == "a");
|
||||
Assert.True(Built((await fx.Db.Sequences.GetAsync(a.Id))!.Status), "built");
|
||||
|
||||
// A desktop sharing the archive imports a photo taken before the others into the sequence: it files the photo and
|
||||
// lists it in the sidecar itself, whose statuses it keeps, so the scan finds no photo to reconcile.
|
||||
var (desktop, archive) = await fx.OtherWriterAsync();
|
||||
var staged = fx.Stage(Start.AddMinutes(-5), seed: 90);
|
||||
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, a.Id, null, null, staged.Path, "IMG_0000.JPG", staged.Size, staged.Sha256, null, null),
|
||||
fx.Ct);
|
||||
var sidecar = fx.Paths.Sequence("a").ManifestPath;
|
||||
Assert.True(Built(ManifestStore.ReadSequence(sidecar).Status), "the other writer keeps the statuses");
|
||||
|
||||
var again = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
|
||||
await fx.WaitForAsync(again.Id, JobState.Done);
|
||||
|
||||
// The frames changed behind the index: the statuses no longer hold, and the sequence gets a pass of each kind.
|
||||
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
|
||||
{
|
||||
var pass = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items.Single(j => !first.Contains(j.Id));
|
||||
Assert.SequenceEqual([a.Id], pass.TargetIds);
|
||||
await fx.WaitForAsync(pass.Id, JobState.Done);
|
||||
}
|
||||
var row = (await fx.Db.Sequences.GetAsync(a.Id))!;
|
||||
Assert.Equal(7, row.FrameCount);
|
||||
Assert.True(Built(row.Status), "built again");
|
||||
// Every frame has its own thumbnail and proxy: the new first one, and the others it renumbered.
|
||||
var folder = fx.Paths.Sequence("a");
|
||||
using var pack = ThumbPackReader.Open(folder.ThumbsPackPath);
|
||||
foreach (var frame in ManifestStore.ReadSequence(sidecar).Frames)
|
||||
{
|
||||
Assert.True(pack.Entries.GetValueOrDefault(frame.I) is { } entry
|
||||
&& entry.SourceHash.AsSpan().SequenceEqual(ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size)), $"thumbnail of {frame.File}");
|
||||
Assert.True(ArchiveImages.IsProxyOf(folder.Proxy(frame.I), frame.CapturedAt, frame.W, frame.H, null), $"proxy of {frame.File}");
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnImagePassMarksTheSequenceBuiltOnlyWhenItsSidecarStillListsTheFramesThePassRead()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
// The pass commits its pack and checkpoints after 200 frames.
|
||||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 201);
|
||||
await fx.ScanAsync();
|
||||
var a = (await LocationsAsync(fx)).Single();
|
||||
var sequence = (await fx.Db.Sequences.GetAsync(a.Id))!;
|
||||
|
||||
// While it runs, another writer copies in a photo taken before the others, and a scan reconciles it.
|
||||
var scanned = false;
|
||||
var result = await fx.Service.Images.BuildThumbnailsAsync(fx.Archive, sequence, false, 0, async (_, _) =>
|
||||
{
|
||||
if (scanned) return;
|
||||
scanned = true;
|
||||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start.AddMinutes(-5), 1, firstSeed: 900, firstNumber: 900);
|
||||
Assert.SequenceEqual([a.Id], (await fx.ScanAsync()).Sequences);
|
||||
}, fx.Ct);
|
||||
|
||||
// The pass drew every frame it read, but not the new photo: the sequence is not built.
|
||||
Assert.True(scanned, "scanned during the pass");
|
||||
Assert.Equal((201, 0), (result.Built, result.Undecodable));
|
||||
var row = (await fx.Db.Sequences.GetAsync(a.Id))!;
|
||||
Assert.Equal(202, row.FrameCount);
|
||||
Assert.False(row.Status.HasFlag(SequenceStatus.ThumbsBuilt), row.Status.ToString());
|
||||
Assert.False(ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath).Status.HasFlag(SequenceStatus.ThumbsBuilt), "sidecar flag");
|
||||
|
||||
// So a scan's follow-up (here a resumed one, which lists the whole archive) queues the next pass, which makes them.
|
||||
await using var engine = ServerEngine(fx);
|
||||
var resumed = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Scan(fx.Archive), "after:a");
|
||||
await engine.StartAsync(fx.Ct);
|
||||
await fx.WaitForAsync(resumed.Id, JobState.Done);
|
||||
var pass = (await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items.Single();
|
||||
Assert.SequenceEqual([a.Id], pass.TargetIds);
|
||||
await fx.WaitForAsync(pass.Id, JobState.Done);
|
||||
Assert.True((await fx.Db.Sequences.GetAsync(a.Id))!.Status.HasFlag(SequenceStatus.ThumbsBuilt), "built");
|
||||
using var pack = ThumbPackReader.Open(fx.Paths.Sequence("a").ThumbsPackPath);
|
||||
Assert.Count(202, pack.Entries);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AFolderGoneBeforeItsImagePassesFailsThemAloneAndTheOtherSequencesGetTheirImages()
|
||||
{
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Data.Repositories;
|
||||
using Titano.Data.Tests.Infrastructure;
|
||||
using Titano.Testing;
|
||||
using static Titano.Data.Tests.Archiving.ArchiveFixture;
|
||||
|
||||
namespace Titano.Data.Tests.Archiving;
|
||||
|
||||
/// <summary>
|
||||
/// Writers that change part of a sequence's sidecar and index row (decode tiers, the verify stamp) while a settings edit
|
||||
/// or another writer commits, on both dialects: they write only what they own, under the sequence lease.
|
||||
/// </summary>
|
||||
[Category("archive")]
|
||||
public abstract class PartialRowWriterTests(TestContext context) : DatabaseTest(context)
|
||||
{
|
||||
private async Task<ArchiveFixture> FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true)));
|
||||
|
||||
/// <summary>One indexed sequence of three photos in <c>a</c>.</summary>
|
||||
private static async Task<Sequence> IndexedAsync(ArchiveFixture fx)
|
||||
{
|
||||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 3);
|
||||
await fx.ScanAsync();
|
||||
var location = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single();
|
||||
return (await fx.Db.Sequences.GetAsync(location.Id, fx.Ct))!;
|
||||
}
|
||||
|
||||
/// <summary>Records decode tiers as analyses, image passes and renders do.</summary>
|
||||
private static FrameTiers Tiers(ArchiveFixture fx)
|
||||
{
|
||||
var writer = new SequenceWriter(new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) });
|
||||
return new FrameTiers(fx.Db, writer, new ArchiveIndexer(fx.Db, writer));
|
||||
}
|
||||
|
||||
/// <summary>The frame decodes only through its 48 × 32 embedded preview.</summary>
|
||||
private static Dictionary<Guid, FileTier> Preview(Guid frameId) =>
|
||||
new() { [frameId] = new FileTier(ImageFileFormat.Jpeg, DecodeTier.EmbeddedPreview, "test-preview", null, 48, 32) };
|
||||
|
||||
/// <summary>Commits <paramref name="tx"/> once the writer under test had time to read the row and wait for it.</summary>
|
||||
private static async Task CommitLaterAsync(TitanoTransaction tx, CancellationToken cancellationToken)
|
||||
{
|
||||
await using (tx)
|
||||
{
|
||||
await Task.Delay(500, cancellationToken);
|
||||
await tx.CommitAsync(cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TiersRecordedWhileASettingsEditCommitsLeaveTheEdit()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var sequence = await IndexedAsync(fx);
|
||||
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||||
|
||||
// The server's PATCH holds the row (and SQLite's write lock) while an analysis records a tier.
|
||||
Task record;
|
||||
await using (var tx = await fx.Db.BeginAsync(fx.Ct))
|
||||
{
|
||||
var edit = await tx.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":2}""" }, sequence.SettingsVersion, fx.Ct);
|
||||
Assert.Equal(SequenceEditOutcome.Updated, edit.Outcome);
|
||||
record = Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
|
||||
await Task.WhenAny(record, Task.Delay(500, fx.Ct));
|
||||
Assert.False(record.IsCompleted, "the record waits for the row");
|
||||
await tx.CommitAsync(fx.Ct);
|
||||
}
|
||||
await record;
|
||||
|
||||
// The tier and what follows from it are recorded, and the edit committed meanwhile stays.
|
||||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
Assert.Contains("heroFrame", row.SettingsJson);
|
||||
Assert.Equal((sequence.SettingsVersion + 1, 1), (row.SettingsVersion, row.PreviewOnlyCount));
|
||||
Assert.Equal(FrameFlags.PreviewOnly, (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct))[0].Flags);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TiersRecordedAfterAnotherWriterAddedFramesLeaveThoseFramesToTheNextScan()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var sequence = await IndexedAsync(fx);
|
||||
var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct);
|
||||
|
||||
// A desktop sharing the archive imports a photo into the sequence: its own index follows the sidecar, this one not yet.
|
||||
var (desktop, archive) = await fx.OtherWriterAsync();
|
||||
var staged = fx.Stage(Start.AddMinutes(1), seed: 90);
|
||||
await desktop.Ingest.CommitAsync(new IngestRequest(archive.Id, sequence.Id, null, null, staged.Path, "IMG_0004.JPG", staged.Size, staged.Sha256, null, null),
|
||||
fx.Ct);
|
||||
|
||||
// A tier recorded meanwhile changes the tiers here, not the frames this index holds.
|
||||
await Tiers(fx).RecordAsync(fx.Archive, sequence, Preview(frames[0].Id), proxies: false, fx.Ct);
|
||||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
Assert.Equal((3, sequence.FramesVersion, 1), (row.FrameCount, row.FramesVersion, row.PreviewOnlyCount));
|
||||
|
||||
// The next scan still finds the folder changed, and indexes the photo.
|
||||
Assert.SequenceEqual([sequence.Id], (await fx.ScanAsync()).Sequences);
|
||||
var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence("a").ManifestPath);
|
||||
row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
Assert.Equal((4, sidecar.Versions.Frames), (row.FrameCount, row.FramesVersion));
|
||||
Assert.SequenceEqual(sidecar.Frames.Select(f => f.Id), (await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct)).Select(f => f.Id));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AVerifyStampedWhileASettingsEditCommitsLeavesTheEdit()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var sequence = await IndexedAsync(fx);
|
||||
// Hashed by a first verify, so the next one changes no frame and only stamps the sequence.
|
||||
await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
|
||||
sequence = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
|
||||
// The server's PATCH takes the row as the last photo is checked, and commits while the verify stamps the sequence.
|
||||
Task? commit = null;
|
||||
await fx.Service.Verifier.VerifyAsync(fx.Archive, [sequence.Id], default, (_, _, _) => ValueTask.CompletedTask, async (done, total) =>
|
||||
{
|
||||
if (done < total || commit is not null) return;
|
||||
var tx = await fx.Db.BeginAsync(fx.Ct);
|
||||
await tx.Sequences.EditAsync(sequence.Id, new SequenceEdit { SettingsJson = """{"heroFrame":1}""" }, sequence.SettingsVersion, fx.Ct);
|
||||
commit = CommitLaterAsync(tx, fx.Ct);
|
||||
}, fx.Ct);
|
||||
await Assert.NotNull(commit);
|
||||
|
||||
var row = (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!;
|
||||
Assert.Contains("heroFrame", row.SettingsJson);
|
||||
Assert.Equal(sequence.SettingsVersion + 1, row.SettingsVersion);
|
||||
Assert.True(row.Status.HasFlag(SequenceStatus.Verified), row.Status.ToString());
|
||||
Assert.True(row.VerifiedAt > sequence.VerifiedAt, "stamped again");
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class SqlitePartialRowWriterTests(TestContext context) : PartialRowWriterTests(context)
|
||||
{
|
||||
protected override Backend Backend => Backend.SqliteFile;
|
||||
}
|
||||
|
||||
[RequiresEnvironment(PgTestServer.Variable)]
|
||||
public sealed class PostgresPartialRowWriterTests(TestContext context) : PartialRowWriterTests(context)
|
||||
{
|
||||
protected override Backend Backend => Backend.Postgres;
|
||||
}
|
||||
@@ -1,8 +1,11 @@
|
||||
using System.Net;
|
||||
using System.Security.Cryptography;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Codecs.Jpeg;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Protocol;
|
||||
using Titano.Protocol.Client;
|
||||
using Titano.Server.Jobs;
|
||||
@@ -106,6 +109,45 @@ public sealed class ArchiveModuleTests(TestContext context)
|
||||
await UntilImagesAsync(server, editor, archive.Id, "garden");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task FramesADesktopSharingTheArchiveAddsToASequenceGetThumbnailsAndProxiesAfterThePoll()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context, options =>
|
||||
{
|
||||
Directory.CreateDirectory(Path.Combine(options.ArchivePath, "roof"));
|
||||
File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0002.JPG"), Photo());
|
||||
}, ArchiveModule.Register);
|
||||
var archive = server.Services.GetRequiredService<ServerArchive>();
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
await UntilIndexedAsync(server, archive.Id, "roof");
|
||||
await UntilScannedAsync(server);
|
||||
await UntilImagesAsync(server, editor, archive.Id, "roof");
|
||||
await UntilIdleAsync(server);
|
||||
var roof = (await server.Database.Sequences.ListLocationsAsync(archive.Id)).Single(l => l.Folder == "roof");
|
||||
|
||||
// A desktop sharing the archive, with an index of its own, imports a photo taken earlier into the sequence: it files
|
||||
// the photo and lists it in the sidecar itself, keeping the sidecar's ThumbsBuilt and ProxiesBuilt.
|
||||
var desktopFolder = context.CreateTempDirectory("desktop");
|
||||
await using (var desktopIndex = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(desktopFolder, "titano.db")), context.Cancellation))
|
||||
{
|
||||
var desktop = new ArchiveService(desktopIndex, new ArchiveServiceOptions { Owner = "desktop" });
|
||||
var shared = await desktop.OpenAsync(server.Options.ArchivePath, context.Cancellation);
|
||||
await desktop.Indexer.ScanAsync(shared, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, context.Cancellation);
|
||||
var photo = Photo(seed: 5);
|
||||
var staged = Path.Combine(desktopFolder, "IMG_0001.JPG");
|
||||
await File.WriteAllBytesAsync(staged, photo);
|
||||
await desktop.Ingest.CommitAsync(new IngestRequest(shared.Id, roof.Id, null, null, staged, "IMG_0001.JPG", photo.Length,
|
||||
Convert.ToHexStringLower(SHA256.HashData(photo)), new DateTime(2026, 9, 28, 18, 0, 0), null), context.Cancellation);
|
||||
}
|
||||
Assert.Equal(1, Assert.NotNull(await server.Database.Sequences.GetAsync(roof.Id)).FrameCount);
|
||||
|
||||
// The next poll indexes it and queues the images: the new frame and the one it moved get theirs.
|
||||
Assert.NotNull(await server.Services.GetRequiredService<ArchivePolling>().PollAsync(context.Cancellation));
|
||||
await UntilScannedAsync(server);
|
||||
Assert.Equal(2, Assert.NotNull(await server.Database.Sequences.GetAsync(roof.Id)).FrameCount);
|
||||
await UntilImagesAsync(server, editor, archive.Id, "roof");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task SettingsADesktopChangedInArchiveJsonReachUploadsAndTheAdminPanel()
|
||||
{
|
||||
@@ -157,28 +199,51 @@ public sealed class ArchiveModuleTests(TestContext context)
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Waits until the first frame of the sequence in <paramref name="folder"/> has its thumbnail and proxy served.</summary>
|
||||
/// <summary>
|
||||
/// Waits until every frame of the sequence in <paramref name="folder"/> has its thumbnail and proxy served and the
|
||||
/// passes that made them have marked the sequence (a pass writes its files before it marks).
|
||||
/// </summary>
|
||||
private static async Task UntilImagesAsync(TestServer server, HttpClient client, Guid archiveId, string folder)
|
||||
{
|
||||
var sequence = (await server.Database.Sequences.ListLocationsAsync(archiveId)).Single(l => l.Folder == folder);
|
||||
var frame = (await server.Database.Frames.ListBySequenceAsync(sequence.Id)).First();
|
||||
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
||||
foreach (var asset in new[] { "thumb", "proxy" })
|
||||
foreach (var frame in await server.Database.Frames.ListBySequenceAsync(sequence.Id))
|
||||
{
|
||||
while (true)
|
||||
foreach (var asset in new[] { "thumb", "proxy" })
|
||||
{
|
||||
using var response = await client.GetAsync($"/api/v1/frames/{frame.Id}/{asset}");
|
||||
if (response.StatusCode == HttpStatusCode.OK)
|
||||
while (true)
|
||||
{
|
||||
Assert.True(await response.Content.ReadAsByteArrayAsync() is [0xFF, 0xD8, ..], $"a JPEG {asset} of {folder}");
|
||||
break;
|
||||
using var response = await client.GetAsync($"/api/v1/frames/{frame.Id}/{asset}");
|
||||
if (response.StatusCode == HttpStatusCode.OK)
|
||||
{
|
||||
Assert.True(await response.Content.ReadAsByteArrayAsync() is [0xFF, 0xD8, ..], $"a JPEG {asset} of {frame.FileName}");
|
||||
break;
|
||||
}
|
||||
Assert.True(DateTime.UtcNow < until, $"no {asset} of {frame.FileName} in {folder}: {(int)response.StatusCode}");
|
||||
await Task.Delay(100);
|
||||
}
|
||||
Assert.True(DateTime.UtcNow < until, $"no {asset} of {folder}: {(int)response.StatusCode}");
|
||||
await Task.Delay(100);
|
||||
}
|
||||
}
|
||||
var built = Assert.NotNull(await server.Database.Sequences.GetAsync(sequence.Id)).Status;
|
||||
Assert.True(built.HasFlag(SequenceStatus.ThumbsBuilt) && built.HasFlag(SequenceStatus.ProxiesBuilt), $"{folder}: {built}");
|
||||
while (Assert.NotNull(await server.Database.Sequences.GetAsync(sequence.Id)).Status is var built
|
||||
&& !(built.HasFlag(SequenceStatus.ThumbsBuilt) && built.HasFlag(SequenceStatus.ProxiesBuilt)))
|
||||
{
|
||||
Assert.True(DateTime.UtcNow < until, $"{folder}: {built}");
|
||||
await Task.Delay(50);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Waits until no job is waiting or running.</summary>
|
||||
private static async Task UntilIdleAsync(TestServer server)
|
||||
{
|
||||
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
||||
foreach (var state in new[] { JobState.Queued, JobState.Running, JobState.Queued })
|
||||
{
|
||||
while ((await server.Database.Jobs.ListAsync(state)).Items.Count > 0)
|
||||
{
|
||||
Assert.True(DateTime.UtcNow < until, $"jobs still {state}");
|
||||
await Task.Delay(50);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Waits for the scan queued at start to finish: its sequences are indexed before it ends.</summary>
|
||||
|
||||
Reference in new issue
Block a user