Files
Chronos/src/Chronos.Data/Archiving/ArchiveIndexer.cs
T
Alby96 eb607d47aa fix(archive): crash-safe photo restore, hidden half deletions, tombstones for every photo
The review of owner review 2 phase 3 found that the photo trash could destroy or
strand originals outside the Empty job and let deleted photos come back.

- A restore into a new sequence stages each original inside its own entry
  (<entry>/.restoring), never in a staging folder other restores share or
  maintenance sweeps; any failure moves it back, the commit runs to its end once
  the file has left, a cancel stops between photos with the record kept true,
  and the next restore of the entry puts back what a crash left staged. Nothing
  erases a staging folder any more.
- A restore into an existing sequence writes .chronos/restore.pending.json before
  its first move; SequenceWriter.ReadLocked settles it (files back into the entry
  before the sidecar was written, record rewritten after), cancellation is
  checked once before the journal, and a companion that cannot follow puts its
  photo back whole. Only an intact frame keeps a trashed copy out.
- A deletion's record keeps a .pending name until its sidecar is written, so the
  trash neither lists, restores nor empties half an entry; a failed sidecar
  write undoes the moves, and scans settle any of the three journals.
- Photos never hashed are hashed as they go and get their tombstone; a photo
  with neither hash nor original blocks the plan (FrameDeleteBlock.Unhashable).
  Sidecars rebuilt from the photos, and sequences restored without one, keep
  their tombstones; the index drops a tombstone whose content a frame holds.
- Emptying re-reads and re-measures each entry under its lease before erasing.
- Once the sidecar is written a deletion or restore stands when the index cannot
  follow (logged; the next scan resyncs it).
- XMP companions are listed once per directory, case-insensitively; a restore
  into a new sequence picks a name its companions can follow, and one that still
  cannot is kept in the record as a leftover.
- thumbs.pack is re-keyed with the frames on delete and restore, so the queued
  pass reuses the thumbnails; the server checks the fingerprint of thumbnails of
  frames never hashed too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit 0bb631dd83d082e2569b98d127d827adc2e7f899)
2026-10-27 12:45:33 +01:00

562 lines
35 KiB
C#

using Chronos.Core.Archiving;
using Chronos.Core.Diagnostics;
using Chronos.Core.IO;
using Chronos.Core.Model;
using Chronos.Data.Repositories;
namespace Chronos.Data.Archiving;
public enum ScanMode
{
/// <summary>Only folders whose sidecar or photo folder changed size or time since the last scan.</summary>
Incremental,
/// <summary>Every folder: sidecars re-read and folders reconciled with their photos (manual rescan).</summary>
Full,
/// <summary>The index rebuilt from sidecars alone; rows without a sidecar are dropped (BRIEF §12.5).</summary>
Rebuild,
}
public sealed record ScanProgress(int FoldersDone, int FoldersTotal, string Folder, bool Changed);
/// <param name="Failed">Folders that could not be indexed, with the reason; they are retried by the next scan.</param>
/// <param name="Sequences">The sequences this run indexed (<paramref name="Indexed"/> of them), in folder order.</param>
public sealed record ScanSummary(int Folders, int Indexed, int Adopted, int Removed, IReadOnlyList<(string Folder, string Error)> Failed,
IReadOnlyList<Guid> Sequences);
/// <summary>
/// Keeps the database index in line with the archive on disk (BRIEF §7, §12.5): scans by size/time stamps, adopts
/// folders without a sidecar, rebuilds from sidecars (using the index snapshot when it is fresh) and syncs one
/// sequence after any service changed its sidecar. The sidecars are the truth; the index is derived.
/// </summary>
public sealed class ArchiveIndexer
{
/// <summary>What covers the frames as they were: a scan that finds them changed clears it, and its follow-ups make it again.</summary>
private const SequenceStatus CoveringStatus = SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt;
private readonly ChronosDatabase _db;
private readonly SequenceWriter _writer;
internal ArchiveIndexer(ChronosDatabase db, SequenceWriter writer)
{
_db = db;
_writer = writer;
}
/// <param name="resumeAfter">Last folder completed by an interrupted run (job checkpoint), or null.</param>
/// <param name="progress">Called after each folder; persist <see cref="ScanProgress.Folder"/> as the resume point.</param>
/// <exception cref="ArchiveOfflineException">The archive is offline or read-only; the index is left as it is.</exception>
public async Task<ScanSummary> ScanAsync(Archive archive, ScanMode mode, string? resumeAfter, Func<ScanProgress, ValueTask> progress,
CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
if (mode == ScanMode.Rebuild && resumeAfter is null) await _db.Archives.ClearFolderStatesAsync(archive.Id, cancellationToken).ConfigureAwait(false);
var folders = ArchiveScanner.FindSequenceFolders(paths, cancellationToken);
var states = await _db.Archives.GetFolderStatesAsync(archive.Id, cancellationToken).ConfigureAwait(false);
var rows = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
// A sequence waiting for its first frame has no folder yet: the archive root, adopted or re-adopted, never takes its id.
var awaiting = await AwaitingFolderAsync(paths, rows, cancellationToken).ConfigureAwait(false);
var locations = new Locations(rows.Where(r => !awaiting.Contains(r.Id)));
var snapshot = mode == ScanMode.Rebuild ? IndexSnapshot.TryLoad(paths.SnapshotPath) : null;
using var snapshotWriter = mode != ScanMode.Incremental && resumeAfter is null ? new IndexSnapshot.Writer(paths.SnapshotPath) : null;
int adopted = 0;
var indexed = new List<Guid>();
var failed = new List<(string, string)>();
for (var i = 0; i < folders.Count; i++)
{
cancellationToken.ThrowIfCancellationRequested();
var folder = folders[i];
if (resumeAfter is not null && string.CompareOrdinal(folder.RelativeFolder, resumeAfter) <= 0) continue;
states.TryGetValue(folder.RelativeFolder, out var state);
var changed = mode != ScanMode.Incremental || state is null || state.Sidecar != folder.Sidecar || state.Photos != folder.Photos;
if (changed)
{
try
{
var reconcile = mode == ScanMode.Full || (state is not null && state.Photos != folder.Photos);
var (id, bytes) = await IndexFolderAsync(archive, paths, folder, reconcile, snapshot, locations, cancellationToken).ConfigureAwait(false);
if (snapshotWriter is not null)
{
// Bytes go under the stamp they were read at (the listed one for the bytes indexed, else one taken before
// reading): a sidecar rewritten meanwhile is never filed under its new stamp with its old content.
var sidecar = paths.Sequence(folder.RelativeFolder).ManifestPath;
var read = bytes is not null ? folder.Sidecar!.Value : FileStamp.Of(sidecar)!.Value;
snapshotWriter.Add(folder.RelativeFolder, read, bytes ?? await AtomicFile.ReadAllBytesAsync(sidecar, cancellationToken).ConfigureAwait(false));
}
indexed.Add(id);
if (folder.Sidecar is null) adopted++;
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Warning("archive", $"Indexing {folder.RelativeFolder} failed", ex);
failed.Add((folder.RelativeFolder, ex.Message));
}
}
await progress(new ScanProgress(i + 1, folders.Count, folder.RelativeFolder, changed)).ConfigureAwait(false);
}
var removed = mode == ScanMode.Rebuild ? await RemoveUnbackedAsync(archive, paths, folders, locations, cancellationToken).ConfigureAwait(false) : 0;
if (failed.Count == 0) snapshotWriter?.Commit();
return new ScanSummary(folders.Count, indexed.Count, adopted, removed, failed, indexed);
}
/// <summary>
/// Writes a sequence's sidecar into the index (row, frames, session keys) and records the folder stamps. Frames
/// inserted before indexed ones take the project's frame positions along (<see cref="ProjectPositions"/>). The frames
/// version never goes back, nor names other frames than it did: a sidecar read under the lease that is behind the
/// locked row (an older copy put back by hand or by a file-sync tool), or at the row's version over other frames
/// (another index of the archive bumped a copy put back to it, <see cref="OtherFramesAsync"/>), is written again one
/// version past the row, so editors and clients holding the row's version see the change.
/// </summary>
/// <param name="listed">The folder as the scan listed it before reading <paramref name="manifest"/> without the sequence
/// lease; null when the caller holds the lease and read the sidecar under it.</param>
/// <param name="clear">Statuses cleared in this index's row only, never in the sidecar (<see cref="Uncovered"/>).</param>
/// <param name="photosChanged">The caller, holding the lease, found the photo directory changed since this index recorded
/// its stamp before it changed the folder itself (<see cref="PhotosChangedAsync"/>). This sync indexes the sidecar, not
/// the photos, so the recorded photo stamp stays (none when none was) and the next incremental scan reconciles them.</param>
/// <returns>The manifest indexed; null, with nothing written, when <paramref name="listed"/> is given and the sidecar is no
/// longer the one listed or the locked row holds another frames version. Another writer then committed since the read
/// (an ingest's frames, a tier or a verify), and the caller reads the sidecar again under the lease
/// (<see cref="UpdateAndSyncAsync"/>) rather than write the older read over it and record stamps that hide it.</returns>
internal async Task<SequenceManifest?> SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, SequenceFolder? listed,
SequenceStatus clear, bool photosChanged, CancellationToken cancellationToken)
{
PositionMove? positions = null;
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
{
// Locked: a settings edit committed after this read (the server's PATCH takes no lease) would be written back over,
// and writers of the sequence's frames commit before or after this, never in between (GetForUpdateAsync).
var existing = await tx.Sequences.GetForUpdateAsync(manifest.Id, cancellationToken).ConfigureAwait(false);
if (listed is not null && (FileStamp.Of(sequence.ManifestPath) != listed.Sidecar
|| (existing is not null && existing.FramesVersion != manifest.Versions.Frames)))
return null;
// Under the lease, the row's own version may also name other frames: another index of the archive bumped a copy
// put back to it (OtherFramesAsync). A caller without the lease compared them before its read (FramesMovedAsync).
if (existing is not null && (existing.FramesVersion > manifest.Versions.Frames
|| (listed is null && existing.FramesVersion == manifest.Versions.Frames
&& await OtherFramesAsync(tx, existing, manifest, cancellationToken).ConfigureAwait(false))))
{
manifest = manifest with
{
Versions = manifest.Versions with { Frames = existing.FramesVersion + 1 },
UpdatedAt = DateTimeOffset.UtcNow,
};
ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
}
if (existing is not null)
{
positions = await ProjectPositions.MoveAsync(tx, existing, manifest, cancellationToken).ConfigureAwait(false);
if (positions is not null) existing = positions.ApplyTo(existing);
}
var row = SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing);
// A hero frame the sidecar no longer lists (a photo deleted) names nothing: the project's hero waits for its photo.
if (row.HeroFrameId is { } hero && !manifest.Frames.Any(f => f.Id == hero)) row = row with { HeroFrameId = null };
await tx.Sequences.UpsertAsync(row with { Status = row.Status & ~clear }, cancellationToken).ConfigureAwait(false);
await tx.Frames.SyncSequenceAsync(manifest.Id, SequenceMapping.ToFrames(manifest, sequence), cancellationToken).ConfigureAwait(false);
await tx.FrameRemovals.SyncSequenceAsync(manifest.Id, SequenceMapping.ToRemovals(manifest), cancellationToken).ConfigureAwait(false);
foreach (var key in manifest.SessionKeys)
await tx.Sequences.MapSessionKeyAsync(archive.Id, key, manifest.Id, cancellationToken).ConfigureAwait(false);
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
}
if (positions is { } moved) ProjectPositions.WriteProject(sequence, moved.Before, moved.Moved);
// Unleased, the stamps the sync was checked against: a sidecar or photos changed since keep the folder changed.
if (listed is not null)
{
await RecordStampsAsync(archive, sequence.RelativeFolder, manifest.Id, listed.Sidecar, listed.Photos, cancellationToken).ConfigureAwait(false);
}
else if (photosChanged)
{
// The photo stamp as recorded: the photos this sync did not index keep the folder changed for the next scan.
var recorded = await _db.Archives.GetFolderStateAsync(archive.Id, sequence.RelativeFolder, cancellationToken).ConfigureAwait(false);
await RecordStampsAsync(archive, sequence.RelativeFolder, manifest.Id, FileStamp.Of(sequence.ManifestPath), recorded?.Photos, cancellationToken)
.ConfigureAwait(false);
}
else
{
await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
}
return manifest;
}
/// <summary>
/// Remembers the current stamps of a folder whose sidecar and index were just updated together, so the next
/// incremental scan does not re-read what is already indexed.
/// </summary>
internal Task RecordStampsAsync(Archive archive, SequencePaths sequence, Guid sequenceId, CancellationToken cancellationToken) =>
RecordStampsAsync(archive, sequence.RelativeFolder, sequenceId, FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence)),
cancellationToken);
private Task RecordStampsAsync(Archive archive, string folder, Guid sequenceId, FileStamp? sidecar, FileStamp? photos, CancellationToken cancellationToken) =>
_db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(folder, sequenceId, sidecar, photos), cancellationToken);
/// <summary>
/// Whether this index holds the folder of <paramref name="manifest"/> (the sidecar read under the sequence lease) as it
/// is now: the stamps it recorded when it last synced the folder are still the folder's, and its row holds the
/// sidecar's frames. A writer that changes, under the lease, only what it owns of the sidecar and the index row
/// (statuses, decode tiers, frame flags, title, tags, rating and notes, the folder) asks before its own write, and
/// records the folder's new stamps after it only when this held (<see cref="RecordStampsAsync"/>). What another
/// process (a desktop sharing the archive) changed in the sidecar alone, frames or a new title, then keeps the folder
/// changed for the next incremental scan, which writes the whole row from the sidecar. An upload asks too, and
/// appends its frames only when this held; otherwise it writes the whole row and every frame from the sidecar itself.
/// </summary>
internal Task<bool> InStepAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) =>
InStepAsync(archive, sequence.RelativeFolder, sequence, manifest, cancellationToken);
/// <inheritdoc cref="InStepAsync(Archive, SequencePaths, SequenceManifest, CancellationToken)"/>
/// <param name="recordedFolder">The folder the stamps were recorded at: the one a rename just moved the sequence from,
/// whose files keep their stamps.</param>
internal async Task<bool> InStepAsync(Archive archive, string recordedFolder, SequencePaths sequence, SequenceManifest manifest,
CancellationToken cancellationToken)
{
if (await _db.Archives.GetFolderStateAsync(archive.Id, recordedFolder, cancellationToken).ConfigureAwait(false) is not { } state
|| state.SequenceId != manifest.Id || state.Sidecar != FileStamp.Of(sequence.ManifestPath)
|| state.Photos != FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))) return false;
// A sidecar rewritten within its file system's time resolution, at the same size, keeps its stamp.
return await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { } indexed
&& indexed.FramesVersion == manifest.Versions.Frames;
}
/// <summary>
/// Whether the folder's photo directory no longer has the stamp this index recorded for it, or none was recorded:
/// photos copied in or removed by hand or by another process since the last scan, which only a scan's reconcile
/// indexes. An upload asks under the lease before it files its photos, which change the stamp, and its sync from the
/// sidecar then keeps the recorded one (<see cref="SyncAsync"/>), so the next incremental scan still reconciles them.
/// </summary>
internal async Task<bool> PhotosChangedAsync(Archive archive, SequencePaths sequence, CancellationToken cancellationToken) =>
(await _db.Archives.GetFolderStateAsync(archive.Id, sequence.RelativeFolder, cancellationToken).ConfigureAwait(false))?.Photos
!= FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence));
/// <summary>
/// Indexes unindexed photos and flags missing files of one sequence (the orphans fix action), as a scan's reconcile
/// does (<see cref="Reconcile"/>): what covered the frames is cleared in the sidecar and the row.
/// </summary>
/// <returns>Whether photos were added, gone or back: the light analysis, thumbnails and proxies are then to be made again
/// (<see cref="ArchiveService.ReconcileAsync"/>). The folder's stamps are recorded, so no scan comes back to it and
/// queues them.</returns>
public async Task<bool> ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var folder = paths.Sequence(sequence.Folder);
var changed = false;
await UpdateAndSyncAsync(archive, paths, folder, sequence.Id, m =>
{
var reconciled = Reconcile(folder, m, cancellationToken);
changed = reconciled is not null;
return reconciled;
}, cancellationToken).ConfigureAwait(false);
return changed;
}
/// <summary>
/// The sidecar brought in line with the folder's photos (<see cref="SequenceAdopter.Reconcile"/>); null when nothing
/// changed. Photos added, gone or back: the light analysis, thumbnails and proxies no longer cover the frames, as the
/// sidecar then tells every host, so the follow-ups make them again (the images on a server,
/// <see cref="ArchiveServiceOptions.ImagesAfterScan"/>).
/// </summary>
private static SequenceManifest? Reconcile(SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) =>
SequenceAdopter.Reconcile(sequence, manifest, cancellationToken) is { } reconciled
? reconciled with { Status = reconciled.Status & ~CoveringStatus }
: null;
/// <summary>
/// Applies <paramref name="edit"/> to the sidecar (null writes nothing) and indexes the result, both under the sequence
/// lease. Frames inserted before indexed ones move the project's positions (<see cref="ProjectPositions"/>) without a
/// new settings version; an editor saving its project holds the same lease, so it reads them moved and merges onto them
/// rather than writing its old positions back. When the sidecar, as read under the lease, lists other frames than the
/// index row (another process changed them together with their sidecar entries), the row is written without what
/// this host makes for the frames (<see cref="Uncovered"/>).
/// </summary>
private async Task<SequenceManifest> UpdateAndSyncAsync(Archive archive, ArchivePaths paths, SequencePaths sequence, Guid sequenceId,
Func<SequenceManifest, SequenceManifest?> edit, CancellationToken cancellationToken)
{
await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false))
{
var manifest = SequenceWriter.ReadLocked(sequence, sequenceId);
var moved = await FramesMovedAsync(manifest, cancellationToken).ConfigureAwait(false);
if (edit(manifest) is { } updated)
{
ManifestStore.WriteSequence(sequence.ManifestPath, updated);
manifest = updated;
}
return (await SyncAsync(archive, sequence, manifest, null, moved ? Uncovered : SequenceStatus.None, false, cancellationToken).ConfigureAwait(false))!;
}
}
/// <summary>
/// What a scan clears in this host's own row, never in the shared sidecar, when another process added, removed or
/// renumbered frames together with their sidecar entries: on a server (<see cref="ArchiveServiceOptions.ImagesAfterScan"/>),
/// which analyses and draws what its clients load, the light analysis, thumbnails and proxies, so the scan's follow-ups
/// make them for the frames as they are, and their passes set the flags in both places again. Nothing on a desktop:
/// the flags in the sidecar are the writer's that added the frames (which analyses them, and on a server draws them
/// after every upload), and clearing them there would have the server make everything again.
/// </summary>
private SequenceStatus Uncovered => _writer.Options.ImagesAfterScan ? CoveringStatus : SequenceStatus.None;
/// <summary>Indexes one folder; returns its sequence and the sidecar bytes when they were read and not rewritten since (for the snapshot).</summary>
private async Task<(Guid Id, byte[]? Bytes)> IndexFolderAsync(Archive archive, ArchivePaths paths, SequenceFolder folder, bool reconcile, IndexSnapshot? snapshot,
Locations locations, CancellationToken cancellationToken)
{
var sequence = paths.Sequence(folder.RelativeFolder);
SequenceManifest manifest;
byte[]? bytes = null;
if (folder.Sidecar is not { } stamp)
{
manifest = await AdoptAsync(sequence, locations, cancellationToken).ConfigureAwait(false);
}
else
{
bytes = snapshot?.TryGet(folder.RelativeFolder, stamp) ?? await AtomicFile.ReadAllBytesAsync(sequence.ManifestPath, cancellationToken).ConfigureAwait(false);
try
{
manifest = ManifestStore.ParseSequence(bytes, sequence.ManifestPath);
}
catch (InvalidDataException ex)
{
Log.Warning("archive", $"Damaged sidecar in {folder.RelativeFolder}; rebuilding it from the photos", ex);
manifest = await ReplaceDamagedAsync(sequence, locations, cancellationToken).ConfigureAwait(false);
bytes = null;
}
}
if (locations.IsCopy(paths, manifest.Id, folder.RelativeFolder))
{
manifest = Reidentify(manifest);
ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
bytes = null;
}
// A journal of an ingest, a photo deletion or a restore a crash interrupted is settled under the lease first (ReadLocked).
if (reconcile || File.Exists(sequence.IngestJournalPath) || File.Exists(sequence.RemovalJournalPath) || File.Exists(sequence.RestoreJournalPath))
{
// Photos added, gone or back clear what covered the frames, and the scan's follow-ups make it again.
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, m => Reconcile(sequence, m, cancellationToken), cancellationToken)
.ConfigureAwait(false);
bytes = null;
}
else if (await FramesMovedAsync(manifest, cancellationToken).ConfigureAwait(false))
{
// Another process (a desktop sharing the archive, or the server it shares it with) added, removed or renumbered
// frames together with their sidecar entries: read again under the lease, under which the project's positions
// move, and indexed without what this host makes for the frames (Uncovered).
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false);
bytes = null;
}
else if (await SyncAsync(archive, sequence, manifest, folder, SequenceStatus.None, false, cancellationToken).ConfigureAwait(false) is null)
{
// Another writer committed between the read and the sync: read again under the lease, as it is now.
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false);
bytes = null;
}
locations.Set(manifest.Id, folder.RelativeFolder);
return (manifest.Id, bytes);
}
/// <summary>
/// The sequences among <paramref name="locations"/> created without photos that still wait for their first frame: no
/// folder, no frames, and not the sequence the archive root's sidecar names. Their empty folder is not the archive
/// root: scans do not adopt or rebuild the root under their id, a rebuild keeps them although they have no sidecar yet,
/// orphan checks do not count them missing, and maintenance leaves the root's files to the root's own sequence. Unlike
/// <see cref="ArchiveIngest.AwaitsFolder"/>, whose commit reports it, a root sidecar that cannot be read names nobody
/// here: a scan rebuilds a damaged one under the id of the sequence that holds the root's frames, and one it may not
/// read fails the root folder alone rather than the whole scan or rebuild.
/// </summary>
internal async Task<HashSet<Guid>> AwaitingFolderAsync(ArchivePaths paths, IEnumerable<SequenceLocation> locations, CancellationToken cancellationToken)
{
var awaiting = new HashSet<Guid>();
(Guid? Id, bool Read) root = (null, false);
foreach (var location in locations.Where(l => l.Folder.Length == 0))
{
if (await _db.Sequences.GetAsync(location.Id, cancellationToken).ConfigureAwait(false) is not { Folder.Length: 0, FrameCount: 0 } sequence) continue;
// Read once, and only when a row may be waiting: a root sidecar can list a hundred thousand frames.
if (!root.Read) root = (RootSidecarId(paths), true);
if (root.Id != sequence.Id) awaiting.Add(sequence.Id);
}
return awaiting;
}
/// <summary>The id the archive root's sidecar names; null when there is none or it cannot be read.</summary>
private static Guid? RootSidecarId(ArchivePaths paths)
{
var sidecar = paths.Sequence(string.Empty).ManifestPath;
try
{
return File.Exists(sidecar) ? ManifestStore.ReadSequence(sidecar).Id : null;
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
// Indexing the root reports it (a damaged sidecar is rebuilt, an unreadable one fails that folder alone).
return null;
}
}
/// <summary>
/// The index holds this sequence with other frames than the sidecar: another process changed them (its project's
/// positions may move, and what covered the frames no longer does).
/// </summary>
private async Task<bool> FramesMovedAsync(SequenceManifest manifest, CancellationToken cancellationToken) =>
await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { } indexed
&& await OtherFramesAsync(_db, indexed, manifest, cancellationToken).ConfigureAwait(false);
/// <summary>
/// Whether the index row <paramref name="indexed"/> holds other frames than <paramref name="manifest"/>: another frames
/// version, or the same version over other frames (ids, or their order). Each index of a shared archive bumps a sidecar
/// put back behind its own row one version past that row (<see cref="SyncAsync"/>), and another index may hold that
/// very version for the frames as they were before: that index still sees the change, and bumps past it in turn.
/// </summary>
private static async Task<bool> OtherFramesAsync(ChronosRepositories db, Sequence indexed, SequenceManifest manifest, CancellationToken cancellationToken)
{
if (indexed.FramesVersion != manifest.Versions.Frames) return true;
var rows = await db.Frames.ListPositionsAsync(indexed.Id, cancellationToken).ConfigureAwait(false);
if (rows.Count != manifest.Frames.Count) return true;
var listed = new Dictionary<int, Guid>(manifest.Frames.Count);
foreach (var frame in manifest.Frames) listed[frame.I] = frame.Id;
return rows.Any(row => !listed.TryGetValue(row.Index, out var id) || id != row.FrameId);
}
/// <summary>
/// A sidecar for a folder that has none; ids and tombstones of a sequence the index already knew at this folder are
/// kept, so a photo deleted from it is not offered back once the index is synced from the new sidecar.
/// </summary>
private async Task<SequenceManifest> AdoptAsync(SequencePaths sequence, Locations locations, CancellationToken cancellationToken)
{
var known = locations.AtFolder(sequence.RelativeFolder);
var manifest = await WithKnownFrameIdsAsync(SequenceAdopter.Build(known ?? Guid.CreateVersion7(), sequence, _writer.Options.Owner, cancellationToken),
sequence, cancellationToken).ConfigureAwait(false);
if (known is { } id && await _db.FrameRemovals.ListBySequenceAsync(id, cancellationToken).ConfigureAwait(false) is { Count: > 0 } removals)
manifest = manifest with { Removed = SequenceMapping.ToTombstones(removals) };
Directory.CreateDirectory(sequence.MetaDirectory);
// Another process may adopt the same folder at the same moment: the first sidecar wins.
return ManifestStore.TryCreateSequence(sequence.ManifestPath, manifest) ? manifest : ManifestStore.ReadSequence(sequence.ManifestPath);
}
/// <summary>Keeps the unreadable sidecar aside (never deleted) and builds a new one from the photos.</summary>
private async Task<SequenceManifest> ReplaceDamagedAsync(SequencePaths sequence, Locations locations, CancellationToken cancellationToken)
{
File.Move(sequence.ManifestPath, $"{sequence.ManifestPath}.damaged-{DateTime.UtcNow:yyyyMMddHHmmss}");
return await AdoptAsync(sequence, locations, cancellationToken).ConfigureAwait(false);
}
private async Task<SequenceManifest> WithKnownFrameIdsAsync(SequenceManifest manifest, SequencePaths sequence, CancellationToken cancellationToken)
{
var known = new Dictionary<string, Guid>(StringComparer.Ordinal);
var prefix = sequence.ArchiveRelative(string.Empty);
for (var after = -1; ;)
{
var page = await _db.Frames.ListBySequenceAsync(manifest.Id, after, 1000, cancellationToken).ConfigureAwait(false);
foreach (var f in page)
{
if (f.RelativePath.StartsWith(prefix, StringComparison.Ordinal)) known[f.RelativePath[prefix.Length..]] = f.Id;
}
if (page.Count < 1000) break;
after = page[^1].Index;
}
if (known.Count == 0) return manifest;
return manifest with { Frames = manifest.Frames.Select(f => known.TryGetValue(f.File, out var id) ? f with { Id = id } : f).ToList() };
}
/// <summary>A copied sequence folder carries the ids of its source; the copy gets ids of its own.</summary>
private static SequenceManifest Reidentify(SequenceManifest manifest) => manifest with
{
Id = Guid.CreateVersion7(),
Frames = manifest.Frames.Select(f => f with { Id = Guid.CreateVersion7() }).ToList(),
SessionKeys = [],
UpdatedAt = DateTimeOffset.UtcNow,
};
/// <summary>
/// Rebuild only: rows whose folder no longer holds a sidecar leave the index (their files are untouched). A sequence
/// waiting for its first frame has no sidecar yet and stays. A row the folders listed when the rebuild began do not
/// back is looked at again under its lease, as it and its folder are when it would go (<see cref="BackedAsync"/>):
/// uploads go on during a rebuild, which can take minutes, so a sequence that got its first frame or was created
/// meanwhile has a folder the listing does not hold.
/// </summary>
private async Task<int> RemoveUnbackedAsync(Archive archive, ArchivePaths paths, IReadOnlyList<SequenceFolder> folders, Locations locations,
CancellationToken cancellationToken)
{
ArchiveVolume.EnsureWritable(paths);
var present = folders.Where(f => File.Exists(paths.Sequence(f.RelativeFolder).ManifestPath)).Select(f => f.RelativeFolder).ToHashSet(StringComparer.Ordinal);
var rows = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
var awaiting = await AwaitingFolderAsync(paths, rows, cancellationToken).ConfigureAwait(false);
var removed = 0;
foreach (var location in rows)
{
if (awaiting.Contains(location.Id) || (present.Contains(location.Folder) && locations.AtFolder(location.Folder) == location.Id)) continue;
IAsyncDisposable held;
try
{
held = await _writer.LockAsync(paths, location.Id, cancellationToken).ConfigureAwait(false);
}
catch (LeaseUnavailableException)
{
// Another process is writing the sequence, so it is there.
continue;
}
await using (held)
{
if (await BackedAsync(paths, location.Id, cancellationToken).ConfigureAwait(false)) continue;
if (await _db.Sequences.DeleteAsync(location.Id, cancellationToken).ConfigureAwait(false)) removed++;
await _db.Archives.DeleteFolderStateAsync(archive.Id, location.Folder, cancellationToken).ConfigureAwait(false);
}
}
return removed;
}
/// <summary>
/// Whether the index row of <paramref name="sequenceId"/> keeps its place as it is now: it waits for its first frame,
/// its folder holds a sidecar naming it (one that cannot be read proves nothing), or it is gone already.
/// </summary>
private async Task<bool> BackedAsync(ArchivePaths paths, Guid sequenceId, CancellationToken cancellationToken)
{
if (await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) is not { } row) return true;
if ((await AwaitingFolderAsync(paths, [new SequenceLocation(row.Id, row.Folder, row.Title)], cancellationToken).ConfigureAwait(false)).Count > 0)
return true;
var sidecar = paths.Sequence(row.Folder).ManifestPath;
try
{
return File.Exists(sidecar) && ManifestStore.ReadSequence(sidecar).Id == sequenceId;
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
return true;
}
}
/// <summary>Which folder each indexed sequence id lives in, updated as the scan goes.</summary>
private sealed class Locations
{
private readonly Dictionary<Guid, string> _byId = [];
private readonly Dictionary<string, Guid> _byFolder = new(StringComparer.Ordinal);
public Locations(IEnumerable<SequenceLocation> rows)
{
foreach (var row in rows) Set(row.Id, row.Folder);
}
public Guid? AtFolder(string folder) => _byFolder.TryGetValue(folder, out var id) ? id : null;
/// <summary>The id is indexed at another folder whose sidecar still carries it: this folder is a copy.</summary>
public bool IsCopy(ArchivePaths paths, Guid id, string folder)
{
if (!_byId.TryGetValue(id, out var other) || other == folder) return false;
var otherSidecar = paths.Sequence(other).ManifestPath;
try
{
return File.Exists(otherSidecar) && ManifestStore.ReadSequence(otherSidecar).Id == id;
}
catch (InvalidDataException)
{
return false;
}
}
public void Set(Guid id, string folder)
{
if (_byId.TryGetValue(id, out var previous) && _byFolder.GetValueOrDefault(previous) == id) _byFolder.Remove(previous);
_byId[id] = folder;
_byFolder[folder] = id;
}
}
}