The review of owner review 2 phase 3 found that the photo trash could destroy or strand originals outside the Empty job and let deleted photos come back. - A restore into a new sequence stages each original inside its own entry (<entry>/.restoring), never in a staging folder other restores share or maintenance sweeps; any failure moves it back, the commit runs to its end once the file has left, a cancel stops between photos with the record kept true, and the next restore of the entry puts back what a crash left staged. Nothing erases a staging folder any more. - A restore into an existing sequence writes .chronos/restore.pending.json before its first move; SequenceWriter.ReadLocked settles it (files back into the entry before the sidecar was written, record rewritten after), cancellation is checked once before the journal, and a companion that cannot follow puts its photo back whole. Only an intact frame keeps a trashed copy out. - A deletion's record keeps a .pending name until its sidecar is written, so the trash neither lists, restores nor empties half an entry; a failed sidecar write undoes the moves, and scans settle any of the three journals. - Photos never hashed are hashed as they go and get their tombstone; a photo with neither hash nor original blocks the plan (FrameDeleteBlock.Unhashable). Sidecars rebuilt from the photos, and sequences restored without one, keep their tombstones; the index drops a tombstone whose content a frame holds. - Emptying re-reads and re-measures each entry under its lease before erasing. - Once the sidecar is written a deletion or restore stands when the index cannot follow (logged; the next scan resyncs it). - XMP companions are listed once per directory, case-insensitively; a restore into a new sequence picks a name its companions can follow, and one that still cannot is kept in the record as a leftover. - thumbs.pack is re-keyed with the frames on delete and restore, so the queued pass reuses the thumbnails; the server checks the fingerprint of thumbnails of frames never hashed too. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> (cherry picked from commit 0bb631dd83d082e2569b98d127d827adc2e7f899)
562 lines
35 KiB
C#
562 lines
35 KiB
C#
using Chronos.Core.Archiving;
|
|
using Chronos.Core.Diagnostics;
|
|
using Chronos.Core.IO;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Data.Repositories;
|
|
|
|
namespace Chronos.Data.Archiving;
|
|
|
|
public enum ScanMode
|
|
{
|
|
/// <summary>Only folders whose sidecar or photo folder changed size or time since the last scan.</summary>
|
|
Incremental,
|
|
|
|
/// <summary>Every folder: sidecars re-read and folders reconciled with their photos (manual rescan).</summary>
|
|
Full,
|
|
|
|
/// <summary>The index rebuilt from sidecars alone; rows without a sidecar are dropped (BRIEF §12.5).</summary>
|
|
Rebuild,
|
|
}
|
|
|
|
public sealed record ScanProgress(int FoldersDone, int FoldersTotal, string Folder, bool Changed);
|
|
|
|
/// <param name="Failed">Folders that could not be indexed, with the reason; they are retried by the next scan.</param>
|
|
/// <param name="Sequences">The sequences this run indexed (<paramref name="Indexed"/> of them), in folder order.</param>
|
|
public sealed record ScanSummary(int Folders, int Indexed, int Adopted, int Removed, IReadOnlyList<(string Folder, string Error)> Failed,
|
|
IReadOnlyList<Guid> Sequences);
|
|
|
|
/// <summary>
|
|
/// Keeps the database index in line with the archive on disk (BRIEF §7, §12.5): scans by size/time stamps, adopts
|
|
/// folders without a sidecar, rebuilds from sidecars (using the index snapshot when it is fresh) and syncs one
|
|
/// sequence after any service changed its sidecar. The sidecars are the truth; the index is derived.
|
|
/// </summary>
|
|
public sealed class ArchiveIndexer
|
|
{
|
|
/// <summary>What covers the frames as they were: a scan that finds them changed clears it, and its follow-ups make it again.</summary>
|
|
private const SequenceStatus CoveringStatus = SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt;
|
|
|
|
private readonly ChronosDatabase _db;
|
|
private readonly SequenceWriter _writer;
|
|
|
|
internal ArchiveIndexer(ChronosDatabase db, SequenceWriter writer)
|
|
{
|
|
_db = db;
|
|
_writer = writer;
|
|
}
|
|
|
|
/// <param name="resumeAfter">Last folder completed by an interrupted run (job checkpoint), or null.</param>
|
|
/// <param name="progress">Called after each folder; persist <see cref="ScanProgress.Folder"/> as the resume point.</param>
|
|
/// <exception cref="ArchiveOfflineException">The archive is offline or read-only; the index is left as it is.</exception>
|
|
public async Task<ScanSummary> ScanAsync(Archive archive, ScanMode mode, string? resumeAfter, Func<ScanProgress, ValueTask> progress,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var paths = new ArchivePaths(archive.RootPath);
|
|
ArchiveVolume.EnsureWritable(paths);
|
|
if (mode == ScanMode.Rebuild && resumeAfter is null) await _db.Archives.ClearFolderStatesAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
|
var folders = ArchiveScanner.FindSequenceFolders(paths, cancellationToken);
|
|
var states = await _db.Archives.GetFolderStatesAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
|
var rows = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
|
// A sequence waiting for its first frame has no folder yet: the archive root, adopted or re-adopted, never takes its id.
|
|
var awaiting = await AwaitingFolderAsync(paths, rows, cancellationToken).ConfigureAwait(false);
|
|
var locations = new Locations(rows.Where(r => !awaiting.Contains(r.Id)));
|
|
var snapshot = mode == ScanMode.Rebuild ? IndexSnapshot.TryLoad(paths.SnapshotPath) : null;
|
|
using var snapshotWriter = mode != ScanMode.Incremental && resumeAfter is null ? new IndexSnapshot.Writer(paths.SnapshotPath) : null;
|
|
|
|
int adopted = 0;
|
|
var indexed = new List<Guid>();
|
|
var failed = new List<(string, string)>();
|
|
for (var i = 0; i < folders.Count; i++)
|
|
{
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
var folder = folders[i];
|
|
if (resumeAfter is not null && string.CompareOrdinal(folder.RelativeFolder, resumeAfter) <= 0) continue;
|
|
states.TryGetValue(folder.RelativeFolder, out var state);
|
|
var changed = mode != ScanMode.Incremental || state is null || state.Sidecar != folder.Sidecar || state.Photos != folder.Photos;
|
|
if (changed)
|
|
{
|
|
try
|
|
{
|
|
var reconcile = mode == ScanMode.Full || (state is not null && state.Photos != folder.Photos);
|
|
var (id, bytes) = await IndexFolderAsync(archive, paths, folder, reconcile, snapshot, locations, cancellationToken).ConfigureAwait(false);
|
|
if (snapshotWriter is not null)
|
|
{
|
|
// Bytes go under the stamp they were read at (the listed one for the bytes indexed, else one taken before
|
|
// reading): a sidecar rewritten meanwhile is never filed under its new stamp with its old content.
|
|
var sidecar = paths.Sequence(folder.RelativeFolder).ManifestPath;
|
|
var read = bytes is not null ? folder.Sidecar!.Value : FileStamp.Of(sidecar)!.Value;
|
|
snapshotWriter.Add(folder.RelativeFolder, read, bytes ?? await AtomicFile.ReadAllBytesAsync(sidecar, cancellationToken).ConfigureAwait(false));
|
|
}
|
|
indexed.Add(id);
|
|
if (folder.Sidecar is null) adopted++;
|
|
}
|
|
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
|
|
{
|
|
Log.Warning("archive", $"Indexing {folder.RelativeFolder} failed", ex);
|
|
failed.Add((folder.RelativeFolder, ex.Message));
|
|
}
|
|
}
|
|
await progress(new ScanProgress(i + 1, folders.Count, folder.RelativeFolder, changed)).ConfigureAwait(false);
|
|
}
|
|
|
|
var removed = mode == ScanMode.Rebuild ? await RemoveUnbackedAsync(archive, paths, folders, locations, cancellationToken).ConfigureAwait(false) : 0;
|
|
if (failed.Count == 0) snapshotWriter?.Commit();
|
|
return new ScanSummary(folders.Count, indexed.Count, adopted, removed, failed, indexed);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Writes a sequence's sidecar into the index (row, frames, session keys) and records the folder stamps. Frames
|
|
/// inserted before indexed ones take the project's frame positions along (<see cref="ProjectPositions"/>). The frames
|
|
/// version never goes back, nor names other frames than it did: a sidecar read under the lease that is behind the
|
|
/// locked row (an older copy put back by hand or by a file-sync tool), or at the row's version over other frames
|
|
/// (another index of the archive bumped a copy put back to it, <see cref="OtherFramesAsync"/>), is written again one
|
|
/// version past the row, so editors and clients holding the row's version see the change.
|
|
/// </summary>
|
|
/// <param name="listed">The folder as the scan listed it before reading <paramref name="manifest"/> without the sequence
|
|
/// lease; null when the caller holds the lease and read the sidecar under it.</param>
|
|
/// <param name="clear">Statuses cleared in this index's row only, never in the sidecar (<see cref="Uncovered"/>).</param>
|
|
/// <param name="photosChanged">The caller, holding the lease, found the photo directory changed since this index recorded
|
|
/// its stamp before it changed the folder itself (<see cref="PhotosChangedAsync"/>). This sync indexes the sidecar, not
|
|
/// the photos, so the recorded photo stamp stays (none when none was) and the next incremental scan reconciles them.</param>
|
|
/// <returns>The manifest indexed; null, with nothing written, when <paramref name="listed"/> is given and the sidecar is no
|
|
/// longer the one listed or the locked row holds another frames version. Another writer then committed since the read
|
|
/// (an ingest's frames, a tier or a verify), and the caller reads the sidecar again under the lease
|
|
/// (<see cref="UpdateAndSyncAsync"/>) rather than write the older read over it and record stamps that hide it.</returns>
|
|
internal async Task<SequenceManifest?> SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, SequenceFolder? listed,
|
|
SequenceStatus clear, bool photosChanged, CancellationToken cancellationToken)
|
|
{
|
|
PositionMove? positions = null;
|
|
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
|
|
{
|
|
// Locked: a settings edit committed after this read (the server's PATCH takes no lease) would be written back over,
|
|
// and writers of the sequence's frames commit before or after this, never in between (GetForUpdateAsync).
|
|
var existing = await tx.Sequences.GetForUpdateAsync(manifest.Id, cancellationToken).ConfigureAwait(false);
|
|
if (listed is not null && (FileStamp.Of(sequence.ManifestPath) != listed.Sidecar
|
|
|| (existing is not null && existing.FramesVersion != manifest.Versions.Frames)))
|
|
return null;
|
|
// Under the lease, the row's own version may also name other frames: another index of the archive bumped a copy
|
|
// put back to it (OtherFramesAsync). A caller without the lease compared them before its read (FramesMovedAsync).
|
|
if (existing is not null && (existing.FramesVersion > manifest.Versions.Frames
|
|
|| (listed is null && existing.FramesVersion == manifest.Versions.Frames
|
|
&& await OtherFramesAsync(tx, existing, manifest, cancellationToken).ConfigureAwait(false))))
|
|
{
|
|
manifest = manifest with
|
|
{
|
|
Versions = manifest.Versions with { Frames = existing.FramesVersion + 1 },
|
|
UpdatedAt = DateTimeOffset.UtcNow,
|
|
};
|
|
ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
|
|
}
|
|
if (existing is not null)
|
|
{
|
|
positions = await ProjectPositions.MoveAsync(tx, existing, manifest, cancellationToken).ConfigureAwait(false);
|
|
if (positions is not null) existing = positions.ApplyTo(existing);
|
|
}
|
|
var row = SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing);
|
|
// A hero frame the sidecar no longer lists (a photo deleted) names nothing: the project's hero waits for its photo.
|
|
if (row.HeroFrameId is { } hero && !manifest.Frames.Any(f => f.Id == hero)) row = row with { HeroFrameId = null };
|
|
await tx.Sequences.UpsertAsync(row with { Status = row.Status & ~clear }, cancellationToken).ConfigureAwait(false);
|
|
await tx.Frames.SyncSequenceAsync(manifest.Id, SequenceMapping.ToFrames(manifest, sequence), cancellationToken).ConfigureAwait(false);
|
|
await tx.FrameRemovals.SyncSequenceAsync(manifest.Id, SequenceMapping.ToRemovals(manifest), cancellationToken).ConfigureAwait(false);
|
|
foreach (var key in manifest.SessionKeys)
|
|
await tx.Sequences.MapSessionKeyAsync(archive.Id, key, manifest.Id, cancellationToken).ConfigureAwait(false);
|
|
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
|
}
|
|
if (positions is { } moved) ProjectPositions.WriteProject(sequence, moved.Before, moved.Moved);
|
|
// Unleased, the stamps the sync was checked against: a sidecar or photos changed since keep the folder changed.
|
|
if (listed is not null)
|
|
{
|
|
await RecordStampsAsync(archive, sequence.RelativeFolder, manifest.Id, listed.Sidecar, listed.Photos, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
else if (photosChanged)
|
|
{
|
|
// The photo stamp as recorded: the photos this sync did not index keep the folder changed for the next scan.
|
|
var recorded = await _db.Archives.GetFolderStateAsync(archive.Id, sequence.RelativeFolder, cancellationToken).ConfigureAwait(false);
|
|
await RecordStampsAsync(archive, sequence.RelativeFolder, manifest.Id, FileStamp.Of(sequence.ManifestPath), recorded?.Photos, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
}
|
|
else
|
|
{
|
|
await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
return manifest;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Remembers the current stamps of a folder whose sidecar and index were just updated together, so the next
|
|
/// incremental scan does not re-read what is already indexed.
|
|
/// </summary>
|
|
internal Task RecordStampsAsync(Archive archive, SequencePaths sequence, Guid sequenceId, CancellationToken cancellationToken) =>
|
|
RecordStampsAsync(archive, sequence.RelativeFolder, sequenceId, FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence)),
|
|
cancellationToken);
|
|
|
|
private Task RecordStampsAsync(Archive archive, string folder, Guid sequenceId, FileStamp? sidecar, FileStamp? photos, CancellationToken cancellationToken) =>
|
|
_db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(folder, sequenceId, sidecar, photos), cancellationToken);
|
|
|
|
/// <summary>
|
|
/// Whether this index holds the folder of <paramref name="manifest"/> (the sidecar read under the sequence lease) as it
|
|
/// is now: the stamps it recorded when it last synced the folder are still the folder's, and its row holds the
|
|
/// sidecar's frames. A writer that changes, under the lease, only what it owns of the sidecar and the index row
|
|
/// (statuses, decode tiers, frame flags, title, tags, rating and notes, the folder) asks before its own write, and
|
|
/// records the folder's new stamps after it only when this held (<see cref="RecordStampsAsync"/>). What another
|
|
/// process (a desktop sharing the archive) changed in the sidecar alone, frames or a new title, then keeps the folder
|
|
/// changed for the next incremental scan, which writes the whole row from the sidecar. An upload asks too, and
|
|
/// appends its frames only when this held; otherwise it writes the whole row and every frame from the sidecar itself.
|
|
/// </summary>
|
|
internal Task<bool> InStepAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) =>
|
|
InStepAsync(archive, sequence.RelativeFolder, sequence, manifest, cancellationToken);
|
|
|
|
/// <inheritdoc cref="InStepAsync(Archive, SequencePaths, SequenceManifest, CancellationToken)"/>
|
|
/// <param name="recordedFolder">The folder the stamps were recorded at: the one a rename just moved the sequence from,
|
|
/// whose files keep their stamps.</param>
|
|
internal async Task<bool> InStepAsync(Archive archive, string recordedFolder, SequencePaths sequence, SequenceManifest manifest,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
if (await _db.Archives.GetFolderStateAsync(archive.Id, recordedFolder, cancellationToken).ConfigureAwait(false) is not { } state
|
|
|| state.SequenceId != manifest.Id || state.Sidecar != FileStamp.Of(sequence.ManifestPath)
|
|
|| state.Photos != FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))) return false;
|
|
// A sidecar rewritten within its file system's time resolution, at the same size, keeps its stamp.
|
|
return await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { } indexed
|
|
&& indexed.FramesVersion == manifest.Versions.Frames;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Whether the folder's photo directory no longer has the stamp this index recorded for it, or none was recorded:
|
|
/// photos copied in or removed by hand or by another process since the last scan, which only a scan's reconcile
|
|
/// indexes. An upload asks under the lease before it files its photos, which change the stamp, and its sync from the
|
|
/// sidecar then keeps the recorded one (<see cref="SyncAsync"/>), so the next incremental scan still reconciles them.
|
|
/// </summary>
|
|
internal async Task<bool> PhotosChangedAsync(Archive archive, SequencePaths sequence, CancellationToken cancellationToken) =>
|
|
(await _db.Archives.GetFolderStateAsync(archive.Id, sequence.RelativeFolder, cancellationToken).ConfigureAwait(false))?.Photos
|
|
!= FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence));
|
|
|
|
/// <summary>
|
|
/// Indexes unindexed photos and flags missing files of one sequence (the orphans fix action), as a scan's reconcile
|
|
/// does (<see cref="Reconcile"/>): what covered the frames is cleared in the sidecar and the row.
|
|
/// </summary>
|
|
/// <returns>Whether photos were added, gone or back: the light analysis, thumbnails and proxies are then to be made again
|
|
/// (<see cref="ArchiveService.ReconcileAsync"/>). The folder's stamps are recorded, so no scan comes back to it and
|
|
/// queues them.</returns>
|
|
public async Task<bool> ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default)
|
|
{
|
|
var paths = new ArchivePaths(archive.RootPath);
|
|
ArchiveVolume.EnsureWritable(paths);
|
|
var folder = paths.Sequence(sequence.Folder);
|
|
var changed = false;
|
|
await UpdateAndSyncAsync(archive, paths, folder, sequence.Id, m =>
|
|
{
|
|
var reconciled = Reconcile(folder, m, cancellationToken);
|
|
changed = reconciled is not null;
|
|
return reconciled;
|
|
}, cancellationToken).ConfigureAwait(false);
|
|
return changed;
|
|
}
|
|
|
|
/// <summary>
|
|
/// The sidecar brought in line with the folder's photos (<see cref="SequenceAdopter.Reconcile"/>); null when nothing
|
|
/// changed. Photos added, gone or back: the light analysis, thumbnails and proxies no longer cover the frames, as the
|
|
/// sidecar then tells every host, so the follow-ups make them again (the images on a server,
|
|
/// <see cref="ArchiveServiceOptions.ImagesAfterScan"/>).
|
|
/// </summary>
|
|
private static SequenceManifest? Reconcile(SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) =>
|
|
SequenceAdopter.Reconcile(sequence, manifest, cancellationToken) is { } reconciled
|
|
? reconciled with { Status = reconciled.Status & ~CoveringStatus }
|
|
: null;
|
|
|
|
/// <summary>
|
|
/// Applies <paramref name="edit"/> to the sidecar (null writes nothing) and indexes the result, both under the sequence
|
|
/// lease. Frames inserted before indexed ones move the project's positions (<see cref="ProjectPositions"/>) without a
|
|
/// new settings version; an editor saving its project holds the same lease, so it reads them moved and merges onto them
|
|
/// rather than writing its old positions back. When the sidecar, as read under the lease, lists other frames than the
|
|
/// index row (another process changed them together with their sidecar entries), the row is written without what
|
|
/// this host makes for the frames (<see cref="Uncovered"/>).
|
|
/// </summary>
|
|
private async Task<SequenceManifest> UpdateAndSyncAsync(Archive archive, ArchivePaths paths, SequencePaths sequence, Guid sequenceId,
|
|
Func<SequenceManifest, SequenceManifest?> edit, CancellationToken cancellationToken)
|
|
{
|
|
await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false))
|
|
{
|
|
var manifest = SequenceWriter.ReadLocked(sequence, sequenceId);
|
|
var moved = await FramesMovedAsync(manifest, cancellationToken).ConfigureAwait(false);
|
|
if (edit(manifest) is { } updated)
|
|
{
|
|
ManifestStore.WriteSequence(sequence.ManifestPath, updated);
|
|
manifest = updated;
|
|
}
|
|
return (await SyncAsync(archive, sequence, manifest, null, moved ? Uncovered : SequenceStatus.None, false, cancellationToken).ConfigureAwait(false))!;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// What a scan clears in this host's own row, never in the shared sidecar, when another process added, removed or
|
|
/// renumbered frames together with their sidecar entries: on a server (<see cref="ArchiveServiceOptions.ImagesAfterScan"/>),
|
|
/// which analyses and draws what its clients load, the light analysis, thumbnails and proxies, so the scan's follow-ups
|
|
/// make them for the frames as they are, and their passes set the flags in both places again. Nothing on a desktop:
|
|
/// the flags in the sidecar are the writer's that added the frames (which analyses them, and on a server draws them
|
|
/// after every upload), and clearing them there would have the server make everything again.
|
|
/// </summary>
|
|
private SequenceStatus Uncovered => _writer.Options.ImagesAfterScan ? CoveringStatus : SequenceStatus.None;
|
|
|
|
/// <summary>Indexes one folder; returns its sequence and the sidecar bytes when they were read and not rewritten since (for the snapshot).</summary>
|
|
private async Task<(Guid Id, byte[]? Bytes)> IndexFolderAsync(Archive archive, ArchivePaths paths, SequenceFolder folder, bool reconcile, IndexSnapshot? snapshot,
|
|
Locations locations, CancellationToken cancellationToken)
|
|
{
|
|
var sequence = paths.Sequence(folder.RelativeFolder);
|
|
SequenceManifest manifest;
|
|
byte[]? bytes = null;
|
|
if (folder.Sidecar is not { } stamp)
|
|
{
|
|
manifest = await AdoptAsync(sequence, locations, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
else
|
|
{
|
|
bytes = snapshot?.TryGet(folder.RelativeFolder, stamp) ?? await AtomicFile.ReadAllBytesAsync(sequence.ManifestPath, cancellationToken).ConfigureAwait(false);
|
|
try
|
|
{
|
|
manifest = ManifestStore.ParseSequence(bytes, sequence.ManifestPath);
|
|
}
|
|
catch (InvalidDataException ex)
|
|
{
|
|
Log.Warning("archive", $"Damaged sidecar in {folder.RelativeFolder}; rebuilding it from the photos", ex);
|
|
manifest = await ReplaceDamagedAsync(sequence, locations, cancellationToken).ConfigureAwait(false);
|
|
bytes = null;
|
|
}
|
|
}
|
|
if (locations.IsCopy(paths, manifest.Id, folder.RelativeFolder))
|
|
{
|
|
manifest = Reidentify(manifest);
|
|
ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
|
|
bytes = null;
|
|
}
|
|
// A journal of an ingest, a photo deletion or a restore a crash interrupted is settled under the lease first (ReadLocked).
|
|
if (reconcile || File.Exists(sequence.IngestJournalPath) || File.Exists(sequence.RemovalJournalPath) || File.Exists(sequence.RestoreJournalPath))
|
|
{
|
|
// Photos added, gone or back clear what covered the frames, and the scan's follow-ups make it again.
|
|
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, m => Reconcile(sequence, m, cancellationToken), cancellationToken)
|
|
.ConfigureAwait(false);
|
|
bytes = null;
|
|
}
|
|
else if (await FramesMovedAsync(manifest, cancellationToken).ConfigureAwait(false))
|
|
{
|
|
// Another process (a desktop sharing the archive, or the server it shares it with) added, removed or renumbered
|
|
// frames together with their sidecar entries: read again under the lease, under which the project's positions
|
|
// move, and indexed without what this host makes for the frames (Uncovered).
|
|
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false);
|
|
bytes = null;
|
|
}
|
|
else if (await SyncAsync(archive, sequence, manifest, folder, SequenceStatus.None, false, cancellationToken).ConfigureAwait(false) is null)
|
|
{
|
|
// Another writer committed between the read and the sync: read again under the lease, as it is now.
|
|
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, _ => null, cancellationToken).ConfigureAwait(false);
|
|
bytes = null;
|
|
}
|
|
locations.Set(manifest.Id, folder.RelativeFolder);
|
|
return (manifest.Id, bytes);
|
|
}
|
|
|
|
/// <summary>
|
|
/// The sequences among <paramref name="locations"/> created without photos that still wait for their first frame: no
|
|
/// folder, no frames, and not the sequence the archive root's sidecar names. Their empty folder is not the archive
|
|
/// root: scans do not adopt or rebuild the root under their id, a rebuild keeps them although they have no sidecar yet,
|
|
/// orphan checks do not count them missing, and maintenance leaves the root's files to the root's own sequence. Unlike
|
|
/// <see cref="ArchiveIngest.AwaitsFolder"/>, whose commit reports it, a root sidecar that cannot be read names nobody
|
|
/// here: a scan rebuilds a damaged one under the id of the sequence that holds the root's frames, and one it may not
|
|
/// read fails the root folder alone rather than the whole scan or rebuild.
|
|
/// </summary>
|
|
internal async Task<HashSet<Guid>> AwaitingFolderAsync(ArchivePaths paths, IEnumerable<SequenceLocation> locations, CancellationToken cancellationToken)
|
|
{
|
|
var awaiting = new HashSet<Guid>();
|
|
(Guid? Id, bool Read) root = (null, false);
|
|
foreach (var location in locations.Where(l => l.Folder.Length == 0))
|
|
{
|
|
if (await _db.Sequences.GetAsync(location.Id, cancellationToken).ConfigureAwait(false) is not { Folder.Length: 0, FrameCount: 0 } sequence) continue;
|
|
// Read once, and only when a row may be waiting: a root sidecar can list a hundred thousand frames.
|
|
if (!root.Read) root = (RootSidecarId(paths), true);
|
|
if (root.Id != sequence.Id) awaiting.Add(sequence.Id);
|
|
}
|
|
return awaiting;
|
|
}
|
|
|
|
/// <summary>The id the archive root's sidecar names; null when there is none or it cannot be read.</summary>
|
|
private static Guid? RootSidecarId(ArchivePaths paths)
|
|
{
|
|
var sidecar = paths.Sequence(string.Empty).ManifestPath;
|
|
try
|
|
{
|
|
return File.Exists(sidecar) ? ManifestStore.ReadSequence(sidecar).Id : null;
|
|
}
|
|
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
|
|
{
|
|
// Indexing the root reports it (a damaged sidecar is rebuilt, an unreadable one fails that folder alone).
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// The index holds this sequence with other frames than the sidecar: another process changed them (its project's
|
|
/// positions may move, and what covered the frames no longer does).
|
|
/// </summary>
|
|
private async Task<bool> FramesMovedAsync(SequenceManifest manifest, CancellationToken cancellationToken) =>
|
|
await _db.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false) is { } indexed
|
|
&& await OtherFramesAsync(_db, indexed, manifest, cancellationToken).ConfigureAwait(false);
|
|
|
|
/// <summary>
|
|
/// Whether the index row <paramref name="indexed"/> holds other frames than <paramref name="manifest"/>: another frames
|
|
/// version, or the same version over other frames (ids, or their order). Each index of a shared archive bumps a sidecar
|
|
/// put back behind its own row one version past that row (<see cref="SyncAsync"/>), and another index may hold that
|
|
/// very version for the frames as they were before: that index still sees the change, and bumps past it in turn.
|
|
/// </summary>
|
|
private static async Task<bool> OtherFramesAsync(ChronosRepositories db, Sequence indexed, SequenceManifest manifest, CancellationToken cancellationToken)
|
|
{
|
|
if (indexed.FramesVersion != manifest.Versions.Frames) return true;
|
|
var rows = await db.Frames.ListPositionsAsync(indexed.Id, cancellationToken).ConfigureAwait(false);
|
|
if (rows.Count != manifest.Frames.Count) return true;
|
|
var listed = new Dictionary<int, Guid>(manifest.Frames.Count);
|
|
foreach (var frame in manifest.Frames) listed[frame.I] = frame.Id;
|
|
return rows.Any(row => !listed.TryGetValue(row.Index, out var id) || id != row.FrameId);
|
|
}
|
|
|
|
/// <summary>
|
|
/// A sidecar for a folder that has none; ids and tombstones of a sequence the index already knew at this folder are
|
|
/// kept, so a photo deleted from it is not offered back once the index is synced from the new sidecar.
|
|
/// </summary>
|
|
private async Task<SequenceManifest> AdoptAsync(SequencePaths sequence, Locations locations, CancellationToken cancellationToken)
|
|
{
|
|
var known = locations.AtFolder(sequence.RelativeFolder);
|
|
var manifest = await WithKnownFrameIdsAsync(SequenceAdopter.Build(known ?? Guid.CreateVersion7(), sequence, _writer.Options.Owner, cancellationToken),
|
|
sequence, cancellationToken).ConfigureAwait(false);
|
|
if (known is { } id && await _db.FrameRemovals.ListBySequenceAsync(id, cancellationToken).ConfigureAwait(false) is { Count: > 0 } removals)
|
|
manifest = manifest with { Removed = SequenceMapping.ToTombstones(removals) };
|
|
Directory.CreateDirectory(sequence.MetaDirectory);
|
|
// Another process may adopt the same folder at the same moment: the first sidecar wins.
|
|
return ManifestStore.TryCreateSequence(sequence.ManifestPath, manifest) ? manifest : ManifestStore.ReadSequence(sequence.ManifestPath);
|
|
}
|
|
|
|
/// <summary>Keeps the unreadable sidecar aside (never deleted) and builds a new one from the photos.</summary>
|
|
private async Task<SequenceManifest> ReplaceDamagedAsync(SequencePaths sequence, Locations locations, CancellationToken cancellationToken)
|
|
{
|
|
File.Move(sequence.ManifestPath, $"{sequence.ManifestPath}.damaged-{DateTime.UtcNow:yyyyMMddHHmmss}");
|
|
return await AdoptAsync(sequence, locations, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
private async Task<SequenceManifest> WithKnownFrameIdsAsync(SequenceManifest manifest, SequencePaths sequence, CancellationToken cancellationToken)
|
|
{
|
|
var known = new Dictionary<string, Guid>(StringComparer.Ordinal);
|
|
var prefix = sequence.ArchiveRelative(string.Empty);
|
|
for (var after = -1; ;)
|
|
{
|
|
var page = await _db.Frames.ListBySequenceAsync(manifest.Id, after, 1000, cancellationToken).ConfigureAwait(false);
|
|
foreach (var f in page)
|
|
{
|
|
if (f.RelativePath.StartsWith(prefix, StringComparison.Ordinal)) known[f.RelativePath[prefix.Length..]] = f.Id;
|
|
}
|
|
if (page.Count < 1000) break;
|
|
after = page[^1].Index;
|
|
}
|
|
if (known.Count == 0) return manifest;
|
|
return manifest with { Frames = manifest.Frames.Select(f => known.TryGetValue(f.File, out var id) ? f with { Id = id } : f).ToList() };
|
|
}
|
|
|
|
/// <summary>A copied sequence folder carries the ids of its source; the copy gets ids of its own.</summary>
|
|
private static SequenceManifest Reidentify(SequenceManifest manifest) => manifest with
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
Frames = manifest.Frames.Select(f => f with { Id = Guid.CreateVersion7() }).ToList(),
|
|
SessionKeys = [],
|
|
UpdatedAt = DateTimeOffset.UtcNow,
|
|
};
|
|
|
|
/// <summary>
|
|
/// Rebuild only: rows whose folder no longer holds a sidecar leave the index (their files are untouched). A sequence
|
|
/// waiting for its first frame has no sidecar yet and stays. A row the folders listed when the rebuild began do not
|
|
/// back is looked at again under its lease, as it and its folder are when it would go (<see cref="BackedAsync"/>):
|
|
/// uploads go on during a rebuild, which can take minutes, so a sequence that got its first frame or was created
|
|
/// meanwhile has a folder the listing does not hold.
|
|
/// </summary>
|
|
private async Task<int> RemoveUnbackedAsync(Archive archive, ArchivePaths paths, IReadOnlyList<SequenceFolder> folders, Locations locations,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
ArchiveVolume.EnsureWritable(paths);
|
|
var present = folders.Where(f => File.Exists(paths.Sequence(f.RelativeFolder).ManifestPath)).Select(f => f.RelativeFolder).ToHashSet(StringComparer.Ordinal);
|
|
var rows = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
|
var awaiting = await AwaitingFolderAsync(paths, rows, cancellationToken).ConfigureAwait(false);
|
|
var removed = 0;
|
|
foreach (var location in rows)
|
|
{
|
|
if (awaiting.Contains(location.Id) || (present.Contains(location.Folder) && locations.AtFolder(location.Folder) == location.Id)) continue;
|
|
IAsyncDisposable held;
|
|
try
|
|
{
|
|
held = await _writer.LockAsync(paths, location.Id, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
catch (LeaseUnavailableException)
|
|
{
|
|
// Another process is writing the sequence, so it is there.
|
|
continue;
|
|
}
|
|
await using (held)
|
|
{
|
|
if (await BackedAsync(paths, location.Id, cancellationToken).ConfigureAwait(false)) continue;
|
|
if (await _db.Sequences.DeleteAsync(location.Id, cancellationToken).ConfigureAwait(false)) removed++;
|
|
await _db.Archives.DeleteFolderStateAsync(archive.Id, location.Folder, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
}
|
|
return removed;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Whether the index row of <paramref name="sequenceId"/> keeps its place as it is now: it waits for its first frame,
|
|
/// its folder holds a sidecar naming it (one that cannot be read proves nothing), or it is gone already.
|
|
/// </summary>
|
|
private async Task<bool> BackedAsync(ArchivePaths paths, Guid sequenceId, CancellationToken cancellationToken)
|
|
{
|
|
if (await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) is not { } row) return true;
|
|
if ((await AwaitingFolderAsync(paths, [new SequenceLocation(row.Id, row.Folder, row.Title)], cancellationToken).ConfigureAwait(false)).Count > 0)
|
|
return true;
|
|
var sidecar = paths.Sequence(row.Folder).ManifestPath;
|
|
try
|
|
{
|
|
return File.Exists(sidecar) && ManifestStore.ReadSequence(sidecar).Id == sequenceId;
|
|
}
|
|
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
|
|
{
|
|
return true;
|
|
}
|
|
}
|
|
|
|
/// <summary>Which folder each indexed sequence id lives in, updated as the scan goes.</summary>
|
|
private sealed class Locations
|
|
{
|
|
private readonly Dictionary<Guid, string> _byId = [];
|
|
private readonly Dictionary<string, Guid> _byFolder = new(StringComparer.Ordinal);
|
|
|
|
public Locations(IEnumerable<SequenceLocation> rows)
|
|
{
|
|
foreach (var row in rows) Set(row.Id, row.Folder);
|
|
}
|
|
|
|
public Guid? AtFolder(string folder) => _byFolder.TryGetValue(folder, out var id) ? id : null;
|
|
|
|
/// <summary>The id is indexed at another folder whose sidecar still carries it: this folder is a copy.</summary>
|
|
public bool IsCopy(ArchivePaths paths, Guid id, string folder)
|
|
{
|
|
if (!_byId.TryGetValue(id, out var other) || other == folder) return false;
|
|
var otherSidecar = paths.Sequence(other).ManifestPath;
|
|
try
|
|
{
|
|
return File.Exists(otherSidecar) && ManifestStore.ReadSequence(otherSidecar).Id == id;
|
|
}
|
|
catch (InvalidDataException)
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
|
|
public void Set(Guid id, string folder)
|
|
{
|
|
if (_byId.TryGetValue(id, out var previous) && _byFolder.GetValueOrDefault(previous) == id) _byFolder.Remove(previous);
|
|
_byId[id] = folder;
|
|
_byFolder[folder] = id;
|
|
}
|
|
}
|
|
}
|