Files
Chronos/src/Chronos.Data/Archiving/ArchiveIngest.cs
T
Alby96 4518ffe655 fix(archive): settle trash journals under their leases and say why restored photos stayed
Second fix round of owner review 2, phase 3 (engine, server, desktop).

- A photo never hashed whose original is missing can be deleted again: it goes
  into its trash record without a file and without a tombstone (nothing is left
  to name), stays listed until Empty erases it, and a restore keeps it with the
  reason "no file". FrameDeleteBlock.Unhashable now blocks only a present
  original that cannot be read; the API answers it as 409 photo_unreadable
  naming the photo, and every block has its own code and message.
- Deleting a whole sequence first settles an interrupted photo deletion,
  restore or ingest of it (ReadLocked under its lease), so no journal or half
  entry travels into the sequence's trash entry; it refuses while a restore
  into it waits for its busy entry.
- FrameTrash.Settle takes the photo entry's lease and leaves the journal for
  later while the entry is busy (an emptying), so it never races EraseAsync;
  meanwhile no other restore writes its journal over it and scans do not adopt
  its files.
- The hashes computed for frames that stay (to find a twin of a deleted photo)
  go into the sidecar and index, and only a frame whose first and last 64 KiB
  match a deleted photo's is read whole.
- A restore into a new sequence rewrites its record after each photo filed, so
  a crash or a database failure never leaves filed photos listed.
- An upload completion no longer has a window before the commit: the ingest
  checks the sidecar's tombstones under the sequence lease a deletion takes
  (IngestRequest.RefuseRemoved), unless the upload asked to bring it back.
- A restore reports each kept photo with its reason (no longer matches its
  record, could not be moved back, already in the sequence, had no file); the
  desktop line and the API message say so in en and it, and the tooltip holds
  the whole sentence. The desktop tells TrashMismatchException by its type.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit 1858f1f450a35fb8383ede8d80ce5f8de9b7ceac)
2026-10-27 13:53:41 +01:00

571 lines
32 KiB
C#

using System.Collections.Concurrent;
using System.Globalization;
using Chronos.Core.Archiving;
using Chronos.Core.Diagnostics;
using Chronos.Core.IO;
using Chronos.Core.Localization;
using Chronos.Core.Metadata;
using Chronos.Core.Model;
using Chronos.Core.Naming;
namespace Chronos.Data.Archiving;
/// <summary>A verified staged file and how to name it.</summary>
/// <param name="PlannedName">Name chosen by an import plan; null = the archive's file template.</param>
/// <param name="RefuseRemoved">See <see cref="IngestRequest.RefuseRemoved"/>.</param>
internal sealed record StagedFrame(string StagedPath, string OriginalFileName, long Size, string Sha256, FrameMetadata Metadata,
DateTime? ClientCapturedAt, string? PlannedName, bool RefuseRemoved = false);
/// <summary>A sequence found or created by key (upload session, card session, or a new title).</summary>
/// <param name="PreferredFolder">Folder of an import plan; null = the archive's folder template.</param>
internal sealed record SequenceTarget(string Key, string? Title, string? PreferredFolder, string? CreatedBy);
/// <summary>
/// Commits staged, verified files into sequences (BRIEF §4.3, §14.4): the file is renamed into the sequence folder
/// under the naming template, the sidecar is rewritten atomically in capture order and the index follows. Commits
/// to one sequence are serialised and batched (concurrent uploads share one sidecar write); a lease file keeps
/// other processes out meanwhile, and a write-ahead journal repairs a batch interrupted by a crash. A file whose
/// SHA-256 an intact frame of the sequence already holds is reported as a duplicate and left in staging; a frame with
/// that SHA-256 that is flagged missing or corrupt, or whose file is gone or has another size, is repaired with it
/// instead (its damaged file kept beside it). A sequence created without photos gets its folder with its first frame.
/// </summary>
public sealed class ArchiveIngest : IArchiveIngest
{
private readonly ChronosDatabase _db;
private readonly SequenceWriter _writer;
private readonly ArchiveIndexer _indexer;
private readonly ConcurrentDictionary<Guid, CommitQueue> _queues = new();
private readonly SemaphoreSlim _creation = new(1, 1);
internal ArchiveIngest(ChronosDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
{
_db = db;
_writer = writer;
_indexer = indexer;
}
public async Task<IngestResult> CommitAsync(IngestRequest request, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(request);
if ((request.SequenceId is null ? 0 : 1) + (request.NewSequenceTitle is null ? 0 : 1) + (request.SessionKey is null ? 0 : 1) != 1)
throw new ArgumentException("Exactly one of SequenceId, NewSequenceTitle or SessionKey selects the target.", nameof(request));
var archive = await _db.Archives.GetAsync(request.ArchiveId, cancellationToken).ConfigureAwait(false)
?? throw new ArgumentException($"Unknown archive {request.ArchiveId}.", nameof(request));
var staged = new FileInfo(request.StagedPath);
if (!staged.Exists || staged.Length != request.Size)
throw new FileNotFoundException("The staged file is missing or has another size.", request.StagedPath);
var frame = new StagedFrame(request.StagedPath, request.OriginalFileName, request.Size, request.Sha256,
MetadataReader.Read(request.StagedPath, cancellationToken), request.ClientCapturedAt, null, request.RefuseRemoved);
Guid sequenceId;
if (request.SequenceId is { } id)
{
var sequence = await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false);
if (sequence?.ArchiveId != archive.Id) throw new ArgumentException($"Sequence {id} is not in archive {archive.Id}.", nameof(request));
if (AwaitsFolder(sequence, new ArchivePaths(archive.RootPath))) await PlaceAsync(archive, id, frame, cancellationToken).ConfigureAwait(false);
sequenceId = id;
}
else
{
// A title names one sequence per user: parallel uploads for "new sequence X" all land in the same one.
var key = request.SessionKey ?? $"title:{request.UserId:N}:{request.NewSequenceTitle!.Trim().ToLowerInvariant()}";
sequenceId = await FindOrCreateAsync(archive, new SequenceTarget(key, request.NewSequenceTitle?.Trim(), null, request.UserId?.ToString()),
frame, cancellationToken).ConfigureAwait(false);
}
return (await EnqueueAsync(archive, sequenceId, [frame], cancellationToken).ConfigureAwait(false))[0];
}
/// <summary>Commits files of one import session into the sequence of its key (created on first use).</summary>
internal async Task<IReadOnlyList<IngestResult>> CommitSessionAsync(Archive archive, SequenceTarget target, IReadOnlyList<StagedFrame> frames,
CancellationToken cancellationToken)
{
if (frames.Count == 0) return [];
var sequenceId = await FindOrCreateAsync(archive, target, frames[0], cancellationToken).ConfigureAwait(false);
return await EnqueueAsync(archive, sequenceId, frames, cancellationToken).ConfigureAwait(false);
}
private async Task<Guid> FindOrCreateAsync(Archive archive, SequenceTarget target, StagedFrame first, CancellationToken cancellationToken)
{
if (await _db.Sequences.FindBySessionKeyAsync(archive.Id, target.Key, cancellationToken).ConfigureAwait(false) is { } found) return found;
await _creation.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (await _db.Sequences.FindBySessionKeyAsync(archive.Id, target.Key, cancellationToken).ConfigureAwait(false) is { } again) return again;
return await CreateAsync(archive, target, first, cancellationToken).ConfigureAwait(false);
}
finally
{
_creation.Release();
}
}
private async Task<Guid> CreateAsync(Archive archive, SequenceTarget target, StagedFrame first, CancellationToken cancellationToken)
{
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var now = DateTimeOffset.UtcNow;
var m = first.Metadata;
var manifest = new SequenceManifest
{
Id = Guid.CreateVersion7(),
Title = target.Title,
Camera = new SidecarCamera { Make = m.Make, Model = m.Model, Serial = m.BodySerial },
Lens = m.LensModel,
SessionKeys = [target.Key],
CreatedBy = target.CreatedBy,
Created = now,
UpdatedAt = now,
};
var sequence = await ClaimNewFolderAsync(archive, paths, target.PreferredFolder, first, manifest, cancellationToken).ConfigureAwait(false);
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
{
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, null), cancellationToken).ConfigureAwait(false);
var winner = await tx.Sequences.MapSessionKeyAsync(archive.Id, target.Key, manifest.Id, cancellationToken).ConfigureAwait(false);
if (winner != manifest.Id)
{
// Another process bound the key first: its sequence wins, ours was never used.
await tx.RollbackAsync(cancellationToken).ConfigureAwait(false);
ReleaseFolder(sequence);
return winner;
}
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
}
await _indexer.RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
return manifest.Id;
}
/// <summary>
/// A sequence created without photos (<c>POST /api/v1/sequences</c>, BRIEF §14.3) has no folder until its first frame.
/// An empty folder otherwise means the archive root itself, adopted in place, whose sidecar names the sequence; one
/// that holds frames is never moved elsewhere, even while its root sidecar is missing.
/// </summary>
internal static bool AwaitsFolder(Sequence sequence, ArchivePaths paths)
{
if (sequence.Folder.Length > 0 || sequence.FrameCount > 0) return false;
var root = paths.Sequence(string.Empty);
if (!File.Exists(root.ManifestPath)) return true;
try
{
return ManifestStore.ReadSequence(root.ManifestPath).Id != sequence.Id;
}
catch (Exception ex) when (ex is IOException or InvalidDataException)
{
// An unreadable root sidecar: the commit reports it rather than placing the sequence elsewhere.
return false;
}
}
/// <summary>
/// Gives a sequence created without photos its folder, named from its first frame and title by the same rules as a
/// sequence created by an upload, with a sidecar carrying the row's id, title, tags, rating and notes (and its project
/// when one was saved); the index row takes the folder in one transaction. Serialised with the creation of sequences.
/// </summary>
private async Task PlaceAsync(Archive archive, Guid sequenceId, StagedFrame first, CancellationToken cancellationToken)
{
await _creation.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
var paths = new ArchivePaths(archive.RootPath);
var row = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
?? throw new InvalidOperationException($"Sequence {sequenceId} is not indexed.");
// A concurrent first frame placed it meanwhile.
if (!AwaitsFolder(row, paths)) return;
ArchiveVolume.EnsureWritable(paths);
var now = DateTimeOffset.UtcNow;
var m = first.Metadata;
var manifest = new SequenceManifest
{
Id = row.Id,
Title = row.Title,
// The photo names its camera; the creator's hint stands in when it does not.
Camera = m.Make is null && m.Model is null && row.CameraModel is not null
? new SidecarCamera { Model = row.CameraModel }
: new SidecarCamera { Make = m.Make, Model = m.Model, Serial = m.BodySerial },
Lens = m.LensModel,
Tags = row.Tags,
Rating = row.Rating,
Notes = row.Notes,
Versions = new ManifestVersions { Settings = row.SettingsVersion, Frames = row.FramesVersion },
CreatedBy = row.OwnerUserId?.ToString(),
Created = now,
UpdatedAt = now,
};
var sequence = await ClaimNewFolderAsync(archive, paths, null, first, manifest, cancellationToken).ConfigureAwait(false);
try
{
if (row.SettingsJson is { } settings) AtomicFile.WriteAllText(sequence.ProjectPath, settings);
await using var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false);
// Locked, so an edit of the new sequence committed meanwhile is not written back over.
var current = await tx.Sequences.GetForUpdateAsync(sequenceId, cancellationToken).ConfigureAwait(false)
?? throw new InvalidOperationException($"Sequence {sequenceId} was deleted before its first frame.");
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, current), cancellationToken).ConfigureAwait(false);
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
}
catch
{
// Nothing was filed into the folder yet: it goes, and the next first frame claims one again.
if (File.Exists(sequence.ProjectPath)) File.Delete(sequence.ProjectPath);
ReleaseFolder(sequence);
throw;
}
await _indexer.RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
}
finally
{
_creation.Release();
}
}
/// <summary>A new folder for <paramref name="manifest"/>: the plan's folder or the archive's folder template, made unique.</summary>
private async Task<SequencePaths> ClaimNewFolderAsync(Archive archive, ArchivePaths paths, string? preferredFolder, StagedFrame first,
SequenceManifest manifest, CancellationToken cancellationToken)
{
var captured = CaptureTime(first);
var wanted = preferredFolder ?? PathTemplate.Parse(archive.FolderTemplate).ExpandFolder(Naming(first, captured, captured, 1, manifest.Title));
if (wanted.Length == 0) wanted = "Sequence";
var taken = (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false))
.Select(l => l.Folder).ToHashSet(StringComparer.OrdinalIgnoreCase);
return ClaimFolder(paths, wanted, taken, manifest);
}
/// <summary>The first free variant of <paramref name="wanted"/>, claimed by creating its sidecar (first writer wins).</summary>
private static SequencePaths ClaimFolder(ArchivePaths paths, string wanted, HashSet<string> taken, SequenceManifest manifest)
{
for (var n = 1; ; n++)
{
var folder = n == 1 ? wanted : string.Create(CultureInfo.InvariantCulture, $"{wanted} ({n})");
if (taken.Contains(folder)) continue;
var sequence = paths.Sequence(folder);
if (Directory.Exists(sequence.Folder) && Directory.EnumerateFileSystemEntries(sequence.Folder).Any()) continue;
Directory.CreateDirectory(sequence.MetaDirectory);
if (ManifestStore.TryCreateSequence(sequence.ManifestPath, manifest)) return sequence;
}
}
/// <summary>Removes the empty sequence this process created and lost to a concurrent creator.</summary>
private static void ReleaseFolder(SequencePaths sequence)
{
File.Delete(sequence.ManifestPath);
if (!Directory.EnumerateFileSystemEntries(sequence.MetaDirectory).Any()) Directory.Delete(sequence.MetaDirectory);
if (!Directory.EnumerateFileSystemEntries(sequence.Folder).Any()) Directory.Delete(sequence.Folder);
}
/// <summary>Group commit: whoever gets the sequence's gate commits everything queued for it.</summary>
private async Task<IngestResult[]> EnqueueAsync(Archive archive, Guid sequenceId, IReadOnlyList<StagedFrame> frames, CancellationToken cancellationToken)
{
var queue = _queues.GetOrAdd(sequenceId, _ => new CommitQueue());
var pending = frames.Select(f => new Pending(f)).ToList();
foreach (var p in pending) queue.Items.Enqueue(p);
await queue.Gate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (pending.Any(p => !p.Completion.Task.IsCompleted))
{
var batch = new List<Pending>();
while (queue.Items.TryDequeue(out var p)) batch.Add(p);
try
{
await CommitBatchAsync(archive, sequenceId, batch, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
foreach (var p in batch) p.Completion.TrySetException(ex);
}
}
}
finally
{
queue.Gate.Release();
}
return await Task.WhenAll(pending.Select(p => p.Completion.Task)).ConfigureAwait(false);
}
private async Task CommitBatchAsync(Archive archive, Guid sequenceId, List<Pending> batch, CancellationToken cancellationToken)
{
var row = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
?? throw new InvalidOperationException($"Sequence {sequenceId} is not indexed.");
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var sequence = paths.Sequence(row.Folder);
await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false))
{
var manifest = SequenceWriter.ReadLocked(sequence, sequenceId);
// Asked before this batch changes the folder: whether this index holds it as it is (ArchiveIndexer.InStepAsync), and
// when it does not, whether photos were copied in or removed by hand since the last scan, which this batch's
// resync leaves to the next one (ArchiveIndexer.PhotosChangedAsync).
var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
var photosChanged = !inStep && await _indexer.PhotosChangedAsync(archive, sequence, cancellationToken).ConfigureAwait(false);
var before = manifest.Frames.Count;
var held = manifest.Frames.Where(f => f.Sha256 is not null).ToLookup(f => f.Sha256!, StringComparer.OrdinalIgnoreCase);
var tombstoned = (manifest.Removed ?? []).Select(r => r.Sha256).ToHashSet(StringComparer.OrdinalIgnoreCase);
// Content this batch files (new frames and repairs): another copy of it in the batch is a duplicate of that frame.
var filed = new Dictionary<string, Guid>(StringComparer.OrdinalIgnoreCase);
var accepted = new List<(Pending Pending, SidecarFrame Frame)>();
var repairs = new List<(Pending Pending, SidecarFrame Frame)>();
var duplicates = new List<(Pending Pending, Guid FrameId)>();
foreach (var p in batch)
{
// Deleted from the sequence (its tombstone, and no frame holds the content) and not asked back: checked under the
// lease a deletion takes, so none lands between this check and the commit (owner review 2, R6).
if (p.Frame.RefuseRemoved && tombstoned.Contains(p.Frame.Sha256) && !held[p.Frame.Sha256].Any())
{
p.Completion.TrySetException(new PhotoRemovedException(sequenceId, p.Frame.Sha256));
continue;
}
if (filed.TryGetValue(p.Frame.Sha256, out var same))
{
duplicates.Add((p, same));
continue;
}
// Only a frame that still holds the content makes this copy a duplicate; a missing or damaged one is repaired with it.
var holders = held[p.Frame.Sha256];
if (holders.FirstOrDefault(f => IsIntact(sequence, f)) is { } intact)
{
duplicates.Add((p, intact.Id));
continue;
}
if (holders.FirstOrDefault() is { } damaged)
{
filed[p.Frame.Sha256] = damaged.Id;
repairs.Add((p, damaged));
continue;
}
var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), p.Frame.OriginalFileName, p.Frame.OriginalFileName, p.Frame.Size,
p.Frame.Sha256, p.Frame.Metadata, p.Frame.ClientCapturedAt);
filed[p.Frame.Sha256] = frame.Id;
accepted.Add((p, frame));
}
var named = Name(archive, sequence, manifest, accepted);
IngestJournal.Write(sequence, new IngestJournal(named.Values.ToList()));
var moved = new List<SidecarFrame>();
foreach (var (p, frame) in accepted)
{
var final = named[frame.Id];
try
{
Directory.CreateDirectory(Path.GetDirectoryName(sequence.File(final.File))!);
File.Move(p.Frame.StagedPath, sequence.File(final.File), overwrite: false);
moved.Add(final);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
p.Completion.TrySetException(ex);
}
}
var repaired = new HashSet<Guid>();
foreach (var (p, frame) in repairs)
{
if (Repair(sequence, frame, p.Frame.StagedPath) is { } error) p.Completion.TrySetException(error);
else repaired.Add(frame.Id);
}
var updated = SequenceFrames.Insert(manifest, moved);
if (repaired.Count > 0)
{
updated = SequenceFrames.Update(updated, repaired.ToDictionary(id => id,
_ => (Func<SidecarFrame, SidecarFrame>)(f => f with { Flags = f.Flags & ~(FrameFlags.Missing | FrameFlags.Corrupt) })));
}
// A photo deleted from the sequence and filed into it again (a card imported again, a sync download) is back: its
// tombstone goes, so sync offers it like any other.
var back = Readded(updated, moved);
if (back) updated = updated with { Removed = Remaining(updated.Removed!, moved) };
if (moved.Count > 0 || repaired.Count > 0) ManifestStore.WriteSequence(sequence.ManifestPath, updated);
IngestJournal.Delete(sequence);
var byId = updated.Frames.ToDictionary(f => f.Id);
if (moved.Count > 0 || repaired.Count > 0)
{
await IndexAsync(archive, sequence, updated, inStep, photosChanged, before, moved.ConvertAll(m => byId[m.Id]), repaired.Count > 0 || back,
cancellationToken).ConfigureAwait(false);
}
foreach (var (p, frame) in accepted.Where(a => byId.ContainsKey(a.Frame.Id)))
p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false));
foreach (var (p, frame) in repairs.Where(r => repaired.Contains(r.Frame.Id)))
p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false) with { Repaired = true });
var unrepaired = repairs.Select(r => r.Frame.Id).Where(id => !repaired.Contains(id)).ToHashSet();
foreach (var (p, frameId) in duplicates)
{
// A copy of content this batch failed to file has nothing intact to stand for it.
if (byId.TryGetValue(frameId, out var original) && !unrepaired.Contains(frameId))
p.Completion.TrySetResult(Result(sequenceId, sequence, original, true));
else p.Completion.TrySetException(new IOException(Loc.L("Archive.Error.DuplicateLost", p.Frame.OriginalFileName)));
}
}
}
/// <summary>Whether a frame filed by the batch holds content the sequence's tombstones name.</summary>
private static bool Readded(SequenceManifest manifest, List<SidecarFrame> moved)
{
if (manifest.Removed is not { Count: > 0 } removed || moved.Count == 0) return false;
var tombstoned = removed.Select(r => r.Sha256).ToHashSet(StringComparer.OrdinalIgnoreCase);
return moved.Any(f => f.Sha256 is { } sha && tombstoned.Contains(sha));
}
/// <summary>The tombstones without those of the content filed again; null when none is left.</summary>
private static IReadOnlyList<RemovedPhoto>? Remaining(IReadOnlyList<RemovedPhoto> removed, List<SidecarFrame> moved)
{
var filed = moved.Where(f => f.Sha256 is not null).Select(f => f.Sha256!).ToHashSet(StringComparer.OrdinalIgnoreCase);
var left = removed.Where(r => !filed.Contains(r.Sha256)).ToList();
return left.Count == 0 ? null : left;
}
/// <summary>Final file names, in capture order: the plan's name or the template with <c>{n}</c> = index + 1, made unique.</summary>
private static Dictionary<Guid, SidecarFrame> Name(Archive archive, SequencePaths sequence, SequenceManifest manifest,
List<(Pending Pending, SidecarFrame Frame)> accepted)
{
var merged = SequenceFrames.Insert(manifest, accepted.ConvertAll(a => a.Frame));
var incoming = accepted.ToDictionary(a => a.Frame.Id);
// Photos adopted in place stay in place; everything else goes to originals/.
var directory = Directory.Exists(sequence.Originals) || manifest.Frames.Count == 0 || manifest.Frames.Any(f => f.File.Contains('/'))
? ArchiveLayout.OriginalsFolder + "/"
: string.Empty;
var taken = new HashSet<string>(manifest.Frames.Select(f => f.File), StringComparer.OrdinalIgnoreCase);
var onDisk = Path.Combine(sequence.Folder, directory);
if (Directory.Exists(onDisk))
{
foreach (var file in Directory.EnumerateFiles(onDisk)) taken.Add(directory + Path.GetFileName(file));
}
var template = PathTemplate.Parse(archive.FileTemplate);
var start = merged.Frames.Count > 0 ? merged.Frames[0].CapturedAt : null;
var named = new Dictionary<Guid, SidecarFrame>();
foreach (var frame in merged.Frames)
{
if (!incoming.TryGetValue(frame.Id, out var entry)) continue;
var staged = entry.Pending.Frame;
var captured = frame.CapturedAt ?? DateTime.Now;
var name = staged.PlannedName ?? template.ExpandFile(Naming(staged, captured, start ?? captured, frame.I + 1, manifest.Title));
var file = directory + name;
var stem = Path.GetFileNameWithoutExtension(name);
for (var n = 2; !taken.Add(file); n++) file = string.Create(CultureInfo.InvariantCulture, $"{directory}{stem}_{n}{Path.GetExtension(name)}");
named[frame.Id] = entry.Frame with { File = file };
}
return named;
}
/// <summary>The frame still holds its content: not flagged missing or corrupt, and its file is there with its size.</summary>
internal static bool IsIntact(SequencePaths sequence, SidecarFrame frame)
{
if ((frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0) return false;
var file = new FileInfo(sequence.File(frame.File));
return file.Exists && file.Length == frame.Size;
}
/// <summary>
/// Puts a verified copy where a damaged frame's file belongs. A file still there (corrupt, or another size) moves aside
/// as <c>name.damaged-&lt;time&gt;</c>, which no scan takes for a photo, and is never deleted (BRIEF §0.2 rule 8); it comes
/// back when the copy cannot be moved in. Returns the failure, or null.
/// </summary>
private static Exception? Repair(SequencePaths sequence, SidecarFrame frame, string stagedPath)
{
var target = sequence.File(frame.File);
string? aside = null;
try
{
Directory.CreateDirectory(Path.GetDirectoryName(target)!);
if (File.Exists(target))
{
aside = string.Create(CultureInfo.InvariantCulture, $"{target}.damaged-{DateTime.UtcNow:yyyyMMddHHmmss}");
File.Move(target, aside, overwrite: false);
}
File.Move(stagedPath, target, overwrite: false);
Log.Info("archive", $"Repaired {sequence.ArchiveRelative(frame.File)} with a verified copy");
return null;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
if (aside is not null && !File.Exists(target) && File.Exists(aside))
{
try
{
File.Move(aside, target);
}
catch (Exception again) when (again is IOException or UnauthorizedAccessException)
{
Log.Warning("archive", $"{aside} stays beside {target}", again);
}
}
return ex;
}
}
/// <summary>
/// Appends the new rows when the index held the folder as it was before this batch and the batch only appended frames;
/// otherwise resyncs the whole sequence from the sidecar, which also moves the project's frame positions after an
/// insertion. That covers frames repaired (their flags change) and what another process changed in the sidecar alone
/// (a desktop's verify, decode tiers, duplicates or title): the appended rows and the scan stamps recorded after them
/// would otherwise keep it out of this index until a full scan. A project waiting for photos taken from another
/// copy's settings places those that arrived either way.
/// </summary>
/// <param name="inStep">Whether the index held the folder as it was when the batch read the sidecar under the lease
/// (<see cref="ArchiveIndexer.InStepAsync(Archive, SequencePaths, SequenceManifest, CancellationToken)"/>).</param>
/// <param name="photosChanged">Whether the photo directory had changed since the last scan when the batch began
/// (<see cref="ArchiveIndexer.PhotosChangedAsync"/>): the resync, which indexes the sidecar, keeps the recorded photo
/// stamp so the next incremental scan reconciles those photos.</param>
private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, bool inStep, bool photosChanged, int before,
List<SidecarFrame> appended, bool repaired, CancellationToken cancellationToken)
{
var append = inStep && !repaired && appended.All(f => f.I >= before)
&& await _db.Frames.CountBySequenceAsync(manifest.Id, cancellationToken).ConfigureAwait(false) == before;
if (!append)
{
await _indexer.SyncAsync(archive, sequence, manifest, null, SequenceStatus.None, photosChanged, cancellationToken).ConfigureAwait(false);
return;
}
PositionMove? placed = null;
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
{
// Locked: a settings edit committed after this read (the server's PATCH takes no lease) would be written back over.
var existing = await tx.Sequences.GetForUpdateAsync(manifest.Id, cancellationToken).ConfigureAwait(false);
if (existing is not null && ProjectPositions.Place(existing, manifest) is { } positions)
{
placed = positions;
existing = positions.ApplyTo(existing);
}
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
await tx.Frames.BulkInsertAsync(appended.Select(f => SequenceMapping.ToFrame(f, manifest.Id, sequence)), cancellationToken).ConfigureAwait(false);
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
}
if (placed is not null) ProjectPositions.WriteProject(sequence, placed.Before, placed.Moved);
await _indexer.RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
}
private static IngestResult Result(Guid sequenceId, SequencePaths sequence, SidecarFrame frame, bool duplicate) =>
new(sequenceId, frame.Id, frame.I, sequence.ArchiveRelative(frame.File), duplicate);
private static DateTime CaptureTime(StagedFrame frame) =>
frame.Metadata.TimestampSource is TimestampSource.FileSystem or TimestampSource.Unknown && frame.ClientCapturedAt is { } client
? client
: frame.Metadata.CapturedAt ?? frame.ClientCapturedAt ?? DateTime.Now;
private static NamingContext Naming(StagedFrame frame, DateTime captured, DateTime sessionStart, int number, string? title) => new()
{
CapturedAt = captured,
SessionStart = sessionStart,
Number = number,
NumberInSession = number,
CameraMake = frame.Metadata.Make,
CameraModel = frame.Metadata.Model,
Lens = frame.Metadata.LensModel,
Iso = frame.Metadata.Iso,
ExposureTime = frame.Metadata.ExposureTime,
FNumber = frame.Metadata.FNumber,
OriginalName = Path.GetFileNameWithoutExtension(frame.OriginalFileName),
Extension = Path.GetExtension(frame.OriginalFileName).TrimStart('.'),
Title = title,
};
private sealed class CommitQueue
{
public SemaphoreSlim Gate { get; } = new(1, 1);
public ConcurrentQueue<Pending> Items { get; } = new();
}
private sealed class Pending(StagedFrame frame)
{
public StagedFrame Frame { get; } = frame;
public TaskCompletionSource<IngestResult> Completion { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
}
}