Second fix round of owner review 2, phase 3 (engine, server, desktop). - A photo never hashed whose original is missing can be deleted again: it goes into its trash record without a file and without a tombstone (nothing is left to name), stays listed until Empty erases it, and a restore keeps it with the reason "no file". FrameDeleteBlock.Unhashable now blocks only a present original that cannot be read; the API answers it as 409 photo_unreadable naming the photo, and every block has its own code and message. - Deleting a whole sequence first settles an interrupted photo deletion, restore or ingest of it (ReadLocked under its lease), so no journal or half entry travels into the sequence's trash entry; it refuses while a restore into it waits for its busy entry. - FrameTrash.Settle takes the photo entry's lease and leaves the journal for later while the entry is busy (an emptying), so it never races EraseAsync; meanwhile no other restore writes its journal over it and scans do not adopt its files. - The hashes computed for frames that stay (to find a twin of a deleted photo) go into the sidecar and index, and only a frame whose first and last 64 KiB match a deleted photo's is read whole. - A restore into a new sequence rewrites its record after each photo filed, so a crash or a database failure never leaves filed photos listed. - An upload completion no longer has a window before the commit: the ingest checks the sidecar's tombstones under the sequence lease a deletion takes (IngestRequest.RefuseRemoved), unless the upload asked to bring it back. - A restore reports each kept photo with its reason (no longer matches its record, could not be moved back, already in the sequence, had no file); the desktop line and the API message say so in en and it, and the tooltip holds the whole sentence. The desktop tells TrashMismatchException by its type. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> (cherry picked from commit 1858f1f450a35fb8383ede8d80ce5f8de9b7ceac)
571 lines
32 KiB
C#
571 lines
32 KiB
C#
using System.Collections.Concurrent;
|
|
using System.Globalization;
|
|
using Chronos.Core.Archiving;
|
|
using Chronos.Core.Diagnostics;
|
|
using Chronos.Core.IO;
|
|
using Chronos.Core.Localization;
|
|
using Chronos.Core.Metadata;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Core.Naming;
|
|
|
|
namespace Chronos.Data.Archiving;
|
|
|
|
/// <summary>A verified staged file and how to name it.</summary>
|
|
/// <param name="PlannedName">Name chosen by an import plan; null = the archive's file template.</param>
|
|
/// <param name="RefuseRemoved">See <see cref="IngestRequest.RefuseRemoved"/>.</param>
|
|
internal sealed record StagedFrame(string StagedPath, string OriginalFileName, long Size, string Sha256, FrameMetadata Metadata,
|
|
DateTime? ClientCapturedAt, string? PlannedName, bool RefuseRemoved = false);
|
|
|
|
/// <summary>A sequence found or created by key (upload session, card session, or a new title).</summary>
|
|
/// <param name="PreferredFolder">Folder of an import plan; null = the archive's folder template.</param>
|
|
internal sealed record SequenceTarget(string Key, string? Title, string? PreferredFolder, string? CreatedBy);
|
|
|
|
/// <summary>
|
|
/// Commits staged, verified files into sequences (BRIEF §4.3, §14.4): the file is renamed into the sequence folder
|
|
/// under the naming template, the sidecar is rewritten atomically in capture order and the index follows. Commits
|
|
/// to one sequence are serialised and batched (concurrent uploads share one sidecar write); a lease file keeps
|
|
/// other processes out meanwhile, and a write-ahead journal repairs a batch interrupted by a crash. A file whose
|
|
/// SHA-256 an intact frame of the sequence already holds is reported as a duplicate and left in staging; a frame with
|
|
/// that SHA-256 that is flagged missing or corrupt, or whose file is gone or has another size, is repaired with it
|
|
/// instead (its damaged file kept beside it). A sequence created without photos gets its folder with its first frame.
|
|
/// </summary>
|
|
public sealed class ArchiveIngest : IArchiveIngest
|
|
{
|
|
private readonly ChronosDatabase _db;
|
|
private readonly SequenceWriter _writer;
|
|
private readonly ArchiveIndexer _indexer;
|
|
private readonly ConcurrentDictionary<Guid, CommitQueue> _queues = new();
|
|
private readonly SemaphoreSlim _creation = new(1, 1);
|
|
|
|
internal ArchiveIngest(ChronosDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
|
|
{
|
|
_db = db;
|
|
_writer = writer;
|
|
_indexer = indexer;
|
|
}
|
|
|
|
public async Task<IngestResult> CommitAsync(IngestRequest request, CancellationToken cancellationToken = default)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(request);
|
|
if ((request.SequenceId is null ? 0 : 1) + (request.NewSequenceTitle is null ? 0 : 1) + (request.SessionKey is null ? 0 : 1) != 1)
|
|
throw new ArgumentException("Exactly one of SequenceId, NewSequenceTitle or SessionKey selects the target.", nameof(request));
|
|
var archive = await _db.Archives.GetAsync(request.ArchiveId, cancellationToken).ConfigureAwait(false)
|
|
?? throw new ArgumentException($"Unknown archive {request.ArchiveId}.", nameof(request));
|
|
var staged = new FileInfo(request.StagedPath);
|
|
if (!staged.Exists || staged.Length != request.Size)
|
|
throw new FileNotFoundException("The staged file is missing or has another size.", request.StagedPath);
|
|
var frame = new StagedFrame(request.StagedPath, request.OriginalFileName, request.Size, request.Sha256,
|
|
MetadataReader.Read(request.StagedPath, cancellationToken), request.ClientCapturedAt, null, request.RefuseRemoved);
|
|
|
|
Guid sequenceId;
|
|
if (request.SequenceId is { } id)
|
|
{
|
|
var sequence = await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false);
|
|
if (sequence?.ArchiveId != archive.Id) throw new ArgumentException($"Sequence {id} is not in archive {archive.Id}.", nameof(request));
|
|
if (AwaitsFolder(sequence, new ArchivePaths(archive.RootPath))) await PlaceAsync(archive, id, frame, cancellationToken).ConfigureAwait(false);
|
|
sequenceId = id;
|
|
}
|
|
else
|
|
{
|
|
// A title names one sequence per user: parallel uploads for "new sequence X" all land in the same one.
|
|
var key = request.SessionKey ?? $"title:{request.UserId:N}:{request.NewSequenceTitle!.Trim().ToLowerInvariant()}";
|
|
sequenceId = await FindOrCreateAsync(archive, new SequenceTarget(key, request.NewSequenceTitle?.Trim(), null, request.UserId?.ToString()),
|
|
frame, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
return (await EnqueueAsync(archive, sequenceId, [frame], cancellationToken).ConfigureAwait(false))[0];
|
|
}
|
|
|
|
/// <summary>Commits files of one import session into the sequence of its key (created on first use).</summary>
|
|
internal async Task<IReadOnlyList<IngestResult>> CommitSessionAsync(Archive archive, SequenceTarget target, IReadOnlyList<StagedFrame> frames,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
if (frames.Count == 0) return [];
|
|
var sequenceId = await FindOrCreateAsync(archive, target, frames[0], cancellationToken).ConfigureAwait(false);
|
|
return await EnqueueAsync(archive, sequenceId, frames, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
private async Task<Guid> FindOrCreateAsync(Archive archive, SequenceTarget target, StagedFrame first, CancellationToken cancellationToken)
|
|
{
|
|
if (await _db.Sequences.FindBySessionKeyAsync(archive.Id, target.Key, cancellationToken).ConfigureAwait(false) is { } found) return found;
|
|
await _creation.WaitAsync(cancellationToken).ConfigureAwait(false);
|
|
try
|
|
{
|
|
if (await _db.Sequences.FindBySessionKeyAsync(archive.Id, target.Key, cancellationToken).ConfigureAwait(false) is { } again) return again;
|
|
return await CreateAsync(archive, target, first, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
finally
|
|
{
|
|
_creation.Release();
|
|
}
|
|
}
|
|
|
|
private async Task<Guid> CreateAsync(Archive archive, SequenceTarget target, StagedFrame first, CancellationToken cancellationToken)
|
|
{
|
|
var paths = new ArchivePaths(archive.RootPath);
|
|
ArchiveVolume.EnsureWritable(paths);
|
|
var now = DateTimeOffset.UtcNow;
|
|
var m = first.Metadata;
|
|
var manifest = new SequenceManifest
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
Title = target.Title,
|
|
Camera = new SidecarCamera { Make = m.Make, Model = m.Model, Serial = m.BodySerial },
|
|
Lens = m.LensModel,
|
|
SessionKeys = [target.Key],
|
|
CreatedBy = target.CreatedBy,
|
|
Created = now,
|
|
UpdatedAt = now,
|
|
};
|
|
var sequence = await ClaimNewFolderAsync(archive, paths, target.PreferredFolder, first, manifest, cancellationToken).ConfigureAwait(false);
|
|
|
|
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
|
|
{
|
|
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, null), cancellationToken).ConfigureAwait(false);
|
|
var winner = await tx.Sequences.MapSessionKeyAsync(archive.Id, target.Key, manifest.Id, cancellationToken).ConfigureAwait(false);
|
|
if (winner != manifest.Id)
|
|
{
|
|
// Another process bound the key first: its sequence wins, ours was never used.
|
|
await tx.RollbackAsync(cancellationToken).ConfigureAwait(false);
|
|
ReleaseFolder(sequence);
|
|
return winner;
|
|
}
|
|
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
|
}
|
|
await _indexer.RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
|
|
return manifest.Id;
|
|
}
|
|
|
|
/// <summary>
|
|
/// A sequence created without photos (<c>POST /api/v1/sequences</c>, BRIEF §14.3) has no folder until its first frame.
|
|
/// An empty folder otherwise means the archive root itself, adopted in place, whose sidecar names the sequence; one
|
|
/// that holds frames is never moved elsewhere, even while its root sidecar is missing.
|
|
/// </summary>
|
|
internal static bool AwaitsFolder(Sequence sequence, ArchivePaths paths)
|
|
{
|
|
if (sequence.Folder.Length > 0 || sequence.FrameCount > 0) return false;
|
|
var root = paths.Sequence(string.Empty);
|
|
if (!File.Exists(root.ManifestPath)) return true;
|
|
try
|
|
{
|
|
return ManifestStore.ReadSequence(root.ManifestPath).Id != sequence.Id;
|
|
}
|
|
catch (Exception ex) when (ex is IOException or InvalidDataException)
|
|
{
|
|
// An unreadable root sidecar: the commit reports it rather than placing the sequence elsewhere.
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gives a sequence created without photos its folder, named from its first frame and title by the same rules as a
|
|
/// sequence created by an upload, with a sidecar carrying the row's id, title, tags, rating and notes (and its project
|
|
/// when one was saved); the index row takes the folder in one transaction. Serialised with the creation of sequences.
|
|
/// </summary>
|
|
private async Task PlaceAsync(Archive archive, Guid sequenceId, StagedFrame first, CancellationToken cancellationToken)
|
|
{
|
|
await _creation.WaitAsync(cancellationToken).ConfigureAwait(false);
|
|
try
|
|
{
|
|
var paths = new ArchivePaths(archive.RootPath);
|
|
var row = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
|
|
?? throw new InvalidOperationException($"Sequence {sequenceId} is not indexed.");
|
|
// A concurrent first frame placed it meanwhile.
|
|
if (!AwaitsFolder(row, paths)) return;
|
|
ArchiveVolume.EnsureWritable(paths);
|
|
var now = DateTimeOffset.UtcNow;
|
|
var m = first.Metadata;
|
|
var manifest = new SequenceManifest
|
|
{
|
|
Id = row.Id,
|
|
Title = row.Title,
|
|
// The photo names its camera; the creator's hint stands in when it does not.
|
|
Camera = m.Make is null && m.Model is null && row.CameraModel is not null
|
|
? new SidecarCamera { Model = row.CameraModel }
|
|
: new SidecarCamera { Make = m.Make, Model = m.Model, Serial = m.BodySerial },
|
|
Lens = m.LensModel,
|
|
Tags = row.Tags,
|
|
Rating = row.Rating,
|
|
Notes = row.Notes,
|
|
Versions = new ManifestVersions { Settings = row.SettingsVersion, Frames = row.FramesVersion },
|
|
CreatedBy = row.OwnerUserId?.ToString(),
|
|
Created = now,
|
|
UpdatedAt = now,
|
|
};
|
|
var sequence = await ClaimNewFolderAsync(archive, paths, null, first, manifest, cancellationToken).ConfigureAwait(false);
|
|
try
|
|
{
|
|
if (row.SettingsJson is { } settings) AtomicFile.WriteAllText(sequence.ProjectPath, settings);
|
|
await using var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false);
|
|
// Locked, so an edit of the new sequence committed meanwhile is not written back over.
|
|
var current = await tx.Sequences.GetForUpdateAsync(sequenceId, cancellationToken).ConfigureAwait(false)
|
|
?? throw new InvalidOperationException($"Sequence {sequenceId} was deleted before its first frame.");
|
|
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, current), cancellationToken).ConfigureAwait(false);
|
|
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
|
}
|
|
catch
|
|
{
|
|
// Nothing was filed into the folder yet: it goes, and the next first frame claims one again.
|
|
if (File.Exists(sequence.ProjectPath)) File.Delete(sequence.ProjectPath);
|
|
ReleaseFolder(sequence);
|
|
throw;
|
|
}
|
|
await _indexer.RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
finally
|
|
{
|
|
_creation.Release();
|
|
}
|
|
}
|
|
|
|
/// <summary>A new folder for <paramref name="manifest"/>: the plan's folder or the archive's folder template, made unique.</summary>
|
|
private async Task<SequencePaths> ClaimNewFolderAsync(Archive archive, ArchivePaths paths, string? preferredFolder, StagedFrame first,
|
|
SequenceManifest manifest, CancellationToken cancellationToken)
|
|
{
|
|
var captured = CaptureTime(first);
|
|
var wanted = preferredFolder ?? PathTemplate.Parse(archive.FolderTemplate).ExpandFolder(Naming(first, captured, captured, 1, manifest.Title));
|
|
if (wanted.Length == 0) wanted = "Sequence";
|
|
var taken = (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false))
|
|
.Select(l => l.Folder).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
|
return ClaimFolder(paths, wanted, taken, manifest);
|
|
}
|
|
|
|
/// <summary>The first free variant of <paramref name="wanted"/>, claimed by creating its sidecar (first writer wins).</summary>
|
|
private static SequencePaths ClaimFolder(ArchivePaths paths, string wanted, HashSet<string> taken, SequenceManifest manifest)
|
|
{
|
|
for (var n = 1; ; n++)
|
|
{
|
|
var folder = n == 1 ? wanted : string.Create(CultureInfo.InvariantCulture, $"{wanted} ({n})");
|
|
if (taken.Contains(folder)) continue;
|
|
var sequence = paths.Sequence(folder);
|
|
if (Directory.Exists(sequence.Folder) && Directory.EnumerateFileSystemEntries(sequence.Folder).Any()) continue;
|
|
Directory.CreateDirectory(sequence.MetaDirectory);
|
|
if (ManifestStore.TryCreateSequence(sequence.ManifestPath, manifest)) return sequence;
|
|
}
|
|
}
|
|
|
|
/// <summary>Removes the empty sequence this process created and lost to a concurrent creator.</summary>
|
|
private static void ReleaseFolder(SequencePaths sequence)
|
|
{
|
|
File.Delete(sequence.ManifestPath);
|
|
if (!Directory.EnumerateFileSystemEntries(sequence.MetaDirectory).Any()) Directory.Delete(sequence.MetaDirectory);
|
|
if (!Directory.EnumerateFileSystemEntries(sequence.Folder).Any()) Directory.Delete(sequence.Folder);
|
|
}
|
|
|
|
/// <summary>Group commit: whoever gets the sequence's gate commits everything queued for it.</summary>
|
|
private async Task<IngestResult[]> EnqueueAsync(Archive archive, Guid sequenceId, IReadOnlyList<StagedFrame> frames, CancellationToken cancellationToken)
|
|
{
|
|
var queue = _queues.GetOrAdd(sequenceId, _ => new CommitQueue());
|
|
var pending = frames.Select(f => new Pending(f)).ToList();
|
|
foreach (var p in pending) queue.Items.Enqueue(p);
|
|
await queue.Gate.WaitAsync(cancellationToken).ConfigureAwait(false);
|
|
try
|
|
{
|
|
if (pending.Any(p => !p.Completion.Task.IsCompleted))
|
|
{
|
|
var batch = new List<Pending>();
|
|
while (queue.Items.TryDequeue(out var p)) batch.Add(p);
|
|
try
|
|
{
|
|
await CommitBatchAsync(archive, sequenceId, batch, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
foreach (var p in batch) p.Completion.TrySetException(ex);
|
|
}
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
queue.Gate.Release();
|
|
}
|
|
return await Task.WhenAll(pending.Select(p => p.Completion.Task)).ConfigureAwait(false);
|
|
}
|
|
|
|
private async Task CommitBatchAsync(Archive archive, Guid sequenceId, List<Pending> batch, CancellationToken cancellationToken)
|
|
{
|
|
var row = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
|
|
?? throw new InvalidOperationException($"Sequence {sequenceId} is not indexed.");
|
|
var paths = new ArchivePaths(archive.RootPath);
|
|
ArchiveVolume.EnsureWritable(paths);
|
|
var sequence = paths.Sequence(row.Folder);
|
|
await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false))
|
|
{
|
|
var manifest = SequenceWriter.ReadLocked(sequence, sequenceId);
|
|
// Asked before this batch changes the folder: whether this index holds it as it is (ArchiveIndexer.InStepAsync), and
|
|
// when it does not, whether photos were copied in or removed by hand since the last scan, which this batch's
|
|
// resync leaves to the next one (ArchiveIndexer.PhotosChangedAsync).
|
|
var inStep = await _indexer.InStepAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
|
|
var photosChanged = !inStep && await _indexer.PhotosChangedAsync(archive, sequence, cancellationToken).ConfigureAwait(false);
|
|
var before = manifest.Frames.Count;
|
|
var held = manifest.Frames.Where(f => f.Sha256 is not null).ToLookup(f => f.Sha256!, StringComparer.OrdinalIgnoreCase);
|
|
var tombstoned = (manifest.Removed ?? []).Select(r => r.Sha256).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
|
// Content this batch files (new frames and repairs): another copy of it in the batch is a duplicate of that frame.
|
|
var filed = new Dictionary<string, Guid>(StringComparer.OrdinalIgnoreCase);
|
|
|
|
var accepted = new List<(Pending Pending, SidecarFrame Frame)>();
|
|
var repairs = new List<(Pending Pending, SidecarFrame Frame)>();
|
|
var duplicates = new List<(Pending Pending, Guid FrameId)>();
|
|
foreach (var p in batch)
|
|
{
|
|
// Deleted from the sequence (its tombstone, and no frame holds the content) and not asked back: checked under the
|
|
// lease a deletion takes, so none lands between this check and the commit (owner review 2, R6).
|
|
if (p.Frame.RefuseRemoved && tombstoned.Contains(p.Frame.Sha256) && !held[p.Frame.Sha256].Any())
|
|
{
|
|
p.Completion.TrySetException(new PhotoRemovedException(sequenceId, p.Frame.Sha256));
|
|
continue;
|
|
}
|
|
if (filed.TryGetValue(p.Frame.Sha256, out var same))
|
|
{
|
|
duplicates.Add((p, same));
|
|
continue;
|
|
}
|
|
// Only a frame that still holds the content makes this copy a duplicate; a missing or damaged one is repaired with it.
|
|
var holders = held[p.Frame.Sha256];
|
|
if (holders.FirstOrDefault(f => IsIntact(sequence, f)) is { } intact)
|
|
{
|
|
duplicates.Add((p, intact.Id));
|
|
continue;
|
|
}
|
|
if (holders.FirstOrDefault() is { } damaged)
|
|
{
|
|
filed[p.Frame.Sha256] = damaged.Id;
|
|
repairs.Add((p, damaged));
|
|
continue;
|
|
}
|
|
var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), p.Frame.OriginalFileName, p.Frame.OriginalFileName, p.Frame.Size,
|
|
p.Frame.Sha256, p.Frame.Metadata, p.Frame.ClientCapturedAt);
|
|
filed[p.Frame.Sha256] = frame.Id;
|
|
accepted.Add((p, frame));
|
|
}
|
|
|
|
var named = Name(archive, sequence, manifest, accepted);
|
|
IngestJournal.Write(sequence, new IngestJournal(named.Values.ToList()));
|
|
var moved = new List<SidecarFrame>();
|
|
foreach (var (p, frame) in accepted)
|
|
{
|
|
var final = named[frame.Id];
|
|
try
|
|
{
|
|
Directory.CreateDirectory(Path.GetDirectoryName(sequence.File(final.File))!);
|
|
File.Move(p.Frame.StagedPath, sequence.File(final.File), overwrite: false);
|
|
moved.Add(final);
|
|
}
|
|
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
|
{
|
|
p.Completion.TrySetException(ex);
|
|
}
|
|
}
|
|
var repaired = new HashSet<Guid>();
|
|
foreach (var (p, frame) in repairs)
|
|
{
|
|
if (Repair(sequence, frame, p.Frame.StagedPath) is { } error) p.Completion.TrySetException(error);
|
|
else repaired.Add(frame.Id);
|
|
}
|
|
var updated = SequenceFrames.Insert(manifest, moved);
|
|
if (repaired.Count > 0)
|
|
{
|
|
updated = SequenceFrames.Update(updated, repaired.ToDictionary(id => id,
|
|
_ => (Func<SidecarFrame, SidecarFrame>)(f => f with { Flags = f.Flags & ~(FrameFlags.Missing | FrameFlags.Corrupt) })));
|
|
}
|
|
// A photo deleted from the sequence and filed into it again (a card imported again, a sync download) is back: its
|
|
// tombstone goes, so sync offers it like any other.
|
|
var back = Readded(updated, moved);
|
|
if (back) updated = updated with { Removed = Remaining(updated.Removed!, moved) };
|
|
if (moved.Count > 0 || repaired.Count > 0) ManifestStore.WriteSequence(sequence.ManifestPath, updated);
|
|
IngestJournal.Delete(sequence);
|
|
|
|
var byId = updated.Frames.ToDictionary(f => f.Id);
|
|
if (moved.Count > 0 || repaired.Count > 0)
|
|
{
|
|
await IndexAsync(archive, sequence, updated, inStep, photosChanged, before, moved.ConvertAll(m => byId[m.Id]), repaired.Count > 0 || back,
|
|
cancellationToken).ConfigureAwait(false);
|
|
}
|
|
foreach (var (p, frame) in accepted.Where(a => byId.ContainsKey(a.Frame.Id)))
|
|
p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false));
|
|
foreach (var (p, frame) in repairs.Where(r => repaired.Contains(r.Frame.Id)))
|
|
p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false) with { Repaired = true });
|
|
var unrepaired = repairs.Select(r => r.Frame.Id).Where(id => !repaired.Contains(id)).ToHashSet();
|
|
foreach (var (p, frameId) in duplicates)
|
|
{
|
|
// A copy of content this batch failed to file has nothing intact to stand for it.
|
|
if (byId.TryGetValue(frameId, out var original) && !unrepaired.Contains(frameId))
|
|
p.Completion.TrySetResult(Result(sequenceId, sequence, original, true));
|
|
else p.Completion.TrySetException(new IOException(Loc.L("Archive.Error.DuplicateLost", p.Frame.OriginalFileName)));
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>Whether a frame filed by the batch holds content the sequence's tombstones name.</summary>
|
|
private static bool Readded(SequenceManifest manifest, List<SidecarFrame> moved)
|
|
{
|
|
if (manifest.Removed is not { Count: > 0 } removed || moved.Count == 0) return false;
|
|
var tombstoned = removed.Select(r => r.Sha256).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
|
return moved.Any(f => f.Sha256 is { } sha && tombstoned.Contains(sha));
|
|
}
|
|
|
|
/// <summary>The tombstones without those of the content filed again; null when none is left.</summary>
|
|
private static IReadOnlyList<RemovedPhoto>? Remaining(IReadOnlyList<RemovedPhoto> removed, List<SidecarFrame> moved)
|
|
{
|
|
var filed = moved.Where(f => f.Sha256 is not null).Select(f => f.Sha256!).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
|
var left = removed.Where(r => !filed.Contains(r.Sha256)).ToList();
|
|
return left.Count == 0 ? null : left;
|
|
}
|
|
|
|
/// <summary>Final file names, in capture order: the plan's name or the template with <c>{n}</c> = index + 1, made unique.</summary>
|
|
private static Dictionary<Guid, SidecarFrame> Name(Archive archive, SequencePaths sequence, SequenceManifest manifest,
|
|
List<(Pending Pending, SidecarFrame Frame)> accepted)
|
|
{
|
|
var merged = SequenceFrames.Insert(manifest, accepted.ConvertAll(a => a.Frame));
|
|
var incoming = accepted.ToDictionary(a => a.Frame.Id);
|
|
// Photos adopted in place stay in place; everything else goes to originals/.
|
|
var directory = Directory.Exists(sequence.Originals) || manifest.Frames.Count == 0 || manifest.Frames.Any(f => f.File.Contains('/'))
|
|
? ArchiveLayout.OriginalsFolder + "/"
|
|
: string.Empty;
|
|
var taken = new HashSet<string>(manifest.Frames.Select(f => f.File), StringComparer.OrdinalIgnoreCase);
|
|
var onDisk = Path.Combine(sequence.Folder, directory);
|
|
if (Directory.Exists(onDisk))
|
|
{
|
|
foreach (var file in Directory.EnumerateFiles(onDisk)) taken.Add(directory + Path.GetFileName(file));
|
|
}
|
|
var template = PathTemplate.Parse(archive.FileTemplate);
|
|
var start = merged.Frames.Count > 0 ? merged.Frames[0].CapturedAt : null;
|
|
var named = new Dictionary<Guid, SidecarFrame>();
|
|
foreach (var frame in merged.Frames)
|
|
{
|
|
if (!incoming.TryGetValue(frame.Id, out var entry)) continue;
|
|
var staged = entry.Pending.Frame;
|
|
var captured = frame.CapturedAt ?? DateTime.Now;
|
|
var name = staged.PlannedName ?? template.ExpandFile(Naming(staged, captured, start ?? captured, frame.I + 1, manifest.Title));
|
|
var file = directory + name;
|
|
var stem = Path.GetFileNameWithoutExtension(name);
|
|
for (var n = 2; !taken.Add(file); n++) file = string.Create(CultureInfo.InvariantCulture, $"{directory}{stem}_{n}{Path.GetExtension(name)}");
|
|
named[frame.Id] = entry.Frame with { File = file };
|
|
}
|
|
return named;
|
|
}
|
|
|
|
/// <summary>The frame still holds its content: not flagged missing or corrupt, and its file is there with its size.</summary>
|
|
internal static bool IsIntact(SequencePaths sequence, SidecarFrame frame)
|
|
{
|
|
if ((frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0) return false;
|
|
var file = new FileInfo(sequence.File(frame.File));
|
|
return file.Exists && file.Length == frame.Size;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Puts a verified copy where a damaged frame's file belongs. A file still there (corrupt, or another size) moves aside
|
|
/// as <c>name.damaged-<time></c>, which no scan takes for a photo, and is never deleted (BRIEF §0.2 rule 8); it comes
|
|
/// back when the copy cannot be moved in. Returns the failure, or null.
|
|
/// </summary>
|
|
private static Exception? Repair(SequencePaths sequence, SidecarFrame frame, string stagedPath)
|
|
{
|
|
var target = sequence.File(frame.File);
|
|
string? aside = null;
|
|
try
|
|
{
|
|
Directory.CreateDirectory(Path.GetDirectoryName(target)!);
|
|
if (File.Exists(target))
|
|
{
|
|
aside = string.Create(CultureInfo.InvariantCulture, $"{target}.damaged-{DateTime.UtcNow:yyyyMMddHHmmss}");
|
|
File.Move(target, aside, overwrite: false);
|
|
}
|
|
File.Move(stagedPath, target, overwrite: false);
|
|
Log.Info("archive", $"Repaired {sequence.ArchiveRelative(frame.File)} with a verified copy");
|
|
return null;
|
|
}
|
|
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
|
{
|
|
if (aside is not null && !File.Exists(target) && File.Exists(aside))
|
|
{
|
|
try
|
|
{
|
|
File.Move(aside, target);
|
|
}
|
|
catch (Exception again) when (again is IOException or UnauthorizedAccessException)
|
|
{
|
|
Log.Warning("archive", $"{aside} stays beside {target}", again);
|
|
}
|
|
}
|
|
return ex;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Appends the new rows when the index held the folder as it was before this batch and the batch only appended frames;
|
|
/// otherwise resyncs the whole sequence from the sidecar, which also moves the project's frame positions after an
|
|
/// insertion. That covers frames repaired (their flags change) and what another process changed in the sidecar alone
|
|
/// (a desktop's verify, decode tiers, duplicates or title): the appended rows and the scan stamps recorded after them
|
|
/// would otherwise keep it out of this index until a full scan. A project waiting for photos taken from another
|
|
/// copy's settings places those that arrived either way.
|
|
/// </summary>
|
|
/// <param name="inStep">Whether the index held the folder as it was when the batch read the sidecar under the lease
|
|
/// (<see cref="ArchiveIndexer.InStepAsync(Archive, SequencePaths, SequenceManifest, CancellationToken)"/>).</param>
|
|
/// <param name="photosChanged">Whether the photo directory had changed since the last scan when the batch began
|
|
/// (<see cref="ArchiveIndexer.PhotosChangedAsync"/>): the resync, which indexes the sidecar, keeps the recorded photo
|
|
/// stamp so the next incremental scan reconciles those photos.</param>
|
|
private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, bool inStep, bool photosChanged, int before,
|
|
List<SidecarFrame> appended, bool repaired, CancellationToken cancellationToken)
|
|
{
|
|
var append = inStep && !repaired && appended.All(f => f.I >= before)
|
|
&& await _db.Frames.CountBySequenceAsync(manifest.Id, cancellationToken).ConfigureAwait(false) == before;
|
|
if (!append)
|
|
{
|
|
await _indexer.SyncAsync(archive, sequence, manifest, null, SequenceStatus.None, photosChanged, cancellationToken).ConfigureAwait(false);
|
|
return;
|
|
}
|
|
PositionMove? placed = null;
|
|
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
|
|
{
|
|
// Locked: a settings edit committed after this read (the server's PATCH takes no lease) would be written back over.
|
|
var existing = await tx.Sequences.GetForUpdateAsync(manifest.Id, cancellationToken).ConfigureAwait(false);
|
|
if (existing is not null && ProjectPositions.Place(existing, manifest) is { } positions)
|
|
{
|
|
placed = positions;
|
|
existing = positions.ApplyTo(existing);
|
|
}
|
|
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
|
|
await tx.Frames.BulkInsertAsync(appended.Select(f => SequenceMapping.ToFrame(f, manifest.Id, sequence)), cancellationToken).ConfigureAwait(false);
|
|
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
|
}
|
|
if (placed is not null) ProjectPositions.WriteProject(sequence, placed.Before, placed.Moved);
|
|
await _indexer.RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
private static IngestResult Result(Guid sequenceId, SequencePaths sequence, SidecarFrame frame, bool duplicate) =>
|
|
new(sequenceId, frame.Id, frame.I, sequence.ArchiveRelative(frame.File), duplicate);
|
|
|
|
private static DateTime CaptureTime(StagedFrame frame) =>
|
|
frame.Metadata.TimestampSource is TimestampSource.FileSystem or TimestampSource.Unknown && frame.ClientCapturedAt is { } client
|
|
? client
|
|
: frame.Metadata.CapturedAt ?? frame.ClientCapturedAt ?? DateTime.Now;
|
|
|
|
private static NamingContext Naming(StagedFrame frame, DateTime captured, DateTime sessionStart, int number, string? title) => new()
|
|
{
|
|
CapturedAt = captured,
|
|
SessionStart = sessionStart,
|
|
Number = number,
|
|
NumberInSession = number,
|
|
CameraMake = frame.Metadata.Make,
|
|
CameraModel = frame.Metadata.Model,
|
|
Lens = frame.Metadata.LensModel,
|
|
Iso = frame.Metadata.Iso,
|
|
ExposureTime = frame.Metadata.ExposureTime,
|
|
FNumber = frame.Metadata.FNumber,
|
|
OriginalName = Path.GetFileNameWithoutExtension(frame.OriginalFileName),
|
|
Extension = Path.GetExtension(frame.OriginalFileName).TrimStart('.'),
|
|
Title = title,
|
|
};
|
|
|
|
private sealed class CommitQueue
|
|
{
|
|
public SemaphoreSlim Gate { get; } = new(1, 1);
|
|
public ConcurrentQueue<Pending> Items { get; } = new();
|
|
}
|
|
|
|
private sealed class Pending(StagedFrame frame)
|
|
{
|
|
public StagedFrame Frame { get; } = frame;
|
|
public TaskCompletionSource<IngestResult> Completion { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
}
|
|
}
|