feat: owner review 2 phase 4b: mass conversion after import (#13)
CI / Build and test (push) Failing after 3m1s
CI / PostgreSQL 16 integration (push) Skipped
CI / PostgreSQL 17 integration (push) Skipped
CI / Docker image (push) Skipped
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 10s
Release / Build, test and publish (push) Successful in 15m31s
Release / Server image (push) Successful in 1m13s
CI / Build and test (push) Failing after 3m1s
CI / PostgreSQL 16 integration (push) Skipped
CI / PostgreSQL 17 integration (push) Skipped
CI / Docker image (push) Skipped
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 10s
Release / Build, test and publish (push) Successful in 15m31s
Release / Server image (push) Successful in 1m13s
Convert photos already in the archive to JPEG, TIFF-16 or Linear DNG, from the panel's Library and the desktop's Archive and Library. Each run asks Replace (same frames, originals to the trash, recoverable) or New sequence. A dry run lists what converts, what is skipped and why, size before and after, and free space; the job converts exactly the confirmed photos, resumes, and commits all or nothing per sequence. A conversion never changes what Chronos reads from a photo. Replaced originals are erased only on the archive's own word that their conversion is gone. Desktop TIFFs that WIC showed black or as noise now go to the managed reader; the Edit picker no longer spins a core. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
This commit was merged in pull request #13.
This commit is contained in:
1 parent
7e4a3b481d
commit
efa87f68ce
354 files changed
+24530
-843
No files matched your search
@@ -292,11 +292,12 @@ public sealed class ArchiveIndexer
|
||||
/// </summary>
|
||||
private static SequenceManifest? Reconcile(SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken)
|
||||
{
|
||||
// A restore from the trash that could not be settled yet (its entry busy) may have photos in the folder its sidecar
|
||||
// does not list: they are its own, not photos to adopt, until a later writer settles it.
|
||||
if (File.Exists(sequence.RestoreJournalPath))
|
||||
// A restore from the trash or a frame replacement that could not be settled yet (its entry busy, a file that could not
|
||||
// go back) may have photos in the folder its sidecar does not list: they are its own, not photos to adopt, until a
|
||||
// later writer settles it.
|
||||
if (File.Exists(sequence.RestoreJournalPath) || File.Exists(sequence.ReplaceJournalPath))
|
||||
{
|
||||
Log.Info("archive", $"{sequence.RelativeFolder} is not reconciled while a restore from the trash into it waits");
|
||||
Log.Info("archive", $"{sequence.RelativeFolder} is not reconciled while a restore from the trash or a replacement of its frames waits");
|
||||
return null;
|
||||
}
|
||||
return SequenceAdopter.Reconcile(sequence, manifest, cancellationToken) is { } reconciled
|
||||
@@ -369,8 +370,10 @@ public sealed class ArchiveIndexer
|
||||
ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
|
||||
bytes = null;
|
||||
}
|
||||
// A journal of an ingest, a photo deletion or a restore a crash interrupted is settled under the lease first (ReadLocked).
|
||||
if (reconcile || File.Exists(sequence.IngestJournalPath) || File.Exists(sequence.RemovalJournalPath) || File.Exists(sequence.RestoreJournalPath))
|
||||
// A journal of an ingest, a photo deletion, a frame replacement or a restore a crash interrupted is settled under the
|
||||
// lease first (ReadLocked).
|
||||
if (reconcile || File.Exists(sequence.IngestJournalPath) || File.Exists(sequence.RemovalJournalPath) || File.Exists(sequence.RestoreJournalPath)
|
||||
|| File.Exists(sequence.ReplaceJournalPath))
|
||||
{
|
||||
// Photos added, gone or back clear what covered the frames, and the scan's follow-ups make it again.
|
||||
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id, m => Reconcile(sequence, m, cancellationToken), cancellationToken)
|
||||
|
||||
@@ -14,7 +14,18 @@ namespace Chronos.Data.Archiving;
|
||||
/// <param name="PlannedName">Name chosen by an import plan; null = the archive's file template.</param>
|
||||
/// <param name="RefuseRemoved">See <see cref="IngestRequest.RefuseRemoved"/>.</param>
|
||||
internal sealed record StagedFrame(string StagedPath, string OriginalFileName, long Size, string Sha256, FrameMetadata Metadata,
|
||||
DateTime? ClientCapturedAt, string? PlannedName, bool RefuseRemoved = false);
|
||||
DateTime? ClientCapturedAt, string? PlannedName, bool RefuseRemoved = false)
|
||||
{
|
||||
/// <summary>
|
||||
/// The file name the template names the photo from (its <c>{ext}</c>) when it is not <see cref="OriginalFileName"/>,
|
||||
/// which stays its <c>{name}</c> and the frame's original: a conversion's copy, the original's name with the target's
|
||||
/// extension (owner review 2, R7). Null: <see cref="OriginalFileName"/>.
|
||||
/// </summary>
|
||||
public string? NamedAs { get; init; }
|
||||
|
||||
/// <summary>The frame's file before it is named (<see cref="SidecarFrame.File"/> as the templates see it).</summary>
|
||||
public string NamingFile => NamedAs ?? OriginalFileName;
|
||||
}
|
||||
|
||||
/// <summary>A sequence found or created by key (upload session, card session, or a new title).</summary>
|
||||
/// <param name="PreferredFolder">Folder of an import plan; null = the archive's folder template.</param>
|
||||
@@ -27,7 +38,9 @@ internal sealed record SequenceTarget(string Key, string? Title, string? Preferr
|
||||
/// other processes out meanwhile, and a write-ahead journal repairs a batch interrupted by a crash. A file whose
|
||||
/// SHA-256 an intact frame of the sequence already holds is reported as a duplicate and left in staging; a frame with
|
||||
/// that SHA-256 that is flagged missing or corrupt, or whose file is gone or has another size, is repaired with it
|
||||
/// instead (its damaged file kept beside it). A sequence created without photos gets its folder with its first frame.
|
||||
/// instead (its damaged file kept beside it). A file whose content a conversion replaced in the sequence is a duplicate of
|
||||
/// the intact frame that holds what replaced it (owner review 2, R7). A sequence created without photos gets its folder
|
||||
/// with its first frame. A restore from the trash or a frame replacement of the sequence not settled yet refuses the commit.
|
||||
/// </summary>
|
||||
public sealed class ArchiveIngest : IArchiveIngest
|
||||
{
|
||||
@@ -224,7 +237,7 @@ public sealed class ArchiveIngest : IArchiveIngest
|
||||
private async Task<SequencePaths> ClaimNewFolderAsync(Archive archive, ArchivePaths paths, string? preferredFolder, StagedFrame first,
|
||||
SequenceManifest manifest, CancellationToken cancellationToken)
|
||||
{
|
||||
var frame = SequenceFrames.FromMetadata(Guid.Empty, first.OriginalFileName, first.OriginalFileName, first.Size, first.Sha256, first.Metadata,
|
||||
var frame = SequenceFrames.FromMetadata(Guid.Empty, first.NamingFile, first.OriginalFileName, first.Size, first.Sha256, first.Metadata,
|
||||
first.ClientCapturedAt);
|
||||
var wanted = preferredFolder
|
||||
?? PathTemplate.Parse(archive.FolderTemplate).ExpandFolder(FrameNaming.Context(frame, manifest with { Frames = [frame] }, manifest.Title));
|
||||
@@ -296,6 +309,10 @@ public sealed class ArchiveIngest : IArchiveIngest
|
||||
{
|
||||
var sequence = await SequenceWriter.FolderAsync(_db, paths, row, cancellationToken).ConfigureAwait(false);
|
||||
var manifest = SequenceWriter.ReadLocked(sequence, sequenceId);
|
||||
// A restore from the trash or a frame replacement not settled yet (its entry busy) is settled before frames are
|
||||
// filed beside it, as every writer of the frames does: the commit fails and is tried again (an upload's client).
|
||||
FrameTrash.EnsureSettled(sequence);
|
||||
FrameTrash.EnsureReplaced(sequence);
|
||||
// Asked before this batch changes the folder: whether this index holds it as it is (ArchiveIndexer.InStepAsync), and
|
||||
// when it does not, whether photos were copied in or removed by hand since the last scan, which this batch's
|
||||
// resync leaves to the next one (ArchiveIndexer.PhotosChangedAsync).
|
||||
@@ -304,6 +321,7 @@ public sealed class ArchiveIngest : IArchiveIngest
|
||||
var before = manifest.Frames.Count;
|
||||
var held = manifest.Frames.Where(f => f.Sha256 is not null).ToLookup(f => f.Sha256!, StringComparer.OrdinalIgnoreCase);
|
||||
var tombstoned = (manifest.Removed ?? []).Select(r => r.Sha256).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var replacedBy = Replacements(manifest);
|
||||
// Content this batch files (new frames and repairs): another copy of it in the batch is a duplicate of that frame.
|
||||
var filed = new Dictionary<string, Guid>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
@@ -337,7 +355,15 @@ public sealed class ArchiveIngest : IArchiveIngest
|
||||
repairs.Add((p, damaged));
|
||||
continue;
|
||||
}
|
||||
var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), p.Frame.OriginalFileName, p.Frame.OriginalFileName, p.Frame.Size,
|
||||
// A photo a conversion replaced here (owner review 2, R7) is in the sequence as what replaced it: a card or a
|
||||
// folder imported again, or a download of the original, is a duplicate of that frame, and the tombstone that
|
||||
// makes sync count both as one photo stays. A replacement no longer intact leaves the photo to be filed again.
|
||||
if (ReplacedBy(sequence, p.Frame.Sha256, replacedBy, held) is { } replacement)
|
||||
{
|
||||
duplicates.Add((p, replacement.Id));
|
||||
continue;
|
||||
}
|
||||
var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), p.Frame.NamingFile, p.Frame.OriginalFileName, p.Frame.Size,
|
||||
p.Frame.Sha256, p.Frame.Metadata, p.Frame.ClientCapturedAt);
|
||||
filed[p.Frame.Sha256] = frame.Id;
|
||||
accepted.Add((p, frame));
|
||||
@@ -400,6 +426,32 @@ public sealed class ArchiveIngest : IArchiveIngest
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The sequence's Replaced tombstones (owner review 2, R7): each content a conversion replaced, and what replaced it.</summary>
|
||||
private static Dictionary<string, string> Replacements(SequenceManifest manifest)
|
||||
{
|
||||
var replaced = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var removal in manifest.Removed ?? [])
|
||||
{
|
||||
if (removal.Reason == RemovalReason.Replaced && removal.Replacement is { Length: > 0 } replacement) replaced[removal.Sha256] = replacement;
|
||||
}
|
||||
return replaced;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The intact frame that holds what replaced <paramref name="sha256"/> in the sequence, followed through later conversions
|
||||
/// (a conversion of a conversion); null when the content was not replaced, or no frame holds a replacement intact.
|
||||
/// </summary>
|
||||
private static SidecarFrame? ReplacedBy(SequencePaths sequence, string sha256, IReadOnlyDictionary<string, string> replacements,
|
||||
ILookup<string, SidecarFrame> held)
|
||||
{
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase) { sha256 };
|
||||
for (var current = sha256; replacements.TryGetValue(current, out var next) && seen.Add(next); current = next)
|
||||
{
|
||||
if (held[next].FirstOrDefault(f => IsIntact(sequence, f)) is { } frame) return frame;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>Whether a frame filed by the batch holds content the sequence's tombstones name.</summary>
|
||||
private static bool Readded(SequenceManifest manifest, List<SidecarFrame> moved)
|
||||
{
|
||||
|
||||
@@ -40,6 +40,13 @@ public sealed class ArchiveMaintenance
|
||||
/// <summary>Staged copies older than this belong to no running import.</summary>
|
||||
private static readonly TimeSpan StagingAge = TimeSpan.FromHours(24);
|
||||
|
||||
/// <summary>
|
||||
/// The journal and staged conversions of a conversion job this index does not know (another host's, or one its history
|
||||
/// forgot) are left alone until none of them has been written for this long: the job history keeps finished jobs as long
|
||||
/// by default.
|
||||
/// </summary>
|
||||
private static readonly TimeSpan JournalAge = TimeSpan.FromDays(Housekeeping.CleanupSteps.DefaultHistoryDays);
|
||||
|
||||
/// <summary>Temporary files of atomic writes older than this were left by a crash.</summary>
|
||||
private static readonly TimeSpan TempAge = TimeSpan.FromHours(1);
|
||||
|
||||
@@ -141,7 +148,7 @@ public sealed class ArchiveMaintenance
|
||||
leases = LeaseFile.CleanupStale(paths.LocksDirectory, dryRun);
|
||||
break;
|
||||
case 1:
|
||||
temps = PurgeTemporaryFiles(paths, locations, dryRun);
|
||||
temps = await PurgeTemporaryFilesAsync(paths, locations, dryRun, cancellationToken).ConfigureAwait(false);
|
||||
break;
|
||||
case 2:
|
||||
packs = await CompactPacksAsync(paths, locations, dryRun, cancellationToken).ConfigureAwait(false);
|
||||
@@ -172,20 +179,83 @@ public sealed class ArchiveMaintenance
|
||||
return (space.ReclaimableBytes, long.Max(0, before.FileBytes - after.FileBytes));
|
||||
}
|
||||
|
||||
/// <summary>Staging copies and atomic-write temporaries left behind by crashes (never originals).</summary>
|
||||
private static List<string> PurgeTemporaryFiles(ArchivePaths paths, IReadOnlyList<SequenceLocation> locations, bool dryRun)
|
||||
/// <summary>
|
||||
/// Staging copies and atomic-write temporaries left behind by crashes (never originals), and what a conversion job left
|
||||
/// (<see cref="ConvertJournal"/>): its staged conversions stay while the job is queued, running or paused, however old, go
|
||||
/// as soon as it is done, and otherwise once none of them was written for a day (a retry then converts again); its journal
|
||||
/// stays while a retry can use it, and goes once the job is done. A job this index does not know (another host's on a
|
||||
/// shared archive, or one its history forgot) may still be resumed where it is known: its journal and staged conversions
|
||||
/// stay together while any of them was written within as long as the job history keeps jobs, then go together (staged
|
||||
/// conversions without a journal, which no resume can use, once idle for a day).
|
||||
/// </summary>
|
||||
private async Task<List<string>> PurgeTemporaryFilesAsync(ArchivePaths paths, IReadOnlyList<SequenceLocation> locations, bool dryRun,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
var found = new List<string>();
|
||||
var conversions = new List<string>();
|
||||
var states = new Dictionary<Guid, JobState?>();
|
||||
async Task<JobState?> StateAsync(Guid job)
|
||||
{
|
||||
if (!states.TryGetValue(job, out var state)) states[job] = state = (await _db.Jobs.GetAsync(job, cancellationToken).ConfigureAwait(false))?.State;
|
||||
return state;
|
||||
}
|
||||
bool Old(string file, TimeSpan age) => now - File.GetLastWriteTimeUtc(file) > age;
|
||||
// The journal and staged conversions of a conversion job this index does not know, none written for JournalAge: a
|
||||
// resume where the job is known would no longer find them worth keeping, so they go together.
|
||||
bool Expired(Guid job)
|
||||
{
|
||||
var staging = Path.Combine(paths.StagingDirectory, MassConverter.StagingFolder(job));
|
||||
IEnumerable<string> staged = Directory.Exists(staging) ? Directory.EnumerateFiles(staging, "*", SearchOption.AllDirectories) : [];
|
||||
return ConvertJournal.Files(paths, job).Where(File.Exists).Concat(staged).All(f => Old(f, JournalAge));
|
||||
}
|
||||
void Collect(string directory, string pattern, TimeSpan age, bool recursive)
|
||||
{
|
||||
if (!Directory.Exists(directory)) return;
|
||||
foreach (var file in Directory.EnumerateFiles(directory, pattern, recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly))
|
||||
{
|
||||
if (now - File.GetLastWriteTimeUtc(file) > age) found.Add(file);
|
||||
if (Old(file, age)) found.Add(file);
|
||||
}
|
||||
}
|
||||
if (Directory.Exists(paths.StagingDirectory))
|
||||
{
|
||||
foreach (var file in Directory.EnumerateFiles(paths.StagingDirectory))
|
||||
{
|
||||
if (Old(file, StagingAge)) found.Add(file);
|
||||
}
|
||||
foreach (var directory in Directory.EnumerateDirectories(paths.StagingDirectory))
|
||||
{
|
||||
if (ConvertJournal.JobOf(Path.GetFileName(directory)) is not { } job)
|
||||
{
|
||||
Collect(directory, "*", StagingAge, recursive: true);
|
||||
continue;
|
||||
}
|
||||
var state = await StateAsync(job).ConfigureAwait(false);
|
||||
if (state is JobState.Queued or JobState.Running or JobState.Paused) continue;
|
||||
var files = Directory.EnumerateFiles(directory, "*", SearchOption.AllDirectories).ToList();
|
||||
var gone = state switch
|
||||
{
|
||||
JobState.Done => true,
|
||||
// Another host's, or forgotten: kept as long as its journal, and going with it.
|
||||
null when ConvertJournal.Files(paths, job).Any(File.Exists) => Expired(job),
|
||||
_ => files.All(f => Old(f, StagingAge)),
|
||||
};
|
||||
if (gone)
|
||||
{
|
||||
found.AddRange(files);
|
||||
conversions.Add(directory);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (Directory.Exists(paths.JournalDirectory))
|
||||
{
|
||||
foreach (var file in Directory.EnumerateFiles(paths.JournalDirectory, ConvertJournal.Prefix + "*"))
|
||||
{
|
||||
if (ConvertJournal.JobOf(Path.GetFileName(file)) is not { } job) continue;
|
||||
var state = await StateAsync(job).ConfigureAwait(false);
|
||||
if (state == JobState.Done || state is null && Expired(job)) found.Add(file);
|
||||
}
|
||||
}
|
||||
Collect(paths.StagingDirectory, "*", StagingAge, recursive: true);
|
||||
Collect(paths.MetaDirectory, ".*.tmp", TempAge, recursive: false);
|
||||
foreach (var location in locations)
|
||||
{
|
||||
@@ -206,6 +276,7 @@ public sealed class ArchiveMaintenance
|
||||
Log.Warning("archive", $"Could not remove {file}", ex);
|
||||
}
|
||||
}
|
||||
foreach (var directory in conversions) MassConverter.DeleteStaging(directory);
|
||||
}
|
||||
return found.ConvertAll(paths.ToRelative);
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using System.Text.Json;
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.Conversion;
|
||||
using Chronos.Core.Diagnostics;
|
||||
using Chronos.Core.Import;
|
||||
using Chronos.Core.Localization;
|
||||
@@ -14,8 +15,8 @@ namespace Chronos.Data.Archiving;
|
||||
|
||||
/// <summary>
|
||||
/// What the desktop client and the server call to work with archives (BRIEF §7, §12.4, §12.5, §14.4): open, create
|
||||
/// or adopt an archive, read and change its settings, plan and apply mass renames, plan imports and card erases, delete
|
||||
/// sequences into the trash and restore them, query duplicates and orphans, and
|
||||
/// or adopt an archive, read and change its settings, plan and apply mass renames, plan mass conversions, plan imports and
|
||||
/// card erases, delete sequences into the trash and restore them, query duplicates and orphans, and
|
||||
/// the job handlers for the long operations (enqueue them with <see cref="ArchiveJobs"/>).
|
||||
/// </summary>
|
||||
public sealed class ArchiveService : IJobHandlerProvider
|
||||
@@ -42,6 +43,7 @@ public sealed class ArchiveService : IJobHandlerProvider
|
||||
Imports = new ImportService(db, Ingest);
|
||||
Trash = new SequenceTrash(db, _writer, Indexer);
|
||||
FrameTrash = new FrameTrash(db, _writer, Indexer, Ingest, Trash);
|
||||
Converter = new MassConverter(db, _writer, Ingest, FrameTrash, Trash, this);
|
||||
var shared = Housekeeping.CleanupSteps.Shared(Trash);
|
||||
CleanupSteps = [.. shared, .. _writer.Options.CleanupSteps.Where(s => !shared.Any(o => o.Name == s.Name))];
|
||||
}
|
||||
@@ -63,6 +65,9 @@ public sealed class ArchiveService : IJobHandlerProvider
|
||||
/// <summary>Deletes photos of a sequence into the archive's trash and restores them (owner review 2, R6).</summary>
|
||||
public FrameTrash FrameTrash { get; }
|
||||
|
||||
/// <summary>Converts sequences to JPEG, TIFF-16 or linear DNG, replacing their frames or into new sequences (owner review 2, R7).</summary>
|
||||
public MassConverter Converter { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The steps this host's <see cref="JobKind.Cleanup"/> jobs can run: the shared ones (<see cref="Housekeeping.CleanupSteps.Shared"/>),
|
||||
/// then the host's (<see cref="ArchiveServiceOptions.CleanupSteps"/>). A schedule host lists their names.
|
||||
@@ -74,8 +79,9 @@ public sealed class ArchiveService : IJobHandlerProvider
|
||||
|
||||
/// <summary>
|
||||
/// Handlers for scan, rebuild, verify, import, thumbs, proxies and maintenance jobs, the read-only duplicates and orphans
|
||||
/// reports, index backups and clean-ups (owner review 2, R10), the emptying of the trash (R6) and Reorganize (R7,
|
||||
/// <see cref="JobKind.Move"/>); register them on the <see cref="JobEngine"/>.
|
||||
/// reports, index backups and clean-ups (owner review 2, R10), the emptying of the trash (R6), Reorganize (R7,
|
||||
/// <see cref="JobKind.Move"/>) and mass conversion (R7, <see cref="JobKind.Convert"/>); register them on the
|
||||
/// <see cref="JobEngine"/>.
|
||||
/// </summary>
|
||||
public IReadOnlyList<IJobHandler> CreateJobHandlers() =>
|
||||
[
|
||||
@@ -92,6 +98,7 @@ public sealed class ArchiveService : IJobHandlerProvider
|
||||
new Housekeeping.CleanupJobHandler(_db, CleanupSteps, TimeProvider.System),
|
||||
new EmptyTrashJobHandler(_db, Trash),
|
||||
new MoveJobHandler(_db, Renamer, _writer.Options.SequenceChanged),
|
||||
new ConvertJobHandler(_db, Converter, _writer.Options.SequenceChanged),
|
||||
];
|
||||
|
||||
/// <summary>Makes <paramref name="root"/> a new, empty archive and indexes it.</summary>
|
||||
@@ -394,6 +401,21 @@ public sealed class ArchiveService : IJobHandlerProvider
|
||||
return await Renamer.PlanAsync(archive, ids, templates, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The dry run of a mass conversion (owner review 2, R7): <paramref name="sequenceIds"/>, or every sequence of the
|
||||
/// archive (by folder) when null, converted with <paramref name="options"/> in <paramref name="mode"/>. Apply it in a job
|
||||
/// (<see cref="ArchiveJobs.Convert"/>).
|
||||
/// </summary>
|
||||
/// <exception cref="ArgumentException">The archive is unknown, the options are invalid, or a sequence is not in the archive.</exception>
|
||||
public async Task<ConvertPlan> PlanConvertAsync(Guid archiveId, IReadOnlyList<Guid>? sequenceIds, ConvertOptions options, ConvertMode mode,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
var archive = await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false);
|
||||
var ids = sequenceIds ?? [.. (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)).Select(l => l.Id)];
|
||||
return await Converter.PlanAsync(archive, ids, options, mode, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task<RenameResult> ApplyRenameAsync(RenamePlan plan, IProgress<int>? progress = null, CancellationToken cancellationToken = default) =>
|
||||
await Renamer.ApplyAsync(await LoadArchiveAsync(plan.ArchiveId, cancellationToken).ConfigureAwait(false), plan, progress, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
@@ -432,7 +454,7 @@ public sealed class ArchiveService : IJobHandlerProvider
|
||||
if (!Archiving.FrameTrash.IsPhotoEntry(entryName))
|
||||
return new TrashRestoreResult(TrashKind.Sequence, await Trash.RestoreAsync(archive, entryName, cancellationToken).ConfigureAwait(false), 0, [], false);
|
||||
var photos = await FrameTrash.RestoreAsync(archive, entryName, followUps, cancellationToken).ConfigureAwait(false);
|
||||
return new TrashRestoreResult(TrashKind.Photos, photos.Sequence, photos.Restored, photos.Kept, photos.NewSequence);
|
||||
return new TrashRestoreResult(photos.Kind, photos.Sequence, photos.Restored, photos.Kept, photos.NewSequence);
|
||||
}
|
||||
|
||||
/// <summary>The dry run of emptying an archive's trash: <paramref name="names"/> of the entries (null: all), measured, with the plan's fingerprint.</summary>
|
||||
@@ -673,8 +695,37 @@ public static class ArchiveJobs
|
||||
JobPriority.User);
|
||||
}
|
||||
|
||||
public static Task<Job> EnqueueAsync(this JobEngine engine, ArchiveJobRequest request, Guid? createdBy = null, CancellationToken cancellationToken = default) =>
|
||||
engine.EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, createdBy, cancellationToken);
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.Convert"/>: applies the conversion <paramref name="plan"/> (<see cref="ArchiveService.PlanConvertAsync"/>)
|
||||
/// as confirmed (owner review 2, R7): the sequences it converts, in its order with their frames versions and the digest
|
||||
/// of the photos each converts (<see cref="ConvertSequencePlan.Digest"/>), are the job's targets, so jobs that would touch
|
||||
/// them wait. The request names the job's id, under which <see cref="MassConverter.QueueAsync"/> (the way to queue it,
|
||||
/// which refuses a second conversion of the same sequences) writes which photos those are beside the job's journal
|
||||
/// (<see cref="ConfirmedPhotos"/>). The job converts those photos alone, and leaves as it is and reports a sequence where
|
||||
/// one of them no longer holds the content confirmed when the job comes to it.
|
||||
/// </summary>
|
||||
/// <exception cref="ArgumentException">The plan converts nothing.</exception>
|
||||
public static ArchiveJobRequest Convert(Archive archive, ConvertPlan plan)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(archive);
|
||||
ArgumentNullException.ThrowIfNull(plan);
|
||||
var sequences = plan.Converted.Select(s => new ConvertSequence(s.SequenceId, s.FramesVersion, s.Digest())).ToList();
|
||||
if (sequences.Count == 0) throw new ArgumentException("The plan converts nothing.", nameof(plan));
|
||||
var label = sequences.Count == 1 ? plan.Converted.First().Title : archive.Name;
|
||||
return new(JobKind.Convert, [.. sequences.Select(s => s.Id)], label,
|
||||
Json(new ConvertJobParameters(archive.Id, plan.Options.Target, plan.Options.Quality, plan.Mode, sequences, plan.Fingerprint()),
|
||||
ArchiveJobJson.Default.ConvertJobParameters), JobPriority.User, Guid.CreateVersion7());
|
||||
}
|
||||
|
||||
/// <summary>Queues <paramref name="request"/>, under its own id when it names one (<see cref="ArchiveJobRequest.Id"/>).</summary>
|
||||
public static Task<Job> EnqueueAsync(this JobEngine engine, ArchiveJobRequest request, Guid? createdBy = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(engine);
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
return request.Id is { } id
|
||||
? engine.EnqueueAsync(id, request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, createdBy, cancellationToken)
|
||||
: engine.EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, createdBy, cancellationToken);
|
||||
}
|
||||
|
||||
private static string Json<T>(T value, System.Text.Json.Serialization.Metadata.JsonTypeInfo<T> typeInfo) => JsonSerializer.Serialize(value, typeInfo);
|
||||
}
|
||||
@@ -0,0 +1,677 @@
|
||||
using System.Text.Json;
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.Diagnostics;
|
||||
using Chronos.Core.IO;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Core.Naming;
|
||||
using SequenceFrames = Chronos.Core.Archiving.SequenceFrames;
|
||||
|
||||
namespace Chronos.Data.Archiving;
|
||||
|
||||
/// <summary>
|
||||
/// One frame a replacement swaps (owner review 2, R7): the file that comes in, its companions, the name it is filed under
|
||||
/// and the record the frame gets.
|
||||
/// </summary>
|
||||
/// <param name="Incoming">The file that comes in, relative to the archive root: a staged conversion, or an original in a
|
||||
/// trash entry.</param>
|
||||
/// <param name="Companions">Files that come in with it (the XMP companions of an original back from the trash).</param>
|
||||
/// <param name="Name">The name it is filed under, from the frame and the sequence as the sidecar has them under the lease,
|
||||
/// relative to the sequence folder; made unique with <c>_2</c>, <c>_3</c>… when it, or one of its companions' names, is
|
||||
/// taken.</param>
|
||||
/// <param name="Record">The frame's record after the swap, from the frame as the sidecar has it under the lease (its
|
||||
/// <see cref="SidecarFrame.Sha256"/> names the incoming file): the replacement keeps the frame's id and index, files it
|
||||
/// under its name, and keeps the flags a user or the capture set (<see cref="FrameTrash.ContentFlags"/> come from the
|
||||
/// record).</param>
|
||||
internal sealed record FrameSwap(Guid FrameId, string Incoming, IReadOnlyList<IncomingCompanion> Companions, Func<SidecarFrame, SequenceManifest, string> Name,
|
||||
Func<SidecarFrame, SidecarFrame> Record)
|
||||
{
|
||||
/// <summary>
|
||||
/// The frame's original is gone (a frame flagged missing whose photo comes back in another encoding, from the other side
|
||||
/// of a sync link): the incoming file takes its place alone and nothing of the frame goes to the trash. The swap is
|
||||
/// refused while a file is at the frame's place.
|
||||
/// </summary>
|
||||
public bool Missing { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>A companion that comes in with its photo: where it is (relative to the archive root), and what its name adds to the photo's stem.</summary>
|
||||
internal sealed record IncomingCompanion(string From, string Tail);
|
||||
|
||||
/// <summary>
|
||||
/// The trash entry a replacement takes its incoming files from (the undo of a conversion): once the sidecar is written,
|
||||
/// the photos of <see cref="Frames"/> leave its record, and its copy of <c>analysis.json</c> (<see cref="Analysis"/>) goes back
|
||||
/// into the sequence.
|
||||
/// </summary>
|
||||
/// <param name="Record">The entry's record, relative to the archive root.</param>
|
||||
/// <param name="Folder">The entry's folder, relative to the archive root.</param>
|
||||
/// <param name="Lease">The entry's lease, which the replacement's caller holds.</param>
|
||||
/// <param name="Analysis">The entry's copy of <c>analysis.json</c>, relative to the archive root; null when it has none.</param>
|
||||
internal sealed record ConsumedEntry(string Record, string Folder, string Lease, IReadOnlyList<Guid> Frames, string? Analysis);
|
||||
|
||||
/// <summary>One file a replacement moves: its place in the sequence folder and the other end, relative to the archive root.</summary>
|
||||
/// <param name="Place">Relative to the sequence folder (forward slashes).</param>
|
||||
/// <param name="Other">Relative to the archive root: the trash entry an outgoing file goes to, or where an incoming file comes from.</param>
|
||||
internal sealed record SwapMove(string Place, string Other);
|
||||
|
||||
/// <summary>
|
||||
/// What a replacement did: the trash entry of the files that went (null when none went: every frame swapped had lost its
|
||||
/// original), the sequence as it is now, and its statuses before.
|
||||
/// </summary>
|
||||
internal sealed record ReplaceResult(TrashEntry? Entry, Sequence Sequence, SequenceStatus Had);
|
||||
|
||||
/// <summary>How far a frame replacement got (<see cref="FrameReplaceJournal.Phase"/>).</summary>
|
||||
internal enum ReplacePhase
|
||||
{
|
||||
/// <summary>Files are moving: whether the sidecar was written is told by what it lists (<see cref="FrameReplaceJournal.Written"/>).</summary>
|
||||
Moving,
|
||||
|
||||
/// <summary>The sidecar is written: the replacement stands, and only its completion may be left.</summary>
|
||||
Written,
|
||||
|
||||
/// <summary>
|
||||
/// The replacement failed before its sidecar was written and its files are going back: it is only ever undone, whatever
|
||||
/// another writer does to the sidecar meanwhile.
|
||||
/// </summary>
|
||||
Undoing,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Write-ahead record of a frame replacement (<c>.chronos/replace.pending.json</c>, owner review 2, R7): the sidecar's frames
|
||||
/// version before it, the new trash entry (its record pending until the sidecar is written), every file that leaves the
|
||||
/// sequence for the entry and every file that comes in, the name each swapped frame is filed under, whether the entry holds
|
||||
/// a copy of <c>analysis.json</c>, the entry the incoming files come from (an undo), and how far it got
|
||||
/// (<see cref="Phase"/>). Written under the sequence's lease before the first move and removed once the replacement is
|
||||
/// settled, so the next writer of the sequence (<see cref="SequenceWriter.ReadLocked"/>) undoes one a crash or a failure
|
||||
/// interrupted before its sidecar was written, and completes one interrupted after.
|
||||
/// </summary>
|
||||
/// <param name="Record">The new entry's record, relative to the archive root (its name once the replacement is complete); null
|
||||
/// when no file leaves the sequence (no entry).</param>
|
||||
/// <param name="Folder">The new entry's folder, relative to the archive root; null with <paramref name="Record"/>.</param>
|
||||
/// <param name="Out">Files that leave the sequence for the new entry.</param>
|
||||
/// <param name="In">Files that come into the sequence.</param>
|
||||
internal sealed record FrameReplaceJournal(int FramesVersion, string? Record, string? Folder, IReadOnlyList<SwapMove> Out, IReadOnlyList<SwapMove> In,
|
||||
bool Analysis, ConsumedEntry? Consumed)
|
||||
{
|
||||
/// <summary>How far the replacement got; a journal of an earlier build has none (<see cref="ReplacePhase.Moving"/>).</summary>
|
||||
public ReplacePhase Phase { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The name each swapped frame is filed under (frame id → place relative to the sequence folder); null in a journal of an
|
||||
/// earlier build. Each name was claimed free of every file present and every name the sidecar held before, so the
|
||||
/// sidecar before the replacement never lists one of them under its frame.
|
||||
/// </summary>
|
||||
public IReadOnlyDictionary<Guid, string>? Names { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether <paramref name="manifest"/>, read under the sequence's lease, is the sidecar this replacement wrote (or one
|
||||
/// written after it): a swapped frame lists its new name (<see cref="Names"/>; the sidecar is written whole, so one is
|
||||
/// all). Told by content, not by the frames version, which any other writer moves too. A journal of an earlier build,
|
||||
/// without names, still tells by the version.
|
||||
/// </summary>
|
||||
public bool Written(SequenceManifest manifest) => Names is { Count: > 0 } names
|
||||
? manifest.Frames.Any(f => names.TryGetValue(f.Id, out var name) && string.Equals(f.File, name, StringComparison.OrdinalIgnoreCase))
|
||||
: manifest.Versions.Frames != FramesVersion;
|
||||
|
||||
public void Write(SequencePaths sequence)
|
||||
{
|
||||
System.IO.Directory.CreateDirectory(sequence.MetaDirectory);
|
||||
AtomicFile.WriteAllBytes(sequence.ReplaceJournalPath, JsonSerializer.SerializeToUtf8Bytes(this, TrashJson.Default.FrameReplaceJournal));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The journal at <paramref name="phase"/>, written over the sequence's. One that cannot be written (logged) keeps the
|
||||
/// phase before, and the sidecar's content still tells what happened (<see cref="Written"/>).
|
||||
/// </summary>
|
||||
public FrameReplaceJournal Mark(SequencePaths sequence, ReplacePhase phase)
|
||||
{
|
||||
var marked = this with { Phase = phase };
|
||||
try
|
||||
{
|
||||
marked.Write(sequence);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("archive", $"The journal of a frame replacement in {sequence.RelativeFolder} could not record {phase}; the sidecar tells", ex);
|
||||
}
|
||||
return marked;
|
||||
}
|
||||
|
||||
/// <summary>The journal of a sequence; null when it is torn (written by a process that died while writing it: nothing moved yet).</summary>
|
||||
public static FrameReplaceJournal? Read(SequencePaths sequence)
|
||||
{
|
||||
try
|
||||
{
|
||||
return JsonSerializer.Deserialize(AtomicFile.ReadAllBytes(sequence.ReplaceJournalPath), TrashJson.Default.FrameReplaceJournal);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public static void Delete(SequencePaths sequence)
|
||||
{
|
||||
if (File.Exists(sequence.ReplaceJournalPath)) File.Delete(sequence.ReplaceJournalPath);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed partial class FrameTrash
|
||||
{
|
||||
/// <summary>
|
||||
/// Flags that follow a frame's file, not the frame: a replacement takes them from the incoming record and keeps the
|
||||
/// others (excluded, intervalometer, black, white) as the frame has them.
|
||||
/// </summary>
|
||||
internal const FrameFlags ContentFlags = FrameFlags.Missing | FrameFlags.Corrupt | FrameFlags.Duplicate | FrameFlags.PreviewOnly | FrameFlags.Undecodable;
|
||||
|
||||
/// <summary>The name of the copy of a sequence's <c>analysis.json</c> a replaced entry keeps, inside its folder.</summary>
|
||||
internal const string AnalysisCopy = "analysis.json";
|
||||
|
||||
/// <summary>
|
||||
/// Replaces the files of frames of a sequence, keeping the frames (owner review 2, R7; BRIEF §7): under the sequence's
|
||||
/// lease, with its frames version still <paramref name="framesVersion"/> (null: any) and <paramref name="unchanged"/>
|
||||
/// (when given) still true, every swapped frame's original and its XMP companions move into one trash entry of kind
|
||||
/// <see cref="TrashKind.Replaced"/> (its record holds the old frame records and each one's replacement SHA-256, and a copy
|
||||
/// of <c>analysis.json</c>), and the incoming files move in under their names (made unique). The sidecar takes the
|
||||
/// records with the same ids and indices (frames version + 1, the statuses the frames uncovered cleared) and a tombstone
|
||||
/// per old content naming its replacement, so sync counts the old photo as the new one; content that comes back loses
|
||||
/// its tombstone. Then the index follows, the proxies of the swapped frames are deleted (the proxy pass makes them
|
||||
/// again) and the project's positions stay on their frames. A journal (<see cref="FrameReplaceJournal"/>) makes it all
|
||||
/// or nothing: a failure before the sidecar is written puts every file back, and a crash is undone or completed by the
|
||||
/// next writer of the sequence. Returns null, nothing touched, when the sequence is gone, its frames changed, a swapped
|
||||
/// frame's original is not there whole (there at all, for a <see cref="FrameSwap.Missing"/> swap) or was never hashed, or
|
||||
/// <paramref name="unchanged"/> said no. When every frame swapped had lost its original, no entry is made.
|
||||
/// </summary>
|
||||
/// <param name="replacedBy">Who asked (user name or client), kept in the trash record.</param>
|
||||
/// <param name="jobId">The job that asked (a conversion), kept in the trash record so its Undo takes back exactly its entries.</param>
|
||||
/// <param name="consumed">The trash entry the incoming files come from (an undo), settled once the sidecar is written.</param>
|
||||
/// <exception cref="ArchiveOfflineException">The archive is offline or read-only; nothing moved.</exception>
|
||||
/// <exception cref="LeaseUnavailableException">The sequence is busy, or holds a restore or a replacement not settled yet.</exception>
|
||||
/// <exception cref="IOException">An incoming file is missing, or a file could not move; nothing changed.</exception>
|
||||
internal async Task<ReplaceResult?> ReplaceAsync(Archive archive, Guid sequenceId, int? framesVersion, IReadOnlyList<FrameSwap> swaps, string? replacedBy,
|
||||
Guid? jobId, ConsumedEntry? consumed, Func<SequenceManifest, SequencePaths, CancellationToken, Task<bool>>? unchanged, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(archive);
|
||||
ArgumentNullException.ThrowIfNull(swaps);
|
||||
if (swaps.Count == 0) throw new ArgumentException("No frame is swapped.", nameof(swaps));
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
if (await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) is not { } listed || listed.ArchiveId != archive.Id) return null;
|
||||
Sequence row;
|
||||
SequencePaths folder;
|
||||
SequenceStatus had;
|
||||
TrashEntry? entry;
|
||||
SequenceManifest updated;
|
||||
bool indexed;
|
||||
await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
// Where the sequence is now: a Reorganize moves its folder and row under this lease, maybe while this waited for it.
|
||||
if (await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) is not { } moved || moved.ArchiveId != archive.Id) return null;
|
||||
row = moved;
|
||||
folder = paths.Sequence(row.Folder);
|
||||
had = row.Status;
|
||||
var manifest = SequenceWriter.ReadLocked(folder, sequenceId, consumed?.Lease);
|
||||
EnsureSettled(folder);
|
||||
EnsureReplaced(folder);
|
||||
if (framesVersion is { } expected && manifest.Versions.Frames != expected) return null;
|
||||
var byId = manifest.Frames.ToDictionary(f => f.Id);
|
||||
var swapped = new Dictionary<Guid, SidecarFrame>();
|
||||
foreach (var swap in swaps)
|
||||
{
|
||||
if (!byId.TryGetValue(swap.FrameId, out var current) || current.Sha256 is null || swapped.ContainsKey(swap.FrameId)) return null;
|
||||
var file = new FileInfo(folder.File(current.File));
|
||||
// An original gives its place up whole; a frame that lost its original takes the incoming file only while it has none.
|
||||
if (swap.Missing ? file.Exists : !file.Exists || file.Length != current.Size) return null;
|
||||
var next = swap.Record(current);
|
||||
if (next.Sha256 is null) throw new ArgumentException($"The record that replaces {current.File} names no content.", nameof(swaps));
|
||||
swapped[swap.FrameId] = next;
|
||||
}
|
||||
if (unchanged is not null && !await unchanged(manifest, folder, cancellationToken).ConfigureAwait(false)) return null;
|
||||
foreach (var swap in swaps.Where(s => !File.Exists(paths.FromRelative(s.Incoming))))
|
||||
throw new FileNotFoundException($"The file that replaces {byId[swap.FrameId].File} is missing.", swap.Incoming);
|
||||
var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
|
||||
var photosChanged = !inStep && await _indexer.PhotosChangedAsync(archive, folder, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var missing = swaps.Where(s => s.Missing).Select(s => s.FrameId).ToHashSet();
|
||||
var leaving = manifest.Frames.Where(f => swapped.ContainsKey(f.Id)).ToList();
|
||||
var kept = manifest.Frames.Where(f => !swapped.ContainsKey(f.Id)).ToList();
|
||||
var photos = (await TrashedAsync(folder, [.. leaving.Where(f => !missing.Contains(f.Id))], kept, cancellationToken).ConfigureAwait(false))
|
||||
.Select(p => p with { Replacement = swapped[p.Frame.Id].Sha256!.ToLowerInvariant() }).ToList();
|
||||
var (incoming, names) = Incoming(folder, manifest, swaps, byId);
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
updated = SequenceFrames.WithFrames(manifest,
|
||||
[.. manifest.Frames.Select(f => swapped.TryGetValue(f.Id, out var record) ? Swapped(f, record, names[f.Id]) : f)]) with
|
||||
{
|
||||
Status = manifest.Status & ~Uncovered,
|
||||
Removed = WithAliases(manifest.Removed, leaving, swapped, kept, now),
|
||||
};
|
||||
|
||||
// The entry of what leaves; none when nothing does (every frame swapped had lost its original).
|
||||
string? name = null;
|
||||
string? entryFolder = null;
|
||||
string? recordPath = null;
|
||||
PhotoTrashRecord? record = null;
|
||||
var analysis = new FileInfo(folder.AnalysisPath);
|
||||
var copiesAnalysis = false;
|
||||
List<SwapMove> outgoing = [];
|
||||
if (photos.Count > 0)
|
||||
{
|
||||
var trash = Directory(paths);
|
||||
System.IO.Directory.CreateDirectory(trash);
|
||||
name = EntryName(trash, sequenceId);
|
||||
entryFolder = Path.Combine(trash, name);
|
||||
recordPath = entryFolder + RecordExtension;
|
||||
copiesAnalysis = analysis.Exists;
|
||||
List<TrashedFile>? leftovers = copiesAnalysis
|
||||
? [new TrashedFile(ArchiveLayout.MetaFolder + "/" + ArchiveLayout.AnalysisFile, AnalysisCopy, analysis.Length, null)]
|
||||
: null;
|
||||
record = new PhotoTrashRecord(TrashKind.Replaced, sequenceId, row.DisplayTitle, row.Folder, photos,
|
||||
photos.Sum(p => p.Files.Sum(f => f.Size)) + (leftovers?.Sum(f => f.Size) ?? 0), now, replacedBy, leftovers, jobId);
|
||||
var folderPath = entryFolder;
|
||||
outgoing = [.. photos.SelectMany(p => p.Files).Select(f => new SwapMove(f.From, paths.ToRelative(Path.Combine(folderPath, f.Path))))];
|
||||
}
|
||||
var journal = new FrameReplaceJournal(manifest.Versions.Frames, recordPath is null ? null : paths.ToRelative(recordPath),
|
||||
entryFolder is null ? null : paths.ToRelative(entryFolder), outgoing, incoming, copiesAnalysis, consumed)
|
||||
{
|
||||
Names = names,
|
||||
};
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
// The journal first: a crash from here until the sidecar is written puts every file back (ReadLocked).
|
||||
journal.Write(folder);
|
||||
if (record is not null)
|
||||
{
|
||||
// Under a name the trash does not list until the sidecar is written: half an entry is never restored or emptied.
|
||||
AtomicFile.WriteAllText(FrameRemovalJournal.PendingRecord(recordPath!), JsonSerializer.Serialize(record, TrashJson.Default.PhotoTrashRecord));
|
||||
System.IO.Directory.CreateDirectory(entryFolder!);
|
||||
}
|
||||
var movedOut = new List<SwapMove>();
|
||||
var movedIn = new List<SwapMove>();
|
||||
try
|
||||
{
|
||||
if (copiesAnalysis) File.Copy(analysis.FullName, Path.Combine(entryFolder!, AnalysisCopy), overwrite: false);
|
||||
foreach (var move in outgoing)
|
||||
{
|
||||
MoveFile(folder.File(move.Place), paths.FromRelative(move.Other));
|
||||
movedOut.Add(move);
|
||||
}
|
||||
foreach (var move in incoming)
|
||||
{
|
||||
MoveFile(paths.FromRelative(move.Other), folder.File(move.Place));
|
||||
movedIn.Add(move);
|
||||
}
|
||||
ManifestStore.WriteSequence(folder.ManifestPath, updated);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// A file is open elsewhere, or the sidecar cannot be written: what moved goes back, and nothing changed.
|
||||
Log.Warning("archive", $"{row.DisplayTitle}: the frames could not be replaced; the files that moved go back", ex);
|
||||
try
|
||||
{
|
||||
// Undone from here on, whatever happens to the sidecar until it is.
|
||||
UndoReplace(folder, movedOut, movedIn, journal.Mark(folder, ReplacePhase.Undoing));
|
||||
}
|
||||
catch (Exception again) when (again is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Error("archive", $"{row.DisplayTitle}: the files that moved could not all go back; the next writer of the sequence puts them back first", again);
|
||||
}
|
||||
throw;
|
||||
}
|
||||
CompleteReplace(folder, journal.Mark(folder, ReplacePhase.Written), consumed?.Lease);
|
||||
Log.Info("archive", record is null
|
||||
? $"{row.DisplayTitle}: {swaps.Count} missing frame(s) took their photo back in another encoding"
|
||||
: $"{row.DisplayTitle}: {leaving.Count} frame(s) replaced; the files they had are in the trash ({name}, {record.Bytes} bytes)");
|
||||
// The files have moved: the index follows to the end, whatever the caller cancels meanwhile.
|
||||
indexed = await IndexAsync(archive, folder, updated, photosChanged, row.DisplayTitle).ConfigureAwait(false);
|
||||
DropProxies(folder, leaving);
|
||||
entry = record is null ? null : Entry(archive.Id, name!, record);
|
||||
}
|
||||
var sequence = await RowAsync(row, updated, indexed).ConfigureAwait(false);
|
||||
return new ReplaceResult(entry, sequence, had);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Settles the replacement a crash or a failure interrupted (<see cref="FrameReplaceJournal"/>), if any; the caller holds
|
||||
/// the sequence's lease, and <paramref name="heldLease"/> when it holds a trash entry's lease too. A replacement whose
|
||||
/// sidecar was written (<see cref="ReplacePhase.Written"/>, or told by what <paramref name="manifest"/> lists:
|
||||
/// <see cref="FrameReplaceJournal.Written"/>) stands and is completed (<see cref="CompleteReplace"/>). Any other is undone,
|
||||
/// and stays undoing from then on (<see cref="ReplacePhase.Undoing"/>): every file that moved goes back (one whose place is
|
||||
/// taken again stays, logged, in the entry, which is then listed) and the new entry goes.
|
||||
/// </summary>
|
||||
/// <exception cref="IOException">A file could not go back (open elsewhere): the journal stays, and the writer goes no
|
||||
/// further, as a photo deletion's recovery does.</exception>
|
||||
/// <exception cref="UnauthorizedAccessException">The same, for want of access.</exception>
|
||||
internal static void RecoverReplace(SequencePaths sequence, SequenceManifest manifest, string? heldLease = null)
|
||||
{
|
||||
if (!File.Exists(sequence.ReplaceJournalPath)) return;
|
||||
if (FrameReplaceJournal.Read(sequence) is not { } journal)
|
||||
{
|
||||
FrameReplaceJournal.Delete(sequence);
|
||||
return;
|
||||
}
|
||||
if (journal.Phase == ReplacePhase.Written || (journal.Phase == ReplacePhase.Moving && journal.Written(manifest)))
|
||||
{
|
||||
Log.Warning("archive", $"An interrupted frame replacement in {sequence.RelativeFolder} is completed");
|
||||
CompleteReplace(sequence, journal, heldLease);
|
||||
return;
|
||||
}
|
||||
Log.Warning("archive", $"An interrupted frame replacement in {sequence.RelativeFolder} is undone");
|
||||
try
|
||||
{
|
||||
UndoReplace(sequence, journal.Out, journal.In, journal.Phase == ReplacePhase.Undoing ? journal : journal.Mark(sequence, ReplacePhase.Undoing),
|
||||
onlyPresent: true);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Error("archive", $"An interrupted frame replacement in {sequence.RelativeFolder} could not be undone yet; nothing is written to the sequence until it is", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Refuses to go on while a replacement of <paramref name="sequence"/> is not settled after
|
||||
/// <see cref="SequenceWriter.ReadLocked"/> (the entry its files came from was busy): its journal is not written over.
|
||||
/// </summary>
|
||||
/// <exception cref="LeaseUnavailableException">The replacement is not settled; nothing was changed.</exception>
|
||||
internal static void EnsureReplaced(SequencePaths sequence)
|
||||
{
|
||||
if (!File.Exists(sequence.ReplaceJournalPath)) return;
|
||||
var lease = FrameReplaceJournal.Read(sequence)?.Consumed?.Lease ?? "replace";
|
||||
throw new LeaseUnavailableException(lease, LeaseFile.Holder(sequence.Archive.LocksDirectory, lease));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The incoming moves, each file under its name made free: every name the sequence's folders hold stays taken, those of
|
||||
/// the files that leave included, so no incoming file takes the place of one that leaves (a crash between the two is
|
||||
/// then undone without doubt), and so do the files of every frame and the names already given.
|
||||
/// </summary>
|
||||
private static (List<SwapMove> Moves, Dictionary<Guid, string> Names) Incoming(SequencePaths folder, SequenceManifest manifest, IReadOnlyList<FrameSwap> swaps,
|
||||
Dictionary<Guid, SidecarFrame> frames)
|
||||
{
|
||||
var taken = new HashSet<string>(FileRenames.ListPresent(folder, manifest), StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var frame in manifest.Frames) taken.Add(frame.File);
|
||||
var unique = new UniqueNames(taken);
|
||||
var moves = new List<SwapMove>();
|
||||
var names = new Dictionary<Guid, string>();
|
||||
foreach (var swap in swaps)
|
||||
{
|
||||
var file = unique.Claim(swap.Name(frames[swap.FrameId], manifest), c => swap.Companions.All(t => !taken.Contains(StemOf(c) + t.Tail)));
|
||||
names[swap.FrameId] = file;
|
||||
moves.Add(new SwapMove(file, swap.Incoming));
|
||||
foreach (var companion in swap.Companions)
|
||||
{
|
||||
var to = StemOf(file) + companion.Tail;
|
||||
taken.Add(to);
|
||||
moves.Add(new SwapMove(to, companion.From));
|
||||
}
|
||||
}
|
||||
return (moves, names);
|
||||
}
|
||||
|
||||
/// <summary>A path without its extension (<c>originals/IMG_0001</c>): companions add their tail to it.</summary>
|
||||
private static string StemOf(string file) => file[..^Path.GetExtension(file).Length];
|
||||
|
||||
/// <summary>The frame's record after a swap: <paramref name="record"/> at the frame's id and index, under its new name, with its own user flags.</summary>
|
||||
private static SidecarFrame Swapped(SidecarFrame current, SidecarFrame record, string file) => record with
|
||||
{
|
||||
Id = current.Id,
|
||||
I = current.I,
|
||||
File = file,
|
||||
Sha256 = record.Sha256!.ToLowerInvariant(),
|
||||
Flags = (current.Flags & ~ContentFlags) | (record.Flags & ContentFlags),
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// The tombstones after a replacement: those of before but the ones of content that comes back, and one per content
|
||||
/// that leaves naming its replacement (<see cref="RemovalReason.Replaced"/>), unless a frame that stays holds it too.
|
||||
/// </summary>
|
||||
private static List<RemovedPhoto>? WithAliases(IReadOnlyList<RemovedPhoto>? before, List<SidecarFrame> leaving, Dictionary<Guid, SidecarFrame> swapped,
|
||||
List<SidecarFrame> kept, DateTimeOffset at)
|
||||
{
|
||||
var incoming = swapped.Values.Select(s => s.Sha256!).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var held = kept.Where(f => f.Sha256 is not null).Select(f => f.Sha256!).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var tombstones = new Dictionary<string, RemovedPhoto>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var tombstone in before ?? [])
|
||||
{
|
||||
if (!incoming.Contains(tombstone.Sha256)) tombstones[tombstone.Sha256] = tombstone;
|
||||
}
|
||||
foreach (var frame in leaving)
|
||||
{
|
||||
var old = frame.Sha256!.ToLowerInvariant();
|
||||
var replacement = swapped[frame.Id].Sha256!.ToLowerInvariant();
|
||||
if (old == replacement || held.Contains(old)) continue;
|
||||
tombstones[old] = new RemovedPhoto { Sha256 = old, Replacement = replacement, Reason = RemovalReason.Replaced, At = at };
|
||||
}
|
||||
return tombstones.Count == 0 ? null : [.. tombstones.Values];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Puts back what a replacement moved before its sidecar was written (incoming files back where they came from, the
|
||||
/// files that left back into the sequence) and removes its entry and journal. With <paramref name="onlyPresent"/> (a
|
||||
/// recovery) a file not at its new place, or whose old place is taken, is left where it is (logged); the journal then
|
||||
/// goes and the entry stays, listed under its record's own name.
|
||||
/// </summary>
|
||||
/// <exception cref="IOException">A file could not move (open elsewhere): the journal stays for the next writer of the sequence.</exception>
|
||||
/// <exception cref="UnauthorizedAccessException">The same, for want of access.</exception>
|
||||
private static void UndoReplace(SequencePaths folder, IReadOnlyList<SwapMove> movedOut, IReadOnlyList<SwapMove> movedIn, FrameReplaceJournal journal,
|
||||
bool onlyPresent = false)
|
||||
{
|
||||
var archive = folder.Archive;
|
||||
var recordPath = journal.Record is { } record ? archive.FromRelative(record) : null;
|
||||
var entryFolder = journal.Folder is { } entry ? archive.FromRelative(entry) : null;
|
||||
var stuck = 0;
|
||||
foreach (var move in movedIn.Reverse())
|
||||
{
|
||||
if (!Back(folder.File(move.Place), archive.FromRelative(move.Other))) stuck++;
|
||||
}
|
||||
foreach (var move in movedOut.Reverse())
|
||||
{
|
||||
if (!Back(archive.FromRelative(move.Other), folder.File(move.Place))) stuck++;
|
||||
}
|
||||
if (stuck > 0)
|
||||
{
|
||||
// What could not go back is in the entry, listed, restorable and erasable like any entry.
|
||||
if (recordPath is not null) FrameRemovalJournal.Publish(recordPath);
|
||||
FrameReplaceJournal.Delete(folder);
|
||||
return;
|
||||
}
|
||||
if (entryFolder is not null)
|
||||
{
|
||||
var copy = Path.Combine(entryFolder, AnalysisCopy);
|
||||
if (journal.Analysis && File.Exists(copy)) File.Delete(copy);
|
||||
FrameRemovalJournal.RemoveEmpty(entryFolder);
|
||||
}
|
||||
if (recordPath is not null && File.Exists(FrameRemovalJournal.PendingRecord(recordPath))) File.Delete(FrameRemovalJournal.PendingRecord(recordPath));
|
||||
FrameReplaceJournal.Delete(folder);
|
||||
|
||||
bool Back(string from, string to)
|
||||
{
|
||||
if (!File.Exists(from)) return !onlyPresent || File.Exists(to) || Gone(from, to);
|
||||
if (File.Exists(to))
|
||||
{
|
||||
Log.Warning("archive", $"{to} is there again; {from} stays where it is");
|
||||
return false;
|
||||
}
|
||||
MoveFile(from, to);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Neither here nor there: nothing to put back for this file.
|
||||
static bool Gone(string from, string to)
|
||||
{
|
||||
Log.Warning("archive", $"{from} is not there to go back to {to}");
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Completes a replacement whose sidecar is written: its entry is listed under its record's own name, the entry its
|
||||
/// incoming files came from (if any) is settled under its lease (the photos swapped leave its record, its copy of
|
||||
/// <c>analysis.json</c> goes back into the sequence, and an entry left without anything goes), and the journal goes. An
|
||||
/// entry busy now (or a failure, logged) keeps the journal for the next writer of the sequence.
|
||||
/// </summary>
|
||||
private static void CompleteReplace(SequencePaths folder, FrameReplaceJournal journal, string? heldLease)
|
||||
{
|
||||
var archive = folder.Archive;
|
||||
try
|
||||
{
|
||||
if (journal.Record is { } record) FrameRemovalJournal.Publish(archive.FromRelative(record));
|
||||
if (journal.Consumed is { } consumed && !Settle(folder, consumed, heldLease)) return;
|
||||
FrameReplaceJournal.Delete(folder);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("archive", $"A frame replacement in {folder.RelativeFolder} is not settled yet; the next writer of the sequence settles it", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Settles the entry a replacement took its files from (<see cref="CompleteReplace"/>); false while it is busy.</summary>
|
||||
private static bool Settle(SequencePaths folder, ConsumedEntry consumed, string? heldLease)
|
||||
{
|
||||
var archive = folder.Archive;
|
||||
LeaseFile? taken = null;
|
||||
if (consumed.Lease != heldLease)
|
||||
{
|
||||
taken = LeaseFile.TryAcquire(archive.LocksDirectory, consumed.Lease, SettleOwner);
|
||||
if (taken is null)
|
||||
{
|
||||
Log.Info("archive", $"A frame replacement in {folder.RelativeFolder} waits: the trash entry its files came from is busy");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
using var held = taken;
|
||||
var recordPath = archive.FromRelative(consumed.Record);
|
||||
var entryFolder = archive.FromRelative(consumed.Folder);
|
||||
// The copy of the analysis of the frames that came back takes the place of the analysis of the frames that went
|
||||
// (the new entry keeps a copy of that one).
|
||||
if (consumed.Analysis is { } copy && File.Exists(archive.FromRelative(copy)))
|
||||
{
|
||||
System.IO.Directory.CreateDirectory(folder.MetaDirectory);
|
||||
File.Move(archive.FromRelative(copy), folder.AnalysisPath, overwrite: true);
|
||||
}
|
||||
if (ReadRecord(recordPath) is { } record)
|
||||
{
|
||||
var frames = consumed.Frames.ToHashSet();
|
||||
var leftovers = record.Leftovers?.Where(f => File.Exists(Path.Combine(entryFolder, f.Path))).ToList();
|
||||
Rewrite(recordPath, entryFolder, record with
|
||||
{
|
||||
Photos = [.. record.Photos.Where(p => !frames.Contains(p.Frame.Id))],
|
||||
Leftovers = leftovers is { Count: > 0 } ? leftovers : null,
|
||||
});
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes the proxies of frames whose file was replaced: a proxy carries its frame's capture time, which a converted file
|
||||
/// keeps, so the proxy pass would take the old one for current. A failure is logged; the pass then reuses what is left.
|
||||
/// </summary>
|
||||
private static void DropProxies(SequencePaths folder, IReadOnlyList<SidecarFrame> frames)
|
||||
{
|
||||
if (!System.IO.Directory.Exists(folder.Proxies)) return;
|
||||
try
|
||||
{
|
||||
foreach (var frame in frames)
|
||||
{
|
||||
var proxy = folder.Proxy(frame.I);
|
||||
if (File.Exists(proxy)) File.Delete(proxy);
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("archive", $"Proxies of the frames replaced in {folder.RelativeFolder} could not be deleted; the proxy pass may keep them", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether every photo of a replaced entry may swap back into its sequence: each one's frame is there with the content
|
||||
/// that replaced it (its file re-hashed now) and still names the photo as its source.
|
||||
/// </summary>
|
||||
private static async Task<bool> SwappableAsync(PhotoTrashRecord record, SequenceManifest manifest, SequencePaths folder, CancellationToken cancellationToken)
|
||||
{
|
||||
var frames = manifest.Frames.ToDictionary(f => f.Id);
|
||||
foreach (var photo in record.Photos)
|
||||
{
|
||||
if (photo.Replacement is not { } replacement || photo.Frame.Sha256 is not { } original || !frames.TryGetValue(photo.Frame.Id, out var frame)) return false;
|
||||
if (!string.Equals(frame.Sha256, replacement, StringComparison.OrdinalIgnoreCase)
|
||||
|| !string.Equals(frame.Source?.Sha256, original, StringComparison.OrdinalIgnoreCase)) return false;
|
||||
var file = new FileInfo(folder.File(frame.File));
|
||||
if (!file.Exists || file.Length != frame.Size
|
||||
|| !string.Equals(await ContentHash.Sha256Async(file.FullName, null, cancellationToken).ConfigureAwait(false), replacement, StringComparison.OrdinalIgnoreCase))
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The undo of a conversion: the originals of a replaced entry take their frames' place again
|
||||
/// (<see cref="ReplaceAsync"/>), and the files that replaced them go to the trash as a replaced entry in turn. Null,
|
||||
/// nothing touched, when a frame changed since (<see cref="SwappableAsync"/>) or the sequence did.
|
||||
/// </summary>
|
||||
/// <param name="lease">The entry's lease, which the caller holds.</param>
|
||||
private async Task<ReplaceResult?> SwapBackAsync(Archive archive, ArchivePaths paths, Sequence sequence, PhotoTrashRecord record, string recordPath,
|
||||
string entryFolder, string lease, CancellationToken cancellationToken)
|
||||
{
|
||||
if (record.Photos.Count == 0 || record.Photos.Any(p => p.Files.Count == 0 || p.Frame.Sha256 is null || p.Replacement is null)) return null;
|
||||
var swaps = record.Photos.Select(p => new FrameSwap(p.Frame.Id, paths.ToRelative(Path.Combine(entryFolder, p.Files[0].Path)),
|
||||
[.. p.Files.Skip(1).Select(c => new IncomingCompanion(paths.ToRelative(Path.Combine(entryFolder, c.Path)), FrameCompanions.Tail(c.From, p.Files[0].From)))],
|
||||
(_, _) => p.Files[0].From, _ => p.Frame)).ToList();
|
||||
var copy = Path.Combine(entryFolder, AnalysisCopy);
|
||||
var analysis = record.Leftovers?.Any(f => f.Path == AnalysisCopy) == true && File.Exists(copy) ? paths.ToRelative(copy) : null;
|
||||
var consumed = new ConsumedEntry(paths.ToRelative(recordPath), paths.ToRelative(entryFolder), lease, [.. record.Photos.Select(p => p.Frame.Id)], analysis);
|
||||
return await ReplaceAsync(archive, sequence.Id, null, swaps, null, null, consumed, (manifest, folder, ct) => SwappableAsync(record, manifest, folder, ct),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gives a frame whose original is missing its photo back in another encoding (owner review 2, R7): a verified copy of
|
||||
/// what replaced it, or of what it replaced, fetched from the other side of a sync link, where the photo was converted
|
||||
/// (<see cref="Sync.SyncTombstones"/> counts both as one photo). Under the sequence's lease, while the frame still holds
|
||||
/// <paramref name="sha256"/> and has no file, <paramref name="staged"/> takes its place (<see cref="ReplaceAsync"/>,
|
||||
/// <see cref="FrameSwap.Missing"/>): the same id and index, the user's flags, a record read from the file as filing reads
|
||||
/// one (named as the frame with the file's extension), the old content as its source and a tombstone naming the new one,
|
||||
/// so the photo is neither filed twice nor missing. Nothing goes to the trash. Returns the sequence as it is now and its
|
||||
/// statuses before, whose passes (<see cref="FollowUps"/>) the swap cleared as any replacement does; null, nothing
|
||||
/// touched, when the frame changed or its file is back.
|
||||
/// </summary>
|
||||
/// <param name="staged">The verified copy, inside the archive (its staging); moved in.</param>
|
||||
/// <param name="fileName">The copy's name on the other side, whose extension the frame's file takes.</param>
|
||||
internal async Task<ReplaceResult?> ReplaceMissingAsync(Archive archive, Guid sequenceId, Guid frameId, string sha256, string staged, string stagedSha256,
|
||||
string fileName, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(archive);
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
var size = new FileInfo(staged).Length;
|
||||
var metadata = Core.Metadata.MetadataReader.Read(staged, cancellationToken);
|
||||
var extension = Path.GetExtension(fileName);
|
||||
var swap = new FrameSwap(frameId, paths.ToRelative(staged), [], Named, Read) { Missing = true };
|
||||
return await ReplaceAsync(archive, sequenceId, null, [swap], null, null, null,
|
||||
(manifest, _, _) => Task.FromResult(manifest.Frames.Any(f => f.Id == frameId && string.Equals(f.Sha256, sha256, StringComparison.OrdinalIgnoreCase))),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
string Named(SidecarFrame current, SequenceManifest _) => extension.Length > 1 ? Path.ChangeExtension(current.File, extension) : current.File;
|
||||
|
||||
// The file's own facts, at the frame's capture time, with the content it had as its source.
|
||||
SidecarFrame Read(SidecarFrame current)
|
||||
{
|
||||
var read = SequenceFrames.FromMetadata(current.Id, current.File, current.Original, size, stagedSha256, metadata, current.CapturedAt);
|
||||
return read with
|
||||
{
|
||||
CapturedAt = current.CapturedAt ?? read.CapturedAt,
|
||||
UtcOffset = current.UtcOffset ?? read.UtcOffset,
|
||||
TimeSource = current.CapturedAt is null ? read.TimeSource : current.TimeSource,
|
||||
Source = new FrameSource { Sha256 = current.Sha256, Format = current.Format, File = current.File, Size = current.Size },
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A replaced entry's record without its copy of <c>analysis.json</c> once no photo is left in it (all of them filed into
|
||||
/// a new sequence, which is analysed again): the copy, derived from those photos, goes with the entry.
|
||||
/// </summary>
|
||||
private static PhotoTrashRecord WithoutAnalysisCopy(string entryFolder, PhotoTrashRecord record)
|
||||
{
|
||||
if (record.Kind != TrashKind.Replaced || record.Photos.Count > 0 || record.Leftovers?.Any(f => f.Path == AnalysisCopy) != true) return record;
|
||||
var copy = Path.Combine(entryFolder, AnalysisCopy);
|
||||
if (File.Exists(copy)) File.Delete(copy);
|
||||
var leftovers = record.Leftovers.Where(f => f.Path != AnalysisCopy).ToList();
|
||||
return record with { Leftovers = leftovers.Count > 0 ? leftovers : null };
|
||||
}
|
||||
}
|
||||
@@ -32,7 +32,10 @@ public sealed partial class FrameTrash
|
||||
/// (<see cref="FrameRestoreJournal"/>) settles a restore a crash interrupted, and once the first file moves the restore
|
||||
/// runs to its end whatever the caller cancels. When the sequence is gone (deleted, or never indexed in this archive),
|
||||
/// they go into a new sequence through the archive ingest, titled and placed as theirs was, the record rewritten after
|
||||
/// each photo filed; a cancel there stops before the next photo. The entry goes once nothing is left in it.
|
||||
/// each photo filed; a cancel there stops before the next photo. The entry goes once nothing is left in it. An entry of
|
||||
/// <see cref="TrashKind.Replaced"/> (originals a conversion replaced, owner review 2, R7) swaps back instead while every
|
||||
/// converted frame is as the conversion left it (<see cref="SwapBackAsync"/>: the converted files go to the trash as a
|
||||
/// replaced entry in turn); otherwise its originals come back as a new sequence.
|
||||
/// </summary>
|
||||
/// <param name="name">The entry's <see cref="TrashEntry.Name"/> (<c>photos-…</c>).</param>
|
||||
/// <exception cref="DirectoryNotFoundException">The entry is not in the trash.</exception>
|
||||
@@ -59,16 +62,30 @@ public sealed partial class FrameTrash
|
||||
var into = sequence is not null && sequence.ArchiveId == archive.Id ? sequence : null;
|
||||
// What an earlier restore a crash interrupted left half-way is settled first, so the record read next is true.
|
||||
Unstage(entryFolder);
|
||||
if (into is not null && File.Exists(paths.Sequence(into.Folder).RestoreJournalPath))
|
||||
// A replacement too (a conversion or its undo), whose completion may wait for this entry's lease.
|
||||
if (into is not null && (File.Exists(paths.Sequence(into.Folder).RestoreJournalPath) || File.Exists(paths.Sequence(into.Folder).ReplaceJournalPath)))
|
||||
{
|
||||
await using (await _writer.LockAsync(paths, into.Id, cancellationToken).ConfigureAwait(false))
|
||||
SequenceWriter.ReadLocked(paths.Sequence(into.Folder), into.Id, lease);
|
||||
SequenceWriter.ReadLocked(await SequenceWriter.FolderAsync(_db, paths, into, cancellationToken).ConfigureAwait(false), into.Id, lease);
|
||||
}
|
||||
if (!System.IO.Directory.Exists(entryFolder) || ReadRecord(recordPath) is not { } record)
|
||||
throw new DirectoryNotFoundException($"{name} is not in the trash of {archive.Name}.");
|
||||
var (intact, kept) = await CheckAsync(entryFolder, record, cancellationToken).ConfigureAwait(false);
|
||||
// Nothing that matches its record: the entry stays as it is.
|
||||
if (intact.Count == 0) throw new TrashMismatchException();
|
||||
if (record.Kind == TrashKind.Replaced)
|
||||
{
|
||||
// The undo of a conversion: while every converted frame is as the conversion left it, the originals take their
|
||||
// place again; otherwise they come back as a new sequence, beside the converted one.
|
||||
if (into is not null && kept.Count == 0
|
||||
&& await SwapBackAsync(archive, paths, into, record, recordPath, entryFolder, lease, cancellationToken).ConfigureAwait(false) is { } swapped)
|
||||
{
|
||||
Log.Info("archive", $"{record.SequenceTitle}: {record.Photos.Count} converted frame(s) swapped back to their originals");
|
||||
await QueueAsync(followUps, archive, swapped.Sequence, swapped.Had, CancellationToken.None).ConfigureAwait(false);
|
||||
return new PhotoRestoreResult(swapped.Sequence, record.Photos.Count, [], false) { Kind = TrashKind.Replaced };
|
||||
}
|
||||
into = null;
|
||||
}
|
||||
var had = into?.Status ?? SequenceStatus.None;
|
||||
PhotoRestoreResult result;
|
||||
List<Stay> stays;
|
||||
@@ -83,7 +100,8 @@ public sealed partial class FrameTrash
|
||||
{
|
||||
var restore = await IntoNewSequenceAsync(archive, paths, record, recordPath, entryFolder, intact, cancellationToken).ConfigureAwait(false);
|
||||
stays = [.. kept, .. restore.Left];
|
||||
Rewrite(recordPath, entryFolder, record with { Photos = [.. stays.Select(s => s.Photo)], Leftovers = Merge(record.Leftovers, restore.Leftovers) });
|
||||
Rewrite(recordPath, entryFolder, WithoutAnalysisCopy(entryFolder,
|
||||
record with { Photos = [.. stays.Select(s => s.Photo)], Leftovers = Merge(record.Leftovers, restore.Leftovers) }));
|
||||
restore.Failure?.Throw();
|
||||
stopped = restore.Stopped;
|
||||
if (stopped && restore.Restored == 0) throw new OperationCanceledException(cancellationToken);
|
||||
@@ -94,7 +112,11 @@ public sealed partial class FrameTrash
|
||||
result = new PhotoRestoreResult(row, restore.Restored, [], true);
|
||||
}
|
||||
Log.Info("archive", $"{record.SequenceTitle}: {result.Restored} photo(s) restored from the trash, {stays.Count} kept there");
|
||||
var all = result with { Kept = [.. stays.Select(s => new KeptPhoto(s.Photo.Frame.Original ?? Path.GetFileName(s.Photo.Frame.File), s.Reason))] };
|
||||
var all = result with
|
||||
{
|
||||
Kept = [.. stays.Select(s => new KeptPhoto(s.Photo.Frame.Original ?? Path.GetFileName(s.Photo.Frame.File), s.Reason))],
|
||||
Kind = record.Kind,
|
||||
};
|
||||
if (result.Restored > 0) await QueueAsync(followUps, archive, all.Sequence, had, CancellationToken.None).ConfigureAwait(false);
|
||||
// A cancel stopped a restore into a new sequence before its last photo: those not done stay in the trash, listed.
|
||||
if (stopped) throw new OperationCanceledException(cancellationToken);
|
||||
@@ -158,8 +180,9 @@ public sealed partial class FrameTrash
|
||||
{
|
||||
var folder = await SequenceWriter.FolderAsync(_db, paths, sequence, cancellationToken).ConfigureAwait(false);
|
||||
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id, lease);
|
||||
// A restore of another entry, busy now, is not settled yet: its journal is not written over.
|
||||
// A restore of another entry, or a replacement, busy now, is not settled yet: its journal is not written over.
|
||||
EnsureSettled(folder);
|
||||
EnsureReplaced(folder);
|
||||
var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
|
||||
var photosChanged = !inStep && await _indexer.PhotosChangedAsync(archive, folder, cancellationToken).ConfigureAwait(false);
|
||||
var ids = manifest.Frames.Select(f => f.Id).ToHashSet();
|
||||
|
||||
@@ -89,7 +89,11 @@ public sealed record KeptPhoto(string Name, KeptReason Reason)
|
||||
/// What a restore of photos did: the sequence they went back into (a new one when theirs is gone), how many came back, and
|
||||
/// the photos kept in the trash, each with why (<see cref="KeptReason"/>).
|
||||
/// </summary>
|
||||
public sealed record PhotoRestoreResult(Sequence Sequence, int Restored, IReadOnlyList<KeptPhoto> Kept, bool NewSequence);
|
||||
public sealed record PhotoRestoreResult(Sequence Sequence, int Restored, IReadOnlyList<KeptPhoto> Kept, bool NewSequence)
|
||||
{
|
||||
/// <summary>The kind of entry restored: deleted photos, or originals a conversion replaced (<see cref="TrashKind.Replaced"/>).</summary>
|
||||
public TrashKind Kind { get; init; } = TrashKind.Photos;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deleting photos of a sequence without erasing anything (owner review 2, R6; BRIEF §0.2 rule 8): <see cref="PlanDeleteAsync"/>
|
||||
@@ -233,6 +237,8 @@ public sealed partial class FrameTrash
|
||||
if ((await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false))?.Folder != revised.Sequence.Folder)
|
||||
throw new StaleDeletePlanException(id);
|
||||
var manifest = SequenceWriter.ReadLocked(folder, id);
|
||||
// A replacement not settled yet (the entry its files came from busy) is settled first: nothing moves under it.
|
||||
EnsureReplaced(folder);
|
||||
var plannedPaths = plan.Frames.ToDictionary(f => f.Id, f => f.RelativePath);
|
||||
var planned = plannedPaths.Keys.ToHashSet();
|
||||
var removed = manifest.Frames.Where(f => planned.Contains(f.Id)).ToList();
|
||||
@@ -611,10 +617,13 @@ public sealed partial class FrameTrash
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Queues what the frames need again (see <see cref="DeleteAsync"/>), as an orphans fix does; an index that cannot take them is logged.</summary>
|
||||
private async Task QueueAsync(JobEngine? followUps, Archive archive, Sequence sequence, SequenceStatus had, CancellationToken cancellationToken)
|
||||
/// <summary>
|
||||
/// What the frames of <paramref name="sequence"/> need again after they changed, by the statuses it
|
||||
/// <paramref name="had"/>: the light analysis, a background deep analysis when it had one, and the thumbnails and proxies
|
||||
/// the host keeps (always on a server; on a desktop those it had).
|
||||
/// </summary>
|
||||
internal List<ArchiveJobRequest> FollowUps(Archive archive, Sequence sequence, SequenceStatus had)
|
||||
{
|
||||
if (followUps is null) return;
|
||||
var label = sequence.DisplayTitle;
|
||||
List<ArchiveJobRequest> requests = [ProcessingJobs.LightAnalysis([sequence.Id], label)];
|
||||
if ((had & SequenceStatus.AnalyzedDeep) != 0) requests.Add(ProcessingJobs.DeepAnalysis([sequence.Id], label) with { Priority = JobPriority.Background });
|
||||
@@ -622,6 +631,15 @@ public sealed partial class FrameTrash
|
||||
if (images || (had & SequenceStatus.ThumbsBuilt) != 0) requests.Add(ArchiveJobs.Thumbnails([sequence.Id], label));
|
||||
if ((images || (had & SequenceStatus.ProxiesBuilt) != 0) && archive.ProxyLongEdge != ProxyPolicy.Disabled)
|
||||
requests.Add(ArchiveJobs.Proxies([sequence.Id], label));
|
||||
return requests;
|
||||
}
|
||||
|
||||
/// <summary>Queues what the frames need again (<see cref="FollowUps"/>), as an orphans fix does; an index that cannot take them is logged.</summary>
|
||||
private async Task QueueAsync(JobEngine? followUps, Archive archive, Sequence sequence, SequenceStatus had, CancellationToken cancellationToken)
|
||||
{
|
||||
if (followUps is null) return;
|
||||
var label = sequence.DisplayTitle;
|
||||
var requests = FollowUps(archive, sequence, had);
|
||||
try
|
||||
{
|
||||
foreach (var request in requests)
|
||||
@@ -658,7 +676,10 @@ public sealed partial class FrameTrash
|
||||
|
||||
private static TrashEntry Entry(Guid archiveId, string folderName, PhotoTrashRecord record) =>
|
||||
new(archiveId, EntryPrefix + folderName, record.SequenceId, record.SequenceTitle, record.SequenceFolder, record.Photos.Count,
|
||||
record.Photos.Sum(p => p.Files.Count) + (record.Leftovers?.Count ?? 0), record.Bytes, record.DeletedAt, record.DeletedBy, record.Kind);
|
||||
record.Photos.Sum(p => p.Files.Count) + (record.Leftovers?.Count ?? 0), record.Bytes, record.DeletedAt, record.DeletedBy, record.Kind)
|
||||
{
|
||||
JobId = record.JobId,
|
||||
};
|
||||
|
||||
private static PhotoTrashRecord? ReadRecord(string path)
|
||||
{
|
||||
@@ -676,12 +697,15 @@ public sealed partial class FrameTrash
|
||||
|
||||
/// <summary>
|
||||
/// What the trash keeps next to deleted photos (owner review 2, R6): the frame records, where each file was and is, the
|
||||
/// sequence they came from. <see cref="TrashKind.Replaced"/> entries (a later conversion) name each photo's replacement.
|
||||
/// sequence they came from. <see cref="TrashKind.Replaced"/> entries (a conversion, R7) name each photo's replacement and
|
||||
/// keep a copy of the sequence's <c>analysis.json</c> as a leftover (<see cref="FrameTrash.AnalysisCopy"/>).
|
||||
/// </summary>
|
||||
/// <param name="Leftovers">Files of the entry whose photo is back in a sequence without them (a companion whose name was
|
||||
/// taken there): counted and erased with the entry, never restored.</param>
|
||||
/// <param name="JobId">The job that made the entry (a conversion's Replace, owner review 2, R7), whose Undo takes back exactly
|
||||
/// its own entries; null for an entry a user's action made.</param>
|
||||
internal sealed record PhotoTrashRecord(TrashKind Kind, Guid SequenceId, string SequenceTitle, string SequenceFolder, IReadOnlyList<TrashedPhoto> Photos,
|
||||
long Bytes, DateTimeOffset DeletedAt, string? DeletedBy, IReadOnlyList<TrashedFile>? Leftovers = null);
|
||||
long Bytes, DateTimeOffset DeletedAt, string? DeletedBy, IReadOnlyList<TrashedFile>? Leftovers = null, Guid? JobId = null);
|
||||
|
||||
/// <summary>One photo of a photo entry: its frame record, its files (the original first) and, when replaced, the replacement's SHA-256.</summary>
|
||||
internal sealed record TrashedPhoto(SidecarFrame Frame, IReadOnlyList<TrashedFile> Files, string? Replacement);
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Chronos.Core.Conversion;
|
||||
using Chronos.Core.Import;
|
||||
using Chronos.Core.Model;
|
||||
|
||||
@@ -51,8 +52,33 @@ public sealed record MoveJobParameters(string? FolderTemplate, string? FileTempl
|
||||
/// <summary>A sequence of a <see cref="MoveJobParameters"/> (a job queued by an earlier build also names a frames version, which is ignored).</summary>
|
||||
public sealed record MoveSequence(Guid Id);
|
||||
|
||||
/// <summary>A job ready to enqueue.</summary>
|
||||
public sealed record ArchiveJobRequest(JobKind Kind, IReadOnlyList<Guid> Targets, string? Label, string? ParametersJson, JobPriority Priority);
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.Convert"/> (owner review 2, R7): targets = the sequences converted, so the jobs that would touch them
|
||||
/// wait; the archive, the target format and JPEG quality, the mode chosen for this run, the sequences of the confirmed plan
|
||||
/// in its order, each with what it converts, and the plan's fingerprint (<see cref="ConvertPlan.Fingerprint"/>). A sequence
|
||||
/// whose photos to convert are no longer those confirmed is left as it is and reported. Which photos those are is not
|
||||
/// here but in the job's own file beside its journal (<see cref="ConfirmedPhotos"/>), so that the row stays as small as the
|
||||
/// number of sequences, whatever the number of photos, for everything that reads jobs.
|
||||
/// </summary>
|
||||
public sealed record ConvertJobParameters(Guid ArchiveId, ConvertTarget Target, int Quality, ConvertMode Mode, IReadOnlyList<ConvertSequence> Sequences,
|
||||
string Fingerprint);
|
||||
|
||||
/// <summary>
|
||||
/// A sequence of a <see cref="ConvertJobParameters"/>: the frames version it was planned at, and the digest of the photos it
|
||||
/// converts, each with its content (<see cref="ConvertSequencePlan.Digest"/>). Under the sequence's lease, before it
|
||||
/// converts anything, the job takes the photos confirmed (<see cref="ConfirmedPhotos"/>) from its plan made again and goes
|
||||
/// on only while they have that digest: photos that came in since (an upload while the job waited its turn) stay as they
|
||||
/// are, and a photo the plan did not list (one another program held while it was made) is never converted. Without the
|
||||
/// confirmed photos, the whole plan made again must have the digest. Null in a job an earlier build queued, which checks
|
||||
/// the frames version only.
|
||||
/// </summary>
|
||||
public sealed record ConvertSequence(Guid Id, int FramesVersion, string? Frames = null);
|
||||
|
||||
/// <summary>
|
||||
/// A job ready to enqueue; <paramref name="Id"/>, when set, is the id it is queued under (chosen first, for what the job
|
||||
/// reads beside its journal: <see cref="ConfirmedPhotos"/>), otherwise the engine gives it one.
|
||||
/// </summary>
|
||||
public sealed record ArchiveJobRequest(JobKind Kind, IReadOnlyList<Guid> Targets, string? Label, string? ParametersJson, JobPriority Priority, Guid? Id = null);
|
||||
|
||||
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, UseStringEnumConverter = true)]
|
||||
[JsonSerializable(typeof(ScanParameters))]
|
||||
@@ -64,6 +90,11 @@ public sealed record ArchiveJobRequest(JobKind Kind, IReadOnlyList<Guid> Targets
|
||||
[JsonSerializable(typeof(CleanupParameters))]
|
||||
[JsonSerializable(typeof(EmptyTrashParameters))]
|
||||
[JsonSerializable(typeof(MoveJobParameters))]
|
||||
[JsonSerializable(typeof(ConvertJobParameters))]
|
||||
[JsonSerializable(typeof(ConvertJournal))]
|
||||
[JsonSerializable(typeof(StagedLine))]
|
||||
[JsonSerializable(typeof(PinnedSequence))]
|
||||
[JsonSerializable(typeof(ConfirmedLine))]
|
||||
[JsonSerializable(typeof(CardProfile))]
|
||||
internal sealed partial class ArchiveJobJson : JsonSerializerContext
|
||||
{
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
using System.Text.Json;
|
||||
using Chronos.Core.Diagnostics;
|
||||
using Chronos.Core.Localization;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Data.Jobs;
|
||||
|
||||
namespace Chronos.Data.Archiving.Jobs;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.Convert"/>: a mass conversion (owner review 2, R7; BRIEF §12.5) of the sequences of a confirmed plan
|
||||
/// (<see cref="ConvertJobParameters"/>) through <see cref="MassConverter"/>, which settles what a crash left half-way, refuses
|
||||
/// when the volume lacks the space (measured again before each sequence and each photo), and converts one sequence at a
|
||||
/// time, each all or nothing, the photos confirmed alone and only while each holds the content confirmed (photos that came
|
||||
/// in since stay as they are); its journal makes every stage resumable. Progress counts photos; the checkpoint only marks a
|
||||
/// resume, the journal says where. Each sequence whose frames were replaced is told as <c>converted</c>, each new sequence
|
||||
/// as <c>created</c> (<see cref="ArchiveServiceOptions.SequenceChanged"/>), and the passes it needs (light analysis, a deep
|
||||
/// one when the source had it, thumbnails and proxies) are queued as follow-ups, as soon as it is done. Sequences left as
|
||||
/// they were (changed after the plan, busy, a photo another program holds, a photo that failed) are logged and fail the
|
||||
/// job at the end, after the others are converted; a retry tries again those busy, held or failed, with what they staged,
|
||||
/// and leaves one that changed after the plan as it is (its log line says to plan again). A retry of a New sequence
|
||||
/// conversion finishes the copies it made and makes none of a source another conversion copied in the same format since
|
||||
/// this job was queued (its creation time: when the plan was confirmed).
|
||||
/// </summary>
|
||||
internal sealed class ConvertJobHandler(ChronosDatabase db, MassConverter converter, Func<Guid, string, CancellationToken, Task>? sequenceChanged) : IJobHandler
|
||||
{
|
||||
/// <summary>The change told about a sequence whose frames a conversion replaced.</summary>
|
||||
public const string Converted = "converted";
|
||||
|
||||
/// <summary>The change told about a sequence a conversion made.</summary>
|
||||
public const string Created = "created";
|
||||
|
||||
private const string Checkpoint = "convert";
|
||||
|
||||
public JobKind Kind => JobKind.Convert;
|
||||
|
||||
public async Task RunAsync(JobContext context)
|
||||
{
|
||||
var parameters = (context.Job.ParametersJson is { Length: > 0 } json ? JsonSerializer.Deserialize(json, ArchiveJobJson.Default.ConvertJobParameters) : null)
|
||||
?? throw new InvalidOperationException($"Convert job {context.Job.Id} has no plan.");
|
||||
if (parameters.Fingerprint.Length == 0) throw new InvalidOperationException($"Convert job {context.Job.Id} has no confirmed plan.");
|
||||
var archive = await db.Archives.GetAsync(parameters.ArchiveId, context.Cancellation).ConfigureAwait(false)
|
||||
?? throw new InvalidOperationException($"Convert job {context.Job.Id} does not name a known archive.");
|
||||
var name = context.Job.CreatedBy is { } user ? (await db.Users.GetAsync(user, context.Cancellation).ConfigureAwait(false))?.Username : null;
|
||||
var by = new ConvertRequester(name, context.Job.CreatedBy, context.Job.Created);
|
||||
var result = await converter.RunAsync(archive, parameters, context.Job.Id, by, new ConvertCallbacks(
|
||||
(done, total) => context.ReportAsync(done, total),
|
||||
(done, total) => context.CheckpointAsync(Checkpoint, done, total),
|
||||
context.Log, TellAsync, requests => QueueAsync(context, requests)), context.Cancellation).ConfigureAwait(false);
|
||||
context.Log(Loc.L("Archive.Log.ConvertDone", result.Sequences.Count(s => s.Outcome == ConvertOutcome.Converted),
|
||||
result.Sequences.Count(s => s.Outcome == ConvertOutcome.Created), result.Frames));
|
||||
var kept = result.Kept.Count();
|
||||
if (kept > 0) throw new InvalidOperationException(Loc.Plural("Archive.Error.ConvertKept", kept));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queues what a sequence converted or made needs as soon as its outcome is journaled, each kind once per sequence; a
|
||||
/// failure is logged, never passed on.
|
||||
/// </summary>
|
||||
private async ValueTask QueueAsync(JobContext context, IReadOnlyList<ArchiveJobRequest> requests)
|
||||
{
|
||||
foreach (var request in requests)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (request.Targets.Count > 0 && await db.Jobs.IsQueuedAsync(request.Kind, request.Targets[0], CancellationToken.None).ConfigureAwait(false)) continue;
|
||||
await context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority).ConfigureAwait(false);
|
||||
}
|
||||
catch (DatabaseException ex)
|
||||
{
|
||||
Log.Warning("archive", $"{request.Label}: the passes its converted frames need were not queued; the next scan queues them", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tells a sequence whose frames were replaced, or the one made, as soon as its outcome is journaled, while the rest of the
|
||||
/// job runs (an editor open on it reads it again); a failure is logged, never passed on to the job.
|
||||
/// </summary>
|
||||
private async ValueTask TellAsync(ConvertSequenceResult sequence)
|
||||
{
|
||||
if (sequenceChanged is null) return;
|
||||
var (id, change) = sequence.Outcome switch
|
||||
{
|
||||
ConvertOutcome.Converted => (sequence.SequenceId, Converted),
|
||||
ConvertOutcome.Created when sequence.NewSequenceId is { } made => (made, Created),
|
||||
_ => (Guid.Empty, string.Empty),
|
||||
};
|
||||
if (id == Guid.Empty) return;
|
||||
try
|
||||
{
|
||||
await sequenceChanged(id, change, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Warning("archive", $"The change of sequence {id} was not told", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -10,8 +10,9 @@ namespace Chronos.Data.Archiving.Jobs;
|
||||
/// <see cref="JobKind.EmptyTrash"/> (owner review 2, R6): erases exactly the entries of a confirmed plan
|
||||
/// (<see cref="EmptyTrashParameters"/>) and fails, erasing nothing, when the trash no longer matches the plan's
|
||||
/// fingerprint (<see cref="StaleTrashPlanException"/>). Checkpoint: <c>done:<n></c>, the entries done; a resumed job
|
||||
/// checks each remaining entry against its own fingerprint. Entries kept (a replacement that no longer matches, an entry
|
||||
/// changed since) are logged and fail the job at the end, after the others are erased.
|
||||
/// checks each remaining entry against its own fingerprint. Entries kept (an entry changed since, a conversion of replaced
|
||||
/// originals that no longer matches, cannot be located or is busy) are logged each with why, and fail the job at the end,
|
||||
/// after the others are erased.
|
||||
/// </summary>
|
||||
internal sealed class EmptyTrashJobHandler(ChronosDatabase db, SequenceTrash trash) : IJobHandler
|
||||
{
|
||||
@@ -31,7 +32,7 @@ internal sealed class EmptyTrashJobHandler(ChronosDatabase db, SequenceTrash tra
|
||||
var result = await trash.EmptyAsync(archive, parameters, start,
|
||||
(n, total) => context.CheckpointAsync(Prefix + n.ToString(CultureInfo.InvariantCulture), n, total), context.Cancellation).ConfigureAwait(false);
|
||||
context.Log(Loc.L("Archive.Log.TrashEmptied", result.Erased, CleanupSteps.Megabytes(result.Bytes)));
|
||||
foreach (var title in result.Refused) context.Log(Loc.L("Archive.Log.TrashKept", title));
|
||||
foreach (var kept in result.Refused) context.Log(kept.LogLine);
|
||||
if (result.Refused.Count > 0) throw new InvalidOperationException(Loc.Plural("Archive.Error.TrashKept", result.Refused.Count));
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -582,17 +582,19 @@ public sealed class MassRenamer
|
||||
/// <summary>
|
||||
/// Settles, under the step's leases and before anything of it moves, what a crash or a busy trash entry left in the
|
||||
/// sequence, as every writer does (<see cref="SequenceWriter.ReadLocked"/>): an interrupted rename of its files, photo
|
||||
/// deletion, restore from the trash or ingest. A restore whose trash entry is busy is not settled yet, and the step
|
||||
/// waits for it (<see cref="FrameTrash.EnsureSettled"/>) as a deletion of the sequence does: its journal names the
|
||||
/// files it moved by the names they had, so neither the folder nor those names change under it. Returns the sidecar as
|
||||
/// settled.
|
||||
/// deletion, frame replacement (a conversion or its undo), restore from the trash or ingest. A restore or a replacement
|
||||
/// whose trash entry is busy is not settled yet, and the step waits for it (<see cref="FrameTrash.EnsureSettled"/>,
|
||||
/// <see cref="FrameTrash.EnsureReplaced"/>) as a deletion of the sequence does: its journal names the files it moved by
|
||||
/// the names they had, so neither the folder nor those names change under it. Returns the sidecar as settled.
|
||||
/// </summary>
|
||||
/// <exception cref="LeaseUnavailableException">A restore from the trash into the sequence is not settled; nothing of the step changed.</exception>
|
||||
/// <exception cref="LeaseUnavailableException">A restore from the trash into the sequence, or a replacement of its frames,
|
||||
/// is not settled; nothing of the step changed.</exception>
|
||||
private static SequenceManifest Settle(ArchivePaths paths, RenameStep step)
|
||||
{
|
||||
var sequence = Where(paths, step);
|
||||
var manifest = SequenceWriter.ReadLocked(sequence, step.SequenceId);
|
||||
FrameTrash.EnsureSettled(sequence);
|
||||
FrameTrash.EnsureReplaced(sequence);
|
||||
return manifest;
|
||||
}
|
||||
|
||||
|
||||
@@ -57,26 +57,30 @@ internal static class ProjectPositions
|
||||
/// Indexed frames are no longer in the sidecar (photos deleted into the trash, here or by another process sharing the
|
||||
/// archive, owner review 2, R6): the positions are translated by photo from the index's frames to the sidecar's
|
||||
/// (<see cref="ProjectSettings.TranslateFramePositions"/>), so those on the photos that went wait as pending frames
|
||||
/// and take their place again when the photos come back (a restore), and the others follow their photos.
|
||||
/// and take their place again when the photos come back (a restore), and the others follow their photos. A frame still
|
||||
/// in the sidecar is its photo as the sidecar has it, so one whose file a conversion replaced meanwhile (R7: the same
|
||||
/// frame, another SHA-256) keeps its positions.
|
||||
/// </summary>
|
||||
private static async Task<PositionMove?> RemovedAsync(ChronosRepositories db, Guid sequenceId, string project, SequenceManifest manifest,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var current = manifest.Frames.ToDictionary(f => f.Id);
|
||||
var before = new List<(int, FramePhoto)>();
|
||||
for (var after = -1; ;)
|
||||
{
|
||||
var page = await db.Frames.ListBySequenceAsync(sequenceId, after, PageSize, cancellationToken).ConfigureAwait(false);
|
||||
before.AddRange(page.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, f.CapturedAt))));
|
||||
before.AddRange(page.Select(f => (f.Index, current.TryGetValue(f.Id, out var now) ? Photo(now) : new FramePhoto(f.Sha256, f.Size, f.CapturedAt))));
|
||||
if (page.Count < PageSize) break;
|
||||
after = page[^1].Index;
|
||||
}
|
||||
var to = new FramePhotos(manifest.Frames.Select(f => (f.I, new FramePhoto(f.Sha256, f.Size, f.CapturedAt))));
|
||||
var to = new FramePhotos(manifest.Frames.Select(f => (f.I, Photo(f))));
|
||||
var translated = ProjectSettings.TranslateFramePositions(project, new FramePhotos(before), to);
|
||||
if (translated is null) return null;
|
||||
var hero = HeroOf(project) is null && HeroOf(translated) is { } index ? manifest.Frames.FirstOrDefault(f => f.I == index)?.Id : null;
|
||||
return new PositionMove(project, translated, hero);
|
||||
}
|
||||
|
||||
private static FramePhoto Photo(SidecarFrame frame) => new(frame.Sha256, frame.Size, frame.CapturedAt);
|
||||
|
||||
/// <summary>
|
||||
/// The index's copy of the project with the pending frames whose photos <paramref name="manifest"/> now holds put in
|
||||
|
||||
@@ -21,8 +21,71 @@ public sealed record TrashEmptyPlan(Guid ArchiveId, IReadOnlyList<TrashEntry> En
|
||||
public int Photos => Entries.Sum(e => e.Frames);
|
||||
}
|
||||
|
||||
/// <summary>What an emptying erased, and the entries it kept (a replacement that no longer matches, an entry changed since the plan).</summary>
|
||||
public sealed record TrashEmptyResult(int Erased, long Bytes, IReadOnlyList<string> Refused);
|
||||
/// <summary>What an emptying erased, and the entries it kept, each with why (<see cref="TrashKept"/>).</summary>
|
||||
public sealed record TrashEmptyResult(int Erased, long Bytes, IReadOnlyList<TrashKept> Refused);
|
||||
|
||||
/// <summary>An entry an emptying kept in the trash: its title and why.</summary>
|
||||
public sealed record TrashKept(string Title, TrashKeptReason Reason)
|
||||
{
|
||||
/// <summary>
|
||||
/// "Kept in the trash, changed after the plan: Etna", a job log line in the current culture; where something settles the
|
||||
/// reason, the line says what.
|
||||
/// </summary>
|
||||
public string LogLine => Loc.L(Reason switch
|
||||
{
|
||||
TrashKeptReason.Mismatch => "Archive.Log.TrashKeptMismatch",
|
||||
TrashKeptReason.Unknown => "Archive.Log.TrashKeptUnknown",
|
||||
TrashKeptReason.Busy => "Archive.Log.TrashKeptBusy",
|
||||
TrashKeptReason.Unhashed => "Archive.Log.TrashKeptUnhashed",
|
||||
TrashKeptReason.Unreadable => "Archive.Log.TrashKeptUnreadable",
|
||||
TrashKeptReason.StaleIndex => "Archive.Log.TrashKeptStaleIndex",
|
||||
_ => "Archive.Log.TrashKept",
|
||||
}, Title);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Why an emptying kept an entry in the trash. For an entry of replaced originals (owner review 2, R7), all but
|
||||
/// <see cref="Changed"/> and <see cref="Busy"/> are about where the archive shows their conversions: they are erased only
|
||||
/// while each conversion is there and matches, or once the archive itself shows it deleted, as the dry run said.
|
||||
/// </summary>
|
||||
public enum TrashKeptReason
|
||||
{
|
||||
/// <summary>The entry changed since the emptying was planned (restored, or deleted again, meanwhile), or is gone.</summary>
|
||||
Changed,
|
||||
|
||||
/// <summary>
|
||||
/// A conversion of its originals no longer matches its content, in its sequence or in the trash, or its frame holds other
|
||||
/// content now (its file changed since the conversion).
|
||||
/// </summary>
|
||||
Mismatch,
|
||||
|
||||
/// <summary>
|
||||
/// The archive does not show what became of a conversion of its originals: its sequence's sidecar holds none of it and no
|
||||
/// tombstone says it was deleted (a sidecar rebuilt from the photos keeps only what one index knew), or the sequence is
|
||||
/// nowhere in the archive and the trash has no record of erasing it. Nothing in the archive settles it: restoring the
|
||||
/// entry keeps its photos.
|
||||
/// </summary>
|
||||
Unknown,
|
||||
|
||||
/// <summary>Its sequence is busy: another writer holds it, or a change to it is not settled yet.</summary>
|
||||
Busy,
|
||||
|
||||
/// <summary>A frame of its sequence that may hold a conversion of its originals was never hashed: a verify of the sequence settles it.</summary>
|
||||
Unhashed,
|
||||
|
||||
/// <summary>
|
||||
/// A sidecar where its sequence may be cannot be read, or photos there have none: a scan of the archive rebuilds or adopts
|
||||
/// it (what the new sidecar shows then decides).
|
||||
/// </summary>
|
||||
Unreadable,
|
||||
|
||||
/// <summary>
|
||||
/// This index lists its sequence in a folder that no longer holds it, the archive holds it nowhere else and the trash has no
|
||||
/// record of erasing it: a rebuild of the index drops that row (restoring the entry then brings its photos back as a new
|
||||
/// sequence).
|
||||
/// </summary>
|
||||
StaleIndex,
|
||||
}
|
||||
|
||||
/// <summary>The trash changed after the emptying was planned (an entry came, went or changed); nothing was erased.</summary>
|
||||
public sealed class StaleTrashPlanException() : InvalidOperationException(Loc.L("Archive.Error.TrashChanged"));
|
||||
@@ -37,12 +100,20 @@ public sealed partial class SequenceTrash
|
||||
|
||||
/// <summary>
|
||||
/// The dry run of emptying the trash: <paramref name="names"/> of the entries to erase (null: every entry), each
|
||||
/// measured on disk, and the plan's fingerprint (<see cref="Fingerprint"/>). Reads only.
|
||||
/// measured on disk, each replaced entry whose conversion was deleted since said so (<see cref="TrashEntry.ConversionDeleted"/>),
|
||||
/// and the plan's fingerprint (<see cref="Fingerprint"/>). Reads only.
|
||||
/// </summary>
|
||||
public async Task<TrashEmptyPlan> PlanEmptyAsync(Archive archive, IReadOnlyList<string>? names = null, CancellationToken cancellationToken = default)
|
||||
public Task<TrashEmptyPlan> PlanEmptyAsync(Archive archive, IReadOnlyList<string>? names = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(archive);
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
return PlanEmptyAsync(archive, paths, names, new SidecarWalk(paths), cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>The dry run, finding the sequences of replaced entries that moved through <paramref name="walk"/> (made at most once).</summary>
|
||||
private async Task<TrashEmptyPlan> PlanEmptyAsync(Archive archive, ArchivePaths paths, IReadOnlyList<string>? names, SidecarWalk walk,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var listed = await ListAsync(archive, cancellationToken).ConfigureAwait(false);
|
||||
var wanted = names?.ToHashSet(StringComparer.Ordinal);
|
||||
var chosen = wanted is null ? listed : [.. listed.Where(e => wanted.Contains(e.Name))];
|
||||
@@ -52,6 +123,9 @@ public sealed partial class SequenceTrash
|
||||
var (files, bytes) = Directory.Exists(folder) ? Measure(folder, cancellationToken) : (0, 0L);
|
||||
return e with { Files = files, Bytes = bytes };
|
||||
}).ToList(), cancellationToken).ConfigureAwait(false);
|
||||
// The sequences of replaced entries, found once for the whole plan.
|
||||
var located = new Dictionary<Guid, Located>();
|
||||
for (var i = 0; i < measured.Count; i++) measured[i] = await PlannedAsync(archive, paths, measured[i], located, walk, cancellationToken).ConfigureAwait(false);
|
||||
var unknown = names?.Where(n => !listed.Any(e => e.Name == n)).Distinct(StringComparer.Ordinal).ToList() ?? [];
|
||||
return new TrashEmptyPlan(archive.Id, measured, measured.Sum(e => e.Bytes), Fingerprint(measured), unknown);
|
||||
}
|
||||
@@ -68,11 +142,15 @@ public sealed partial class SequenceTrash
|
||||
return Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(text.ToString())));
|
||||
}
|
||||
|
||||
/// <summary>What identifies one entry as planned: its kind, name, files and bytes on disk, and when it was deleted.</summary>
|
||||
/// <summary>
|
||||
/// What identifies one entry as planned: its kind, name, files and bytes on disk, when it was deleted, and for a replaced
|
||||
/// entry whether its conversion was deleted (<see cref="TrashEntry.ConversionDeleted"/>), which the dry run says.
|
||||
/// </summary>
|
||||
public static string EntryFingerprint(TrashEntry entry)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(entry);
|
||||
return string.Create(CultureInfo.InvariantCulture, $"{entry.Kind}|{entry.Name}|{entry.Files}|{entry.Bytes}|{entry.DeletedAt.UtcTicks}");
|
||||
var planned = string.Create(CultureInfo.InvariantCulture, $"{entry.Kind}|{entry.Name}|{entry.Files}|{entry.Bytes}|{entry.DeletedAt.UtcTicks}");
|
||||
return entry.ConversionDeleted ? planned + "|conversion-deleted" : planned;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -81,8 +159,13 @@ public sealed partial class SequenceTrash
|
||||
/// hold exactly the planned entries (<see cref="EmptyTrashParameters.Fingerprint"/>), else nothing is erased; a run
|
||||
/// resumed after a stop checks each remaining entry against its own fingerprint, keeps one that changed and skips one
|
||||
/// that is gone, and each entry is checked against its fingerprint once more under its lease, just before it goes. A
|
||||
/// <see cref="TrashKind.Replaced"/> entry is erased only while every replacement still matches its
|
||||
/// SHA-256 in the sequence (re-hashed now); otherwise it is kept. Leftovers of an interrupted erase are deleted first.
|
||||
/// <see cref="TrashKind.Replaced"/> entry is weighed under its sequence's lease as well, against the archive as it is then
|
||||
/// (<see cref="Survey"/>): it is erased only while every conversion of its originals still matches its SHA-256 where the
|
||||
/// archive shows it (re-hashed now: in the sequence, or a copy the trash holds when its conversion was deleted, as the
|
||||
/// dry run said); otherwise, or when the archive cannot tell, it is kept, with why. Leftovers of an interrupted erase are
|
||||
/// deleted first; the sequences of replaced entries that moved are looked up in one walk of the archive for the whole run
|
||||
/// (<see cref="SidecarWalk"/>), and the records of erased sequences no replaced entry needs any more go at the end
|
||||
/// (<see cref="SweepErasures"/>).
|
||||
/// </summary>
|
||||
/// <param name="progress">Entries done so far (the resume point) and their number.</param>
|
||||
/// <exception cref="StaleTrashPlanException">The trash no longer holds exactly the planned entries; nothing was erased.</exception>
|
||||
@@ -95,9 +178,10 @@ public sealed partial class SequenceTrash
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
SweepErasing(paths);
|
||||
var items = parameters.Entries;
|
||||
var current = await PlanEmptyAsync(archive, [.. items.Skip(start).Select(i => i.Name)], cancellationToken).ConfigureAwait(false);
|
||||
var walk = new SidecarWalk(paths);
|
||||
var current = await PlanEmptyAsync(archive, paths, [.. items.Skip(start).Select(i => i.Name)], walk, cancellationToken).ConfigureAwait(false);
|
||||
if (start == 0 && (current.Fingerprint != parameters.Fingerprint || current.Unknown.Count > 0)) throw new StaleTrashPlanException();
|
||||
var refused = new List<string>();
|
||||
var refused = new List<TrashKept>();
|
||||
int erased = 0;
|
||||
long bytes = 0;
|
||||
for (var i = start; i < items.Count; i++)
|
||||
@@ -108,94 +192,125 @@ public sealed partial class SequenceTrash
|
||||
{
|
||||
if (EntryFingerprint(entry) != item.Fingerprint)
|
||||
{
|
||||
refused.Add(entry.Title);
|
||||
refused.Add(new TrashKept(entry.Title, TrashKeptReason.Changed));
|
||||
Log.Warning("archive", $"Trash entry {entry.Name} changed since the emptying was planned; it is kept");
|
||||
}
|
||||
else if (await EraseAsync(paths, entry, item.Fingerprint, cancellationToken).ConfigureAwait(false))
|
||||
else if (await EraseAsync(archive, paths, entry, item.Fingerprint, walk, cancellationToken).ConfigureAwait(false) is { } reason)
|
||||
{
|
||||
refused.Add(new TrashKept(entry.Title, reason));
|
||||
}
|
||||
else
|
||||
{
|
||||
erased++;
|
||||
bytes += entry.Bytes;
|
||||
}
|
||||
else
|
||||
{
|
||||
refused.Add(entry.Title);
|
||||
}
|
||||
}
|
||||
await progress(i + 1, items.Count).ConfigureAwait(false);
|
||||
}
|
||||
SweepErasures(archive, paths, cancellationToken);
|
||||
return new TrashEmptyResult(erased, bytes, refused);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Erases one entry under its lease, once its record read and its folder measured again under it still give the
|
||||
/// planned <paramref name="fingerprint"/> (another host sharing the archive may have restored the entry and deleted the
|
||||
/// same sequence again since); false (logged) when it changed, or a replaced entry's replacements no longer match.
|
||||
/// same sequence again since). A replaced entry is weighed under its sequence's lease too, so no writer of the sequence
|
||||
/// moves its conversions meanwhile, and only once what a writer left unsettled in the sequence is settled (an undo whose
|
||||
/// completion waited for this entry's lease takes photos out of it): the archive must tell where each conversion is (as
|
||||
/// the dry run said, which the fingerprint holds), and each must still match. A deleted sequence that a replaced entry
|
||||
/// still in the trash came from leaves a record of its erasing (<see cref="RecordErasureAsync"/>). Returns null once erased,
|
||||
/// else why the entry is kept (logged).
|
||||
/// </summary>
|
||||
private async Task<bool> EraseAsync(ArchivePaths paths, TrashEntry entry, string fingerprint, CancellationToken cancellationToken)
|
||||
private async Task<TrashKeptReason?> EraseAsync(Archive archive, ArchivePaths paths, TrashEntry entry, string fingerprint, SidecarWalk walk,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
using (await LeaseFile.AcquireAsync(paths.LocksDirectory, EntryLease(entry.Name), _writer.Options.Owner, _writer.Options.LeaseWait,
|
||||
_writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var (folder, record) = Location(paths, entry);
|
||||
if (!Directory.Exists(folder) || !File.Exists(record)) return false;
|
||||
if (Reread(paths, entry, cancellationToken) is not { } now || EntryFingerprint(now) != fingerprint)
|
||||
if (!Directory.Exists(folder) || !File.Exists(record)) return TrashKeptReason.Changed;
|
||||
IAsyncDisposable? sequence = null;
|
||||
if (entry.Kind == TrashKind.Replaced)
|
||||
{
|
||||
Log.Warning("archive", $"Trash entry {entry.Name} changed since the emptying was planned; it is kept");
|
||||
return false;
|
||||
try
|
||||
{
|
||||
sequence = await _writer.LockAsync(paths, entry.SequenceId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (LeaseUnavailableException ex)
|
||||
{
|
||||
Log.Warning("archive", $"Trash entry {entry.Name} is kept: its sequence is busy", ex);
|
||||
return TrashKeptReason.Busy;
|
||||
}
|
||||
}
|
||||
if (entry.Kind == TrashKind.Replaced && !await ReplacementsHoldAsync(paths, entry, cancellationToken).ConfigureAwait(false))
|
||||
await using (sequence)
|
||||
{
|
||||
Log.Warning("archive", $"Trash entry {entry.Name} is kept: a replacement of its originals no longer matches");
|
||||
return false;
|
||||
// Settled before the entry is read again, so what is read is what an erase takes.
|
||||
var located = sequence is null ? null : await LocateAsync(archive, paths, entry, walk, EntryLease(entry.Name), cancellationToken).ConfigureAwait(false);
|
||||
var (now, survey) = Reread(archive, paths, entry, located, cancellationToken);
|
||||
if (survey?.Kept is { } kept)
|
||||
{
|
||||
Log.Warning("archive", $"Trash entry {entry.Name} is kept ({kept}): the archive does not show the conversions of its originals whole");
|
||||
return kept;
|
||||
}
|
||||
if (now is null || EntryFingerprint(now) != fingerprint)
|
||||
{
|
||||
Log.Warning("archive", $"Trash entry {entry.Name} changed since the emptying was planned; it is kept");
|
||||
return TrashKeptReason.Changed;
|
||||
}
|
||||
if (survey is not null && !await MatchAsync(survey, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
Log.Warning("archive", $"Trash entry {entry.Name} is kept: a conversion of its originals no longer matches");
|
||||
return TrashKeptReason.Mismatch;
|
||||
}
|
||||
var aside = Path.Combine(TrashDirectory(paths), ErasingFolder, Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(aside)!);
|
||||
// Before the folder goes: once it has, the archive alone tells a replaced entry of that sequence what became of it.
|
||||
if (entry.Kind == TrashKind.Sequence) await RecordErasureAsync(archive, paths, now, walk, cancellationToken).ConfigureAwait(false);
|
||||
Directory.Move(folder, aside);
|
||||
File.Delete(record);
|
||||
Erase(aside);
|
||||
Log.Info("archive", $"Trash entry {entry.Name} ({entry.Title}, {entry.Files} files, {entry.Bytes} bytes) erased"
|
||||
+ (now.ConversionDeleted ? "; the conversion of its originals had been deleted" : string.Empty));
|
||||
return null;
|
||||
}
|
||||
var aside = Path.Combine(TrashDirectory(paths), ErasingFolder, Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(aside)!);
|
||||
Directory.Move(folder, aside);
|
||||
File.Delete(record);
|
||||
Erase(aside);
|
||||
Log.Info("archive", $"Trash entry {entry.Name} ({entry.Title}, {entry.Files} files, {entry.Bytes} bytes) erased");
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether every photo of a replaced entry still has its replacement in the sequence: a frame holding the replacement's
|
||||
/// SHA-256 whose file hashes to it now. The originals are erased only then (a later phase's conversion).
|
||||
/// An entry as the trash lists it and an erase plans it, from its record and folder as they are now, and a replaced
|
||||
/// entry's survey against where its sequence was found under its lease (<paramref name="located"/>, which says whether the
|
||||
/// conversion of one of its photos was deleted, <see cref="TrashEntry.ConversionDeleted"/>); a null entry when its record
|
||||
/// cannot be read or its folder is gone.
|
||||
/// </summary>
|
||||
private async Task<bool> ReplacementsHoldAsync(ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken)
|
||||
{
|
||||
if (FrameTrash.ReadEntry(paths, entry.Name) is not { } record) return false;
|
||||
foreach (var photo in record.Photos)
|
||||
{
|
||||
if (photo.Replacement is not { } replacement) return false;
|
||||
var holders = (await _db.Frames.FindBySha256Async(replacement.ToLowerInvariant(), cancellationToken).ConfigureAwait(false))
|
||||
.Where(f => f.SequenceId == record.SequenceId).ToList();
|
||||
var verified = false;
|
||||
foreach (var frame in holders)
|
||||
{
|
||||
var file = paths.FromRelative(frame.RelativePath);
|
||||
if (!File.Exists(file)) continue;
|
||||
if (string.Equals(await ContentHash.Sha256Async(file, null, cancellationToken).ConfigureAwait(false), replacement, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
verified = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!verified) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>An entry as the trash lists it, from its record and folder as they are now; null when its record cannot be read.</summary>
|
||||
private static TrashEntry? Reread(ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken)
|
||||
private static (TrashEntry? Entry, ConversionSurvey? Survey) Reread(Archive archive, ArchivePaths paths, TrashEntry entry, Located? located,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var (folder, recordPath) = Location(paths, entry);
|
||||
var now = FrameTrash.IsPhotoEntry(entry.Name)
|
||||
? FrameTrash.ReadListed(paths, entry.ArchiveId, entry.Name)
|
||||
: ReadRecord(recordPath) is { } record ? Entry(entry.ArchiveId, Path.GetFileName(entry.Name), record) : null;
|
||||
if (now is null) return null;
|
||||
if (now is null || !Directory.Exists(folder)) return (null, null);
|
||||
var (files, bytes) = Measure(folder, cancellationToken);
|
||||
return now with { Files = files, Bytes = bytes };
|
||||
now = now with { Files = files, Bytes = bytes };
|
||||
if (located is null) return (now, null);
|
||||
if (FrameTrash.ReadEntry(paths, entry.Name) is not { } photos) return (null, null);
|
||||
var survey = Survey(archive, paths, entry.Name, photos, located, cancellationToken);
|
||||
return (survey.Deleted ? now with { ConversionDeleted = true } : now, survey);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <paramref name="entry"/> as an emptying plans it: a replaced entry is <see cref="TrashEntry.ConversionDeleted"/> when the
|
||||
/// archive shows the conversion of one of its photos deleted (<see cref="Survey"/>), so the dry run says so. Its sequence
|
||||
/// is found once per plan (<paramref name="located"/>, through <paramref name="walk"/> when it moved). Reads sidecars and
|
||||
/// records; hashes nothing.
|
||||
/// </summary>
|
||||
private async Task<TrashEntry> PlannedAsync(Archive archive, ArchivePaths paths, TrashEntry entry, Dictionary<Guid, Located> located, SidecarWalk walk,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (entry.Kind != TrashKind.Replaced || FrameTrash.ReadEntry(paths, entry.Name) is not { } record) return entry;
|
||||
if (!located.TryGetValue(entry.SequenceId, out var where))
|
||||
located[entry.SequenceId] = where = await LocateAsync(archive, paths, entry, walk, null, cancellationToken).ConfigureAwait(false);
|
||||
return Survey(archive, paths, entry.Name, record, where, cancellationToken).Deleted ? entry with { ConversionDeleted = true } : entry;
|
||||
}
|
||||
|
||||
/// <summary>The folder and record of an entry of either kind.</summary>
|
||||
|
||||
@@ -0,0 +1,582 @@
|
||||
using System.Text.Json;
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.Diagnostics;
|
||||
using Chronos.Core.IO;
|
||||
using Chronos.Core.Model;
|
||||
|
||||
namespace Chronos.Data.Archiving;
|
||||
|
||||
/// <summary>
|
||||
/// Where the conversions of a replaced entry's originals are now (owner review 2, R7), for the emptying of the trash, as the
|
||||
/// archive itself shows it, never as this host's index alone has it (another host sharing the archive may have converted,
|
||||
/// moved or deleted since): the sequence's sidecar, found where the index or the entry says the sequence is, in the trash,
|
||||
/// or among the archive's folders (walked once per plan, <see cref="SidecarWalk"/>), followed through its Replaced tombstones
|
||||
/// (a conversion of a conversion), and the files it names. A conversion counts as deleted only on the archive's own word: a
|
||||
/// Deleted tombstone at the end of the chain in the sequence's sidecar, the sequence in the trash (its trash record), or the
|
||||
/// trash's record of having erased it (<see cref="ErasureRecord"/>). Anything else keeps the entry, with what settles it
|
||||
/// (<see cref="TrashKeptReason"/>): a sidecar that holds none of the conversion and does not say it was deleted (one rebuilt
|
||||
/// from the photos keeps only what one index knew), a sequence found nowhere that the trash has no record of erasing, and a
|
||||
/// frame whose file changed since its conversion.
|
||||
/// </summary>
|
||||
public sealed partial class SequenceTrash
|
||||
{
|
||||
/// <summary>Links a replacement chain may have at most (a conversion of a conversion…), as sync follows them.</summary>
|
||||
private const int ReplacementSteps = 8;
|
||||
|
||||
/// <summary>Folder inside the trash of the records of the deleted sequences it erased (<see cref="ErasureRecord"/>).</summary>
|
||||
private const string ErasedFolder = ".erased";
|
||||
|
||||
/// <summary>Where a replaced entry's sequence is, as the archive shows it.</summary>
|
||||
private enum Whereabouts
|
||||
{
|
||||
/// <summary>In a folder of the archive, whose sidecar names it.</summary>
|
||||
Live,
|
||||
|
||||
/// <summary>Deleted: its folder is in the trash.</summary>
|
||||
Trashed,
|
||||
|
||||
/// <summary>Deleted and erased: the trash recorded erasing it, and no folder of the archive holds it.</summary>
|
||||
Gone,
|
||||
|
||||
/// <summary>The archive cannot tell (<see cref="Located.Reason"/>).</summary>
|
||||
Unknown,
|
||||
}
|
||||
|
||||
/// <summary>What a sidecar shows of a conversion (<see cref="SequenceHoldings.Classify"/>).</summary>
|
||||
private enum Holding
|
||||
{
|
||||
/// <summary>Frames of the sequence hold it, or what replaced it in turn.</summary>
|
||||
Held,
|
||||
|
||||
/// <summary>A Deleted tombstone ends its chain: the sequence no longer holds it.</summary>
|
||||
Deleted,
|
||||
|
||||
/// <summary>Its frame is there with other content: its file changed since the conversion.</summary>
|
||||
Changed,
|
||||
|
||||
/// <summary>A frame that may hold it was never hashed: its frame, or another one, once hashed, may.</summary>
|
||||
Unhashed,
|
||||
|
||||
/// <summary>The sidecar does not say.</summary>
|
||||
Unknown,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A replaced entry's sequence as found: live (its sidecar), in the trash (the sidecar of each deleted folder of it), gone,
|
||||
/// or unknown and why.
|
||||
/// </summary>
|
||||
private sealed record Located(Whereabouts Where, SequenceHoldings? Live = null, IReadOnlyList<SequenceHoldings>? Trashed = null,
|
||||
TrashKeptReason Reason = TrashKeptReason.Unknown)
|
||||
{
|
||||
public static Located Unknown(TrashKeptReason reason = TrashKeptReason.Unknown) => new(Whereabouts.Unknown, Reason: reason);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// What an emptying needs to know of a replaced entry: whether the conversion of one of its photos was deleted (the dry
|
||||
/// run says so, <see cref="TrashEntry.ConversionDeleted"/>), the files each photo's conversion must still hash to one of
|
||||
/// (in the sequence, or the copies the trash holds), and why the entry stays when the archive does not show a conversion
|
||||
/// whole (<see cref="Kept"/>, null when it does; an entry kept is never said to have lost its conversion).
|
||||
/// </summary>
|
||||
private sealed record ConversionSurvey(bool Deleted, IReadOnlyList<IReadOnlyList<(string Path, string Sha256)>> MustMatch, TrashKeptReason? Kept)
|
||||
{
|
||||
public static ConversionSurvey Keeping(TrashKeptReason reason) => new(false, [], reason);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The conversions of <paramref name="record"/>'s photos (the entry <paramref name="name"/>), where the archive shows them
|
||||
/// now: in the sequence as <paramref name="located"/> found it (<see cref="LocateAsync"/>), else among the copies the
|
||||
/// trash holds. Reads the trash's records and sidecars; hashes nothing.
|
||||
/// </summary>
|
||||
private static ConversionSurvey Survey(Archive archive, ArchivePaths paths, string name, PhotoTrashRecord record, Located located,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (located.Where == Whereabouts.Unknown) return ConversionSurvey.Keeping(located.Reason);
|
||||
// Gone with its sequence, deleted or erased since.
|
||||
var deleted = located.Where != Whereabouts.Live;
|
||||
var mustMatch = new List<IReadOnlyList<(string Path, string Sha256)>>();
|
||||
TrashCopies? copies = null;
|
||||
foreach (var photo in record.Photos)
|
||||
{
|
||||
if (photo.Replacement is not { } replacement) return ConversionSurvey.Keeping(TrashKeptReason.Unknown);
|
||||
if (located.Live is { } live)
|
||||
{
|
||||
var (state, held) = live.Classify(photo.Frame.Id, replacement);
|
||||
if (KeptFor(state, inSequence: true) is { } kept) return ConversionSurvey.Keeping(kept);
|
||||
if (state == Holding.Held)
|
||||
{
|
||||
mustMatch.Add(held);
|
||||
continue;
|
||||
}
|
||||
deleted = true;
|
||||
}
|
||||
else if (located.Trashed is { Count: > 0 } trashed)
|
||||
{
|
||||
// Its frames went with the sequence: a deleted folder's sidecar must say clearly what became of it.
|
||||
var states = trashed.Select(t => t.Classify(photo.Frame.Id, replacement).State).ToList();
|
||||
if (!states.Contains(Holding.Held) && states.Select(s => KeptFor(s, inSequence: false)).FirstOrDefault(r => r is not null) is { } kept)
|
||||
return ConversionSurvey.Keeping(kept);
|
||||
}
|
||||
// A copy the trash still holds (the deleted photos, the originals of a later conversion, the deleted sequence) must
|
||||
// match; with none left, nothing keeps the originals.
|
||||
copies ??= Copies(archive, paths, name, record.SequenceId, located, cancellationToken);
|
||||
var found = copies.Of(replacement);
|
||||
if (found.Count > 0) mustMatch.Add(found);
|
||||
}
|
||||
return new ConversionSurvey(deleted, mustMatch, null);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Why an entry stays for what a sidecar shows of one of its conversions; null when that lets it go. A frame never hashed
|
||||
/// in a deleted folder cannot be verified there: like a sidecar that does not say, only restoring the entry settles it.
|
||||
/// </summary>
|
||||
private static TrashKeptReason? KeptFor(Holding state, bool inSequence) => state switch
|
||||
{
|
||||
Holding.Changed => TrashKeptReason.Mismatch,
|
||||
Holding.Unhashed => inSequence ? TrashKeptReason.Unhashed : TrashKeptReason.Unknown,
|
||||
Holding.Unknown => TrashKeptReason.Unknown,
|
||||
_ => null,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Where the sequence of replaced entry <paramref name="entry"/> is, as the archive shows it: in the folder this index has
|
||||
/// for it, or the one the entry recorded, when its sidecar names it there; else its deleted folders in the trash; else in
|
||||
/// any folder of the archive whose sidecar names it (another index moved it), from <paramref name="walk"/>; else gone,
|
||||
/// only when the trash recorded erasing it (<see cref="Erased"/>). Unknown otherwise, with what settles it: a sidecar
|
||||
/// that cannot be read where the sequence may be, or a folder of photos without one there (a scan), a row of this index
|
||||
/// the archive does not bear out (a rebuild of the index), or nothing the archive says (only restoring the entry).
|
||||
/// <paramref name="heldEntryLease"/> (an erase, which holds the entry's lease and the sequence's): the live sidecar is read
|
||||
/// as its writers read it, what a crash left in it settled first (an undo whose completion waited for this entry's lease
|
||||
/// included), and one still unsettled (another entry busy) keeps the entry as busy.
|
||||
/// </summary>
|
||||
private async Task<Located> LocateAsync(Archive archive, ArchivePaths paths, TrashEntry entry, SidecarWalk walk, string? heldEntryLease,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var id = entry.SequenceId;
|
||||
var row = await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false);
|
||||
if (row is not null && row.ArchiveId != archive.Id) row = null;
|
||||
foreach (var folder in new[] { row?.Folder, entry.Folder }.OfType<string>().Distinct(StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
var at = paths.Sequence(folder);
|
||||
switch (Holder(at, id))
|
||||
{
|
||||
case true:
|
||||
return Live(at, id, heldEntryLease);
|
||||
case null:
|
||||
Log.Warning("archive", $"{entry.Title}: {folder} may hold its sequence but its sidecar cannot be read; its replaced originals stay in the trash");
|
||||
return Located.Unknown(TrashKeptReason.Unreadable);
|
||||
}
|
||||
}
|
||||
if (Trashed(paths, id, cancellationToken) is not { } trashed) return Located.Unknown();
|
||||
if (trashed.Count > 0) return new Located(Whereabouts.Trashed, Trashed: trashed);
|
||||
// Another index moved it: wherever a sidecar names it now, from the walk made once for the plan, and made again when a
|
||||
// folder it found no longer holds the sequence.
|
||||
for (var again = false; ; again = true)
|
||||
{
|
||||
var (folders, unreadable) = await walk.FindAsync(id, again, cancellationToken).ConfigureAwait(false);
|
||||
var moved = false;
|
||||
foreach (var folder in folders)
|
||||
{
|
||||
var at = paths.Sequence(folder);
|
||||
switch (Holder(at, id))
|
||||
{
|
||||
case true:
|
||||
return Live(at, id, heldEntryLease);
|
||||
case null:
|
||||
Log.Warning("archive", $"{entry.Title}: the sidecar of {folder} cannot be read; its replaced originals stay in the trash");
|
||||
return Located.Unknown(TrashKeptReason.Unreadable);
|
||||
default:
|
||||
moved = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (moved && !again) continue;
|
||||
if (moved)
|
||||
{
|
||||
Log.Info("archive", $"{entry.Title}: its sequence is moving; its replaced originals stay in the trash");
|
||||
return Located.Unknown(TrashKeptReason.Busy);
|
||||
}
|
||||
if (unreadable is not null)
|
||||
{
|
||||
Log.Warning("archive", $"The sidecar of {unreadable} cannot be read; replaced originals of {entry.Title} stay in the trash");
|
||||
return Located.Unknown(TrashKeptReason.Unreadable);
|
||||
}
|
||||
break;
|
||||
}
|
||||
if (Erased(paths, id)) return new Located(Whereabouts.Gone);
|
||||
if (row is not null)
|
||||
{
|
||||
Log.Info("archive", $"{entry.Title} is in the index but not in the archive; its replaced originals stay in the trash until a rebuild of the index");
|
||||
return Located.Unknown(TrashKeptReason.StaleIndex);
|
||||
}
|
||||
Log.Info("archive", $"{entry.Title} is not in the archive and the trash has no record of erasing it; its replaced originals stay in the trash");
|
||||
return Located.Unknown();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether <paramref name="folder"/> holds sequence <paramref name="id"/>: true when its sidecar names it, false when the
|
||||
/// folder is not there, holds nothing of a sequence, or its sidecar names another one; null when it cannot be told (a
|
||||
/// sidecar that cannot be read, or photos without one).
|
||||
/// </summary>
|
||||
private static bool? Holder(SequencePaths folder, Guid id)
|
||||
{
|
||||
if (!Directory.Exists(folder.Folder)) return false;
|
||||
if (!File.Exists(folder.ManifestPath)) return ArchiveScanner.ListPhotos(folder).Count > 0 ? null : false;
|
||||
return SidecarId(folder.ManifestPath) is { } named ? named == id : null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The live sequence in <paramref name="folder"/> with its sidecar (<see cref="LocateAsync"/>); busy while a change to it is
|
||||
/// not settled or a file of it is held open, unreadable when its sidecar cannot be read or names another sequence now.
|
||||
/// </summary>
|
||||
private static Located Live(SequencePaths folder, Guid id, string? heldEntryLease)
|
||||
{
|
||||
try
|
||||
{
|
||||
SequenceManifest manifest;
|
||||
if (heldEntryLease is not null)
|
||||
{
|
||||
manifest = SequenceWriter.ReadLocked(folder, id, heldEntryLease);
|
||||
FrameTrash.EnsureSettled(folder);
|
||||
FrameTrash.EnsureReplaced(folder);
|
||||
}
|
||||
else
|
||||
{
|
||||
manifest = ManifestStore.ReadSequence(folder.ManifestPath);
|
||||
if (manifest.Id != id) return Located.Unknown(TrashKeptReason.Unreadable);
|
||||
}
|
||||
return new Located(Whereabouts.Live, Live: new SequenceHoldings(folder, manifest));
|
||||
}
|
||||
catch (LeaseUnavailableException ex)
|
||||
{
|
||||
Log.Info("archive", $"{folder.RelativeFolder} holds a change not settled yet; its replaced originals stay in the trash: {ex.Message}");
|
||||
return Located.Unknown(TrashKeptReason.Busy);
|
||||
}
|
||||
catch (Exception ex) when (ex is InvalidDataException or UnauthorizedAccessException or FileNotFoundException or DirectoryNotFoundException)
|
||||
{
|
||||
Log.Warning("archive", $"The sidecar of {folder.RelativeFolder} cannot be read; its replaced originals stay in the trash", ex);
|
||||
return Located.Unknown(TrashKeptReason.Unreadable);
|
||||
}
|
||||
catch (IOException ex)
|
||||
{
|
||||
// Open elsewhere (a change being settled could not move a file yet): another try later.
|
||||
Log.Warning("archive", $"{folder.RelativeFolder} is in use; its replaced originals stay in the trash", ex);
|
||||
return Located.Unknown(TrashKeptReason.Busy);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The deleted folders of sequence <paramref name="id"/> in the trash, each with its sidecar; null when one of them has
|
||||
/// no sidecar that can be read (what it holds of the conversions cannot be told), or an entry's sequence cannot be told.
|
||||
/// </summary>
|
||||
private static List<SequenceHoldings>? Trashed(ArchivePaths paths, Guid id, CancellationToken cancellationToken)
|
||||
{
|
||||
var found = new List<SequenceHoldings>();
|
||||
var trash = TrashDirectory(paths);
|
||||
if (!Directory.Exists(trash)) return found;
|
||||
foreach (var recordPath in Directory.EnumerateFiles(trash, "*" + RecordExtension))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var folder = Path.Combine(trash, Path.GetFileNameWithoutExtension(recordPath));
|
||||
if (!Directory.Exists(folder)) continue;
|
||||
var deleted = paths.Sequence(paths.ToRelative(folder));
|
||||
var owner = ReadRecord(recordPath)?.Sequence.Id ?? (File.Exists(deleted.ManifestPath) ? SidecarId(deleted.ManifestPath) : null);
|
||||
if (owner is null)
|
||||
{
|
||||
Log.Warning("archive", $"Trash entry {Path.GetFileName(folder)} cannot be told apart; replaced originals of its sequence stay in the trash");
|
||||
return null;
|
||||
}
|
||||
if (owner != id) continue;
|
||||
try
|
||||
{
|
||||
found.Add(new SequenceHoldings(deleted, ManifestStore.ReadSequence(deleted.ManifestPath)));
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("archive", $"The sidecar of trash entry {Path.GetFileName(folder)} cannot be read; the replaced originals of its photos stay in the trash", ex);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
/// <summary>The record an emptying keeps of erasing deleted sequence <paramref name="id"/> (<see cref="ErasureRecord"/>).</summary>
|
||||
private static string ErasurePath(ArchivePaths paths, Guid id) => Path.Combine(TrashDirectory(paths), ErasedFolder, id.ToString("N") + RecordExtension);
|
||||
|
||||
/// <summary>Whether the trash recorded erasing deleted sequence <paramref name="id"/>; a record that cannot be read says nothing.</summary>
|
||||
private static bool Erased(ArchivePaths paths, Guid id)
|
||||
{
|
||||
var path = ErasurePath(paths, id);
|
||||
if (!File.Exists(path)) return false;
|
||||
try
|
||||
{
|
||||
return JsonSerializer.Deserialize(File.ReadAllText(path), TrashJson.Default.ErasureRecord)?.SequenceId == id;
|
||||
}
|
||||
catch (Exception ex) when (ex is JsonException or IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("archive", $"The record of erasing sequence {id} cannot be read; its replaced originals stay in the trash", ex);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records that the emptying erases deleted sequence <paramref name="entry"/> (written before its folder goes) when a
|
||||
/// replaced entry of that sequence stays in the trash: its emptying, later or on another host, then knows from the archive
|
||||
/// that the conversions of its originals went with the sequence (<see cref="Whereabouts.Gone"/>). Never while the
|
||||
/// sequence also lives (this index has it, or a folder's sidecar names it: a folder copied back and adopted, a second
|
||||
/// deletion of the same id): the record would then outlive what it says and let a later emptying, whose walk misses the
|
||||
/// live folder, erase originals whose conversion is still there. Without the record such an entry is kept.
|
||||
/// </summary>
|
||||
private async Task RecordErasureAsync(Archive archive, ArchivePaths paths, TrashEntry entry, SidecarWalk walk, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!FrameTrash.List(archive, cancellationToken).Any(e => e.Kind == TrashKind.Replaced && e.SequenceId == entry.SequenceId)) return;
|
||||
if (await _db.Sequences.GetAsync(entry.SequenceId, cancellationToken).ConfigureAwait(false) is { } row && row.ArchiveId == archive.Id) return;
|
||||
if ((await walk.FindAsync(entry.SequenceId, again: false, cancellationToken).ConfigureAwait(false)).Folders.Count > 0) return;
|
||||
var path = ErasurePath(paths, entry.SequenceId);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||
AtomicFile.WriteAllText(path, JsonSerializer.Serialize(new ErasureRecord(entry.SequenceId, entry.Title, entry.Name, DateTimeOffset.UtcNow,
|
||||
_writer.Options.Owner), TrashJson.Default.ErasureRecord));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes the records of erased sequences that no replaced entry in the trash needs any more (all of them erased or
|
||||
/// restored since). One whose name is not a sequence id is left as it is.
|
||||
/// </summary>
|
||||
private static void SweepErasures(Archive archive, ArchivePaths paths, CancellationToken cancellationToken)
|
||||
{
|
||||
var folder = Path.Combine(TrashDirectory(paths), ErasedFolder);
|
||||
if (!Directory.Exists(folder)) return;
|
||||
try
|
||||
{
|
||||
var needed = FrameTrash.List(archive, cancellationToken).Where(e => e.Kind == TrashKind.Replaced).Select(e => e.SequenceId).ToHashSet();
|
||||
foreach (var path in Directory.EnumerateFiles(folder, "*" + RecordExtension).ToList())
|
||||
{
|
||||
if (Guid.TryParseExact(Path.GetFileNameWithoutExtension(path), "N", out var id) && !needed.Contains(id)) File.Delete(path);
|
||||
}
|
||||
if (!Directory.EnumerateFileSystemEntries(folder).Any()) Directory.Delete(folder);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("archive", "Records of erased sequences no trash entry needs could not be deleted; the next emptying tries again", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Adds the Replaced tombstones of a sidecar to <paramref name="copies"/>: each content and what replaced it.</summary>
|
||||
private static void AddLinks(TrashCopies copies, SequenceManifest manifest)
|
||||
{
|
||||
foreach (var removed in manifest.Removed ?? [])
|
||||
{
|
||||
if (removed.Reason == RemovalReason.Replaced && removed.Replacement is { Length: > 0 } replacement) copies.Link(removed.Sha256, replacement);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// What the trash holds of a sequence's photos, but the entry <paramref name="except"/> (the one being erased): the files of
|
||||
/// its photo entries (deleted photos, replaced originals) and of its deleted folders (<paramref name="located"/>), by
|
||||
/// content, and the replacements those entries and the sequence's sidecar name. Read from their records and sidecars; an
|
||||
/// unreadable record adds nothing.
|
||||
/// </summary>
|
||||
private static TrashCopies Copies(Archive archive, ArchivePaths paths, string except, Guid sequenceId, Located located, CancellationToken cancellationToken)
|
||||
{
|
||||
var copies = new TrashCopies();
|
||||
if (located.Live is { } live) AddLinks(copies, live.Manifest);
|
||||
foreach (var listed in FrameTrash.List(archive, cancellationToken).Where(e => e.SequenceId == sequenceId && e.Name != except))
|
||||
{
|
||||
if (FrameTrash.ReadEntry(paths, listed.Name) is not { } record) continue;
|
||||
var (folder, _) = FrameTrash.Location(paths, listed.Name);
|
||||
foreach (var photo in record.Photos)
|
||||
{
|
||||
if (photo.Files.Count > 0 && (photo.Files[0].Sha256 ?? photo.Frame.Sha256) is { } sha) copies.Add(sha, Path.Combine(folder, photo.Files[0].Path));
|
||||
if (photo.Frame.Sha256 is { } original && photo.Replacement is { } replacement) copies.Link(original, replacement);
|
||||
}
|
||||
}
|
||||
foreach (var deleted in located.Trashed ?? [])
|
||||
{
|
||||
foreach (var frame in deleted.Manifest.Frames.Where(f => f.Sha256 is not null)) copies.Add(frame.Sha256!, deleted.Folder.File(frame.File));
|
||||
AddLinks(copies, deleted.Manifest);
|
||||
}
|
||||
return copies;
|
||||
}
|
||||
|
||||
/// <summary>Whether, for each photo of a survey, one of the files its conversion must match is there and hashes to it now.</summary>
|
||||
private static async Task<bool> MatchAsync(ConversionSurvey survey, CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (var files in survey.MustMatch)
|
||||
{
|
||||
if (!await AnyHashesAsync(files, cancellationToken).ConfigureAwait(false)) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>Whether one of <paramref name="files"/> is there and hashes to its SHA-256 now.</summary>
|
||||
private static async Task<bool> AnyHashesAsync(IEnumerable<(string Path, string Sha256)> files, CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (var (path, sha) in files)
|
||||
{
|
||||
if (!File.Exists(path)) continue;
|
||||
if (string.Equals(await ContentHash.Sha256Async(path, null, cancellationToken).ConfigureAwait(false), sha, StringComparison.OrdinalIgnoreCase)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A sequence's sidecar (live, or of a deleted folder) as a survey reads it, indexed once, so an entry of a conversion of
|
||||
/// tens of thousands of photos is weighed in one pass over it: its frames by content and by id, its tombstones by content,
|
||||
/// and whether a frame a verify could hash never was (one whose file is missing holds nothing to hash).
|
||||
/// </summary>
|
||||
private sealed class SequenceHoldings
|
||||
{
|
||||
private readonly Dictionary<string, List<SidecarFrame>> _byContent = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly Dictionary<Guid, SidecarFrame> _byId = [];
|
||||
private readonly Dictionary<string, RemovedPhoto> _tombstones = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly bool _unhashed;
|
||||
|
||||
public SequenceHoldings(SequencePaths folder, SequenceManifest manifest)
|
||||
{
|
||||
Folder = folder;
|
||||
Manifest = manifest;
|
||||
foreach (var frame in manifest.Frames)
|
||||
{
|
||||
_byId.TryAdd(frame.Id, frame);
|
||||
if (frame.Sha256 is not { } sha)
|
||||
{
|
||||
_unhashed |= !Lost(frame);
|
||||
continue;
|
||||
}
|
||||
if (!_byContent.TryGetValue(sha, out var holders)) _byContent[sha] = holders = [];
|
||||
holders.Add(frame);
|
||||
}
|
||||
foreach (var tombstone in manifest.Removed ?? []) _tombstones[tombstone.Sha256] = tombstone;
|
||||
}
|
||||
|
||||
public SequencePaths Folder { get; }
|
||||
|
||||
public SequenceManifest Manifest { get; }
|
||||
|
||||
/// <summary>
|
||||
/// What the sidecar shows of <paramref name="replacement"/>, the conversion that took the place of frame
|
||||
/// <paramref name="frameId"/>'s photo, followed through its Replaced tombstones: the frames holding it, each file with
|
||||
/// the content it must hash to; changed, when the frame is there with other content; deleted, only when a Deleted
|
||||
/// tombstone ends the chain (every writer that takes a frame out of a sidecar leaves one, unless another frame still
|
||||
/// holds the content); otherwise not known: never hashed, when the frame is there unhashed or another frame never
|
||||
/// hashed may hold it (each one a verify can hash, its file there), and unknown when nothing in the sidecar says what
|
||||
/// became of it (a sidecar rebuilt from the photos keeps only the tombstones one index knew; a chain too long or
|
||||
/// looping; the frame there, never hashed, with its file missing).
|
||||
/// </summary>
|
||||
public (Holding State, List<(string Path, string Sha256)> Files) Classify(Guid frameId, string replacement)
|
||||
{
|
||||
var chain = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
RemovedPhoto? end = null;
|
||||
for (var sha = replacement.ToLowerInvariant(); ;)
|
||||
{
|
||||
if (chain.Count > ReplacementSteps || !chain.Add(sha)) return (Holding.Unknown, []);
|
||||
if (_byContent.TryGetValue(sha, out var holders))
|
||||
{
|
||||
var content = sha;
|
||||
return (Holding.Held, [.. holders.Select(f => (Folder.File(f.File), content))]);
|
||||
}
|
||||
if (!_tombstones.TryGetValue(sha, out end) || end.Reason != RemovalReason.Replaced) break;
|
||||
if (end.Replacement is not { Length: > 0 } next) return (Holding.Unknown, []);
|
||||
sha = next.ToLowerInvariant();
|
||||
}
|
||||
if (_byId.TryGetValue(frameId, out var frame))
|
||||
return (frame.Sha256 is not null ? Holding.Changed : Lost(frame) ? Holding.Unknown : Holding.Unhashed, []);
|
||||
if (end is { Reason: RemovalReason.Deleted }) return (Holding.Deleted, []);
|
||||
return (_unhashed ? Holding.Unhashed : Holding.Unknown, []);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether no verify can hash <paramref name="frame"/>: its file is missing, or it is flagged corrupt (a verify does not
|
||||
/// hash a file whose size differs from its record), so advising one would never settle the entry.
|
||||
/// </summary>
|
||||
private static bool Lost(SidecarFrame frame) => (frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The archive's sequence folders by the id their sidecar names, walked once for a whole dry run or emptying (owner
|
||||
/// review 2, R7), not once for each sequence that is not where the index or its entry says, and walked again only when a
|
||||
/// folder it found no longer holds the sequence looked for. Only each sidecar's head is read (<see cref="SidecarId"/>);
|
||||
/// the first folder whose sidecar cannot be read is kept too, as one that may hold the sequence looked for. Folders the
|
||||
/// scanner cannot enter, and photos without a sidecar, name no sequence: one found nowhere is gone only on the trash's
|
||||
/// record of erasing it.
|
||||
/// </summary>
|
||||
private sealed class SidecarWalk(ArchivePaths paths)
|
||||
{
|
||||
private Dictionary<Guid, List<string>>? _folders;
|
||||
private string? _unreadable;
|
||||
|
||||
/// <summary>
|
||||
/// The folders whose sidecar names <paramref name="id"/>, in path order, and a folder whose sidecar cannot be read (null
|
||||
/// when none), from the walk: made now when <paramref name="again"/> or when none was made yet.
|
||||
/// </summary>
|
||||
public async Task<(IReadOnlyList<string> Folders, string? Unreadable)> FindAsync(Guid id, bool again, CancellationToken cancellationToken)
|
||||
{
|
||||
if (_folders is null || again)
|
||||
{
|
||||
var (folders, unreadable) = await Task.Run(() => Walk(cancellationToken), cancellationToken).ConfigureAwait(false);
|
||||
_folders = folders;
|
||||
_unreadable = unreadable;
|
||||
}
|
||||
return (_folders.GetValueOrDefault(id) ?? [], _unreadable);
|
||||
}
|
||||
|
||||
private (Dictionary<Guid, List<string>>, string?) Walk(CancellationToken cancellationToken)
|
||||
{
|
||||
var folders = new Dictionary<Guid, List<string>>();
|
||||
string? unreadable = null;
|
||||
foreach (var found in ArchiveScanner.FindSequenceFolders(paths, cancellationToken))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
if (found.Sidecar is null) continue;
|
||||
if (SidecarId(paths.Sequence(found.RelativeFolder).ManifestPath) is not { } named)
|
||||
{
|
||||
unreadable ??= found.RelativeFolder;
|
||||
continue;
|
||||
}
|
||||
if (!folders.TryGetValue(named, out var holders)) folders[named] = holders = [];
|
||||
holders.Add(found.RelativeFolder);
|
||||
}
|
||||
return (folders, unreadable);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Copies of a sequence's photos in the trash by content, and the replacement links known (<see cref="Copies"/>).</summary>
|
||||
private sealed class TrashCopies
|
||||
{
|
||||
private readonly Dictionary<string, string> _links = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, List<string>> _files = new(StringComparer.Ordinal);
|
||||
|
||||
public void Add(string sha256, string path)
|
||||
{
|
||||
var sha = sha256.ToLowerInvariant();
|
||||
if (!_files.TryGetValue(sha, out var paths)) _files[sha] = paths = [];
|
||||
paths.Add(path);
|
||||
}
|
||||
|
||||
public void Link(string sha256, string replacement) => _links.TryAdd(sha256.ToLowerInvariant(), replacement.ToLowerInvariant());
|
||||
|
||||
/// <summary>The copies of <paramref name="replacement"/> and of what replaced it in turn, each with the content it must hash to.</summary>
|
||||
public List<(string Path, string Sha256)> Of(string replacement)
|
||||
{
|
||||
var found = new List<(string, string)>();
|
||||
var sha = replacement.ToLowerInvariant();
|
||||
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||||
for (var step = 0; step <= ReplacementSteps && seen.Add(sha); step++)
|
||||
{
|
||||
if (_files.TryGetValue(sha, out var paths)) found.AddRange(paths.Select(p => (p, sha)));
|
||||
if (!_links.TryGetValue(sha, out var next)) break;
|
||||
sha = next;
|
||||
}
|
||||
return found;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// What the trash keeps of erasing a deleted sequence while a replaced entry of it stays (owner review 2, R7):
|
||||
/// <c>.chronos/trash/.erased/<sequence>.json</c>, the archive's own word that the sequence, and the conversions of the
|
||||
/// replaced originals it held, went for good, for the emptying of that entry here or on another host sharing the archive.
|
||||
/// It goes once no replaced entry of the sequence is left in the trash.
|
||||
/// </summary>
|
||||
/// <param name="Entry">The name of the trash entry erased.</param>
|
||||
/// <param name="ErasedBy">The host that erased it (<see cref="ArchiveServiceOptions.Owner"/>).</param>
|
||||
internal sealed record ErasureRecord(Guid SequenceId, string Title, string Entry, DateTimeOffset ErasedAt, string? ErasedBy);
|
||||
@@ -6,6 +6,7 @@ using Chronos.Core.IO;
|
||||
using Chronos.Core.Localization;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Data.Archiving.Jobs;
|
||||
using Chronos.Data.Repositories;
|
||||
|
||||
namespace Chronos.Data.Archiving;
|
||||
|
||||
@@ -57,7 +58,7 @@ public enum TrashKind
|
||||
/// <summary>Photos deleted from a sequence (<see cref="FrameTrash"/>).</summary>
|
||||
Photos,
|
||||
|
||||
/// <summary>Originals a conversion replaced (a later phase): erased only while their replacements still match.</summary>
|
||||
/// <summary>Originals a conversion replaced (owner review 2, R7): erased only while their replacements still match.</summary>
|
||||
Replaced,
|
||||
}
|
||||
|
||||
@@ -68,7 +69,19 @@ public enum TrashKind
|
||||
/// <param name="Folder">Archive-relative folder the sequence came from (and returns to when it is free).</param>
|
||||
/// <param name="Frames">Photos the entry holds.</param>
|
||||
public sealed record TrashEntry(Guid ArchiveId, string Name, Guid SequenceId, string Title, string Folder, int Frames, int Files, long Bytes,
|
||||
DateTimeOffset DeletedAt, string? DeletedBy, TrashKind Kind = TrashKind.Sequence);
|
||||
DateTimeOffset DeletedAt, string? DeletedBy, TrashKind Kind = TrashKind.Sequence)
|
||||
{
|
||||
/// <summary>The job that made the entry (a conversion's Replace, owner review 2, R7); null for one a user's action made.</summary>
|
||||
public Guid? JobId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// A <see cref="TrashKind.Replaced"/> entry whose conversion was deleted since: the archive shows a photo's replacement
|
||||
/// (followed through later conversions) deleted from its sequence, or the sequence itself deleted, so erasing the entry may
|
||||
/// erase the last copy of that photo. Set by the dry run of an emptying (<see cref="SequenceTrash.PlanEmptyAsync"/>), which
|
||||
/// says so; false in a plain listing.
|
||||
/// </summary>
|
||||
public bool ConversionDeleted { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>The plan no longer matches the index or the jobs (a sequence moved, vanished or got busy since it was made); nothing was deleted.</summary>
|
||||
public sealed class StaleDeletePlanException(Guid sequenceId) : InvalidOperationException(Loc.L("Archive.Error.StalePlan"))
|
||||
@@ -103,7 +116,6 @@ public sealed partial class SequenceTrash
|
||||
public const string FolderName = "trash";
|
||||
|
||||
private const string RecordExtension = ".json";
|
||||
private const int JobPage = 200;
|
||||
|
||||
/// <summary>Bytes of a sidecar read to find its id (written second, after the schema version).</summary>
|
||||
private const int SidecarHead = 4096;
|
||||
@@ -373,15 +385,17 @@ public sealed partial class SequenceTrash
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
if (!Directory.Exists(folder.Folder) || (File.Exists(folder.ManifestPath) && SidecarId(folder.ManifestPath) != sequence.Id))
|
||||
throw new StaleDeletePlanException(sequence.Id);
|
||||
// A photo deletion, a restore from the trash or an ingest a crash interrupted is settled first, as any writer does
|
||||
// (SequenceWriter.ReadLocked): neither its journal nor the files it moved travel into this entry, where emptying it
|
||||
// would erase them while their own entry still lists them. A restore whose entry is busy keeps the folder here.
|
||||
// A photo deletion, a frame replacement, a restore from the trash or an ingest a crash interrupted is settled first,
|
||||
// as any writer does (SequenceWriter.ReadLocked): neither its journal nor the files it moved travel into this entry,
|
||||
// where emptying it would erase them while their own entry still lists them. A restore or a replacement whose entry
|
||||
// is busy keeps the folder here.
|
||||
var (files, bytes) = (step.Files, step.Bytes);
|
||||
if (File.Exists(folder.ManifestPath) && (File.Exists(folder.RemovalJournalPath) || File.Exists(folder.RestoreJournalPath)
|
||||
|| File.Exists(folder.IngestJournalPath)))
|
||||
|| File.Exists(folder.ReplaceJournalPath) || File.Exists(folder.IngestJournalPath)))
|
||||
{
|
||||
SequenceWriter.ReadLocked(folder, sequence.Id);
|
||||
FrameTrash.EnsureSettled(folder);
|
||||
FrameTrash.EnsureReplaced(folder);
|
||||
// Files came back into the folder, or left it: the record counts what moves.
|
||||
(files, bytes) = Measure(folder.Folder, cancellationToken);
|
||||
}
|
||||
@@ -477,29 +491,25 @@ public sealed partial class SequenceTrash
|
||||
/// archives of archive-wide ones (scan, rebuild, import, maintenance, a verify of everything). An import's folders
|
||||
/// are known only to its plan, so it holds its whole archive; a verify of named sequences holds only those.
|
||||
/// </summary>
|
||||
internal async Task<IReadOnlySet<Guid>> BusyAsync(CancellationToken cancellationToken)
|
||||
internal Task<IReadOnlySet<Guid>> BusyAsync(CancellationToken cancellationToken) => BusyAsync(null, cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// What running and paused jobs hold, but the job <paramref name="except"/> (the one asking), read from their targets
|
||||
/// alone (<see cref="JobRepository.ListHoldsAsync"/>): nothing reads the parameters a conversion or an import keeps.
|
||||
/// </summary>
|
||||
internal async Task<IReadOnlySet<Guid>> BusyAsync(Guid? except, CancellationToken cancellationToken)
|
||||
{
|
||||
var busy = new HashSet<Guid>();
|
||||
foreach (var state in new[] { JobState.Running, JobState.Paused })
|
||||
{
|
||||
string? cursor = null;
|
||||
do
|
||||
{
|
||||
var page = await _db.Jobs.ListAsync(state, JobPage, cursor, cancellationToken).ConfigureAwait(false);
|
||||
foreach (var job in page.Items) busy.UnionWith(Holds(job));
|
||||
cursor = page.NextCursor;
|
||||
}
|
||||
while (cursor is not null);
|
||||
}
|
||||
foreach (var job in await _db.Jobs.ListHoldsAsync(except, cancellationToken).ConfigureAwait(false)) busy.UnionWith(Holds(job));
|
||||
return busy;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<Guid> Holds(Job job)
|
||||
private static IReadOnlyList<Guid> Holds(JobHold job)
|
||||
{
|
||||
if (job.Kind != JobKind.Verify) return job.TargetIds;
|
||||
if (job is not { Kind: JobKind.Verify, ParametersJson: { Length: > 0 } json }) return job.TargetIds;
|
||||
try
|
||||
{
|
||||
return ArchiveJobJson.Read(job, ArchiveJobJson.Default.VerifyParameters, new VerifyParameters(null)).SequenceIds ?? job.TargetIds;
|
||||
return JsonSerializer.Deserialize(json, ArchiveJobJson.Default.VerifyParameters)?.SequenceIds ?? job.TargetIds;
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
@@ -608,4 +618,6 @@ internal sealed record TrashRecord(Sequence Sequence, IReadOnlyList<Render> Rend
|
||||
[JsonSerializable(typeof(TrashRecord))]
|
||||
[JsonSerializable(typeof(PhotoTrashRecord))]
|
||||
[JsonSerializable(typeof(FrameRestoreJournal))]
|
||||
[JsonSerializable(typeof(FrameReplaceJournal))]
|
||||
[JsonSerializable(typeof(ErasureRecord))]
|
||||
internal sealed partial class TrashJson : JsonSerializerContext;
|
||||
@@ -55,12 +55,22 @@ public sealed record ArchiveServiceOptions
|
||||
/// <summary>
|
||||
/// Told after a Reorganize changed a sequence's folder, title or file names (change <c>reorganized</c>), once its step is
|
||||
/// journaled, or once its folder moved and its index row followed when the step then failed: by its job, or by an archive
|
||||
/// open that completed a plan an interruption left (<see cref="ArchiveService.OpenAsync"/>).
|
||||
/// open that completed a plan an interruption left (<see cref="ArchiveService.OpenAsync"/>). Told too after a conversion
|
||||
/// replaced a sequence's frames (<c>converted</c>) or made a new sequence of them (<c>created</c>).
|
||||
/// A server tells its connected clients (BRIEF §14.3); the service it opens its archive with at start sets none, as no
|
||||
/// client is connected yet. The desktop reloads an editor open on the sequence. A failure is logged and never fails the job.
|
||||
/// </summary>
|
||||
public Func<Guid, string, CancellationToken, Task>? SequenceChanged { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The free bytes a write under a folder of an archive (its root, its staging folder) may count on, as the host counts
|
||||
/// them: a server takes the tighter of its space volume (owner review 2, R2) and the folder's own volume, the desktop asks
|
||||
/// Windows, which measures a network share too; null: the folder's drive (<see cref="DriveInfo"/>). A conversion checks its
|
||||
/// estimate against it before it starts, before each sequence and before each photo (R7). The function returns null when
|
||||
/// the space cannot be measured.
|
||||
/// </summary>
|
||||
public Func<string, long?>? FreeSpace { get; init; }
|
||||
|
||||
/// <summary>Half the logical cores, at least one: the parallelism when the host sets none.</summary>
|
||||
public static int DefaultParallelism => Math.Max(1, Environment.ProcessorCount / 2);
|
||||
|
||||
@@ -99,7 +109,8 @@ internal sealed class SequenceWriter(ArchiveServiceOptions options)
|
||||
|
||||
/// <summary>
|
||||
/// Reads the sidecar of a locked sequence, completing an interrupted Reorganize of its files (<see cref="FileRenameJournal"/>),
|
||||
/// undoing an interrupted photo deletion (<see cref="FrameRemovalJournal"/>), settling an interrupted restore from the
|
||||
/// undoing an interrupted photo deletion (<see cref="FrameRemovalJournal"/>), undoing or completing an interrupted frame
|
||||
/// replacement (<see cref="FrameReplaceJournal"/>, a conversion or its undo), settling an interrupted restore from the
|
||||
/// trash (<see cref="FrameRestoreJournal"/>; left for later while its trash entry is busy) and completing an interrupted
|
||||
/// ingest batch first. A sidecar a rename recovery rewrites leaves the index behind it until the rename resumes or the
|
||||
/// next scan reads it.
|
||||
@@ -121,6 +132,7 @@ internal sealed class SequenceWriter(ArchiveServiceOptions options)
|
||||
FileRenameJournal.Delete(sequence);
|
||||
}
|
||||
FrameRemovalJournal.Recover(sequence, manifest);
|
||||
FrameTrash.RecoverReplace(sequence, manifest, heldEntryLease);
|
||||
FrameTrash.RecoverRestore(sequence, manifest, heldEntryLease);
|
||||
if (IngestJournal.Recover(sequence, manifest) is { } recovered)
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user