using Chronos.Core.Archiving; using Chronos.Core.Diagnostics; using Chronos.Core.Import; using Chronos.Core.Localization; using Chronos.Core.Model; using Chronos.Data.Copies; using Chronos.Data.Jobs; using Chronos.Data.Projects; using Chronos.Data.Repositories; using Chronos.Data.Sync; using ProcessingFrames = Chronos.Data.Processing.SequenceFrames; namespace Chronos.Data.Archiving.Jobs; /// /// (owner review 6, #73; design §5d): applies the merge plan the user confirmed /// (). Its first run plans again and refuses a plan whose fingerprint moved; the journal /// () then keeps what that plan decided and how far each stage came, so a crash, a pause or a /// retry resumes where it stopped. Stages: copy the other's photos into the kept sequence (each hash-checked, filed with /// tombstones honoured), move the renders, verify every original of the other is held by the kept one (hashed again on /// both sides) or listed as a last copy, trash the kept sequence's losing versions, write the details, copy the projects, /// move session keys, links and pairs, then move the other to its archive's trash, only while it is still at the frames /// version the verify read and the kept photos verified are still there. Every check that can still abandon the merge runs /// before the links stage changes anything, and again at the trash; a merge abandoned after the links stage (a check that /// fails at the trash, a cancel) gives the other back its keys, links and pairs first (). A /// cancel leaves the photos copied so far in the kept sequence and drops the journal. /// internal sealed class MergeJobHandler(ChronosDatabase db, SequenceMerger merger, ArchiveService archives, IProjectCopier projects, IWorkspaceGuard workspace) : ICancelledJobHandler { /// The change told about the kept sequence. public const string KeepChange = "frames"; /// The change told about the sequence that went to the trash. public const string DropChange = "deleted"; private static readonly CopyPlanContext JobContextPlan = new(string.Empty, null, false, false, false, false, false, false, null, new HashSet(), new HashSet()); public JobKind Kind => JobKind.MergeSequences; private ChronosDatabase Db => db; private ArchiveService Archives => archives; private IProjectCopier Projects => projects; private IWorkspaceGuard Workspace => workspace; private SequenceMerger Merger => merger; /// Called before each stage runs (tests: a crash, a change made between stages). internal Func? BeforeStage { get; set; } public async Task RunAsync(JobContext context) { var job = context.Job; var ct = context.Cancellation; var parameters = MergeJobParameters.Read(job.ParametersJson) ?? throw new InvalidOperationException($"Merge job {job.Id} has no plan."); var keepRow = await db.Sequences.GetAsync(parameters.KeepId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException(); var keepArchive = await db.Archives.GetAsync(keepRow.ArchiveId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException(); var keepPaths = new ArchivePaths(keepArchive.RootPath); try { var journal = MergeJournal.Read(keepPaths, job.Id); if (journal is null) journal = await StartAsync(context, parameters, keepRow, keepPaths).ConfigureAwait(false); else await ResumeAsync(job, parameters, keepPaths, ct).ConfigureAwait(false); var staging = MergeJournal.StagingFor(keepPaths, job.Id); if (Directory.Exists(staging)) Directory.Delete(staging, recursive: true); await new Run(this, context, journal, keepArchive, keepPaths).RunAsync().ConfigureAwait(false); } catch (OperationCanceledException) when (ct.IsCancellationRequested) { if (await db.Jobs.GetAsync(job.Id, CancellationToken.None).ConfigureAwait(false) is { State: JobState.Cancelled }) await ForgetAsync(job, parameters, keepPaths).ConfigureAwait(false); throw; } } public async Task CancelledAsync(Job job, CancellationToken cancellationToken) { try { if (MergeJobParameters.Read(job.ParametersJson) is not { } parameters) return; if (await db.Sequences.GetAsync(parameters.KeepId, cancellationToken).ConfigureAwait(false) is not { } keep) return; if (await db.Archives.GetAsync(keep.ArchiveId, cancellationToken).ConfigureAwait(false) is not { } archive) return; await ForgetAsync(job, parameters, new ArchivePaths(archive.RootPath)).ConfigureAwait(false); } catch (Exception ex) when (ex is not OperationCanceledException) { Log.Warning("archive", $"The cancelled merge {job.Id} left its journal; the archive's next maintenance removes it", ex); } } /// A cancelled merge: its journal and staging go, and its pair returns to the state it had (when it is still this job's). private async Task ForgetAsync(Job job, MergeJobParameters parameters, ArchivePaths keepPaths) { await MoveBackAsync(keepPaths, job.Id).ConfigureAwait(false); try { MergeJournal.Delete(keepPaths, job.Id); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { Log.Warning("archive", $"The journal of the cancelled merge {job.Id} was not removed", ex); } // By the stored row: the engine marked the job Cancelled first, so the row already reads as its state before. await new FoundCopyStore(db).ReleaseAsync(parameters.CopyId, job.Id, null, keepJob: false, CancellationToken.None).ConfigureAwait(false); } /// Gives back what the links stage of job 's journal moved (when it has one and the stage began). private async Task MoveBackAsync(ArchivePaths keepPaths, Guid jobId) { MergeJournal? journal; try { journal = MergeJournal.Read(keepPaths, jobId); } catch (InvalidDataException ex) { Log.Warning("archive", $"The journal of merge {jobId} could not be read: what its links stage moved stays with the kept sequence", ex); return; } if (journal is not null) await MoveBackAsync(journal).ConfigureAwait(false); } /// /// Gives the other sequence back what the links stage took (): its rows (session keys, /// links, pairs), then the session keys of both manifests; the journal is set back to before the links stage (not saved). /// Nothing when the stage had not begun, or the other is gone (the trash ran). False when it could not be done. /// internal async Task MoveBackAsync(MergeJournal journal) { if (journal.LinksUndo is not { } undo) return true; var ct = CancellationToken.None; try { var drop = await db.Sequences.GetAsync(journal.DropId, ct).ConfigureAwait(false); var keep = await db.Sequences.GetAsync(journal.KeepId, ct).ConfigureAwait(false); if (drop is null || keep is null) return true; await new FoundCopyStore(db).MoveBackAsync(journal.DropId, journal.KeepId, keep.ArchiveId, undo.DropArchiveId, undo.Moved, ct).ConfigureAwait(false); var keys = journal.SessionKeys.ToHashSet(StringComparer.Ordinal); var added = keys.Where(k => !undo.KeepKeysBefore.Contains(k, StringComparer.Ordinal)).ToHashSet(StringComparer.Ordinal); var removed = keys.Where(k => undo.DropKeysBefore.Contains(k, StringComparer.Ordinal)).ToList(); if (added.Count > 0 && await db.Archives.GetAsync(keep.ArchiveId, ct).ConfigureAwait(false) is { } keepArchive) { var paths = new ArchivePaths(keepArchive.RootPath); if (File.Exists(paths.Sequence(keep.Folder).ManifestPath)) await archives.Writer.UpdateAsync(paths, keep.Id, keep.Folder, manifest => !manifest.SessionKeys.Any(added.Contains) ? null : manifest with { SessionKeys = [.. manifest.SessionKeys.Where(k => !added.Contains(k))] }, ct).ConfigureAwait(false); } if (removed.Count > 0 && await db.Archives.GetAsync(drop.ArchiveId, ct).ConfigureAwait(false) is { } dropArchive) { var paths = new ArchivePaths(dropArchive.RootPath); if (File.Exists(paths.Sequence(drop.Folder).ManifestPath)) await archives.Writer.UpdateAsync(paths, drop.Id, drop.Folder, manifest => removed.All(manifest.SessionKeys.Contains) ? null : manifest with { SessionKeys = [.. manifest.SessionKeys.Union(removed, StringComparer.Ordinal)] }, ct).ConfigureAwait(false); } journal.LinksUndo = null; journal.KeepKeysWritten = false; journal.DropKeysRemoved = false; journal.LinksMoved = false; if (journal.Stage > MergeStage.Links) journal.Stage = MergeStage.Links; return true; } catch (Exception ex) when (ex is not OperationCanceledException) { Log.Warning("archive", $"Merge {journal.JobId}: what its links stage moved could not be given back to the other sequence", ex); return false; } } /// /// A resumed run (a pause, a crash, or a retry after a failure) goes on only while the pair is still this job's: it reads /// Integrating with this job (a failure keeps both; a Keep apart, an Ask again or another merge since clears them), or /// it is gone with the other sequence (the trash stage ran before the journal said so). Otherwise the journal is stale: /// it and its staging go and nothing more is applied; the pair, no longer this job's, is left as it is. /// private async Task ResumeAsync(Job job, MergeJobParameters parameters, ArchivePaths keepPaths, CancellationToken ct) { var copy = await new FoundCopyStore(db).GetAsync(parameters.CopyId, ct).ConfigureAwait(false); if (copy is { State: CopyState.Integrating } && copy.JobId == job.Id) return; if (copy is null && await db.Sequences.GetAsync(parameters.DropId, ct).ConfigureAwait(false) is null) return; await MoveBackAsync(keepPaths, job.Id).ConfigureAwait(false); try { MergeJournal.Delete(keepPaths, job.Id); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { Log.Warning("archive", $"The stale journal of merge {job.Id} was not removed", ex); } throw new StaleCopyPlanException(); } /// The first run: plans again, refuses a plan that moved or is blocked, and journals it. private async Task StartAsync(JobContext context, MergeJobParameters parameters, Sequence keepRow, ArchivePaths keepPaths) { var ct = context.Cancellation; var copy = await new FoundCopyStore(db).GetAsync(parameters.CopyId, ct).ConfigureAwait(false); if (copy is null || copy.JobId != context.Job.Id) throw new StaleCopyPlanException(); if (await workspace.HasUnsentAsync(parameters.DropId, ct).ConfigureAwait(false)) throw new InvalidOperationException(Loc.L("Errors.Copies.Unsent", (await db.Sequences.GetAsync(parameters.DropId, ct).ConfigureAwait(false))?.DisplayTitle ?? string.Empty)); var plan = await merger.PlanAsync(parameters.CopyId, parameters.KeepId, parameters.DropId, parameters.Choices, JobContextPlan, context.Job.Id, ct) .ConfigureAwait(false); if (plan.KeepId != parameters.KeepId || plan.DropId != parameters.DropId) throw new StaleCopyPlanException(); if (!plan.CanApply) throw new InvalidOperationException(await merger.BlockMessageAsync(plan, ct).ConfigureAwait(false)); if (!string.Equals(plan.Fingerprint, parameters.Fingerprint, StringComparison.Ordinal)) throw new StaleCopyPlanException(); var winners = plan.Photos.Where(p => p.Action == PhotoAction.FetchVersion).ToDictionary(p => p.Key, p => p.Sha256, StringComparer.Ordinal); var journal = new MergeJournal { JobId = context.Job.Id, CopyId = plan.CopyId, KeepId = plan.KeepId, DropId = plan.DropId, Fingerprint = plan.Fingerprint, Stage = MergeStage.Copy, KeepSettingsVersion = plan.KeepSettingsVersion, DropFramesVersion = plan.DropFramesVersion, DropSettingsVersion = plan.DropSettingsVersion, KeepStatus = (int)keepRow.Status, Copies = [.. plan.Photos.Where(p => p.Action is PhotoAction.Fetch or PhotoAction.FetchVersion && p.FrameId is not null) .Select(p => new MergeCopy(p.Key, p.Action, p.FrameId!.Value, p.FileName, p.Bytes, p.Sha256))], Replaces = [.. plan.Photos.Where(p => p.Action == PhotoAction.TrashHere && p.FrameId is not null && winners.GetValueOrDefault(p.Key) is { Length: > 0 }) .Select(p => new MergeReplace(p.Key, p.FrameId!.Value, p.FileName, winners[p.Key]!))], Absent = [.. plan.LastCopies.Where(l => l.Side == CopySide.Other).Select(l => l.Key.ToLowerInvariant())], Renders = [.. plan.Renders], Payload = plan.Payload, Projects = [.. plan.Projects], SessionKeys = [.. plan.SessionKeys], }; journal.Save(keepPaths); return journal; } /// One run of a merge, from the stage its journal reached. private sealed class Run(MergeJobHandler owner, JobContext context, MergeJournal journal, Archive keepArchive, ArchivePaths keepPaths) { private readonly ChronosDatabase _db = owner.Db; private readonly CancellationToken _ct = context.Cancellation; private readonly string _staging = MergeJournal.StagingFor(keepPaths, journal.JobId); private Archive _dropArchive = null!; private ArchivePaths _dropPaths = null!; private string _keepTitle = string.Empty; private string _dropTitle = string.Empty; private bool _forget; private long Total => journal.Copies.Count + journal.Renders.Count + journal.Replaces.Count + journal.Projects.Count + 4; public async Task RunAsync() { var keep = await _db.Sequences.GetAsync(journal.KeepId, _ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException(); _keepTitle = keep.DisplayTitle; var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false); if (drop is null && journal.Stage < MergeStage.Trash) throw new InvalidOperationException(Loc.L("Errors.Copies.StalePlan")); if (drop is not null) { _dropTitle = drop.DisplayTitle; _dropArchive = await _db.Archives.GetAsync(drop.ArchiveId, _ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException(); _dropPaths = new ArchivePaths(_dropArchive.RootPath); } while (journal.Stage != MergeStage.Done) { _ct.ThrowIfCancellationRequested(); if (owner.BeforeStage is { } hook) await hook(journal.Stage).ConfigureAwait(false); try { switch (journal.Stage) { case MergeStage.Copy: await CopyAsync().ConfigureAwait(false); break; case MergeStage.Renders: await RendersAsync().ConfigureAwait(false); break; case MergeStage.Verify: await VerifyAsync().ConfigureAwait(false); break; case MergeStage.Replace: await ReplaceAsync().ConfigureAwait(false); break; case MergeStage.Details: await DetailsAsync().ConfigureAwait(false); break; case MergeStage.Projects: await ProjectsAsync().ConfigureAwait(false); break; case MergeStage.Links: await LinksAsync().ConfigureAwait(false); break; case MergeStage.Trash: await TrashAsync().ConfigureAwait(false); break; } } catch (Exception ex) when (ex is not OperationCanceledException) { await AbandonAsync().ConfigureAwait(false); throw; } journal.Stage++; Save(); await context.CheckpointAsync($"merge:{journal.Stage}", Done(), Total).ConfigureAwait(false); } await FinishAsync().ConfigureAwait(false); } private long Done() => journal.Committed.Count + journal.RendersMoved.Count + journal.Replaced.Count + journal.KeptLosers.Count + journal.ProjectsDone.Count + Math.Min(4, Math.Max(0, journal.Stage - MergeStage.Verify)); private void Save() => journal.Save(keepPaths); private InvalidOperationException Changed(string title) => new(Loc.L("Errors.Copies.Changed", title)); /// Asks for this run's journal and staging to go as it fails (): what it planned no longer holds. private void ForgetJournal() => _forget = true; /// /// A stage failed: what the links stage took from the other goes back to it first (a retry runs that stage again, after /// its checks), then the journal goes when asked for it, else it is saved as set back. When /// nothing could be given back, the journal stays (at the stage it reached) so a retry or a cancel tries again. /// private async Task AbandonAsync() { var linksBegun = journal.LinksUndo is not null; if (!await owner.MoveBackAsync(journal).ConfigureAwait(false)) return; if (_forget) { try { MergeJournal.Delete(keepPaths, journal.JobId); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { Log.Warning("archive", $"The stale journal of merge {journal.JobId} was not removed", ex); } } else if (linksBegun) { Save(); } } /// /// The other sequence's files are where the index says, on a volume that answers: a verify or a trash never reads an /// absent drive or a moved folder as "nothing there to lose". /// private void EnsureDropReadable(Sequence drop) { if (ArchiveVolume.Probe(_dropPaths) != ArchiveAvailability.Online) throw new InvalidOperationException(Loc.L("Errors.Copies.ArchiveOffline", _dropTitle)); if (drop.Folder.Length > 0 && !SequenceFolders.Holds(_dropPaths.Sequence(drop.Folder), drop.Id)) throw new InvalidOperationException(Loc.L("Errors.Copies.FolderMoved", _dropTitle)); } private async Task EnsureDropUnchangedAsync(int expected) { var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false); if (drop is null || drop.FramesVersion != expected) throw Changed(_dropTitle); } // ---- copy ---- private async Task CopyAsync() { var n = 0; foreach (var copy in journal.Copies) { n++; if (journal.Committed.ContainsKey(copy.DropFrameId)) continue; _ct.ThrowIfCancellationRequested(); await EnsureDropUnchangedAsync(journal.DropFramesVersion).ConfigureAwait(false); var frame = await _db.Frames.GetAsync(copy.DropFrameId, _ct).ConfigureAwait(false) ?? throw Changed(_dropTitle); await context.ReportAsync(Done(), Total, new JobDetail(JobStep.CopyPhoto, copy.FileName, n, journal.Copies.Count)).ConfigureAwait(false); var result = await FileAsync(frame, copy.Sha256, planned: copy.Sha256 is { Length: > 0 }).ConfigureAwait(false); journal.Committed[copy.DropFrameId] = result.FrameId; Save(); context.Log(Loc.L("Archive.Log.MergeCopied", copy.FileName)); } } /// /// Copies a frame of the other into staging (hash-checked against ) and files it in the /// kept sequence. : the plan knew the photo's content (else a tombstone of the kept sequence /// may refuse it although the plan listed no last copy: the plan is stale, never this job's to retry). /// private async Task FileAsync(Frame frame, string? expected, bool planned) { Directory.CreateDirectory(_staging); var staged = Path.Combine(_staging, frame.Id.ToString("N") + Path.GetExtension(frame.FileName)); if (File.Exists(staged)) File.Delete(staged); string sha; try { sha = await VerifiedCopy.CopyAsync(_dropPaths.FromRelative(frame.RelativePath), staged, verify: true, null, _ct).ConfigureAwait(false); } catch (Exception ex) when (ex is FileNotFoundException or DirectoryNotFoundException) { throw new InvalidOperationException(Loc.L("Errors.Copies.ChangedFile", frame.FileName), ex); } if (expected is { Length: > 0 } && !string.Equals(sha, expected, StringComparison.OrdinalIgnoreCase)) { File.Delete(staged); throw new InvalidOperationException(Loc.L("Errors.Copies.ChangedFile", frame.FileName)); } var size = new FileInfo(staged).Length; IngestResult result; try { result = await owner.Archives.Ingest.CommitAsync(new IngestRequest(keepArchive.Id, journal.KeepId, null, null, staged, frame.FileName, size, sha.ToLowerInvariant(), frame.CapturedAt, null, RefuseRemoved: true), _ct).ConfigureAwait(false); } catch (PhotoRemovedException) { File.Delete(staged); if (planned) throw Changed(_keepTitle); // deleted from the kept sequence since the plan: a last copy the plan did not list // A photo the plan could not compare by content (unhashed) that the kept sequence deleted: a last copy the // plan did not list. Retrying this journal would fail the same way: it goes, and the user plans again. ForgetJournal(); throw new StaleCopyPlanException(); } if (File.Exists(staged)) File.Delete(staged); return result; } // ---- renders ---- private async Task RendersAsync() { var keep = await _db.Sequences.GetAsync(journal.KeepId, _ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException(); var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false) ?? throw Changed(_dropTitle); var n = 0; foreach (var step in journal.Renders) { n++; if (journal.RendersMoved.Contains(step.RenderId)) continue; _ct.ThrowIfCancellationRequested(); await context.ReportAsync(Done(), Total, new JobDetail(JobStep.CopyVideo, step.FileName, n, journal.Renders.Count)).ConfigureAwait(false); var render = await _db.Renders.GetAsync(step.RenderId, _ct).ConfigureAwait(false); if (render is null || render.SequenceId == journal.KeepId) { journal.RendersMoved.Add(step.RenderId); Save(); continue; } var source = SyncService.RenderFile(_dropArchive, render); var relative = render.OutputPath.Replace('\\', '/'); var inside = !Path.IsPathRooted(render.OutputPath) && drop.Folder.Length > 0 && relative.StartsWith(drop.Folder.TrimEnd('/') + "/", StringComparison.Ordinal); string output; if (inside && (File.Exists(source) || Directory.Exists(source))) { // Inside the folder that goes to the trash: copied (verified) into the kept sequence's exports first. if (!journal.RenderTargets.TryGetValue(step.RenderId, out var target)) { target = keepPaths.ToRelative(Free(keepPaths.Sequence(keep.Folder).Exports, Path.GetFileName(source))); journal.RenderTargets[step.RenderId] = target; Save(); } var destination = keepPaths.FromRelative(target); if (Directory.Exists(source)) await CopyFolderAsync(source, destination).ConfigureAwait(false); else await CopyRenderAsync(source, destination, render.Sha256, step.FileName).ConfigureAwait(false); output = target; } else { // Outside it (or gone): only the row moves, naming the same file wherever the kept sequence's archive is. output = Path.IsPathRooted(render.OutputPath) || keepArchive.Id == _dropArchive.Id ? render.OutputPath : source; } var access = new DbAccess(_db.Database); await access.InTransactionAsync(async session => { var inner = new DbAccess(session); await inner.ExecuteAsync("UPDATE renders SET sequence_id = $2, output_path = $3 WHERE id = $1", [step.RenderId, journal.KeepId, output], _ct) .ConfigureAwait(false); foreach (var id in (Guid[])[journal.KeepId, journal.DropId]) await inner.ExecuteAsync("UPDATE sequences SET rendered_count = (SELECT count(*) FROM renders WHERE sequence_id = $1) WHERE id = $1", [id], _ct) .ConfigureAwait(false); return 0; }, _ct).ConfigureAwait(false); journal.RendersMoved.Add(step.RenderId); Save(); } } private async Task CopyRenderAsync(string source, string destination, string? sha, string name) { if (File.Exists(destination)) { // A run that copied it before its row moved: kept when it is the same file. if (sha is { Length: > 0 } && string.Equals(await ContentHash.Sha256Async(destination, null, _ct).ConfigureAwait(false), sha, StringComparison.OrdinalIgnoreCase)) return; File.Delete(destination); } var copied = await VerifiedCopy.CopyAsync(source, destination, verify: true, null, _ct).ConfigureAwait(false); if (sha is { Length: > 0 } && !string.Equals(copied, sha, StringComparison.OrdinalIgnoreCase)) { File.Delete(destination); throw new InvalidOperationException(Loc.L("Errors.Copies.ChangedFile", name)); } } private async Task CopyFolderAsync(string source, string destination) { foreach (var file in Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories)) { var target = Path.Combine(destination, Path.GetRelativePath(source, file)); if (File.Exists(target) && new FileInfo(target).Length == new FileInfo(file).Length && string.Equals(await ContentHash.Sha256Async(target, null, _ct).ConfigureAwait(false), await ContentHash.Sha256Async(file, null, _ct).ConfigureAwait(false), StringComparison.Ordinal)) continue; if (File.Exists(target)) File.Delete(target); await VerifiedCopy.CopyAsync(file, target, verify: true, null, _ct).ConfigureAwait(false); } } private static string Free(string folder, string name) { var stem = Path.GetFileNameWithoutExtension(name); var extension = Path.GetExtension(name); var candidate = Path.Combine(folder, name); for (var n = 2; File.Exists(candidate) || Directory.Exists(candidate); n++) candidate = Path.Combine(folder, $"{stem} ({n}){extension}"); return candidate; } // ---- verify ---- private async Task VerifyAsync() { var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false) ?? throw Changed(_dropTitle); if (drop.FramesVersion != journal.DropFramesVersion) throw Changed(_dropTitle); EnsureDropReadable(drop); var keepFrames = await ProcessingFrames.ListAsync(_db, journal.KeepId, _ct).ConfigureAwait(false); var dropFrames = (await ProcessingFrames.ListAsync(_db, journal.DropId, _ct).ConfigureAwait(false)) .Where(f => (f.Flags & CopySides.UnusableFlags) == 0).ToList(); var total = keepFrames.Count + dropFrames.Count; var done = 0; // What the kept sequence holds now, hashed again. var held = new Dictionary(StringComparer.Ordinal); foreach (var frame in keepFrames.Where(f => (f.Flags & CopySides.UnusableFlags) == 0)) { _ct.ThrowIfCancellationRequested(); await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Verify, frame.FileName, ++done, total)).ConfigureAwait(false); if (await HashAsync(keepPaths.FromRelative(frame.RelativePath)).ConfigureAwait(false) is { } sha) held.TryAdd(sha, frame.Id); } var keepRemoved = (await _db.FrameRemovals.ListBySequenceAsync(journal.KeepId, _ct).ConfigureAwait(false)) .Select(r => r.Sha256.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal); var failed = new List<(Guid FrameId, string Sha256)>(); foreach (var frame in dropFrames) { _ct.ThrowIfCancellationRequested(); await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Verify, frame.FileName, ++done, total)).ConfigureAwait(false); if (journal.Verified.ContainsKey(frame.Id) || journal.Settled.Contains(frame.Id)) continue; // A usable frame (not flagged Missing) whose file is absent stops the merge (design §5d step 4): it is not // "nothing to lose" until a verify says so. var sha = await HashAsync(_dropPaths.FromRelative(frame.RelativePath)).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Copies.ChangedFile", frame.FileName)); if (frame.Sha256 is { Length: > 0 } recorded && !string.Equals(recorded, sha, StringComparison.OrdinalIgnoreCase)) throw new InvalidOperationException(Loc.L("Errors.Copies.ChangedFile", frame.FileName)); if (held.TryGetValue(sha, out var keepFrame)) { journal.Verified[frame.Id] = new MergeVerified(sha, keepFrame); continue; } if (journal.Absent.Contains(sha)) { // A last copy the plan listed (or left out by the user's choice). journal.Settled.Add(frame.Id); continue; } if (frame.Sha256 is not { Length: > 0 } && !keepRemoved.Contains(sha)) { // Compared by size and time only, and another photo after all: filed in the kept sequence as a second version. var filed = await FileAsync(frame, sha, planned: false).ConfigureAwait(false); journal.SecondVersions.Add(frame.Id); journal.Verified[frame.Id] = new MergeVerified(sha, filed.FrameId); held.TryAdd(sha, filed.FrameId); Save(); continue; } failed.Add((frame.Id, sha)); } if (failed.Count > 0) { // A photo held by another sequence of the index is not lost with the other; any other is. The kept // sequence's index rows prove nothing: its files were hashed again above and did not hold it. var others = await new CopySides(_db).HeldElsewhereAsync([.. failed.Select(f => f.Sha256)], [journal.DropId, journal.KeepId], _ct) .ConfigureAwait(false); var lost = failed.Where(f => !others.Contains(f.Sha256)).Select(f => f.Sha256).Distinct(StringComparer.Ordinal).Count(); if (lost > 0) throw new InvalidOperationException(Loc.Plural("Errors.Copies.VerifyFailed", lost)); journal.Settled.AddRange(failed.Select(f => f.FrameId)); // Held by a third sequence now: asked again before the trash, which that sequence may not wait for. foreach (var sha in failed.Select(f => f.Sha256).Distinct(StringComparer.Ordinal)) if (!journal.SettledElsewhere.Contains(sha)) journal.SettledElsewhere.Add(sha); } journal.VerifiedDropFramesVersion = drop.FramesVersion; Save(); } private async Task HashAsync(string path) { try { return (await ContentHash.Sha256Async(path, null, _ct).ConfigureAwait(false)).ToLowerInvariant(); } catch (Exception ex) when (ex is FileNotFoundException or DirectoryNotFoundException) { return null; } } // ---- replace ---- private async Task ReplaceAsync() { var n = 0; foreach (var replace in journal.Replaces) { n++; if (journal.Replaced.Contains(replace.Key) || journal.KeptLosers.Contains(replace.Key)) continue; _ct.ThrowIfCancellationRequested(); await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Replace, replace.FileName, n, journal.Replaces.Count)).ConfigureAwait(false); var loser = await _db.Frames.GetAsync(replace.KeepFrameId, _ct).ConfigureAwait(false); if (loser is null || loser.SequenceId != journal.KeepId) { journal.Replaced.Add(replace.Key); Save(); continue; } if (!await WinnerInPlaceAsync(replace.WinnerSha256).ConfigureAwait(false)) { Keep(replace, Loc.L("Archive.Log.KeptReason.NotInPlace")); continue; } try { var plan = await owner.Archives.FrameTrash.PlanDeleteAsync(journal.KeepId, [replace.KeepFrameId], null, _ct, journal.JobId).ConfigureAwait(false); if (!plan.CanDelete) { Keep(replace, CopyLog.Reasons(plan.Blocked)); continue; } await owner.Archives.FrameTrash.DeleteAsync(plan, null, null, _ct, journal.JobId).ConfigureAwait(false); journal.Replaced.Add(replace.Key); Save(); context.Log(Loc.L("Archive.Log.CopyReplaced", replace.FileName)); } catch (Exception ex) when (ex is StaleDeletePlanException or IOException or UnauthorizedAccessException) { Keep(replace, CopyLog.Reason(ex)); } } } private void Keep(MergeReplace replace, string why) { journal.KeptLosers.Add(replace.Key); Save(); context.Log(Loc.L("Archive.Log.CopyKept", replace.FileName, why)); } /// Whether a usable frame of the kept sequence holds , hashed again. private async Task WinnerInPlaceAsync(string sha) { foreach (var frame in await _db.Frames.FindBySha256Async(sha.ToLowerInvariant(), _ct).ConfigureAwait(false)) { if (frame.SequenceId != journal.KeepId || (frame.Flags & CopySides.UnusableFlags) != 0) continue; if (string.Equals(await HashAsync(keepPaths.FromRelative(frame.RelativePath)).ConfigureAwait(false), sha, StringComparison.OrdinalIgnoreCase)) return true; } return false; } // ---- details ---- private async Task DetailsAsync() { if (journal.Payload is null || journal.DetailsDone) return; await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Details)).ConfigureAwait(false); await new SequencePayloadWriter(_db, owner.Archives).ApplyAsync(journal.KeepId, journal.Payload, journal.KeepSettingsVersion, _ct).ConfigureAwait(false); journal.DetailsDone = true; Save(); } // ---- projects ---- private async Task ProjectsAsync() { await RenamesHeldRendersAsync().ConfigureAwait(false); if (journal.Projects.Count == 0) return; var from = new ArchiveProjectHome(_db, owner.Archives.Projects, journal.DropId); var to = new ArchiveProjectHome(_db, owner.Archives.Projects, journal.KeepId); var sources = (await _db.Projects.ListBySequenceAsync(journal.DropId, includeDeleted: false, _ct).ConfigureAwait(false)).ToDictionary(p => p.Id); var failed = 0; var n = 0; foreach (var step in journal.Projects) { n++; if (journal.ProjectsDone.Contains(step.SourceProjectId)) continue; _ct.ThrowIfCancellationRequested(); await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Projects, step.TargetName, n, journal.Projects.Count)).ConfigureAwait(false); if (!sources.TryGetValue(step.SourceProjectId, out var source)) { // Deleted from the other since the plan: its own trash keeps it. journal.ProjectsDone.Add(step.SourceProjectId); Save(); continue; } try { var copied = await owner.Projects.CopyAsync(from, source, to, step.TargetName, journal.ProjectSteps.GetValueOrDefault(step.SourceProjectId), saved => { journal.ProjectSteps[step.SourceProjectId] = saved; Save(); return ValueTask.CompletedTask; }, _ct).ConfigureAwait(false); await RenamesRendersAsync(step.SourceProjectId, copied.Id).ConfigureAwait(false); journal.ProjectsDone.Add(step.SourceProjectId); Save(); context.Log(Loc.L("Archive.Log.CopyProject", step.SourceName, step.TargetName)); } catch (Exception ex) when (ex is not OperationCanceledException) { failed++; context.Log(Loc.L("Archive.Log.CopyProjectKept", step.SourceName, ex.Message)); } } if (failed > 0) throw new InvalidOperationException(Loc.Plural("Errors.Copies.ProjectsKept", failed)); } /// /// The renders moved to the kept sequence that name a project of the other the plan left because the kept sequence /// holds its copy already name that copy (and its versions), as those of a copied project do. /// private async Task RenamesHeldRendersAsync() { var planned = journal.Projects.Select(p => p.SourceProjectId).ToHashSet(); IReadOnlyList? keepProjects = null; foreach (var project in await _db.Projects.ListBySequenceAsync(journal.DropId, includeDeleted: false, _ct).ConfigureAwait(false)) { if (project.IsMain || planned.Contains(project.Id)) continue; keepProjects ??= await _db.Projects.ListBySequenceAsync(journal.KeepId, includeDeleted: false, _ct).ConfigureAwait(false); if (await owner.Merger.HeldAsync(project, keepProjects, _dropArchive.Name, _ct).ConfigureAwait(false) is { } held) await RenamesRendersAsync(project.Id, held.KeptId, held.Versions.Select(v => new TransplantMove(v.Key, v.Value))).ConfigureAwait(false); } } /// The renders moved to the kept sequence name the project copy (and its versions) instead of the other's. private Task RenamesRendersAsync(Guid source, Guid copy) => RenamesRendersAsync(source, copy, journal.ProjectSteps.GetValueOrDefault(source)?.Moves ?? []); private async Task RenamesRendersAsync(Guid source, Guid copy, IEnumerable moves) { var access = new DbAccess(_db.Database); foreach (var move in moves) await access.ExecuteAsync("UPDATE renders SET project_id = $2, project_version_id = $3 WHERE sequence_id = $1 AND project_version_id = $4", [journal.KeepId, copy, move.Copy, move.Source], _ct).ConfigureAwait(false); await access.ExecuteAsync("UPDATE renders SET project_id = $2 WHERE sequence_id = $1 AND project_id = $3", [journal.KeepId, copy, source], _ct) .ConfigureAwait(false); } // ---- links ---- private async Task LinksAsync() { await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Process)).ConfigureAwait(false); var keep = await _db.Sequences.GetAsync(journal.KeepId, _ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException(); var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false) ?? throw Changed(_dropTitle); if (journal.LinksUndo is null) { // Nothing of the other is touched while a check before the trash can still abandon the merge. await CheckTrashAsync(drop).ConfigureAwait(false); await using (await owner.Archives.Writer.LockAsync(keepPaths, journal.KeepId, _ct).ConfigureAwait(false)) await PlanTrashAsync().ConfigureAwait(false); journal.LinksUndo = new MergeLinksUndo(KeysOf(keepPaths, keep), KeysOf(_dropPaths, drop), _dropArchive.Id, await new FoundCopyStore(_db).UndoOfMoveAsync(journal.DropId, journal.KeepId, keepArchive.Id, journal.SessionKeys, journal.CopyId, _ct) .ConfigureAwait(false)); Save(); } var keys = journal.SessionKeys.ToHashSet(StringComparer.Ordinal); if (!journal.KeepKeysWritten) { if (keys.Count > 0 && File.Exists(keepPaths.Sequence(keep.Folder).ManifestPath)) { await owner.Archives.Writer.UpdateAsync(keepPaths, keep.Id, keep.Folder, manifest => keys.All(manifest.SessionKeys.Contains) ? null : manifest with { SessionKeys = [.. manifest.SessionKeys.Union(keys, StringComparer.Ordinal)] }, _ct).ConfigureAwait(false); } journal.KeepKeysWritten = true; Save(); } if (!journal.DropKeysRemoved) { if (keys.Count > 0 && File.Exists(_dropPaths.Sequence(drop.Folder).ManifestPath)) { await owner.Archives.Writer.UpdateAsync(_dropPaths, drop.Id, drop.Folder, manifest => !manifest.SessionKeys.Any(keys.Contains) ? null : manifest with { SessionKeys = [.. manifest.SessionKeys.Where(k => !keys.Contains(k))] }, _ct).ConfigureAwait(false); } journal.DropKeysRemoved = true; Save(); } if (!journal.LinksMoved) { MoveResult moved; try { moved = await new FoundCopyStore(_db).MoveToKeptAsync(journal.DropId, journal.KeepId, keepArchive.Id, journal.SessionKeys, journal.CopyId, _ct) .ConfigureAwait(false); } catch (CopyLinkTakenException ex) when (ex.Server is { } key) { throw CopyLinkTakenException.BothLinked(key, owner.Merger.ServerName(key)); } foreach (var key in moved.SkippedKeys) context.Log(key); journal.LinksMoved = true; Save(); } } /// The session keys of the manifest of (none when it has no manifest). private static List KeysOf(ArchivePaths paths, Sequence sequence) { var manifest = paths.Sequence(sequence.Folder).ManifestPath; return sequence.Folder.Length > 0 && File.Exists(manifest) ? [.. ManifestStore.ReadSequence(manifest).SessionKeys] : []; } // ---- trash ---- /// /// The checks before the trash that may abandon the merge, run before the links stage and again at the trash: no /// unsent workspace work, no work saved into the other since the plan, the other at the frames version the verify read /// and readable. Returns that frames version. /// private async Task CheckTrashAsync(Sequence drop) { if (await owner.Workspace.HasUnsentAsync(journal.DropId, _ct).ConfigureAwait(false)) throw new InvalidOperationException(Loc.L("Errors.Copies.Unsent", _dropTitle)); if (await WorkSinceThePlanAsync(drop).ConfigureAwait(false)) { // Work saved into the other since the plan (sent home, rendered) never reached the kept one: planned again. ForgetJournal(); throw Changed(_dropTitle); } var expected = journal.VerifiedDropFramesVersion ?? throw Changed(_dropTitle); if (drop.FramesVersion != expected) throw Changed(_dropTitle); EnsureDropReadable(drop); return expected; } /// Under the kept sequence's lease: every verified photo is still held, and the trash's dry run of the other is not blocked. private async Task PlanTrashAsync() { await EnsureVerifiedHeldAsync().ConfigureAwait(false); var plan = await owner.Archives.Trash.PlanAsync([journal.DropId], null, _ct, journal.JobId).ConfigureAwait(false); if (plan.Blocked.Count > 0) throw new InvalidOperationException(Loc.L("Errors.Copies.TrashBlocked", _dropTitle, Loc.L("Archive.Delete.Reason." + plan.Blocked[0].Reason))); return plan; } private async Task TrashAsync() { var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false); if (drop is null) return; // a run that moved it before its journal said so await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Trash, _dropTitle)).ConfigureAwait(false); var expected = await CheckTrashAsync(drop).ConfigureAwait(false); await using (await owner.Archives.Writer.LockAsync(keepPaths, journal.KeepId, _ct).ConfigureAwait(false)) { var plan = await PlanTrashAsync().ConfigureAwait(false); try { await owner.Archives.Trash.DeleteAsync(plan, null, null, _ct, journal.JobId, new Dictionary { [journal.DropId] = expected }) .ConfigureAwait(false); } catch (StaleDeletePlanException ex) { throw new InvalidOperationException(Loc.L("Errors.Copies.Changed", _dropTitle), ex); } } context.Log(Loc.L("Archive.Log.MergeTrashed", _dropTitle)); } /// /// Whether the other holds work the plan did not see and nothing copied: details or Main settings saved since (its /// settings version moved), a project made or a version added since (missing from the plan, or another head than the /// one it copied), or a render the kept sequence does not hold. /// private async Task WorkSinceThePlanAsync(Sequence drop) { if (journal.DropSettingsVersion is { } settings && drop.SettingsVersion != settings) return true; var planned = journal.Projects.ToDictionary(p => p.SourceProjectId); IReadOnlyList? keepProjects = null; foreach (var project in await _db.Projects.ListBySequenceAsync(journal.DropId, includeDeleted: false, _ct).ConfigureAwait(false)) { if (project.IsMain) continue; if (planned.TryGetValue(project.Id, out var step) && (project.HeadVersionId ?? Guid.Empty) == step.SourceHeadId) continue; // Not planned (or moved on since) is fine only when the kept sequence holds a copy of it already, as the plan saw it. keepProjects ??= await _db.Projects.ListBySequenceAsync(journal.KeepId, includeDeleted: false, _ct).ConfigureAwait(false); if (await owner.Merger.HeldAsync(project, keepProjects, _dropArchive.Name, _ct).ConfigureAwait(false) is null) return true; } var renders = await _db.Renders.ListBySequenceAsync(journal.DropId, _ct).ConfigureAwait(false); if (renders.Count == 0) return false; // A render the plan left because the kept sequence holds the same file stays with the other, as planned. var keepHashes = (await _db.Renders.ListBySequenceAsync(journal.KeepId, _ct).ConfigureAwait(false)).Select(r => r.Sha256).OfType() .ToHashSet(StringComparer.OrdinalIgnoreCase); return renders.Any(r => !journal.RendersMoved.Contains(r.Id) && (r.Sha256 is null || !keepHashes.Contains(r.Sha256))); } /// /// Every usable photo of the other was verified (held by the kept sequence, or a last copy the plan listed, or held by /// a third sequence that still holds it now), and every photo the verify found in the kept sequence is still there and /// usable (read under its lease). /// private async Task EnsureVerifiedHeldAsync() { // Every usable photo of the other was verified in this journal: a verify cut short (an absent drive, a moved // folder) never lets the rest through unread. var dropFrames = await ProcessingFrames.ListAsync(_db, journal.DropId, _ct).ConfigureAwait(false); if (dropFrames.Any(f => (f.Flags & CopySides.UnusableFlags) == 0 && !journal.Verified.ContainsKey(f.Id) && !journal.Settled.Contains(f.Id))) throw Changed(_dropTitle); // The photos a third sequence held at the verify: one it no longer holds would leave only with the other, never // listed as a last copy. Planned again, the plan lists it. if (journal.SettledElsewhere.Count > 0) { var still = await new CopySides(_db).HeldElsewhereAsync(journal.SettledElsewhere, [journal.DropId, journal.KeepId], _ct).ConfigureAwait(false); if (journal.SettledElsewhere.Any(sha => !still.Contains(sha))) { ForgetJournal(); throw Changed(_dropTitle); } } var frames = await ProcessingFrames.ListAsync(_db, journal.KeepId, _ct).ConfigureAwait(false); var byId = frames.Where(f => (f.Flags & CopySides.UnusableFlags) == 0).ToDictionary(f => f.Id); var hashes = byId.Values.Select(f => f.Sha256).OfType().Select(s => s.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal); foreach (var verified in journal.Verified.Values) { if (byId.TryGetValue(verified.KeepFrameId, out var frame) && (frame.Sha256 is null || string.Equals(frame.Sha256, verified.Sha256, StringComparison.OrdinalIgnoreCase))) continue; if (hashes.Contains(verified.Sha256)) continue; throw Changed(_keepTitle); } } // ---- done ---- private async Task FinishAsync() { await context.ReportAsync(Total, Total, new JobDetail(JobStep.FollowUps)).ConfigureAwait(false); await new FoundCopyStore(_db).DeleteAsync(journal.CopyId, CancellationToken.None).ConfigureAwait(false); if (await _db.Sequences.GetAsync(journal.KeepId, CancellationToken.None).ConfigureAwait(false) is { } keep) { foreach (var request in owner.Archives.FrameTrash.FollowUps(keepArchive, keep, (SequenceStatus)journal.KeepStatus)) { try { if (request.Targets.Count > 0 && await _db.Jobs.IsQueuedAsync(request.Kind, request.Targets[0], CancellationToken.None).ConfigureAwait(false)) continue; await context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, CancellationToken.None) .ConfigureAwait(false); } catch (DatabaseException ex) { Log.Warning("archive", $"{request.Label}: the passes the merged photos need were not queued; the next scan queues them", ex); } } } await TellAsync(journal.KeepId, KeepChange).ConfigureAwait(false); await TellAsync(journal.DropId, DropChange).ConfigureAwait(false); var kept = journal.KeptLosers.Count; try { MergeJournal.Delete(keepPaths, journal.JobId); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { Log.Warning("archive", $"The journal of merge {journal.JobId} was not removed", ex); } if (kept > 0) throw new InvalidOperationException(Loc.Plural("Errors.Copies.Kept", kept)); } private async Task TellAsync(Guid id, string change) { if (owner.Archives.Writer.Options.SequenceChanged is not { } changed) return; try { await changed(id, change, CancellationToken.None).ConfigureAwait(false); } catch (Exception ex) { Log.Warning("archive", $"The change of sequence {id} was not told", ex); } } } }