From b4070ddfd49e4a48fa268069ce72234f8f2b68c2 Mon Sep 17 00:00:00 2001 From: Alberto Balbo Date: Tue, 17 Nov 2026 10:24:32 +0100 Subject: [PATCH] feat(archive): merge two local sequences into the kept one (owner review 6, P4) SequenceMerger plans the merge of a local pair (kept sequence, photos copied by policy, last copies, renders, details, projects, session keys, links, blocks and a fingerprint) and queues the MergeSequences job. MergeJobHandler applies the confirmed plan through a journal, so every stage resumes: copy and file the other's photos, move renders, hash-verify every original of the other in the kept one (or list it as a last copy), trash the kept one's losing versions, write the details (SequencePayloadWriter), copy projects, move keys and links, then move the other to its archive's trash only at the verified frames version. A cancel drops the journal and returns the pair to its previous state. Refs #73 Co-Authored-By: Claude Opus 5.5 --- src/Chronos.Data/Archiving/ArchiveService.cs | 3 + .../Archiving/Jobs/MergeJobHandler.cs | 705 ++++++++++++++++++ src/Chronos.Data/Archiving/MergeJournal.cs | 151 ++++ src/Chronos.Data/Archiving/SequenceMerger.cs | 439 +++++++++++ .../Archiving/SequencePayloadWriter.cs | 134 ++++ .../Archiving/SequenceMergerTests.cs | 630 ++++++++++++++++ 6 files changed, 2062 insertions(+) create mode 100644 src/Chronos.Data/Archiving/Jobs/MergeJobHandler.cs create mode 100644 src/Chronos.Data/Archiving/MergeJournal.cs create mode 100644 src/Chronos.Data/Archiving/SequenceMerger.cs create mode 100644 src/Chronos.Data/Archiving/SequencePayloadWriter.cs create mode 100644 tests/Chronos.Data.Tests/Archiving/SequenceMergerTests.cs diff --git a/src/Chronos.Data/Archiving/ArchiveService.cs b/src/Chronos.Data/Archiving/ArchiveService.cs index 78650e1f..85016ee5 100644 --- a/src/Chronos.Data/Archiving/ArchiveService.cs +++ b/src/Chronos.Data/Archiving/ArchiveService.cs @@ -54,6 +54,9 @@ public sealed class ArchiveService : IJobHandlerProvider public ArchiveIndexer Indexer { get; } + /// The sidecar writer this service's writers share (its in-process gates and leases): a merge holds sequences with it. + internal SequenceWriter Writer => _writer; + /// Commits staged files into sequences (server uploads, client card imports). public ArchiveIngest Ingest { get; } diff --git a/src/Chronos.Data/Archiving/Jobs/MergeJobHandler.cs b/src/Chronos.Data/Archiving/Jobs/MergeJobHandler.cs new file mode 100644 index 00000000..97afb39b --- /dev/null +++ b/src/Chronos.Data/Archiving/Jobs/MergeJobHandler.cs @@ -0,0 +1,705 @@ +using Chronos.Core.Archiving; +using Chronos.Core.Diagnostics; +using Chronos.Core.Import; +using Chronos.Core.Localization; +using Chronos.Core.Model; +using Chronos.Data.Copies; +using Chronos.Data.Jobs; +using Chronos.Data.Projects; +using Chronos.Data.Repositories; +using Chronos.Data.Sync; +using ProcessingFrames = Chronos.Data.Processing.SequenceFrames; + +namespace Chronos.Data.Archiving.Jobs; + +/// +/// (owner review 6, #73; design §5d): applies the merge plan the user confirmed +/// (). Its first run plans again and refuses a plan whose fingerprint moved; the journal +/// () then keeps what that plan decided and how far each stage came, so a crash, a pause or a +/// retry resumes where it stopped. Stages: copy the other's photos into the kept sequence (each hash-checked, filed with +/// tombstones honoured), move the renders, verify every original of the other is held by the kept one (hashed again on +/// both sides) or listed as a last copy, trash the kept sequence's losing versions, write the details, copy the projects, +/// move session keys, links and pairs, then move the other to its archive's trash, only while it is still at the frames +/// version the verify read and the kept photos verified are still there. A cancel leaves the photos copied so far in the +/// kept sequence (nothing of the other is touched before the trash stage) and drops the journal. +/// +internal sealed class MergeJobHandler(ChronosDatabase db, SequenceMerger merger, ArchiveService archives, IProjectCopier projects, + IWorkspaceGuard workspace) : ICancelledJobHandler +{ + /// The change told about the kept sequence. + public const string KeepChange = "frames"; + + /// The change told about the sequence that went to the trash. + public const string DropChange = "deleted"; + + private static readonly CopyPlanContext JobContextPlan = new(string.Empty, null, false, false, false, false, false, false, null, + new HashSet(), new HashSet()); + + public JobKind Kind => JobKind.MergeSequences; + + private ChronosDatabase Db => db; + + private ArchiveService Archives => archives; + + private IProjectCopier Projects => projects; + + private IWorkspaceGuard Workspace => workspace; + + /// Called before each stage runs (tests: a crash, a change made between stages). + internal Func? BeforeStage { get; set; } + + public async Task RunAsync(JobContext context) + { + var job = context.Job; + var ct = context.Cancellation; + var parameters = MergeJobParameters.Read(job.ParametersJson) ?? throw new InvalidOperationException($"Merge job {job.Id} has no plan."); + var keepRow = await db.Sequences.GetAsync(parameters.KeepId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException(); + var keepArchive = await db.Archives.GetAsync(keepRow.ArchiveId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException(); + var keepPaths = new ArchivePaths(keepArchive.RootPath); + try + { + var journal = MergeJournal.Read(keepPaths, job.Id) ?? await StartAsync(context, parameters, keepRow, keepPaths).ConfigureAwait(false); + var staging = MergeJournal.StagingFor(keepPaths, job.Id); + if (Directory.Exists(staging)) Directory.Delete(staging, recursive: true); + await new Run(this, context, journal, keepArchive, keepPaths).RunAsync().ConfigureAwait(false); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + if (await db.Jobs.GetAsync(job.Id, CancellationToken.None).ConfigureAwait(false) is { State: JobState.Cancelled }) + await ForgetAsync(job, parameters, keepPaths).ConfigureAwait(false); + throw; + } + } + + public async Task CancelledAsync(Job job, CancellationToken cancellationToken) + { + try + { + if (MergeJobParameters.Read(job.ParametersJson) is not { } parameters) return; + if (await db.Sequences.GetAsync(parameters.KeepId, cancellationToken).ConfigureAwait(false) is not { } keep) return; + if (await db.Archives.GetAsync(keep.ArchiveId, cancellationToken).ConfigureAwait(false) is not { } archive) return; + await ForgetAsync(job, parameters, new ArchivePaths(archive.RootPath)).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + Log.Warning("archive", $"The cancelled merge {job.Id} left its journal; the archive's next maintenance removes it", ex); + } + } + + /// A cancelled merge: its journal and staging go, and its pair returns to the state it had (when it is still this job's). + private async Task ForgetAsync(Job job, MergeJobParameters parameters, ArchivePaths keepPaths) + { + try + { + MergeJournal.Delete(keepPaths, job.Id); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warning("archive", $"The journal of the cancelled merge {job.Id} was not removed", ex); + } + var store = new FoundCopyStore(db); + if (await store.GetAsync(parameters.CopyId, CancellationToken.None).ConfigureAwait(false) is { JobId: { } owner } copy && owner == job.Id + && copy.State != CopyState.Integrating) + await store.SetStateAsync(copy.Id, copy.State, copy.State, null, CancellationToken.None).ConfigureAwait(false); + } + + /// The first run: plans again, refuses a plan that moved or is blocked, and journals it. + private async Task StartAsync(JobContext context, MergeJobParameters parameters, Sequence keepRow, ArchivePaths keepPaths) + { + var ct = context.Cancellation; + var copy = await new FoundCopyStore(db).GetAsync(parameters.CopyId, ct).ConfigureAwait(false); + if (copy is null || copy.JobId != context.Job.Id) throw new StaleCopyPlanException(); + if (await workspace.HasUnsentAsync(parameters.DropId, ct).ConfigureAwait(false)) + throw new InvalidOperationException(Loc.L("Errors.Copies.Unsent", (await db.Sequences.GetAsync(parameters.DropId, ct).ConfigureAwait(false))?.DisplayTitle ?? string.Empty)); + var plan = await merger.PlanAsync(parameters.CopyId, parameters.KeepId, parameters.DropId, parameters.Choices, JobContextPlan, context.Job.Id, ct) + .ConfigureAwait(false); + if (plan.KeepId != parameters.KeepId || plan.DropId != parameters.DropId) throw new StaleCopyPlanException(); + if (!plan.CanApply) throw new InvalidOperationException(await merger.BlockMessageAsync(plan, ct).ConfigureAwait(false)); + if (!string.Equals(plan.Fingerprint, parameters.Fingerprint, StringComparison.Ordinal)) throw new StaleCopyPlanException(); + + var winners = plan.Photos.Where(p => p.Action == PhotoAction.FetchVersion).ToDictionary(p => p.Key, p => p.Sha256, StringComparer.Ordinal); + var journal = new MergeJournal + { + JobId = context.Job.Id, + CopyId = plan.CopyId, + KeepId = plan.KeepId, + DropId = plan.DropId, + Fingerprint = plan.Fingerprint, + Stage = MergeStage.Copy, + KeepSettingsVersion = plan.KeepSettingsVersion, + DropFramesVersion = plan.DropFramesVersion, + KeepStatus = (int)keepRow.Status, + Copies = [.. plan.Photos.Where(p => p.Action is PhotoAction.Fetch or PhotoAction.FetchVersion && p.FrameId is not null) + .Select(p => new MergeCopy(p.Key, p.Action, p.FrameId!.Value, p.FileName, p.Bytes, p.Sha256))], + Replaces = [.. plan.Photos.Where(p => p.Action == PhotoAction.TrashHere && p.FrameId is not null && winners.GetValueOrDefault(p.Key) is { Length: > 0 }) + .Select(p => new MergeReplace(p.Key, p.FrameId!.Value, p.FileName, winners[p.Key]!))], + Absent = [.. plan.LastCopies.Where(l => l.Side == CopySide.Other).Select(l => l.Key.ToLowerInvariant())], + Renders = [.. plan.Renders], + Payload = plan.Payload, + Projects = [.. plan.Projects], + SessionKeys = [.. plan.SessionKeys], + }; + journal.Save(keepPaths); + return journal; + } + + /// One run of a merge, from the stage its journal reached. + private sealed class Run(MergeJobHandler owner, JobContext context, MergeJournal journal, Archive keepArchive, ArchivePaths keepPaths) + { + private readonly ChronosDatabase _db = owner.Db; + private readonly CancellationToken _ct = context.Cancellation; + private readonly string _staging = MergeJournal.StagingFor(keepPaths, journal.JobId); + private Archive _dropArchive = null!; + private ArchivePaths _dropPaths = null!; + private string _keepTitle = string.Empty; + private string _dropTitle = string.Empty; + + private long Total => journal.Copies.Count + journal.Renders.Count + journal.Replaces.Count + journal.Projects.Count + 4; + + public async Task RunAsync() + { + var keep = await _db.Sequences.GetAsync(journal.KeepId, _ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException(); + _keepTitle = keep.DisplayTitle; + var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false); + if (drop is null && journal.Stage < MergeStage.Trash) throw new InvalidOperationException(Loc.L("Errors.Copies.StalePlan")); + if (drop is not null) + { + _dropTitle = drop.DisplayTitle; + _dropArchive = await _db.Archives.GetAsync(drop.ArchiveId, _ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException(); + _dropPaths = new ArchivePaths(_dropArchive.RootPath); + } + + while (journal.Stage != MergeStage.Done) + { + _ct.ThrowIfCancellationRequested(); + if (owner.BeforeStage is { } hook) await hook(journal.Stage).ConfigureAwait(false); + switch (journal.Stage) + { + case MergeStage.Copy: await CopyAsync().ConfigureAwait(false); break; + case MergeStage.Renders: await RendersAsync().ConfigureAwait(false); break; + case MergeStage.Verify: await VerifyAsync().ConfigureAwait(false); break; + case MergeStage.Replace: await ReplaceAsync().ConfigureAwait(false); break; + case MergeStage.Details: await DetailsAsync().ConfigureAwait(false); break; + case MergeStage.Projects: await ProjectsAsync().ConfigureAwait(false); break; + case MergeStage.Links: await LinksAsync().ConfigureAwait(false); break; + case MergeStage.Trash: await TrashAsync().ConfigureAwait(false); break; + } + journal.Stage++; + Save(); + await context.CheckpointAsync($"merge:{journal.Stage}", Done(), Total).ConfigureAwait(false); + } + await FinishAsync().ConfigureAwait(false); + } + + private long Done() => journal.Committed.Count + journal.RendersMoved.Count + journal.Replaced.Count + journal.KeptLosers.Count + + journal.ProjectsDone.Count + Math.Min(4, Math.Max(0, journal.Stage - MergeStage.Verify)); + + private void Save() => journal.Save(keepPaths); + + private InvalidOperationException Changed(string title) => new(Loc.L("Errors.Copies.Changed", title)); + + private async Task EnsureDropUnchangedAsync(int expected) + { + var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false); + if (drop is null || drop.FramesVersion != expected) throw Changed(_dropTitle); + } + + // ---- copy ---- + + private async Task CopyAsync() + { + var n = 0; + foreach (var copy in journal.Copies) + { + n++; + if (journal.Committed.ContainsKey(copy.DropFrameId)) continue; + _ct.ThrowIfCancellationRequested(); + await EnsureDropUnchangedAsync(journal.DropFramesVersion).ConfigureAwait(false); + var frame = await _db.Frames.GetAsync(copy.DropFrameId, _ct).ConfigureAwait(false) ?? throw Changed(_dropTitle); + await context.ReportAsync(Done(), Total, new JobDetail(JobStep.CopyPhoto, copy.FileName, n, journal.Copies.Count)).ConfigureAwait(false); + var result = await FileAsync(frame, copy.Sha256).ConfigureAwait(false); + journal.Committed[copy.DropFrameId] = result.FrameId; + Save(); + context.Log(Loc.L("Archive.Log.MergeCopied", copy.FileName)); + } + } + + /// Copies a frame of the other into staging (hash-checked against ) and files it in the kept sequence. + private async Task FileAsync(Frame frame, string? expected) + { + Directory.CreateDirectory(_staging); + var staged = Path.Combine(_staging, frame.Id.ToString("N") + Path.GetExtension(frame.FileName)); + if (File.Exists(staged)) File.Delete(staged); + string sha; + try + { + sha = await VerifiedCopy.CopyAsync(_dropPaths.FromRelative(frame.RelativePath), staged, verify: true, null, _ct).ConfigureAwait(false); + } + catch (Exception ex) when (ex is FileNotFoundException or DirectoryNotFoundException) + { + throw new InvalidOperationException(Loc.L("Errors.Copies.ChangedFile", frame.FileName), ex); + } + if (expected is { Length: > 0 } && !string.Equals(sha, expected, StringComparison.OrdinalIgnoreCase)) + { + File.Delete(staged); + throw new InvalidOperationException(Loc.L("Errors.Copies.ChangedFile", frame.FileName)); + } + var size = new FileInfo(staged).Length; + IngestResult result; + try + { + result = await owner.Archives.Ingest.CommitAsync(new IngestRequest(keepArchive.Id, journal.KeepId, null, null, staged, frame.FileName, size, + sha.ToLowerInvariant(), frame.CapturedAt, null, RefuseRemoved: true), _ct).ConfigureAwait(false); + } + catch (PhotoRemovedException) + { + // Deleted from the kept sequence since the plan: it would be a last copy the plan did not list. + File.Delete(staged); + throw Changed(_keepTitle); + } + if (File.Exists(staged)) File.Delete(staged); + return result; + } + + // ---- renders ---- + + private async Task RendersAsync() + { + var keep = await _db.Sequences.GetAsync(journal.KeepId, _ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException(); + var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false) ?? throw Changed(_dropTitle); + var n = 0; + foreach (var step in journal.Renders) + { + n++; + if (journal.RendersMoved.Contains(step.RenderId)) continue; + _ct.ThrowIfCancellationRequested(); + await context.ReportAsync(Done(), Total, new JobDetail(JobStep.CopyVideo, step.FileName, n, journal.Renders.Count)).ConfigureAwait(false); + var render = await _db.Renders.GetAsync(step.RenderId, _ct).ConfigureAwait(false); + if (render is null || render.SequenceId == journal.KeepId) + { + journal.RendersMoved.Add(step.RenderId); + Save(); + continue; + } + var source = SyncService.RenderFile(_dropArchive, render); + var relative = render.OutputPath.Replace('\\', '/'); + var inside = !Path.IsPathRooted(render.OutputPath) && drop.Folder.Length > 0 + && relative.StartsWith(drop.Folder.TrimEnd('/') + "/", StringComparison.Ordinal); + string output; + if (inside && (File.Exists(source) || Directory.Exists(source))) + { + // Inside the folder that goes to the trash: copied (verified) into the kept sequence's exports first. + if (!journal.RenderTargets.TryGetValue(step.RenderId, out var target)) + { + target = keepPaths.ToRelative(Free(keepPaths.Sequence(keep.Folder).Exports, Path.GetFileName(source))); + journal.RenderTargets[step.RenderId] = target; + Save(); + } + var destination = keepPaths.FromRelative(target); + if (Directory.Exists(source)) await CopyFolderAsync(source, destination).ConfigureAwait(false); + else await CopyRenderAsync(source, destination, render.Sha256, step.FileName).ConfigureAwait(false); + output = target; + } + else + { + // Outside it (or gone): only the row moves, naming the same file wherever the kept sequence's archive is. + output = Path.IsPathRooted(render.OutputPath) || keepArchive.Id == _dropArchive.Id ? render.OutputPath : source; + } + var access = new DbAccess(_db.Database); + await access.InTransactionAsync(async session => + { + var inner = new DbAccess(session); + await inner.ExecuteAsync("UPDATE renders SET sequence_id = $2, output_path = $3 WHERE id = $1", [step.RenderId, journal.KeepId, output], _ct) + .ConfigureAwait(false); + foreach (var id in (Guid[])[journal.KeepId, journal.DropId]) + await inner.ExecuteAsync("UPDATE sequences SET rendered_count = (SELECT count(*) FROM renders WHERE sequence_id = $1) WHERE id = $1", [id], _ct) + .ConfigureAwait(false); + return 0; + }, _ct).ConfigureAwait(false); + journal.RendersMoved.Add(step.RenderId); + Save(); + } + } + + private async Task CopyRenderAsync(string source, string destination, string? sha, string name) + { + if (File.Exists(destination)) + { + // A run that copied it before its row moved: kept when it is the same file. + if (sha is { Length: > 0 } && string.Equals(await ContentHash.Sha256Async(destination, null, _ct).ConfigureAwait(false), sha, StringComparison.OrdinalIgnoreCase)) + return; + File.Delete(destination); + } + var copied = await VerifiedCopy.CopyAsync(source, destination, verify: true, null, _ct).ConfigureAwait(false); + if (sha is { Length: > 0 } && !string.Equals(copied, sha, StringComparison.OrdinalIgnoreCase)) + { + File.Delete(destination); + throw new InvalidOperationException(Loc.L("Errors.Copies.ChangedFile", name)); + } + } + + private async Task CopyFolderAsync(string source, string destination) + { + foreach (var file in Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories)) + { + var target = Path.Combine(destination, Path.GetRelativePath(source, file)); + if (File.Exists(target) && new FileInfo(target).Length == new FileInfo(file).Length + && string.Equals(await ContentHash.Sha256Async(target, null, _ct).ConfigureAwait(false), + await ContentHash.Sha256Async(file, null, _ct).ConfigureAwait(false), StringComparison.Ordinal)) + continue; + if (File.Exists(target)) File.Delete(target); + await VerifiedCopy.CopyAsync(file, target, verify: true, null, _ct).ConfigureAwait(false); + } + } + + private static string Free(string folder, string name) + { + var stem = Path.GetFileNameWithoutExtension(name); + var extension = Path.GetExtension(name); + var candidate = Path.Combine(folder, name); + for (var n = 2; File.Exists(candidate) || Directory.Exists(candidate); n++) candidate = Path.Combine(folder, $"{stem} ({n}){extension}"); + return candidate; + } + + // ---- verify ---- + + private async Task VerifyAsync() + { + var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false) ?? throw Changed(_dropTitle); + if (drop.FramesVersion != journal.DropFramesVersion) throw Changed(_dropTitle); + var keepFrames = await ProcessingFrames.ListAsync(_db, journal.KeepId, _ct).ConfigureAwait(false); + var dropFrames = (await ProcessingFrames.ListAsync(_db, journal.DropId, _ct).ConfigureAwait(false)) + .Where(f => (f.Flags & CopySides.UnusableFlags) == 0).ToList(); + var total = keepFrames.Count + dropFrames.Count; + var done = 0; + + // What the kept sequence holds now, hashed again. + var held = new Dictionary(StringComparer.Ordinal); + foreach (var frame in keepFrames.Where(f => (f.Flags & CopySides.UnusableFlags) == 0)) + { + _ct.ThrowIfCancellationRequested(); + await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Verify, frame.FileName, ++done, total)).ConfigureAwait(false); + if (await HashAsync(keepPaths.FromRelative(frame.RelativePath)).ConfigureAwait(false) is { } sha) held.TryAdd(sha, frame.Id); + } + + var failed = new List(); + foreach (var frame in dropFrames) + { + _ct.ThrowIfCancellationRequested(); + await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Verify, frame.FileName, ++done, total)).ConfigureAwait(false); + if (journal.Verified.ContainsKey(frame.Id)) continue; + var sha = await HashAsync(_dropPaths.FromRelative(frame.RelativePath)).ConfigureAwait(false); + if (sha is null) + { + // Nothing on disk: nothing of it can be lost. + continue; + } + if (frame.Sha256 is { Length: > 0 } recorded && !string.Equals(recorded, sha, StringComparison.OrdinalIgnoreCase)) + throw new InvalidOperationException(Loc.L("Errors.Copies.ChangedFile", frame.FileName)); + if (held.TryGetValue(sha, out var keepFrame)) + { + journal.Verified[frame.Id] = new MergeVerified(sha, keepFrame); + continue; + } + if (frame.Sha256 is not { Length: > 0 }) + { + // Compared by size and time only, and another photo after all: filed in the kept sequence as a second version. + var filed = await FileAsync(frame, sha).ConfigureAwait(false); + journal.SecondVersions.Add(frame.Id); + journal.Verified[frame.Id] = new MergeVerified(sha, filed.FrameId); + held.TryAdd(sha, filed.FrameId); + Save(); + continue; + } + if (journal.Absent.Contains(sha)) continue; + failed.Add(sha); + } + if (failed.Count > 0) + { + // A photo held by another sequence of the index is not lost with the other; any other is. + var others = await new CopySides(_db).HeldElsewhereAsync(failed, [journal.DropId], _ct).ConfigureAwait(false); + var lost = failed.Count(s => !others.Contains(s)); + if (lost > 0) throw new InvalidOperationException(Loc.Plural("Errors.Copies.VerifyFailed", lost)); + } + journal.VerifiedDropFramesVersion = drop.FramesVersion; + Save(); + } + + private async Task HashAsync(string path) + { + try + { + return (await ContentHash.Sha256Async(path, null, _ct).ConfigureAwait(false)).ToLowerInvariant(); + } + catch (Exception ex) when (ex is FileNotFoundException or DirectoryNotFoundException) + { + return null; + } + } + + // ---- replace ---- + + private async Task ReplaceAsync() + { + var n = 0; + foreach (var replace in journal.Replaces) + { + n++; + if (journal.Replaced.Contains(replace.Key) || journal.KeptLosers.Contains(replace.Key)) continue; + _ct.ThrowIfCancellationRequested(); + await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Replace, replace.FileName, n, journal.Replaces.Count)).ConfigureAwait(false); + var loser = await _db.Frames.GetAsync(replace.KeepFrameId, _ct).ConfigureAwait(false); + if (loser is null || loser.SequenceId != journal.KeepId) + { + journal.Replaced.Add(replace.Key); + Save(); + continue; + } + if (!await WinnerInPlaceAsync(replace.WinnerSha256).ConfigureAwait(false)) + { + Keep(replace, "the version that replaces it is not in place"); + continue; + } + try + { + var plan = await owner.Archives.FrameTrash.PlanDeleteAsync(journal.KeepId, [replace.KeepFrameId], null, _ct, journal.JobId).ConfigureAwait(false); + if (!plan.CanDelete) + { + Keep(replace, string.Join(", ", plan.Blocked)); + continue; + } + await owner.Archives.FrameTrash.DeleteAsync(plan, null, null, _ct, journal.JobId).ConfigureAwait(false); + journal.Replaced.Add(replace.Key); + Save(); + } + catch (Exception ex) when (ex is StaleDeletePlanException or IOException or UnauthorizedAccessException) + { + Keep(replace, ex.Message); + } + } + } + + private void Keep(MergeReplace replace, string why) + { + journal.KeptLosers.Add(replace.Key); + Save(); + context.Log($"{replace.FileName}: {why}"); + } + + /// Whether a usable frame of the kept sequence holds , hashed again. + private async Task WinnerInPlaceAsync(string sha) + { + foreach (var frame in await _db.Frames.FindBySha256Async(sha.ToLowerInvariant(), _ct).ConfigureAwait(false)) + { + if (frame.SequenceId != journal.KeepId || (frame.Flags & CopySides.UnusableFlags) != 0) continue; + if (string.Equals(await HashAsync(keepPaths.FromRelative(frame.RelativePath)).ConfigureAwait(false), sha, StringComparison.OrdinalIgnoreCase)) return true; + } + return false; + } + + // ---- details ---- + + private async Task DetailsAsync() + { + if (journal.Payload is null || journal.DetailsDone) return; + await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Details)).ConfigureAwait(false); + await new SequencePayloadWriter(_db, owner.Archives).ApplyAsync(journal.KeepId, journal.Payload, journal.KeepSettingsVersion, _ct).ConfigureAwait(false); + journal.DetailsDone = true; + Save(); + } + + // ---- projects ---- + + private async Task ProjectsAsync() + { + if (journal.Projects.Count == 0) return; + var from = new ArchiveProjectHome(_db, owner.Archives.Projects, journal.DropId); + var to = new ArchiveProjectHome(_db, owner.Archives.Projects, journal.KeepId); + var sources = (await _db.Projects.ListBySequenceAsync(journal.DropId, includeDeleted: false, _ct).ConfigureAwait(false)).ToDictionary(p => p.Id); + var failed = 0; + var n = 0; + foreach (var step in journal.Projects) + { + n++; + if (journal.ProjectsDone.Contains(step.SourceProjectId)) continue; + _ct.ThrowIfCancellationRequested(); + await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Projects, step.TargetName, n, journal.Projects.Count)).ConfigureAwait(false); + if (!sources.TryGetValue(step.SourceProjectId, out var source)) + { + // Deleted from the other since the plan: its own trash keeps it. + journal.ProjectsDone.Add(step.SourceProjectId); + Save(); + continue; + } + try + { + var copied = await owner.Projects.CopyAsync(from, source, to, step.TargetName, journal.ProjectSteps.GetValueOrDefault(step.SourceProjectId), + saved => + { + journal.ProjectSteps[step.SourceProjectId] = saved; + Save(); + return ValueTask.CompletedTask; + }, _ct).ConfigureAwait(false); + await RenamesRendersAsync(step.SourceProjectId, copied.Id).ConfigureAwait(false); + journal.ProjectsDone.Add(step.SourceProjectId); + Save(); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + failed++; + context.Log($"{step.SourceName}: {ex.Message}"); + } + } + if (failed > 0) throw new InvalidOperationException(Loc.Plural("Errors.Copies.ProjectsKept", failed)); + } + + /// The renders moved to the kept sequence name the project copy (and its versions) instead of the other's. + private async Task RenamesRendersAsync(Guid source, Guid copy) + { + var access = new DbAccess(_db.Database); + if (journal.ProjectSteps.GetValueOrDefault(source) is { } step) + { + foreach (var move in step.Moves) + await access.ExecuteAsync("UPDATE renders SET project_id = $2, project_version_id = $3 WHERE sequence_id = $1 AND project_version_id = $4", + [journal.KeepId, copy, move.Copy, move.Source], _ct).ConfigureAwait(false); + } + await access.ExecuteAsync("UPDATE renders SET project_id = $2 WHERE sequence_id = $1 AND project_id = $3", [journal.KeepId, copy, source], _ct) + .ConfigureAwait(false); + } + + // ---- links ---- + + private async Task LinksAsync() + { + await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Process)).ConfigureAwait(false); + var keep = await _db.Sequences.GetAsync(journal.KeepId, _ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException(); + var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false) ?? throw Changed(_dropTitle); + var keys = journal.SessionKeys.ToHashSet(StringComparer.Ordinal); + if (!journal.KeepKeysWritten) + { + if (keys.Count > 0 && File.Exists(keepPaths.Sequence(keep.Folder).ManifestPath)) + { + await owner.Archives.Writer.UpdateAsync(keepPaths, keep.Id, keep.Folder, manifest => + keys.All(manifest.SessionKeys.Contains) ? null : manifest with { SessionKeys = [.. manifest.SessionKeys.Union(keys, StringComparer.Ordinal)] }, + _ct).ConfigureAwait(false); + } + journal.KeepKeysWritten = true; + Save(); + } + if (!journal.DropKeysRemoved) + { + if (keys.Count > 0 && File.Exists(_dropPaths.Sequence(drop.Folder).ManifestPath)) + { + await owner.Archives.Writer.UpdateAsync(_dropPaths, drop.Id, drop.Folder, manifest => + !manifest.SessionKeys.Any(keys.Contains) ? null : manifest with { SessionKeys = [.. manifest.SessionKeys.Where(k => !keys.Contains(k))] }, + _ct).ConfigureAwait(false); + } + journal.DropKeysRemoved = true; + Save(); + } + if (!journal.LinksMoved) + { + var moved = await new FoundCopyStore(_db).MoveToKeptAsync(journal.DropId, journal.KeepId, keepArchive.Id, journal.SessionKeys, journal.CopyId, _ct) + .ConfigureAwait(false); + foreach (var key in moved.SkippedKeys) context.Log(key); + journal.LinksMoved = true; + Save(); + } + } + + // ---- trash ---- + + private async Task TrashAsync() + { + var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false); + if (drop is null) return; // a run that moved it before its journal said so + await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Trash, _dropTitle)).ConfigureAwait(false); + if (await owner.Workspace.HasUnsentAsync(journal.DropId, _ct).ConfigureAwait(false)) + throw new InvalidOperationException(Loc.L("Errors.Copies.Unsent", _dropTitle)); + var expected = journal.VerifiedDropFramesVersion ?? throw Changed(_dropTitle); + if (drop.FramesVersion != expected) throw Changed(_dropTitle); + await using (await owner.Archives.Writer.LockAsync(keepPaths, journal.KeepId, _ct).ConfigureAwait(false)) + { + await EnsureVerifiedHeldAsync().ConfigureAwait(false); + var plan = await owner.Archives.Trash.PlanAsync([journal.DropId], null, _ct, journal.JobId).ConfigureAwait(false); + if (plan.Blocked.Count > 0) + throw new InvalidOperationException(Loc.L("Errors.Copies.TrashBlocked", _dropTitle, Loc.L("Archive.Delete.Reason." + plan.Blocked[0].Reason))); + try + { + await owner.Archives.Trash.DeleteAsync(plan, null, null, _ct, journal.JobId, new Dictionary { [journal.DropId] = expected }) + .ConfigureAwait(false); + } + catch (StaleDeletePlanException ex) + { + throw new InvalidOperationException(Loc.L("Errors.Copies.Changed", _dropTitle), ex); + } + } + context.Log(Loc.L("Archive.Log.MergeTrashed", _dropTitle)); + } + + /// Every photo the verify found in the kept sequence is still there and usable (read under its lease). + private async Task EnsureVerifiedHeldAsync() + { + var frames = await ProcessingFrames.ListAsync(_db, journal.KeepId, _ct).ConfigureAwait(false); + var byId = frames.Where(f => (f.Flags & CopySides.UnusableFlags) == 0).ToDictionary(f => f.Id); + var hashes = byId.Values.Select(f => f.Sha256).OfType().Select(s => s.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal); + foreach (var verified in journal.Verified.Values) + { + if (byId.TryGetValue(verified.KeepFrameId, out var frame) + && (frame.Sha256 is null || string.Equals(frame.Sha256, verified.Sha256, StringComparison.OrdinalIgnoreCase))) + continue; + if (hashes.Contains(verified.Sha256)) continue; + throw Changed(_keepTitle); + } + } + + // ---- done ---- + + private async Task FinishAsync() + { + await context.ReportAsync(Total, Total, new JobDetail(JobStep.FollowUps)).ConfigureAwait(false); + await new FoundCopyStore(_db).DeleteAsync(journal.CopyId, CancellationToken.None).ConfigureAwait(false); + if (await _db.Sequences.GetAsync(journal.KeepId, CancellationToken.None).ConfigureAwait(false) is { } keep) + { + foreach (var request in owner.Archives.FrameTrash.FollowUps(keepArchive, keep, (SequenceStatus)journal.KeepStatus)) + { + try + { + if (request.Targets.Count > 0 && await _db.Jobs.IsQueuedAsync(request.Kind, request.Targets[0], CancellationToken.None).ConfigureAwait(false)) continue; + await context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, CancellationToken.None) + .ConfigureAwait(false); + } + catch (DatabaseException ex) + { + Log.Warning("archive", $"{request.Label}: the passes the merged photos need were not queued; the next scan queues them", ex); + } + } + } + await TellAsync(journal.KeepId, KeepChange).ConfigureAwait(false); + await TellAsync(journal.DropId, DropChange).ConfigureAwait(false); + var kept = journal.KeptLosers.Count; + try + { + MergeJournal.Delete(keepPaths, journal.JobId); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warning("archive", $"The journal of merge {journal.JobId} was not removed", ex); + } + if (kept > 0) throw new InvalidOperationException(Loc.Plural("Errors.Copies.Kept", kept)); + } + + private async Task TellAsync(Guid id, string change) + { + if (owner.Archives.Writer.Options.SequenceChanged is not { } changed) return; + try + { + await changed(id, change, CancellationToken.None).ConfigureAwait(false); + } + catch (Exception ex) + { + Log.Warning("archive", $"The change of sequence {id} was not told", ex); + } + } + } +} diff --git a/src/Chronos.Data/Archiving/MergeJournal.cs b/src/Chronos.Data/Archiving/MergeJournal.cs new file mode 100644 index 00000000..4ee70aeb --- /dev/null +++ b/src/Chronos.Data/Archiving/MergeJournal.cs @@ -0,0 +1,151 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using Chronos.Core.Archiving; +using Chronos.Core.IO; +using Chronos.Data.Copies; +using Chronos.Data.Sync; + +namespace Chronos.Data.Archiving; + +/// The stages of a merge, in the order they run (design §5d). +internal enum MergeStage +{ + Copy, + Renders, + Verify, + Replace, + Details, + Projects, + Links, + Trash, + Done, +} + +/// A photo of the dropped sequence a merge copies into the kept one: its key, the drop's frame, name, size and recorded SHA-256. +internal sealed record MergeCopy(string Key, PhotoAction Action, Guid DropFrameId, string FileName, long Bytes, string? Sha256); + +/// A losing version of the kept sequence that goes to its photo trash once the winner (the drop's version) is in place. +internal sealed record MergeReplace(string Key, Guid KeepFrameId, string FileName, string WinnerSha256); + +/// A drop original the verify found in the kept sequence: its SHA-256 now and the keep frame that holds it. +internal sealed record MergeVerified(string Sha256, Guid KeepFrameId); + +/// +/// The journal of a merge (owner review 6, #73; design §5d): <keep archive>/.chronos/journal/merge-<job:N>.json, +/// written atomically (temp + rename) after each step. It keeps what the confirmed plan decided (a resumed run never plans +/// again: its own steps moved the versions) and how far each stage came, so every stage resumes and nothing is done twice. +/// +internal sealed class MergeJournal +{ + public Guid JobId { get; set; } + + public Guid CopyId { get; set; } + + public Guid KeepId { get; set; } + + public Guid DropId { get; set; } + + public string Fingerprint { get; set; } = string.Empty; + + public MergeStage Stage { get; set; } + + /// The kept sequence's settings version the plan read (the details are written only at it). + public int KeepSettingsVersion { get; set; } + + /// The dropped sequence's frames version the plan read (checked before each photo and at the verify). + public int DropFramesVersion { get; set; } + + /// The kept sequence's statuses before the merge (what its follow-ups make again). + public int KeepStatus { get; set; } + + public List Copies { get; set; } = []; + + public List Replaces { get; set; } = []; + + /// Raw SHA-256 of drop originals the plan leaves out of the kept sequence (last copies, losers, tombstoned in keep, converted). + public List Absent { get; set; } = []; + + public List Renders { get; set; } = []; + + public SyncPayload? Payload { get; set; } + + public List Projects { get; set; } = []; + + public List SessionKeys { get; set; } = []; + + /// Drop frame → the keep frame it was filed as (or that already held it). + public Dictionary Committed { get; set; } = []; + + /// Render → its archive-relative path in the kept sequence, written before the copy. + public Dictionary RenderTargets { get; set; } = []; + + public List RendersMoved { get; set; } = []; + + /// Drop frame → its SHA-256 as hashed by the verify and the keep frame that holds it. + public Dictionary Verified { get; set; } = []; + + /// Drop frames the verify found to be another photo than keep's (unhashed, same size and time) and filed as a second version. + public List SecondVersions { get; set; } = []; + + /// The dropped sequence's frames version the verify read: the trash refuses another. + public int? VerifiedDropFramesVersion { get; set; } + + public List Replaced { get; set; } = []; + + /// Keys of losing versions a refused trash step kept (reported at the end). + public List KeptLosers { get; set; } = []; + + public bool DetailsDone { get; set; } + + public Dictionary ProjectSteps { get; set; } = []; + + public List ProjectsDone { get; set; } = []; + + public bool KeepKeysWritten { get; set; } + + public bool DropKeysRemoved { get; set; } + + public bool LinksMoved { get; set; } + + /// The journal of job in the archive at . + public static string PathFor(ArchivePaths paths, Guid jobId) => Path.Combine(paths.JournalDirectory, $"merge-{jobId:N}.json"); + + /// Where the job stages the files it copies. + public static string StagingFor(ArchivePaths paths, Guid jobId) => Path.Combine(paths.StagingDirectory, $"merge-{jobId:N}"); + + /// The journal of job ; null when there is none (a first run), or it cannot be read. + /// The journal is damaged: the merge stops rather than start again over half a merge. + public static MergeJournal? Read(ArchivePaths paths, Guid jobId) + { + var path = PathFor(paths, jobId); + if (!File.Exists(path)) return null; + try + { + return JsonSerializer.Deserialize(AtomicFile.ReadAllText(path), MergeJournalJson.Default.MergeJournal); + } + catch (JsonException ex) + { + throw new InvalidDataException($"The merge journal {path} is damaged.", ex); + } + } + + /// Writes the journal atomically. + public void Save(ArchivePaths paths) + { + Directory.CreateDirectory(paths.JournalDirectory); + AtomicFile.WriteAllText(PathFor(paths, JobId), JsonSerializer.Serialize(this, MergeJournalJson.Default.MergeJournal)); + } + + /// Deletes the journal and the staging folder of job . + public static void Delete(ArchivePaths paths, Guid jobId) + { + var staging = StagingFor(paths, jobId); + if (Directory.Exists(staging)) Directory.Delete(staging, recursive: true); + File.Delete(PathFor(paths, jobId)); + } +} + +[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, UseStringEnumConverter = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] +[JsonSerializable(typeof(MergeJournal))] +internal sealed partial class MergeJournalJson : JsonSerializerContext; diff --git a/src/Chronos.Data/Archiving/SequenceMerger.cs b/src/Chronos.Data/Archiving/SequenceMerger.cs new file mode 100644 index 00000000..2c0fa5b6 --- /dev/null +++ b/src/Chronos.Data/Archiving/SequenceMerger.cs @@ -0,0 +1,439 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Chronos.Core.Archiving; +using Chronos.Core.Localization; +using Chronos.Core.Model; +using Chronos.Data.Archiving.Jobs; +using Chronos.Data.Copies; +using Chronos.Data.Jobs; +using Chronos.Data.Projects; +using Chronos.Data.Repositories; +using Chronos.Data.Sync; + +namespace Chronos.Data.Archiving; + +/// +/// Merges two local copies of one time-lapse into the one kept (owner review 6, #73; design §5d; BRIEF §12.5 Merge): the +/// dry run () says which photos +/// of the other copy are copied into the kept one, which go to the trash with it and are held nowhere else (last copies), +/// the renders and projects that move, the details written, the session keys and links that follow, and what blocks it; +/// queues the job that applies exactly that plan +/// (), every photo hash-verified in the kept sequence before the other goes to its archive's trash. +/// +public sealed class SequenceMerger(ChronosDatabase db, ArchiveService archives, CopySides sides) +{ + /// The free space a merge leaves on the kept sequence's volume beyond what it copies. + public const long SpaceReserve = 512L << 20; + + private readonly ChronosDatabase _db = db ?? throw new ArgumentNullException(nameof(db)); + private readonly ArchiveService _archives = archives ?? throw new ArgumentNullException(nameof(archives)); + private readonly CopySides _sides = sides ?? throw new ArgumentNullException(nameof(sides)); + private readonly DbAccess _access = new(db.Database); + + internal ChronosDatabase Db => _db; + + internal ArchiveService Archives => _archives; + + /// The handler of jobs; register it on the . + public static IJobHandler CreateHandler(ChronosDatabase db, SequenceMerger merger, ArchiveService archives, IProjectCopier projects, + IWorkspaceGuard workspace) => new MergeJobHandler(db, merger, archives, projects, workspace); + + /// + /// The dry run of merging the pair : is kept and the other goes to the + /// trash. With null the kept one is preselected only when exactly one of the two is linked to a + /// server and lies in a server's originals archive; otherwise the plan is made keeping the one that is not + /// and blocks with . + /// + /// The pair is not a local pair of these sequences, or a sequence is not in the index. + public Task PlanAsync(Guid copyId, Guid? keepId, Guid otherId, MergeChoices choices, CopyPlanContext context, CancellationToken ct) => + PlanAsync(copyId, keepId, otherId, choices, context, null, ct); + + /// The dry run, with the jobs of (the merge job planning again) not counted as busy. + internal async Task PlanAsync(Guid copyId, Guid? keepId, Guid otherId, MergeChoices choices, CopyPlanContext context, Guid? exceptJob, + CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(choices); + ArgumentNullException.ThrowIfNull(context); + var copy = await new FoundCopyStore(_db).GetAsync(copyId, ct).ConfigureAwait(false) + ?? throw new ArgumentException($"Unknown pair {copyId}.", nameof(copyId)); + if (copy is not { Kind: CopyKind.LocalLocal, OtherId: { } second } || !copy.Names(otherId)) + throw new ArgumentException($"Pair {copyId} is not a local pair of sequence {otherId}.", nameof(otherId)); + var first = copy.LocalId; + var blocks = new List(); + + Guid keep, drop; + if (keepId is { } chosen) + { + if (!copy.Names(chosen)) throw new ArgumentException($"Sequence {chosen} is not one of pair {copyId}.", nameof(keepId)); + (keep, drop) = (chosen, chosen == first ? second : first); + } + else if (await PreselectAsync(first, second, context, ct).ConfigureAwait(false) is { } preselected) + { + (keep, drop) = (preselected, preselected == first ? second : first); + } + else + { + (keep, drop) = (otherId == first ? second : first, otherId); + blocks.Add(CopyBlock.KeptUnchosen); + } + + var keepRow = await _db.Sequences.GetAsync(keep, ct).ConfigureAwait(false) ?? throw new ArgumentException($"Unknown sequence {keep}.", nameof(copyId)); + var dropRow = await _db.Sequences.GetAsync(drop, ct).ConfigureAwait(false) ?? throw new ArgumentException($"Unknown sequence {drop}.", nameof(copyId)); + var keepArchive = await _db.Archives.GetAsync(keepRow.ArchiveId, ct).ConfigureAwait(false) + ?? throw new ArgumentException($"Unknown archive {keepRow.ArchiveId}.", nameof(copyId)); + var dropArchive = await _db.Archives.GetAsync(dropRow.ArchiveId, ct).ConfigureAwait(false) + ?? throw new ArgumentException($"Unknown archive {dropRow.ArchiveId}.", nameof(copyId)); + var keepPaths = new ArchivePaths(keepArchive.RootPath); + var dropPaths = new ArchivePaths(dropArchive.RootPath); + + // Where the two are: online, in the folders the index has, not worked on by a job. + var online = ArchiveVolume.Probe(keepPaths) == ArchiveAvailability.Online && ArchiveVolume.Probe(dropPaths) == ArchiveAvailability.Online; + if (!online) blocks.Add(CopyBlock.ArchiveOffline); + else if (Moved(keepPaths, keepRow) || Moved(dropPaths, dropRow)) blocks.Add(CopyBlock.FolderMoved); + if (await BusyAsync([keep, drop, keepRow.ArchiveId, dropRow.ArchiveId], exceptJob, ct).ConfigureAwait(false)) blocks.Add(CopyBlock.Busy); + if (context.UnsentWorkspaceSequences.Contains(drop)) blocks.Add(CopyBlock.UnsentWorkspace); + + // Photos: keep is "here", drop is "there". + var here = await _sides.LocalAsync(keep, ct).ConfigureAwait(false); + var there = await _sides.LocalAsync(drop, ct).ConfigureAwait(false); + var comparison = CopyComparer.Compare(here, there); + var steps = new List(); + var excluded = new List(); + foreach (var photo in comparison.Photos) + { + switch (photo.Class) + { + case PhotoClass.OnlyThere: + // Every photo only the other holds is copied: it goes to the trash, nothing it alone holds stays behind. + steps.Add(Step(photo.Key, PhotoAction.Fetch, photo.There!)); + break; + case PhotoClass.Differs: + var choice = choices.Photos switch + { + CopyPhotoPolicy.Union => PhotoChoice.KeepBoth, + CopyPhotoPolicy.ThisPcWins => PhotoChoice.TakeHere, + CopyPhotoPolicy.OtherWins => PhotoChoice.TakeThere, + _ => choices.PerPhoto.GetValueOrDefault(photo.Key, PhotoChoice.Leave), + }; + if (choice is PhotoChoice.KeepBoth or PhotoChoice.TakeThere) steps.Add(Step(photo.Key, PhotoAction.FetchVersion, photo.There!)); + if (choice == PhotoChoice.KeepBoth && photo.There!.Sha256 is { Length: > 0 } version) excluded.Add(version.ToLowerInvariant()); + if (choice == PhotoChoice.TakeThere) steps.Add(Step(photo.Key, PhotoAction.TrashHere, photo.Here!)); + break; + } + } + + // Last copies: drop originals nothing will hold once the copies are made (losers, tombstoned in keep, converted), and + // keep's losing versions nothing else holds. + var fetched = steps.Where(s => s.Action is PhotoAction.Fetch or PhotoAction.FetchVersion).Select(s => s.Sha256).OfType() + .Select(s => s.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal); + var losers = steps.Where(s => s.Action == PhotoAction.TrashHere).Select(s => s.FrameId!.Value).ToHashSet(); + var keepHolds = here.Photos.Where(p => !p.Unusable && p.Sha256 is { Length: > 0 } && !losers.Contains(p.FrameId)) + .Select(p => p.Sha256!.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal); + var absent = new List(); + foreach (var frame in there.Photos.Where(p => !p.Unusable && p.Sha256 is { Length: > 0 })) + { + var sha = frame.Sha256!.ToLowerInvariant(); + if (keepHolds.Contains(sha) || fetched.Contains(sha) || absent.Any(a => string.Equals(a.Sha256, sha, StringComparison.OrdinalIgnoreCase))) continue; + absent.Add(frame with { Sha256 = sha }); + } + var lastCopies = new List(); + var heldElsewhere = await _sides.HeldElsewhereAsync([.. absent.Select(a => a.Sha256!)], [drop], ct).ConfigureAwait(false); + foreach (var frame in absent.Where(a => !heldElsewhere.Contains(a.Sha256!))) + lastCopies.Add(new LastCopy(frame.Sha256!, CopySide.Other, frame.FileName, frame.Size)); + var loserSteps = steps.Where(s => s.Action == PhotoAction.TrashHere && s.Sha256 is { Length: > 0 }).ToList(); + if (loserSteps.Count > 0) + { + var loserHeld = await _sides.HeldElsewhereAsync([.. loserSteps.Select(s => s.Sha256!)], [keep, drop], ct).ConfigureAwait(false); + foreach (var step in loserSteps) + { + var sha = step.Sha256!.ToLowerInvariant(); + if (loserHeld.Contains(sha) || keepHolds.Contains(sha)) continue; + lastCopies.Add(new LastCopy(sha, CopySide.ThisPc, step.FileName, step.Bytes)); + } + } + + // Renders the kept sequence lacks (by id and SHA-256) move to it. + var keepRenders = await _db.Renders.ListBySequenceAsync(keep, ct).ConfigureAwait(false); + var keepRenderHashes = keepRenders.Select(r => r.Sha256).OfType().ToHashSet(StringComparer.OrdinalIgnoreCase); + var renders = (await _db.Renders.ListBySequenceAsync(drop, ct).ConfigureAwait(false)) + .Where(r => keepRenders.All(k => k.Id != r.Id) && (r.Sha256 is null || !keepRenderHashes.Contains(r.Sha256))) + .OrderBy(r => r.Created).ThenBy(r => r.Id) + .Select(r => new RenderStep(r.Id, Path.GetFileName(r.OutputPath.Replace('\\', '/').TrimEnd('/')), r.Bytes, r.Sha256)).ToList(); + + // Details, line by line, applied to the kept sequence only. + var keepRemovals = await _db.FrameRemovals.ListBySequenceAsync(keep, ct).ConfigureAwait(false); + var dropRemovals = await _db.FrameRemovals.ListBySequenceAsync(drop, ct).ConfigureAwait(false); + var keepPhotos = await SequencePayloadWriter.PhotosAsync(_db, keep, dropRemovals, ct).ConfigureAwait(false); + var dropPhotos = await SequencePayloadWriter.PhotosAsync(_db, drop, keepRemovals, ct).ConfigureAwait(false); + var lines = CopyPayload.Lines(SyncPayload.Of(keepRow), keepPhotos, SyncPayload.Of(dropRow), dropPhotos); + SyncPayload? payload = null; + var changes = 0; + if (lines.Any(l => !choices.Details.ContainsKey(l.Path))) + { + blocks.Add(CopyBlock.DetailsUnchosen); + } + else + { + changes = lines.DistinctBy(l => l.Path, StringComparer.Ordinal).Count(l => choices.Details[l.Path] == CopySide.Other); + if (changes > 0 || excluded.Count > 0) + { + var mine = CopyPayload.Portable(SyncPayload.Of(keepRow), keepPhotos); + var merged = changes > 0 ? CopyPayload.Merge(mine, CopyPayload.Portable(SyncPayload.Of(dropRow), dropPhotos), choices.Details, lines) : mine; + try + { + merged = CopyPayload.ExcludeVersions(merged, excluded); + } + catch (Exception ex) when (ex is JsonException or InvalidDataException) + { + // Settings that are not a project of this Chronos: the incoming versions stay included. + } + payload = merged.SameAs(mine) && changes == 0 ? null : merged; + } + } + + // Projects are always copied across: the other's home goes to the trash. + var projects = await ProjectStepsAsync(keep, drop, dropArchive.Name, ct).ConfigureAwait(false); + + // Session keys and links follow the kept sequence. + var (sessionKeys, skippedKeys) = await SessionKeysAsync(keep, drop, keepRow.ArchiveId, ct).ConfigureAwait(false); + var links = await _access.ListAsync("SELECT server, local_id FROM sync_links WHERE local_id = $1 OR local_id = $2", [keep, drop], + r => (Server: r.GetString(0), Local: r.GetGuid(1)), ct).ConfigureAwait(false); + var moved = links.Where(l => l.Local == drop).Select(l => l.Server).Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal).ToList(); + if (links.GroupBy(l => l.Server, StringComparer.Ordinal).Any(g => g.Select(l => l.Local).Distinct().Count() > 1)) blocks.Add(CopyBlock.BothLinked); + + // The other goes to the trash. + var trash = await _archives.Trash.PlanAsync([drop], null, ct, exceptJob).ConfigureAwait(false); + if (trash.Blocked.Any(b => b.Reason is not (DeleteBlock.Busy or DeleteBlock.Offline))) blocks.Add(CopyBlock.TrashBlocked); + else if (trash.Blocked.Any(b => b.Reason == DeleteBlock.Busy) && !blocks.Contains(CopyBlock.Busy)) blocks.Add(CopyBlock.Busy); + + // Space on the kept sequence's volume. + var copyBytes = steps.Where(s => s.Action is PhotoAction.Fetch or PhotoAction.FetchVersion).Sum(s => s.Bytes) + renders.Sum(r => r.Bytes); + var free = online ? FreeSpace(keepPaths) : null; + if (free is { } bytes && copyBytes + SpaceReserve > bytes) blocks.Add(CopyBlock.NoSpace); + + // Details are in the fingerprint as they read now: an edit that keeps the versions (a title) still moves it. + var keepStamp = await StampAsync(keep, ct).ConfigureAwait(false) + ":" + PayloadHash(SyncPayload.Of(keepRow)); + var dropStamp = await StampAsync(drop, ct).ConfigureAwait(false) + ":" + PayloadHash(SyncPayload.Of(dropRow)); + var fingerprint = CopyRules.PlanFingerprint(FingerprintLines(keepRow, dropRow, keepStamp, dropStamp, choices, steps, lastCopies, renders, payload, + projects, sessionKeys, skippedKeys)); + return new MergePlan(copyId, keep, drop, choices, steps, lastCopies, renders, payload, changes, projects, sessionKeys, skippedKeys, moved, trash, + copyBytes, free ?? -1, [.. blocks.Distinct()], keepRow.FramesVersion, keepRow.SettingsVersion, dropRow.FramesVersion, dropRow.SettingsVersion, + fingerprint); + } + + /// + /// Queues the job that applies : the pair goes to (its state + /// before kept as the previous one), the job targets the kept sequence, then the other (both held from other jobs). + /// + /// The kept sequence's title, for the job's label ("Merge into {title}"). + /// The plan is blocked, or the pair is already being merged or integrated. + /// The pair changed state since the plan. + public async Task EnqueueAsync(JobEngine engine, FoundCopyStore store, MergePlan plan, string title, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(engine); + ArgumentNullException.ThrowIfNull(store); + ArgumentNullException.ThrowIfNull(plan); + if (!plan.CanApply) throw new InvalidOperationException($"The merge plan is blocked: {string.Join(", ", plan.Blocks)}."); + var copy = await store.GetAsync(plan.CopyId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException(); + if (copy.State == CopyState.Integrating) throw new InvalidOperationException(Loc.L("Errors.Copies.Busy")); + var id = Guid.CreateVersion7(); + if (!await store.SetStateAsync(plan.CopyId, copy.State, CopyState.Integrating, new CopyAnswer(null, null, id, null, DateTimeOffset.UtcNow), ct) + .ConfigureAwait(false)) + throw new StaleCopyPlanException(); + var parameters = new MergeJobParameters(plan.CopyId, plan.KeepId, plan.DropId, plan.Choices, plan.Fingerprint); + try + { + return await engine.EnqueueAsync(id, JobKind.MergeSequences, [plan.KeepId, plan.DropId], Loc.L("Jobs.Target.Merge", title), parameters.ToJson(), + JobPriority.User, null, ct).ConfigureAwait(false); + } + catch + { + await store.SetStateAsync(plan.CopyId, CopyState.Integrating, copy.State, null, CancellationToken.None).ConfigureAwait(false); + throw; + } + } + + /// The sentence a job fails with for the first block of a plan it made again. + internal async Task BlockMessageAsync(MergePlan plan, CancellationToken ct) + { + var keep = await _db.Sequences.GetAsync(plan.KeepId, ct).ConfigureAwait(false); + var drop = await _db.Sequences.GetAsync(plan.DropId, ct).ConfigureAwait(false); + var (keepTitle, dropTitle) = (keep?.DisplayTitle ?? string.Empty, drop?.DisplayTitle ?? string.Empty); + return plan.Blocks[0] switch + { + CopyBlock.Busy => Loc.L("Errors.Copies.Busy"), + CopyBlock.ArchiveOffline => Loc.L("Errors.Copies.ArchiveOffline", dropTitle), + CopyBlock.FolderMoved => Loc.L("Errors.Copies.FolderMoved", dropTitle), + CopyBlock.NoSpace => Loc.L("Errors.Copies.NoSpace", keepTitle, Size(plan.CopyBytes + SpaceReserve)), + CopyBlock.BothLinked => Loc.L("Errors.Copies.BothLinked", string.Empty), + CopyBlock.UnsentWorkspace => Loc.L("Errors.Copies.Unsent", dropTitle), + CopyBlock.TrashBlocked => Loc.L("Errors.Copies.TrashBlocked", dropTitle, + plan.Trash.Blocked.Select(b => Loc.L("Archive.Delete.Reason." + b.Reason)).FirstOrDefault() ?? string.Empty), + CopyBlock.DetailsUnchosen => Loc.Plural("Errors.Copies.DetailsUnchosen", 1), + CopyBlock.KeptUnchosen => Loc.L("Errors.Copies.KeptUnchosen"), + _ => Loc.L("Errors.Copies.StalePlan"), + }; + } + + /// Free bytes on the kept archive's volume (its root and its staging folder, the tighter); null when they cannot be measured. + internal long? FreeSpace(ArchivePaths paths) + { + long? free = null; + foreach (var folder in (string[])[paths.Root, paths.StagingDirectory]) + { + if (!Directory.Exists(folder) || Measure(folder) is not { } bytes) continue; + free = free is { } known ? Math.Min(known, bytes) : bytes; + } + return free; + + long? Measure(string folder) + { + if (_archives.FreeSpace is { } measure) return measure(folder); + try + { + return new DriveInfo(OperatingSystem.IsWindows() ? Path.GetPathRoot(folder)! : folder).AvailableFreeSpace; + } + catch (Exception ex) when (ex is ArgumentException or IOException or UnauthorizedAccessException) + { + return null; + } + } + } + + private static string Size(long bytes) => bytes >= 1_000_000_000 + ? Loc.L("Common.Size.GB", (bytes / 1e9).ToString("0.0", Loc.Culture)) + : Loc.L("Common.Size.MB", (bytes / 1e6).ToString("0", Loc.Culture)); + + private static PhotoStep Step(string key, PhotoAction action, CopyPhotoInput frame) => + new(key, action, frame.FrameId, frame.FileName, frame.Size, string.IsNullOrEmpty(frame.Sha256) ? null : frame.Sha256.ToLowerInvariant()); + + private static bool Moved(ArchivePaths paths, Sequence sequence) => + sequence.Folder.Length > 0 && !SequenceFolders.Holds(paths.Sequence(sequence.Folder), sequence.Id); + + /// + /// The kept sequence preselected: the one of the two that is linked to a server and lies in a server's originals archive, + /// when exactly one is (owner review 5 homes: a server's originals stay in its archive); null otherwise. + /// + private async Task PreselectAsync(Guid a, Guid b, CopyPlanContext context, CancellationToken ct) + { + if (context.ServerOriginalsArchives.Count == 0) return null; + var linked = await _access.ListAsync("SELECT DISTINCT local_id FROM sync_links WHERE local_id = $1 OR local_id = $2", [a, b], r => r.GetGuid(0), ct) + .ConfigureAwait(false); + var candidates = new List(); + foreach (var id in linked) + { + if (await _db.Sequences.GetAsync(id, ct).ConfigureAwait(false) is { } row && context.ServerOriginalsArchives.Contains(row.ArchiveId)) candidates.Add(id); + } + return candidates.Count == 1 ? candidates[0] : null; + } + + /// Whether a queued, running or paused job (but ) targets any of . + private async Task BusyAsync(IReadOnlyCollection targets, Guid? exceptJob, CancellationToken ct) + { + var rows = await _access.ListAsync( + $"SELECT id, target_ids FROM jobs WHERE state IN ({(int)JobState.Queued}, {(int)JobState.Running}, {(int)JobState.Paused})", [], + r => (Id: r.GetGuid(0), Targets: r.GetString(1)), ct).ConfigureAwait(false); + var wanted = targets.ToHashSet(); + foreach (var (id, ids) in rows) + { + if (id == exceptJob || ids.Length == 0) continue; + foreach (var part in ids.Split(',')) + { + if (Guid.TryParse(part, out var target) && wanted.Contains(target)) return true; + } + } + return false; + } + + /// The drop's non-Main projects, each with the name it gets at the kept sequence's home. + private async Task> ProjectStepsAsync(Guid keep, Guid drop, string dropArchiveName, CancellationToken ct) + { + var taken = (await _db.Projects.ListBySequenceAsync(keep, includeDeleted: false, ct).ConfigureAwait(false)).Select(p => p.Name) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + var steps = new List(); + foreach (var project in (await _db.Projects.ListBySequenceAsync(drop, includeDeleted: false, ct).ConfigureAwait(false)) + .Where(p => !p.IsMain).OrderBy(p => p.CreatedAt).ThenBy(p => p.Id)) + { + var name = FreeName(project.Name, dropArchiveName, taken); + taken.Add(name); + steps.Add(new ProjectStep(CopySide.Other, project.Id, project.Name, name, project.HeadNumber, project.HeadVersionId ?? Guid.Empty)); + } + return steps; + } + + /// The name itself when free, else "<name> (<archive>)", then " (2)"…, each within . + internal static string FreeName(string name, string archiveName, IReadOnlySet taken) + { + if (!taken.Contains(name)) return name; + var stem = Fit(name, $" ({archiveName})"); + if (!taken.Contains(stem)) return stem; + for (var n = 2; ; n++) + { + var candidate = Fit(stem, string.Create(CultureInfo.InvariantCulture, $" ({n})")); + if (!taken.Contains(candidate)) return candidate; + } + + static string Fit(string stem, string suffix) + { + const int max = ProjectService.MaxNameLength; + if (suffix.Length >= max) suffix = suffix[..(max / 2)]; + return (stem.Length + suffix.Length > max ? stem[..(max - suffix.Length)].TrimEnd() : stem) + suffix; + } + } + + /// The drop's session keys that move to the kept sequence, and those the kept archive maps to a third sequence already. + private async Task<(List Moved, List Skipped)> SessionKeysAsync(Guid keep, Guid drop, Guid keepArchiveId, CancellationToken ct) + { + var keys = await _access.ListAsync("SELECT session_key FROM sequence_sessions WHERE sequence_id = $1 ORDER BY session_key", [drop], + r => r.GetString(0), ct).ConfigureAwait(false); + var (moved, skipped) = (new List(), new List()); + foreach (var key in keys.Distinct(StringComparer.Ordinal)) + { + var holder = await _db.Sequences.FindBySessionKeyAsync(keepArchiveId, key, ct).ConfigureAwait(false); + if (holder is null || holder == keep || holder == drop) moved.Add(key); + else skipped.Add(key); + } + return (moved, skipped); + } + + /// + /// What a sequence's photos are in the index, beyond its versions: how many, their bytes, how many are hashed, flagged + /// missing, corrupt, read from the preview only or undecodable (a verify or a decode records those without another + /// frames version), and its tombstones. Of this sequence alone: an import elsewhere in its archive leaves it as it was. + /// + private async Task StampAsync(Guid sequenceId, CancellationToken ct) + { + var flags = string.Join(", ", new[] { FrameFlags.Missing, FrameFlags.Corrupt, FrameFlags.PreviewOnly, FrameFlags.Undecodable } + .Select(flag => $"CAST(COALESCE(SUM(CASE WHEN (flags & {(int)flag}) <> 0 THEN 1 ELSE 0 END), 0) AS BIGINT)")); + var photos = await _access.SingleAsync( + $"SELECT CAST(count(*) AS BIGINT), CAST(COALESCE(SUM(size), 0) AS BIGINT), CAST(COALESCE(SUM(CASE WHEN sha256 IS NULL OR sha256 = '' THEN 0 ELSE 1 END), 0) AS BIGINT), {flags} FROM frames WHERE sequence_id = $1", + [sequenceId], r => string.Join(':', Enumerable.Range(0, 7).Select(i => r.GetInt64(i).ToString(CultureInfo.InvariantCulture))), ct).ConfigureAwait(false); + var removals = await _access.ScalarAsync("SELECT count(*) FROM frame_removals WHERE sequence_id = $1", [sequenceId], ct).ConfigureAwait(false); + return string.Create(CultureInfo.InvariantCulture, $"{photos}:{removals}"); + } + + private static string PayloadHash(SyncPayload payload) => + Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(payload, SyncPayloadJson.Default.SyncPayload)))); + + private static IEnumerable FingerprintLines(Sequence keep, Sequence drop, string keepStamp, string dropStamp, MergeChoices choices, + List steps, List lastCopies, List renders, SyncPayload? payload, List projects, + List sessionKeys, List skippedKeys) + { + var inv = CultureInfo.InvariantCulture; + yield return string.Create(inv, $"keep:{keep.Id:N}:{keep.FramesVersion}:{keep.SettingsVersion}:{keepStamp}"); + yield return string.Create(inv, $"drop:{drop.Id:N}:{drop.FramesVersion}:{drop.SettingsVersion}:{dropStamp}"); + yield return $"policy:{choices.Photos}"; + foreach (var (key, choice) in choices.PerPhoto.OrderBy(p => p.Key, StringComparer.Ordinal)) yield return $"choice:{key}:{choice}"; + foreach (var (path, side) in choices.Details.OrderBy(p => p.Key, StringComparer.Ordinal)) yield return $"detail:{path}:{side}"; + foreach (var line in steps.Select(s => $"{s.Action}:{s.Key}").Order(StringComparer.Ordinal)) yield return line; + foreach (var line in lastCopies.Select(l => $"last:{l.Side}:{l.Key}").Order(StringComparer.Ordinal)) yield return line; + foreach (var render in renders) yield return string.Create(inv, $"render:{render.RenderId:N}:{render.Sha256}"); + if (payload is not null) + yield return "payload:" + PayloadHash(payload); + foreach (var project in projects) yield return string.Create(inv, $"{project.From}:{project.SourceProjectId:N}:{project.SourceHeadId:N}:{project.TargetName}"); + foreach (var key in sessionKeys) yield return "session:" + key; + foreach (var key in skippedKeys) yield return "skipped:" + key; + } +} diff --git a/src/Chronos.Data/Archiving/SequencePayloadWriter.cs b/src/Chronos.Data/Archiving/SequencePayloadWriter.cs new file mode 100644 index 00000000..55c730df --- /dev/null +++ b/src/Chronos.Data/Archiving/SequencePayloadWriter.cs @@ -0,0 +1,134 @@ +using System.Text.Json; +using Chronos.Core.Archiving; +using Chronos.Core.IO; +using Chronos.Core.Localization; +using Chronos.Core.Model; +using Chronos.Core.Project; +using Chronos.Data.Processing; +using Chronos.Data.Repositories; +using Chronos.Data.Sync; + +namespace Chronos.Data.Archiving; + +/// +/// Writes the details a merge settled (owner review 6, #73; design §5d step 6) into the kept sequence in one step: title, +/// tags, rating, notes and settings, under the sequence's lease, only while its settings are still at the version the plan +/// read. The payload comes in portable form (frame positions by photo); its positions are placed on the sequence's own +/// frames as they are under the lease, then the index, project.json and the sidecar are written as +/// applies a server's payload. A payload the sequence already holds writes nothing (a resumed run). +/// +internal sealed class SequencePayloadWriter(ChronosDatabase db, ArchiveService archives) +{ + /// + /// Applies to at ; + /// returns the sequence as it is then. + /// + /// The settings moved since the plan (an edit, an analysis): nothing was written. + /// The sequence is not in the folder the index has: nothing was written. + /// The sequence or its archive is not in the index. + public async Task ApplyAsync(Guid sequenceId, SyncPayload portable, int expectedSettingsVersion, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(portable); + var sequence = await db.Sequences.GetAsync(sequenceId, ct).ConfigureAwait(false) + ?? throw new ArgumentException($"Unknown sequence {sequenceId}.", nameof(sequenceId)); + var archive = await db.Archives.GetAsync(sequence.ArchiveId, ct).ConfigureAwait(false) + ?? throw new ArgumentException($"Unknown archive {sequence.ArchiveId}.", nameof(sequenceId)); + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + await using (await archives.Writer.LockAsync(paths, sequenceId, ct).ConfigureAwait(false)) + { + var row = await db.Sequences.GetAsync(sequenceId, ct).ConfigureAwait(false) + ?? throw new ArgumentException($"Unknown sequence {sequenceId}.", nameof(sequenceId)); + var folder = paths.Sequence(row.Folder); + if (row.Folder.Length > 0 && !SequenceFolders.Holds(folder, row.Id)) throw new SequenceFolderMovedException(); + // The frames cannot change under the lease: the positions are placed on them as they are now. + var payload = portable with { SettingsJson = await PlaceAsync(row.Id, portable.SettingsJson, ct).ConfigureAwait(false) }; + if (row.SettingsVersion != expectedSettingsVersion) + { + // A resumed run that wrote the payload before its journal said so finds it in place. + if (payload.SameAs(SyncPayload.Of(row))) return row; + throw new SyncConflictChangedException(Loc.L("Errors.Copies.DetailsChanged")); + } + var metadata = !payload.SameMetadata(SyncPayload.Of(row)); + var settings = !SyncRules.SameSettings(payload.SettingsJson, row.SettingsJson); + if (!metadata && !settings) return row; + + string? project = null; + Guid? hero = null; + if (settings) + { + var edited = ProjectStore.Parse(SyncRules.NormalizeSettings(payload.SettingsJson)); + project = edited.ToJson(); + hero = await HeroFrame.EditAsync(db, row.Id, ProjectStore.Read(row, folder).HeroFrame, edited.HeroFrame, ct).ConfigureAwait(false); + } + var edit = new SequenceEdit + { + Title = metadata ? payload.Title?.Trim() ?? string.Empty : null, + Tags = metadata ? payload.Tags : null, + Rating = metadata ? payload.Rating : null, + Notes = metadata ? payload.Notes ?? string.Empty : null, + SettingsJson = project, + HeroFrameId = hero, + }; + // The index first, at the version read under the lease: a writer that skips the lease still cannot be overwritten. + var result = await db.Sequences.EditAsync(row.Id, edit, expectedSettingsVersion, ct).ConfigureAwait(false); + if (result is not { Outcome: SequenceEditOutcome.Updated, Sequence: { } updated }) + throw new SyncConflictChangedException(Loc.L("Errors.Copies.DetailsChanged")); + if (row.Folder.Length == 0 && !File.Exists(folder.ManifestPath)) return updated; + if (project is not null) + { + Directory.CreateDirectory(folder.MetaDirectory); + AtomicFile.WriteAllText(folder.ProjectPath, project); + } + if (metadata) + { + var manifest = SequenceWriter.ReadLocked(folder, row.Id); + // Asked before the sidecar is written: frames another process wrote there since this index last synced the + // folder keep it changed for the next scan. + var inStep = await archives.Indexer.InStepAsync(archive, folder, manifest, ct).ConfigureAwait(false); + ManifestStore.WriteSequence(folder.ManifestPath, manifest with + { + Title = updated.Title, + Tags = updated.Tags, + Rating = updated.Rating, + Notes = updated.Notes, + Versions = manifest.Versions with { Settings = Math.Max(manifest.Versions.Settings, updated.SettingsVersion) }, + UpdatedAt = DateTimeOffset.UtcNow, + }); + if (inStep) await archives.Indexer.RecordStampsAsync(archive, folder, row.Id, ct).ConfigureAwait(false); + } + return updated; + } + } + + /// + /// The photos of a sequence's frames by index, as a merge names them (raw SHA-256, else size and capture time), with the + /// encodings its conversions joined (), so a position parked on a converted photo finds its frame. + /// + internal static async Task PhotosAsync(ChronosDatabase db, Guid sequenceId, IEnumerable? extraRemovals, CancellationToken ct) + { + var frames = await Processing.SequenceFrames.ListAsync(db, sequenceId, ct).ConfigureAwait(false); + var removals = await db.FrameRemovals.ListBySequenceAsync(sequenceId, ct).ConfigureAwait(false); + var aliases = removals.Concat(extraRemovals ?? []) + .Where(r => r.Reason == RemovalReason.Replaced && !string.IsNullOrEmpty(r.ReplacementSha256)) + .Select(r => new PhotoAlias(r.Sha256, r.ReplacementSha256!)); + return new FramePhotos(frames.Select(f => (f.Index, new FramePhoto(string.IsNullOrEmpty(f.Sha256) ? null : f.Sha256.ToLowerInvariant(), f.Size, f.CapturedAt))), + aliases); + } + + /// The portable settings placed on the sequence's frames; settings that are not a project stay as they are. + private async Task PlaceAsync(Guid sequenceId, string? portableSettings, CancellationToken ct) + { + var normalized = SyncRules.NormalizeSettings(portableSettings); + if (!ProjectSettings.NamesFrames(normalized)) return portableSettings; + var photos = await PhotosAsync(db, sequenceId, null, ct).ConfigureAwait(false); + try + { + return ProjectContent.Place(normalized, photos); + } + catch (JsonException) + { + return portableSettings; + } + } +} diff --git a/tests/Chronos.Data.Tests/Archiving/SequenceMergerTests.cs b/tests/Chronos.Data.Tests/Archiving/SequenceMergerTests.cs new file mode 100644 index 00000000..c0b07311 --- /dev/null +++ b/tests/Chronos.Data.Tests/Archiving/SequenceMergerTests.cs @@ -0,0 +1,630 @@ +using System.Security.Cryptography; +using Chronos.Core.Archiving; +using Chronos.Core.Model; +using Chronos.Data.Archiving; +using Chronos.Data.Archiving.Jobs; +using Chronos.Data.Copies; +using Chronos.Data.Jobs; +using Chronos.Data.Repositories; +using Chronos.Data.Sync; +using Chronos.Data.Tests.Infrastructure; +using Chronos.Testing; +using static Chronos.Data.Tests.Archiving.ArchiveFixture; + +namespace Chronos.Data.Tests.Archiving; + +/// +/// Merging two local copies of one time-lapse (owner review 6, #73; design §5d, §11 Merge): the dry run (kept sequence, +/// photos copied, last copies, session keys, links, blocks, fingerprint) and the job that applies it, verifies every photo +/// of the other in the kept one and only then moves the other to its archive's trash. +/// +[Category("archive")] +public abstract class SequenceMergerTests(TestContext context) : DatabaseTest(context) +{ + private const string Nas = "nas.local"; + + private static readonly CopyPlanContext NoContext = PlanContext(); + + private async Task FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true))); + + private static CopyPlanContext PlanContext(params Guid[] originals) => + new("PC", null, false, false, false, false, false, false, null, new HashSet(), originals.ToHashSet()); + + private static SequenceMerger Merger(ArchiveFixture fx) => new(fx.Db, fx.Service, new CopySides(fx.Db)); + + /// A sequence uploaded photo by photo into : (seed, capture offset in ms, file name). + private static async Task SequenceAsync(ArchiveFixture fx, string session, IReadOnlyList<(int Seed, int Ms, string Name)> photos, Archive? archive = null) + { + Guid? id = null; + var staging = Path.Combine(new ArchivePaths((archive ?? fx.Archive).RootPath).StagingDirectory, "uploads"); + Directory.CreateDirectory(staging); + foreach (var (seed, ms, name) in photos) + { + var bytes = Photos.Jpeg(Start.AddMilliseconds(ms), seed); + var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part"); + File.WriteAllBytes(path, bytes); + var result = await fx.Service.Ingest.CommitAsync(new IngestRequest((archive ?? fx.Archive).Id, id, null, id is null ? session : null, path, name, + bytes.Length, Sha(bytes), null, null), fx.Ct); + id = result.SequenceId; + } + return (await fx.Db.Sequences.GetAsync(id!.Value))!; + } + + private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes)); + + private static string Sha(int seed, int ms) => Sha(Photos.Jpeg(Start.AddMilliseconds(ms), seed)); + + private static async Task PairAsync(ArchiveFixture fx, Guid a, Guid b) => + await new FoundCopyStore(fx.Db).UpsertFoundAsync(new FoundCopy + { + Id = Guid.NewGuid(), + Kind = CopyKind.LocalLocal, + LocalId = a, + OtherId = b, + Evidence = CopyEvidence.Hashes, + Fingerprint = "f", + }, fx.Ct); + + /// Choices with every detail line on the kept side but those in . + private static async Task ChoicesAsync(ArchiveFixture fx, Guid keep, Guid drop, CopyPhotoPolicy policy, + IReadOnlyDictionary? perPhoto = null, params string[] other) + { + var keepRow = (await fx.Db.Sequences.GetAsync(keep))!; + var dropRow = (await fx.Db.Sequences.GetAsync(drop))!; + var lines = CopyPayload.Lines(SyncPayload.Of(keepRow), await SequencePayloadWriter.PhotosAsync(fx.Db, keep, null, fx.Ct), + SyncPayload.Of(dropRow), await SequencePayloadWriter.PhotosAsync(fx.Db, drop, null, fx.Ct)); + var details = lines.Select(l => l.Path).Distinct().ToDictionary(p => p, p => other.Contains(p) ? CopySide.Other : CopySide.ThisPc); + return new MergeChoices(policy, perPhoto ?? new Dictionary(), details); + } + + private static async Task TitledAsync(ArchiveFixture fx, Sequence sequence, string title) + { + var result = await fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { Title = title }, null, fx.Ct); + return result.Sequence!; + } + + private static Task RunningAsync(ArchiveFixture fx, JobKind kind, params Guid[] targets) => EnqueuedAsync(fx, new Job + { + Id = Guid.CreateVersion7(), + Kind = kind, + TargetIds = targets, + State = JobState.Running, + NodeId = "test#0", + Created = DateTimeOffset.UtcNow, + Started = DateTimeOffset.UtcNow, + }); + + private static async Task EnqueuedAsync(ArchiveFixture fx, Job job) + { + await fx.Db.Jobs.EnqueueAsync(job); + return job; + } + + private (JobEngine Engine, MergeJobHandler Handler) Engine(ArchiveFixture fx, SequenceMerger merger, FakeProjects projects, FakeWorkspace workspace) + { + var handler = (MergeJobHandler)SequenceMerger.CreateHandler(fx.Db, merger, fx.Service, projects, workspace); + var engine = Own(new JobEngine(fx.Db, [.. fx.Service.CreateJobHandlers(), handler], + new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) })); + return (engine, handler); + } + + private static async Task> HashesAsync(ArchiveFixture fx, Guid sequenceId) => + (await fx.Db.Frames.ListBySequenceAsync(sequenceId)).Select(f => f.Sha256!).ToHashSet(StringComparer.Ordinal); + + /// + /// A pair with something in every stage: shared photo 1, photo 2 kept only, photo 3 only in the other, photo 4 different at + /// one instant (same name), a render inside the other's folder, a project, a session key, a link and a title. + /// + private static async Task ScenarioAsync(ArchiveFixture fx, int n) + { + var b = 100 * n; + var keep = await SequenceAsync(fx, $"keep-{n}", [(b + 1, 0, "IMG_0001.JPG"), (b + 2, 10_000, "IMG_0002.JPG"), (b + 4, 30_250, "IMG_0004.JPG")]); + var drop = await SequenceAsync(fx, $"drop-{n}", [(b + 1, 0, "IMG_0001.JPG"), (b + 3, 20_000, "IMG_0003.JPG"), (b + 5, 30_250, "IMG_0004.JPG")]); + drop = await TitledAsync(fx, drop, $"Harbour {n}"); + var exports = fx.Paths.Sequence(drop.Folder).Exports; + Directory.CreateDirectory(exports); + var clip = Path.Combine(exports, "clip.mp4"); + var bytes = RandomNumberGenerator.GetBytes(4096); + File.WriteAllBytes(clip, bytes); + var render = new Render + { + Id = Guid.CreateVersion7(), + SequenceId = drop.Id, + PresetName = "1080p", + OutputPath = fx.Paths.ToRelative(clip), + Bytes = bytes.Length, + Sha256 = Sha(bytes), + Created = DateTimeOffset.UtcNow, + }; + await fx.Db.Renders.InsertAsync(render, fx.Ct); + var project = await fx.Service.Projects.CreateAsync(drop.Id, "Night cut", cancellationToken: fx.Ct); + await new SyncLinkStore(fx.Db).UpsertAsync(new SyncLink { Server = Nas, RemoteId = Guid.CreateVersion7(), LocalId = drop.Id, Title = drop.DisplayTitle }, fx.Ct); + var copy = await PairAsync(fx, keep.Id, drop.Id); + var differs = Sha(b + 4, 30_250); + var choices = await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.OtherWins, null, SyncRules.MetadataPath + "title"); + return new Scenario(keep, drop, copy, render, project, choices, differs); + } + + private sealed record Scenario(Sequence Keep, Sequence Drop, FoundCopy Copy, Render Render, SequenceProject Project, MergeChoices Choices, string KeepLoser); + + // ---- the dry run ---- + + [Test] + public async Task TheKeptOneIsPreselectedOnlyWhenOneIsTheLinkedOriginalsCopy() + { + var fx = await FixtureAsync(); + var a = await SequenceAsync(fx, "a", [(1, 0, "IMG_0001.JPG")]); + var b = await SequenceAsync(fx, "b", [(1, 0, "IMG_0001.JPG")]); + var copy = await PairAsync(fx, a.Id, b.Id); + var merger = Merger(fx); + var choices = await ChoicesAsync(fx, a.Id, b.Id, CopyPhotoPolicy.Union); + + // Nothing linked: the user chooses; the plan keeps the one the dialog does not name. + var plan = await merger.PlanAsync(copy.Id, null, b.Id, choices, PlanContext(fx.Archive.Id), Ct); + Assert.Contains(CopyBlock.KeptUnchosen, plan.Blocks); + Assert.Equal((a.Id, b.Id), (plan.KeepId, plan.DropId)); + + // One linked, in a server's originals archive: preselected (also when the dialog names it as the other). + await new SyncLinkStore(fx.Db).UpsertAsync(new SyncLink { Server = Nas, RemoteId = Guid.CreateVersion7(), LocalId = b.Id, Title = "b" }, Ct); + plan = await merger.PlanAsync(copy.Id, null, b.Id, choices, PlanContext(fx.Archive.Id), Ct); + Assert.DoesNotContain(CopyBlock.KeptUnchosen, plan.Blocks); + Assert.Equal((b.Id, a.Id), (plan.KeepId, plan.DropId)); + + // Linked but not in an originals archive: asked. + plan = await merger.PlanAsync(copy.Id, null, b.Id, choices, NoContext, Ct); + Assert.Contains(CopyBlock.KeptUnchosen, plan.Blocks); + + // Both linked: asked again. + await new SyncLinkStore(fx.Db).UpsertAsync(new SyncLink { Server = "other.local", RemoteId = Guid.CreateVersion7(), LocalId = a.Id, Title = "a" }, Ct); + plan = await merger.PlanAsync(copy.Id, null, b.Id, choices, PlanContext(fx.Archive.Id), Ct); + Assert.Contains(CopyBlock.KeptUnchosen, plan.Blocks); + + // Chosen: never asked. + plan = await merger.PlanAsync(copy.Id, a.Id, b.Id, choices, NoContext, Ct); + Assert.Empty(plan.Blocks); + Assert.Equal((a.Id, b.Id), (plan.KeepId, plan.DropId)); + } + + [Test] + public async Task OnlyThereIsAlwaysCopiedAndThePolicyDecidesDifferentPhotosOnly() + { + var fx = await FixtureAsync(); + var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG"), (4, 30_250, "IMG_0004.JPG")]); + var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (3, 20_000, "IMG_0003.JPG"), (5, 30_250, "IMG_0004.JPG")]); + var copy = await PairAsync(fx, keep.Id, drop.Id); + var merger = Merger(fx); + var (only, theirs, mine) = (Sha(3, 20_000), Sha(5, 30_250), Sha(4, 30_250)); + + // Choose each with nothing chosen: the only-there photo still comes; the other's different version is a last copy. + var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.ChoosePerPhoto), NoContext, Ct); + Assert.Empty(plan.Blocks); + Assert.SequenceEqual([(PhotoAction.Fetch, only)], plan.Photos.Select(p => (p.Action, p.Sha256!))); + Assert.SequenceEqual([(CopySide.Other, theirs)], plan.LastCopies.Select(l => (l.Side, l.Key))); + + // Choose each, Take the other's: its version comes, the kept one's goes (a last copy of this PC). + var key = mine; + plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, + await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.ChoosePerPhoto, new Dictionary { [key] = PhotoChoice.TakeThere }), NoContext, Ct); + Assert.SequenceEqual([(PhotoAction.Fetch, only), (PhotoAction.FetchVersion, theirs), (PhotoAction.TrashHere, mine)], + plan.Photos.Select(p => (p.Action, p.Sha256!)).OrderBy(p => p.Action)); + Assert.SequenceEqual([(CopySide.ThisPc, mine)], plan.LastCopies.Select(l => (l.Side, l.Key))); + + // Union keeps both versions: nothing is a last copy. + plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct); + Assert.SequenceEqual([(PhotoAction.Fetch, only), (PhotoAction.FetchVersion, theirs)], plan.Photos.Select(p => (p.Action, p.Sha256!)).OrderBy(p => p.Action)); + Assert.Empty(plan.LastCopies); + Assert.Equal(plan.Photos.Sum(p => p.Bytes), plan.CopyBytes); + + // The kept one wins: nothing of the different photo moves; the other's version is a last copy. + plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.ThisPcWins), NoContext, Ct); + Assert.SequenceEqual([(PhotoAction.Fetch, only)], plan.Photos.Select(p => (p.Action, p.Sha256!))); + Assert.SequenceEqual([theirs], plan.LastCopies.Select(l => l.Key)); + } + + [Test] + public async Task APhotoTombstonedInTheKeptOneIsALastCopyUnlessAnotherSequenceHoldsIt() + { + var fx = await FixtureAsync(); + var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]); + var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]); + var gone = Sha(2, 10_000); + await fx.Db.FrameRemovals.SyncSequenceAsync(keep.Id, [new FrameRemoval(keep.Id, gone, null, RemovalReason.Deleted, DateTimeOffset.UtcNow)], Ct); + var copy = await PairAsync(fx, keep.Id, drop.Id); + var merger = Merger(fx); + var choices = await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union); + + var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct); + Assert.Empty(plan.Photos); + Assert.SequenceEqual([(CopySide.Other, gone)], plan.LastCopies.Select(l => (l.Side, l.Key))); + + await SequenceAsync(fx, "third", [(2, 10_000, "IMG_0002.JPG")]); + plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct); + Assert.Empty(plan.LastCopies); + } + + [Test] + public async Task SessionKeysFollowTheKeptOneButOneItsArchiveMapsElsewhereAndLinksArePlanned() + { + var fx = await FixtureAsync(); + var second = await fx.Service.AdoptAsync(Context.CreateTempDirectory("second"), "Second", Ct); + var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]); + var third = await SequenceAsync(fx, "card-7", [(9, 0, "IMG_0009.JPG")]); + var drop = await SequenceAsync(fx, "card-7", [(1, 0, "IMG_0001.JPG")], second); + await fx.Db.Sequences.MapSessionKeyAsync(second.Id, "card-8", drop.Id, Ct); + await new SyncLinkStore(fx.Db).UpsertAsync(new SyncLink { Server = Nas, RemoteId = Guid.CreateVersion7(), LocalId = drop.Id, Title = "d" }, Ct); + var copy = await PairAsync(fx, keep.Id, drop.Id); + + var plan = await Merger(fx).PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct); + Assert.Empty(plan.Blocks); + Assert.SequenceEqual(["card-8"], plan.SessionKeys); + Assert.SequenceEqual(["card-7"], plan.SkippedSessionKeys); + Assert.SequenceEqual([Nas], plan.Links); + Assert.Equal(third.Id, await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, "card-7", Ct)); + } + + [Test] + public async Task AnotherJobOnEitherSequenceAndBothLinkedToOneServerBlockThePlan() + { + var fx = await FixtureAsync(); + var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]); + var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG")]); + var copy = await PairAsync(fx, keep.Id, drop.Id); + var merger = Merger(fx); + var choices = await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union); + + var render = await RunningAsync(fx, JobKind.Render, keep.Id); + var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct); + Assert.Contains(CopyBlock.Busy, plan.Blocks); + await fx.Db.Jobs.SetStateAsync(render.Id, JobState.Done, null, Ct); + Assert.Empty((await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct)).Blocks); + + var links = new SyncLinkStore(fx.Db); + await links.UpsertAsync(new SyncLink { Server = Nas, RemoteId = Guid.CreateVersion7(), LocalId = keep.Id, Title = "k" }, Ct); + await links.UpsertAsync(new SyncLink { Server = Nas, RemoteId = Guid.CreateVersion7(), LocalId = drop.Id, Title = "d" }, Ct); + Assert.Contains(CopyBlock.BothLinked, (await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct)).Blocks); + + var unsent = PlanContext() with { UnsentWorkspaceSequences = new HashSet { drop.Id } }; + Assert.Contains(CopyBlock.UnsentWorkspace, (await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, unsent, Ct)).Blocks); + // A detail that differs and is not chosen. + await TitledAsync(fx, drop, "Other title"); + var unchosen = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct); + Assert.Contains(CopyBlock.DetailsUnchosen, unchosen.Blocks); + Assert.Null(unchosen.Payload); + } + + [Test] + public async Task TheFingerprintMovesWithADetailOfTheKeptOneNotWithAnImportElsewhere() + { + var fx = await FixtureAsync(); + var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]); + var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]); + var copy = await PairAsync(fx, keep.Id, drop.Id); + var merger = Merger(fx); + var choices = await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union); + var first = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct); + + await SequenceAsync(fx, "elsewhere", [(7, 0, "IMG_0007.JPG")]); + Assert.Equal(first.Fingerprint, (await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct)).Fingerprint); + + await fx.Db.Sequences.EditAsync(keep.Id, new SequenceEdit { Rating = 4 }, null, Ct); + Assert.NotEqual(first.Fingerprint, (await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct)).Fingerprint); + } + + // ---- the job ---- + + [Test] + public async Task AMergeCopiesVerifiesAndMovesEverythingThenTrashesTheOther() + { + var fx = await FixtureAsync(); + var s = await ScenarioAsync(fx, 0); + var merger = Merger(fx); + var projects = new FakeProjects(); + var (engine, _) = Engine(fx, merger, projects, new FakeWorkspace()); + await engine.StartAsync(Ct); + + var plan = await merger.PlanAsync(s.Copy.Id, s.Keep.Id, s.Drop.Id, s.Choices, NoContext, Ct); + Assert.Empty(plan.Blocks); + Assert.SequenceEqual([(CopySide.ThisPc, s.KeepLoser)], plan.LastCopies.Select(l => (l.Side, l.Key))); + Assert.SequenceEqual([s.Render.Id], plan.Renders.Select(r => r.RenderId)); + Assert.SequenceEqual([("Night cut", "Night cut")], plan.Projects.Select(p => (p.SourceName, p.TargetName))); + Assert.Equal(1, plan.DetailChanges); + Assert.Equal("Harbour 0", plan.Payload!.Title); + var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, s.Keep.DisplayTitle, Ct); + Assert.Equal(CopyState.Integrating, (await new FoundCopyStore(fx.Db).GetAsync(s.Copy.Id, Ct))!.State); + await fx.WaitForAsync(job.Id, JobState.Done); + + // Photos: shared, kept only, only in the other and the other's winning version; the kept one's losing version went. + Assert.True((await HashesAsync(fx, s.Keep.Id)).SetEquals([Sha(1, 0), Sha(2, 10_000), Sha(3, 20_000), Sha(5, 30_250)])); + var keep = (await fx.Db.Sequences.GetAsync(s.Keep.Id))!; + Assert.Equal("Harbour 0", keep.Title); + // The render keeps its id, its file copied into the kept sequence. + var render = (await fx.Db.Renders.GetAsync(s.Render.Id))!; + Assert.Equal(s.Keep.Id, render.SequenceId); + Assert.Equal(s.Render.Sha256, Sha(File.ReadAllBytes(fx.Paths.FromRelative(render.OutputPath)))); + Assert.Equal(1, keep.RenderedCount); + Assert.SequenceEqual([("Night cut", s.Project.Id)], projects.Copied.Select(c => (c.Name, c.Source))); + // Keys, links and the pair. + Assert.Equal(s.Keep.Id, await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, "drop-0", Ct)); + Assert.Contains("drop-0", ManifestStore.ReadSequence(fx.Paths.Sequence(keep.Folder).ManifestPath).SessionKeys); + Assert.Equal(s.Keep.Id, (await new SyncLinkStore(fx.Db).ListAsync(Nas, Ct)).Single().LocalId); + Assert.Null(await new FoundCopyStore(fx.Db).GetAsync(s.Copy.Id, Ct)); + Assert.False(File.Exists(MergeJournal.PathFor(fx.Paths, job.Id))); + + // The other is in its archive's trash, restorable, without the keys it gave away. + Assert.Null(await fx.Db.Sequences.GetAsync(s.Drop.Id)); + var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, Ct)).Single(e => e.SequenceId == s.Drop.Id && e.Kind == TrashKind.Sequence); + var restored = await fx.Service.RestoreAsync(fx.Archive.Id, entry.Name, Ct); + Assert.Equal(s.Drop.Id, restored.Id); + Assert.Equal(3, restored.FrameCount); + Assert.DoesNotContain("drop-0", ManifestStore.ReadSequence(fx.Paths.Sequence(restored.Folder).ManifestPath).SessionKeys); + } + + [Test] + public async Task ACrashAtEachStageResumesWhereItStopped() + { + var fx = await FixtureAsync(); + var merger = Merger(fx); + var projects = new FakeProjects(); + var (engine, handler) = Engine(fx, merger, projects, new FakeWorkspace()); + await engine.StartAsync(Ct); + var stages = Enum.GetValues().Where(s => s != MergeStage.Done).ToList(); + for (var i = 0; i < stages.Count; i++) + { + var s = await ScenarioAsync(fx, i + 1); + var crashed = false; + var at = stages[i]; + handler.BeforeStage = stage => + { + if (stage != at || crashed) return Task.CompletedTask; + crashed = true; + throw new IOException($"crash before {at}"); + }; + var plan = await merger.PlanAsync(s.Copy.Id, s.Keep.Id, s.Drop.Id, s.Choices, NoContext, Ct); + var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, s.Keep.DisplayTitle, Ct); + await fx.WaitForAsync(job.Id, JobState.Failed); + Assert.NotNull(await fx.Db.Sequences.GetAsync(s.Drop.Id)); + Assert.Equal(CopyState.Candidate, (await new FoundCopyStore(fx.Db).GetAsync(s.Copy.Id, Ct))!.State); + + await engine.ResumeAsync(job.Id, Ct); + await fx.WaitForAsync(job.Id, JobState.Done); + var b = 100 * (i + 1); + Assert.True((await HashesAsync(fx, s.Keep.Id)).SetEquals([Sha(b + 1, 0), Sha(b + 2, 10_000), Sha(b + 3, 20_000), Sha(b + 5, 30_250)]), at.ToString()); + Assert.Null(await fx.Db.Sequences.GetAsync(s.Drop.Id)); + Assert.Equal(s.Keep.Id, (await fx.Db.Renders.GetAsync(s.Render.Id))!.SequenceId); + } + // Each project copied once, whatever stage the run stopped at. + Assert.Equal(stages.Count, projects.Copied.Count); + } + + [Test] + public async Task APhotoOfTheOtherThatChangedOnDiskFailsTheVerifyAndNothingGoesToTheTrash() + { + var fx = await FixtureAsync(); + var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]); + var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]); + var copy = await PairAsync(fx, keep.Id, drop.Id); + var merger = Merger(fx); + var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace()); + await engine.StartAsync(Ct); + var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct); + var shared = (await fx.Db.Frames.ListBySequenceAsync(drop.Id)).First(f => f.Sha256 == Sha(1, 0)); + File.WriteAllBytes(fx.Paths.FromRelative(shared.RelativePath), Photos.Jpeg(Start, 77)); + + var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct); + var failed = await fx.WaitForAsync(job.Id, JobState.Failed); + Assert.Contains(shared.FileName, failed.Error!); + Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id)); + Assert.Count(2, await fx.Db.Frames.ListBySequenceAsync(drop.Id)); + Assert.Equal(CopyState.Candidate, (await new FoundCopyStore(fx.Db).GetAsync(copy.Id, Ct))!.State); + } + + [Test] + public async Task AnUnhashedPhotoThatIsAnotherPhotoAfterAllIsKeptAsASecondVersion() + { + var fx = await FixtureAsync(); + var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]); + var drop = await SequenceAsync(fx, "drop", [(2, 0, "IMG_0001.JPG")]); + // Neither hashed, the same size and time: compared as the same photo. + var access = new DbAccess(fx.Db.Database); + await access.ExecuteAsync("UPDATE frames SET sha256 = NULL, size = 1000 WHERE sequence_id = $1 OR sequence_id = $2", [keep.Id, drop.Id], Ct); + var copy = await PairAsync(fx, keep.Id, drop.Id); + var merger = Merger(fx); + var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace()); + await engine.StartAsync(Ct); + var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct); + Assert.Empty(plan.Photos); + + var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct); + await fx.WaitForAsync(job.Id, JobState.Done); + var frames = await fx.Db.Frames.ListBySequenceAsync(keep.Id); + Assert.Count(2, frames); + Assert.Contains(Sha(2, 0), frames.Select(f => f.Sha256)); + Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id)); + } + + [Test] + public async Task ADamagedPhotoOfTheKeptOneIsRepairedFromTheOther() + { + var fx = await FixtureAsync(); + var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]); + var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]); + var damaged = (await fx.Db.Frames.ListBySequenceAsync(keep.Id)).First(f => f.Sha256 == Sha(2, 10_000)); + File.WriteAllBytes(fx.Paths.FromRelative(damaged.RelativePath), [1, 2, 3]); + await fx.Db.Frames.UpdateFlagsAsync([damaged.Id], FrameFlags.Corrupt, FrameFlags.None, Ct); + var copy = await PairAsync(fx, keep.Id, drop.Id); + var merger = Merger(fx); + var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace()); + await engine.StartAsync(Ct); + var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct); + Assert.SequenceEqual([(PhotoAction.Fetch, Sha(2, 10_000))], plan.Photos.Select(p => (p.Action, p.Sha256!))); + + var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct); + await fx.WaitForAsync(job.Id, JobState.Done); + var frames = await fx.Db.Frames.ListBySequenceAsync(keep.Id); + Assert.Count(2, frames); + var repaired = frames.Single(f => f.Sha256 == Sha(2, 10_000)); + Assert.Equal(Sha(2, 10_000), Sha(File.ReadAllBytes(fx.Paths.FromRelative(repaired.RelativePath)))); + Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id)); + } + + [Test] + public async Task APhotoAddedToTheOtherOrDeletedFromTheKeptOneMeanwhileKeepsTheOtherOutOfTheTrash() + { + var fx = await FixtureAsync(); + var merger = Merger(fx); + var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace()); + await engine.StartAsync(Ct); + + // A photo added to the other after the verify. + var keep = await SequenceAsync(fx, "keep-a", [(1, 0, "IMG_0001.JPG")]); + var drop = await SequenceAsync(fx, "drop-a", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]); + var copy = await PairAsync(fx, keep.Id, drop.Id); + handler.BeforeStage = async stage => + { + if (stage != MergeStage.Trash) return; + var (path, size, sha) = fx.Stage(Start.AddSeconds(40), 3); + await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, drop.Id, null, null, path, "IMG_0003.JPG", size, sha, null, null), fx.Ct); + }; + var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct); + var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct); + await fx.WaitForAsync(job.Id, JobState.Failed); + Assert.Count(3, await fx.Db.Frames.ListBySequenceAsync(drop.Id)); + + // A verified photo deleted from the kept one before the trash. + keep = await SequenceAsync(fx, "keep-b", [(11, 0, "IMG_0001.JPG")]); + drop = await SequenceAsync(fx, "drop-b", [(11, 0, "IMG_0001.JPG"), (12, 10_000, "IMG_0002.JPG")]); + copy = await PairAsync(fx, keep.Id, drop.Id); + var keepId = keep.Id; + Guid? running = null; + handler.BeforeStage = async stage => + { + if (stage != MergeStage.Trash) return; + // Another hand deletes it (the job's own hold does not count, so the deletion is not refused as busy). + var gone = (await fx.Db.Frames.ListBySequenceAsync(keepId)).First(f => f.Sha256 == Sha(12, 10_000)); + var trash = await fx.Service.FrameTrash.PlanDeleteAsync(keepId, [gone.Id], null, fx.Ct, running); + await fx.Service.FrameTrash.DeleteAsync(trash, null, null, fx.Ct, running); + }; + plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct); + job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct); + running = job.Id; + var failed = await fx.WaitForAsync(job.Id, JobState.Failed); + Assert.Contains(keep.DisplayTitle, failed.Error!); + Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id)); + } + + [Test] + public async Task UnsentWorkspaceWorkStopsTheTrashAndTheTrashStepIsNotBusyWithItsOwnJob() + { + var fx = await FixtureAsync(); + var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]); + var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]); + var copy = await PairAsync(fx, keep.Id, drop.Id); + var merger = Merger(fx); + var workspace = new FakeWorkspace(); + var (engine, handler) = Engine(fx, merger, new FakeProjects(), workspace); + await engine.StartAsync(Ct); + handler.BeforeStage = stage => + { + if (stage == MergeStage.Trash) workspace.Unsent.Add(drop.Id); + return Task.CompletedTask; + }; + var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct); + var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct); + await fx.WaitForAsync(job.Id, JobState.Failed); + Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id)); + // The photos are in the kept one already. + Assert.True((await HashesAsync(fx, keep.Id)).SetEquals([Sha(1, 0), Sha(2, 10_000)])); + + handler.BeforeStage = null; + workspace.Unsent.Clear(); + await engine.ResumeAsync(job.Id, Ct); + await fx.WaitForAsync(job.Id, JobState.Done); + Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id)); + } + + [Test] + public async Task ACancelReturnsThePairToItsStateAndLeavesTheOtherAlone() + { + var fx = await FixtureAsync(); + var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]); + var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]); + var copy = await PairAsync(fx, keep.Id, drop.Id); + var merger = Merger(fx); + var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace()); + await engine.StartAsync(Ct); + Guid? id = null; + handler.BeforeStage = async stage => + { + if (stage == MergeStage.Verify) await engine.CancelAsync(id!.Value, CancellationToken.None); + }; + var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct); + var store = new FoundCopyStore(fx.Db); + var job = await merger.EnqueueAsync(engine, store, plan, keep.DisplayTitle, Ct); + id = job.Id; + await fx.WaitForAsync(job.Id, JobState.Cancelled); + var until = DateTime.UtcNow.AddSeconds(10); + while (File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)) && DateTime.UtcNow < until) await Task.Delay(20, Ct); + Assert.False(File.Exists(MergeJournal.PathFor(fx.Paths, job.Id))); + var pair = (await store.GetAsync(copy.Id, Ct))!; + Assert.Equal(CopyState.Candidate, pair.State); + Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id)); + Assert.Count(2, await fx.Db.Frames.ListBySequenceAsync(drop.Id)); + + // Planned and merged again from the start. + plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct); + handler.BeforeStage = null; + job = await merger.EnqueueAsync(engine, store, plan, keep.DisplayTitle, Ct); + await fx.WaitForAsync(job.Id, JobState.Done); + Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id)); + } + + [Test] + public async Task AJobWhosePlanMovedSinceTheDialogRefusesAndChangesNothing() + { + var fx = await FixtureAsync(); + var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]); + var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]); + var copy = await PairAsync(fx, keep.Id, drop.Id); + var merger = Merger(fx); + var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace()); + var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct); + var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct); + await SequenceAsync(fx, "drop", [(3, 20_000, "IMG_0003.JPG")]); // the session key files it into the other + await engine.StartAsync(Ct); + var failed = await fx.WaitForAsync(job.Id, JobState.Failed); + Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.StalePlan"), failed.Error); + Assert.Count(1, await fx.Db.Frames.ListBySequenceAsync(keep.Id)); + Assert.Count(3, await fx.Db.Frames.ListBySequenceAsync(drop.Id)); + } + + private sealed class FakeProjects : IProjectCopier + { + public List<(Guid Source, string Name)> Copied { get; } = []; + + public async Task CopyAsync(Chronos.Data.Projects.IProjectHome from, SequenceProject source, Chronos.Data.Projects.IProjectHome to, + string name, TransplantStep? journal, Func save, CancellationToken ct) + { + await save(new TransplantStep(name, DateTimeOffset.UtcNow, null, null, 0, [])); + lock (Copied) Copied.Add((source.Id, name)); + return source with { Id = Guid.CreateVersion7(), Name = name }; + } + } + + private sealed class FakeWorkspace : IWorkspaceGuard + { + public HashSet Unsent { get; } = []; + + public Task HasUnsentAsync(Guid sequenceId, CancellationToken ct) => Task.FromResult(Unsent.Contains(sequenceId)); + } +} + +public sealed class SqliteSequenceMergerTests(TestContext context) : SequenceMergerTests(context) +{ + protected override Backend Backend => Backend.SqliteFile; +} + +[RequiresEnvironment(PgTestServer.Variable)] +public sealed class PostgresSequenceMergerTests(TestContext context) : SequenceMergerTests(context) +{ + protected override Backend Backend => Backend.Postgres; +}