merge: P4 merge two local sequences into the kept one
Brings in the owner review 6 parcel P4: SequenceMerger plan and fingerprint, the journaled merge job (copy, verify, move, trash) and the sequence payload writer. Merges without conflicts on top of P1b; build has no warnings, Core, Data and Server suites, the copy-related App tests, i18n check, xaml-lint and package-guard pass. Refs #73 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
2a27ef488c
6 files changed
+2062
No files matched your search
@@ -54,6 +54,9 @@ public sealed class ArchiveService : IJobHandlerProvider
|
||||
|
||||
public ArchiveIndexer Indexer { get; }
|
||||
|
||||
/// <summary>The sidecar writer this service's writers share (its in-process gates and leases): a merge holds sequences with it.</summary>
|
||||
internal SequenceWriter Writer => _writer;
|
||||
|
||||
/// <summary>Commits staged files into sequences (server uploads, client card imports).</summary>
|
||||
public ArchiveIngest Ingest { get; }
|
||||
|
||||
|
||||
@@ -0,0 +1,705 @@
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.Diagnostics;
|
||||
using Chronos.Core.Import;
|
||||
using Chronos.Core.Localization;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Data.Copies;
|
||||
using Chronos.Data.Jobs;
|
||||
using Chronos.Data.Projects;
|
||||
using Chronos.Data.Repositories;
|
||||
using Chronos.Data.Sync;
|
||||
using ProcessingFrames = Chronos.Data.Processing.SequenceFrames;
|
||||
|
||||
namespace Chronos.Data.Archiving.Jobs;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.MergeSequences"/> (owner review 6, #73; design §5d): applies the merge plan the user confirmed
|
||||
/// (<see cref="MergeJobParameters"/>). Its first run plans again and refuses a plan whose fingerprint moved; the journal
|
||||
/// (<see cref="MergeJournal"/>) then keeps what that plan decided and how far each stage came, so a crash, a pause or a
|
||||
/// retry resumes where it stopped. Stages: copy the other's photos into the kept sequence (each hash-checked, filed with
|
||||
/// tombstones honoured), move the renders, verify every original of the other is held by the kept one (hashed again on
|
||||
/// both sides) or listed as a last copy, trash the kept sequence's losing versions, write the details, copy the projects,
|
||||
/// move session keys, links and pairs, then move the other to its archive's trash, only while it is still at the frames
|
||||
/// version the verify read and the kept photos verified are still there. A cancel leaves the photos copied so far in the
|
||||
/// kept sequence (nothing of the other is touched before the trash stage) and drops the journal.
|
||||
/// </summary>
|
||||
internal sealed class MergeJobHandler(ChronosDatabase db, SequenceMerger merger, ArchiveService archives, IProjectCopier projects,
|
||||
IWorkspaceGuard workspace) : ICancelledJobHandler
|
||||
{
|
||||
/// <summary>The change told about the kept sequence.</summary>
|
||||
public const string KeepChange = "frames";
|
||||
|
||||
/// <summary>The change told about the sequence that went to the trash.</summary>
|
||||
public const string DropChange = "deleted";
|
||||
|
||||
private static readonly CopyPlanContext JobContextPlan = new(string.Empty, null, false, false, false, false, false, false, null,
|
||||
new HashSet<Guid>(), new HashSet<Guid>());
|
||||
|
||||
public JobKind Kind => JobKind.MergeSequences;
|
||||
|
||||
private ChronosDatabase Db => db;
|
||||
|
||||
private ArchiveService Archives => archives;
|
||||
|
||||
private IProjectCopier Projects => projects;
|
||||
|
||||
private IWorkspaceGuard Workspace => workspace;
|
||||
|
||||
/// <summary>Called before each stage runs (tests: a crash, a change made between stages).</summary>
|
||||
internal Func<MergeStage, Task>? BeforeStage { get; set; }
|
||||
|
||||
public async Task RunAsync(JobContext context)
|
||||
{
|
||||
var job = context.Job;
|
||||
var ct = context.Cancellation;
|
||||
var parameters = MergeJobParameters.Read(job.ParametersJson) ?? throw new InvalidOperationException($"Merge job {job.Id} has no plan.");
|
||||
var keepRow = await db.Sequences.GetAsync(parameters.KeepId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
|
||||
var keepArchive = await db.Archives.GetAsync(keepRow.ArchiveId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
|
||||
var keepPaths = new ArchivePaths(keepArchive.RootPath);
|
||||
try
|
||||
{
|
||||
var journal = MergeJournal.Read(keepPaths, job.Id) ?? await StartAsync(context, parameters, keepRow, keepPaths).ConfigureAwait(false);
|
||||
var staging = MergeJournal.StagingFor(keepPaths, job.Id);
|
||||
if (Directory.Exists(staging)) Directory.Delete(staging, recursive: true);
|
||||
await new Run(this, context, journal, keepArchive, keepPaths).RunAsync().ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
if (await db.Jobs.GetAsync(job.Id, CancellationToken.None).ConfigureAwait(false) is { State: JobState.Cancelled })
|
||||
await ForgetAsync(job, parameters, keepPaths).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task CancelledAsync(Job job, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (MergeJobParameters.Read(job.ParametersJson) is not { } parameters) return;
|
||||
if (await db.Sequences.GetAsync(parameters.KeepId, cancellationToken).ConfigureAwait(false) is not { } keep) return;
|
||||
if (await db.Archives.GetAsync(keep.ArchiveId, cancellationToken).ConfigureAwait(false) is not { } archive) return;
|
||||
await ForgetAsync(job, parameters, new ArchivePaths(archive.RootPath)).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
Log.Warning("archive", $"The cancelled merge {job.Id} left its journal; the archive's next maintenance removes it", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A cancelled merge: its journal and staging go, and its pair returns to the state it had (when it is still this job's).</summary>
|
||||
private async Task ForgetAsync(Job job, MergeJobParameters parameters, ArchivePaths keepPaths)
|
||||
{
|
||||
try
|
||||
{
|
||||
MergeJournal.Delete(keepPaths, job.Id);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("archive", $"The journal of the cancelled merge {job.Id} was not removed", ex);
|
||||
}
|
||||
var store = new FoundCopyStore(db);
|
||||
if (await store.GetAsync(parameters.CopyId, CancellationToken.None).ConfigureAwait(false) is { JobId: { } owner } copy && owner == job.Id
|
||||
&& copy.State != CopyState.Integrating)
|
||||
await store.SetStateAsync(copy.Id, copy.State, copy.State, null, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>The first run: plans again, refuses a plan that moved or is blocked, and journals it.</summary>
|
||||
private async Task<MergeJournal> StartAsync(JobContext context, MergeJobParameters parameters, Sequence keepRow, ArchivePaths keepPaths)
|
||||
{
|
||||
var ct = context.Cancellation;
|
||||
var copy = await new FoundCopyStore(db).GetAsync(parameters.CopyId, ct).ConfigureAwait(false);
|
||||
if (copy is null || copy.JobId != context.Job.Id) throw new StaleCopyPlanException();
|
||||
if (await workspace.HasUnsentAsync(parameters.DropId, ct).ConfigureAwait(false))
|
||||
throw new InvalidOperationException(Loc.L("Errors.Copies.Unsent", (await db.Sequences.GetAsync(parameters.DropId, ct).ConfigureAwait(false))?.DisplayTitle ?? string.Empty));
|
||||
var plan = await merger.PlanAsync(parameters.CopyId, parameters.KeepId, parameters.DropId, parameters.Choices, JobContextPlan, context.Job.Id, ct)
|
||||
.ConfigureAwait(false);
|
||||
if (plan.KeepId != parameters.KeepId || plan.DropId != parameters.DropId) throw new StaleCopyPlanException();
|
||||
if (!plan.CanApply) throw new InvalidOperationException(await merger.BlockMessageAsync(plan, ct).ConfigureAwait(false));
|
||||
if (!string.Equals(plan.Fingerprint, parameters.Fingerprint, StringComparison.Ordinal)) throw new StaleCopyPlanException();
|
||||
|
||||
var winners = plan.Photos.Where(p => p.Action == PhotoAction.FetchVersion).ToDictionary(p => p.Key, p => p.Sha256, StringComparer.Ordinal);
|
||||
var journal = new MergeJournal
|
||||
{
|
||||
JobId = context.Job.Id,
|
||||
CopyId = plan.CopyId,
|
||||
KeepId = plan.KeepId,
|
||||
DropId = plan.DropId,
|
||||
Fingerprint = plan.Fingerprint,
|
||||
Stage = MergeStage.Copy,
|
||||
KeepSettingsVersion = plan.KeepSettingsVersion,
|
||||
DropFramesVersion = plan.DropFramesVersion,
|
||||
KeepStatus = (int)keepRow.Status,
|
||||
Copies = [.. plan.Photos.Where(p => p.Action is PhotoAction.Fetch or PhotoAction.FetchVersion && p.FrameId is not null)
|
||||
.Select(p => new MergeCopy(p.Key, p.Action, p.FrameId!.Value, p.FileName, p.Bytes, p.Sha256))],
|
||||
Replaces = [.. plan.Photos.Where(p => p.Action == PhotoAction.TrashHere && p.FrameId is not null && winners.GetValueOrDefault(p.Key) is { Length: > 0 })
|
||||
.Select(p => new MergeReplace(p.Key, p.FrameId!.Value, p.FileName, winners[p.Key]!))],
|
||||
Absent = [.. plan.LastCopies.Where(l => l.Side == CopySide.Other).Select(l => l.Key.ToLowerInvariant())],
|
||||
Renders = [.. plan.Renders],
|
||||
Payload = plan.Payload,
|
||||
Projects = [.. plan.Projects],
|
||||
SessionKeys = [.. plan.SessionKeys],
|
||||
};
|
||||
journal.Save(keepPaths);
|
||||
return journal;
|
||||
}
|
||||
|
||||
/// <summary>One run of a merge, from the stage its journal reached.</summary>
|
||||
private sealed class Run(MergeJobHandler owner, JobContext context, MergeJournal journal, Archive keepArchive, ArchivePaths keepPaths)
|
||||
{
|
||||
private readonly ChronosDatabase _db = owner.Db;
|
||||
private readonly CancellationToken _ct = context.Cancellation;
|
||||
private readonly string _staging = MergeJournal.StagingFor(keepPaths, journal.JobId);
|
||||
private Archive _dropArchive = null!;
|
||||
private ArchivePaths _dropPaths = null!;
|
||||
private string _keepTitle = string.Empty;
|
||||
private string _dropTitle = string.Empty;
|
||||
|
||||
private long Total => journal.Copies.Count + journal.Renders.Count + journal.Replaces.Count + journal.Projects.Count + 4;
|
||||
|
||||
public async Task RunAsync()
|
||||
{
|
||||
var keep = await _db.Sequences.GetAsync(journal.KeepId, _ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
|
||||
_keepTitle = keep.DisplayTitle;
|
||||
var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false);
|
||||
if (drop is null && journal.Stage < MergeStage.Trash) throw new InvalidOperationException(Loc.L("Errors.Copies.StalePlan"));
|
||||
if (drop is not null)
|
||||
{
|
||||
_dropTitle = drop.DisplayTitle;
|
||||
_dropArchive = await _db.Archives.GetAsync(drop.ArchiveId, _ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
|
||||
_dropPaths = new ArchivePaths(_dropArchive.RootPath);
|
||||
}
|
||||
|
||||
while (journal.Stage != MergeStage.Done)
|
||||
{
|
||||
_ct.ThrowIfCancellationRequested();
|
||||
if (owner.BeforeStage is { } hook) await hook(journal.Stage).ConfigureAwait(false);
|
||||
switch (journal.Stage)
|
||||
{
|
||||
case MergeStage.Copy: await CopyAsync().ConfigureAwait(false); break;
|
||||
case MergeStage.Renders: await RendersAsync().ConfigureAwait(false); break;
|
||||
case MergeStage.Verify: await VerifyAsync().ConfigureAwait(false); break;
|
||||
case MergeStage.Replace: await ReplaceAsync().ConfigureAwait(false); break;
|
||||
case MergeStage.Details: await DetailsAsync().ConfigureAwait(false); break;
|
||||
case MergeStage.Projects: await ProjectsAsync().ConfigureAwait(false); break;
|
||||
case MergeStage.Links: await LinksAsync().ConfigureAwait(false); break;
|
||||
case MergeStage.Trash: await TrashAsync().ConfigureAwait(false); break;
|
||||
}
|
||||
journal.Stage++;
|
||||
Save();
|
||||
await context.CheckpointAsync($"merge:{journal.Stage}", Done(), Total).ConfigureAwait(false);
|
||||
}
|
||||
await FinishAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private long Done() => journal.Committed.Count + journal.RendersMoved.Count + journal.Replaced.Count + journal.KeptLosers.Count
|
||||
+ journal.ProjectsDone.Count + Math.Min(4, Math.Max(0, journal.Stage - MergeStage.Verify));
|
||||
|
||||
private void Save() => journal.Save(keepPaths);
|
||||
|
||||
private InvalidOperationException Changed(string title) => new(Loc.L("Errors.Copies.Changed", title));
|
||||
|
||||
private async Task EnsureDropUnchangedAsync(int expected)
|
||||
{
|
||||
var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false);
|
||||
if (drop is null || drop.FramesVersion != expected) throw Changed(_dropTitle);
|
||||
}
|
||||
|
||||
// ---- copy ----
|
||||
|
||||
private async Task CopyAsync()
|
||||
{
|
||||
var n = 0;
|
||||
foreach (var copy in journal.Copies)
|
||||
{
|
||||
n++;
|
||||
if (journal.Committed.ContainsKey(copy.DropFrameId)) continue;
|
||||
_ct.ThrowIfCancellationRequested();
|
||||
await EnsureDropUnchangedAsync(journal.DropFramesVersion).ConfigureAwait(false);
|
||||
var frame = await _db.Frames.GetAsync(copy.DropFrameId, _ct).ConfigureAwait(false) ?? throw Changed(_dropTitle);
|
||||
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.CopyPhoto, copy.FileName, n, journal.Copies.Count)).ConfigureAwait(false);
|
||||
var result = await FileAsync(frame, copy.Sha256).ConfigureAwait(false);
|
||||
journal.Committed[copy.DropFrameId] = result.FrameId;
|
||||
Save();
|
||||
context.Log(Loc.L("Archive.Log.MergeCopied", copy.FileName));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Copies a frame of the other into staging (hash-checked against <paramref name="expected"/>) and files it in the kept sequence.</summary>
|
||||
private async Task<IngestResult> FileAsync(Frame frame, string? expected)
|
||||
{
|
||||
Directory.CreateDirectory(_staging);
|
||||
var staged = Path.Combine(_staging, frame.Id.ToString("N") + Path.GetExtension(frame.FileName));
|
||||
if (File.Exists(staged)) File.Delete(staged);
|
||||
string sha;
|
||||
try
|
||||
{
|
||||
sha = await VerifiedCopy.CopyAsync(_dropPaths.FromRelative(frame.RelativePath), staged, verify: true, null, _ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is FileNotFoundException or DirectoryNotFoundException)
|
||||
{
|
||||
throw new InvalidOperationException(Loc.L("Errors.Copies.ChangedFile", frame.FileName), ex);
|
||||
}
|
||||
if (expected is { Length: > 0 } && !string.Equals(sha, expected, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
File.Delete(staged);
|
||||
throw new InvalidOperationException(Loc.L("Errors.Copies.ChangedFile", frame.FileName));
|
||||
}
|
||||
var size = new FileInfo(staged).Length;
|
||||
IngestResult result;
|
||||
try
|
||||
{
|
||||
result = await owner.Archives.Ingest.CommitAsync(new IngestRequest(keepArchive.Id, journal.KeepId, null, null, staged, frame.FileName, size,
|
||||
sha.ToLowerInvariant(), frame.CapturedAt, null, RefuseRemoved: true), _ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (PhotoRemovedException)
|
||||
{
|
||||
// Deleted from the kept sequence since the plan: it would be a last copy the plan did not list.
|
||||
File.Delete(staged);
|
||||
throw Changed(_keepTitle);
|
||||
}
|
||||
if (File.Exists(staged)) File.Delete(staged);
|
||||
return result;
|
||||
}
|
||||
|
||||
// ---- renders ----
|
||||
|
||||
private async Task RendersAsync()
|
||||
{
|
||||
var keep = await _db.Sequences.GetAsync(journal.KeepId, _ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
|
||||
var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false) ?? throw Changed(_dropTitle);
|
||||
var n = 0;
|
||||
foreach (var step in journal.Renders)
|
||||
{
|
||||
n++;
|
||||
if (journal.RendersMoved.Contains(step.RenderId)) continue;
|
||||
_ct.ThrowIfCancellationRequested();
|
||||
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.CopyVideo, step.FileName, n, journal.Renders.Count)).ConfigureAwait(false);
|
||||
var render = await _db.Renders.GetAsync(step.RenderId, _ct).ConfigureAwait(false);
|
||||
if (render is null || render.SequenceId == journal.KeepId)
|
||||
{
|
||||
journal.RendersMoved.Add(step.RenderId);
|
||||
Save();
|
||||
continue;
|
||||
}
|
||||
var source = SyncService.RenderFile(_dropArchive, render);
|
||||
var relative = render.OutputPath.Replace('\\', '/');
|
||||
var inside = !Path.IsPathRooted(render.OutputPath) && drop.Folder.Length > 0
|
||||
&& relative.StartsWith(drop.Folder.TrimEnd('/') + "/", StringComparison.Ordinal);
|
||||
string output;
|
||||
if (inside && (File.Exists(source) || Directory.Exists(source)))
|
||||
{
|
||||
// Inside the folder that goes to the trash: copied (verified) into the kept sequence's exports first.
|
||||
if (!journal.RenderTargets.TryGetValue(step.RenderId, out var target))
|
||||
{
|
||||
target = keepPaths.ToRelative(Free(keepPaths.Sequence(keep.Folder).Exports, Path.GetFileName(source)));
|
||||
journal.RenderTargets[step.RenderId] = target;
|
||||
Save();
|
||||
}
|
||||
var destination = keepPaths.FromRelative(target);
|
||||
if (Directory.Exists(source)) await CopyFolderAsync(source, destination).ConfigureAwait(false);
|
||||
else await CopyRenderAsync(source, destination, render.Sha256, step.FileName).ConfigureAwait(false);
|
||||
output = target;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Outside it (or gone): only the row moves, naming the same file wherever the kept sequence's archive is.
|
||||
output = Path.IsPathRooted(render.OutputPath) || keepArchive.Id == _dropArchive.Id ? render.OutputPath : source;
|
||||
}
|
||||
var access = new DbAccess(_db.Database);
|
||||
await access.InTransactionAsync(async session =>
|
||||
{
|
||||
var inner = new DbAccess(session);
|
||||
await inner.ExecuteAsync("UPDATE renders SET sequence_id = $2, output_path = $3 WHERE id = $1", [step.RenderId, journal.KeepId, output], _ct)
|
||||
.ConfigureAwait(false);
|
||||
foreach (var id in (Guid[])[journal.KeepId, journal.DropId])
|
||||
await inner.ExecuteAsync("UPDATE sequences SET rendered_count = (SELECT count(*) FROM renders WHERE sequence_id = $1) WHERE id = $1", [id], _ct)
|
||||
.ConfigureAwait(false);
|
||||
return 0;
|
||||
}, _ct).ConfigureAwait(false);
|
||||
journal.RendersMoved.Add(step.RenderId);
|
||||
Save();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task CopyRenderAsync(string source, string destination, string? sha, string name)
|
||||
{
|
||||
if (File.Exists(destination))
|
||||
{
|
||||
// A run that copied it before its row moved: kept when it is the same file.
|
||||
if (sha is { Length: > 0 } && string.Equals(await ContentHash.Sha256Async(destination, null, _ct).ConfigureAwait(false), sha, StringComparison.OrdinalIgnoreCase))
|
||||
return;
|
||||
File.Delete(destination);
|
||||
}
|
||||
var copied = await VerifiedCopy.CopyAsync(source, destination, verify: true, null, _ct).ConfigureAwait(false);
|
||||
if (sha is { Length: > 0 } && !string.Equals(copied, sha, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
File.Delete(destination);
|
||||
throw new InvalidOperationException(Loc.L("Errors.Copies.ChangedFile", name));
|
||||
}
|
||||
}
|
||||
|
||||
private async Task CopyFolderAsync(string source, string destination)
|
||||
{
|
||||
foreach (var file in Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories))
|
||||
{
|
||||
var target = Path.Combine(destination, Path.GetRelativePath(source, file));
|
||||
if (File.Exists(target) && new FileInfo(target).Length == new FileInfo(file).Length
|
||||
&& string.Equals(await ContentHash.Sha256Async(target, null, _ct).ConfigureAwait(false),
|
||||
await ContentHash.Sha256Async(file, null, _ct).ConfigureAwait(false), StringComparison.Ordinal))
|
||||
continue;
|
||||
if (File.Exists(target)) File.Delete(target);
|
||||
await VerifiedCopy.CopyAsync(file, target, verify: true, null, _ct).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private static string Free(string folder, string name)
|
||||
{
|
||||
var stem = Path.GetFileNameWithoutExtension(name);
|
||||
var extension = Path.GetExtension(name);
|
||||
var candidate = Path.Combine(folder, name);
|
||||
for (var n = 2; File.Exists(candidate) || Directory.Exists(candidate); n++) candidate = Path.Combine(folder, $"{stem} ({n}){extension}");
|
||||
return candidate;
|
||||
}
|
||||
|
||||
// ---- verify ----
|
||||
|
||||
private async Task VerifyAsync()
|
||||
{
|
||||
var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false) ?? throw Changed(_dropTitle);
|
||||
if (drop.FramesVersion != journal.DropFramesVersion) throw Changed(_dropTitle);
|
||||
var keepFrames = await ProcessingFrames.ListAsync(_db, journal.KeepId, _ct).ConfigureAwait(false);
|
||||
var dropFrames = (await ProcessingFrames.ListAsync(_db, journal.DropId, _ct).ConfigureAwait(false))
|
||||
.Where(f => (f.Flags & CopySides.UnusableFlags) == 0).ToList();
|
||||
var total = keepFrames.Count + dropFrames.Count;
|
||||
var done = 0;
|
||||
|
||||
// What the kept sequence holds now, hashed again.
|
||||
var held = new Dictionary<string, Guid>(StringComparer.Ordinal);
|
||||
foreach (var frame in keepFrames.Where(f => (f.Flags & CopySides.UnusableFlags) == 0))
|
||||
{
|
||||
_ct.ThrowIfCancellationRequested();
|
||||
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Verify, frame.FileName, ++done, total)).ConfigureAwait(false);
|
||||
if (await HashAsync(keepPaths.FromRelative(frame.RelativePath)).ConfigureAwait(false) is { } sha) held.TryAdd(sha, frame.Id);
|
||||
}
|
||||
|
||||
var failed = new List<string>();
|
||||
foreach (var frame in dropFrames)
|
||||
{
|
||||
_ct.ThrowIfCancellationRequested();
|
||||
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Verify, frame.FileName, ++done, total)).ConfigureAwait(false);
|
||||
if (journal.Verified.ContainsKey(frame.Id)) continue;
|
||||
var sha = await HashAsync(_dropPaths.FromRelative(frame.RelativePath)).ConfigureAwait(false);
|
||||
if (sha is null)
|
||||
{
|
||||
// Nothing on disk: nothing of it can be lost.
|
||||
continue;
|
||||
}
|
||||
if (frame.Sha256 is { Length: > 0 } recorded && !string.Equals(recorded, sha, StringComparison.OrdinalIgnoreCase))
|
||||
throw new InvalidOperationException(Loc.L("Errors.Copies.ChangedFile", frame.FileName));
|
||||
if (held.TryGetValue(sha, out var keepFrame))
|
||||
{
|
||||
journal.Verified[frame.Id] = new MergeVerified(sha, keepFrame);
|
||||
continue;
|
||||
}
|
||||
if (frame.Sha256 is not { Length: > 0 })
|
||||
{
|
||||
// Compared by size and time only, and another photo after all: filed in the kept sequence as a second version.
|
||||
var filed = await FileAsync(frame, sha).ConfigureAwait(false);
|
||||
journal.SecondVersions.Add(frame.Id);
|
||||
journal.Verified[frame.Id] = new MergeVerified(sha, filed.FrameId);
|
||||
held.TryAdd(sha, filed.FrameId);
|
||||
Save();
|
||||
continue;
|
||||
}
|
||||
if (journal.Absent.Contains(sha)) continue;
|
||||
failed.Add(sha);
|
||||
}
|
||||
if (failed.Count > 0)
|
||||
{
|
||||
// A photo held by another sequence of the index is not lost with the other; any other is.
|
||||
var others = await new CopySides(_db).HeldElsewhereAsync(failed, [journal.DropId], _ct).ConfigureAwait(false);
|
||||
var lost = failed.Count(s => !others.Contains(s));
|
||||
if (lost > 0) throw new InvalidOperationException(Loc.Plural("Errors.Copies.VerifyFailed", lost));
|
||||
}
|
||||
journal.VerifiedDropFramesVersion = drop.FramesVersion;
|
||||
Save();
|
||||
}
|
||||
|
||||
private async Task<string?> HashAsync(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
return (await ContentHash.Sha256Async(path, null, _ct).ConfigureAwait(false)).ToLowerInvariant();
|
||||
}
|
||||
catch (Exception ex) when (ex is FileNotFoundException or DirectoryNotFoundException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ---- replace ----
|
||||
|
||||
private async Task ReplaceAsync()
|
||||
{
|
||||
var n = 0;
|
||||
foreach (var replace in journal.Replaces)
|
||||
{
|
||||
n++;
|
||||
if (journal.Replaced.Contains(replace.Key) || journal.KeptLosers.Contains(replace.Key)) continue;
|
||||
_ct.ThrowIfCancellationRequested();
|
||||
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Replace, replace.FileName, n, journal.Replaces.Count)).ConfigureAwait(false);
|
||||
var loser = await _db.Frames.GetAsync(replace.KeepFrameId, _ct).ConfigureAwait(false);
|
||||
if (loser is null || loser.SequenceId != journal.KeepId)
|
||||
{
|
||||
journal.Replaced.Add(replace.Key);
|
||||
Save();
|
||||
continue;
|
||||
}
|
||||
if (!await WinnerInPlaceAsync(replace.WinnerSha256).ConfigureAwait(false))
|
||||
{
|
||||
Keep(replace, "the version that replaces it is not in place");
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
var plan = await owner.Archives.FrameTrash.PlanDeleteAsync(journal.KeepId, [replace.KeepFrameId], null, _ct, journal.JobId).ConfigureAwait(false);
|
||||
if (!plan.CanDelete)
|
||||
{
|
||||
Keep(replace, string.Join(", ", plan.Blocked));
|
||||
continue;
|
||||
}
|
||||
await owner.Archives.FrameTrash.DeleteAsync(plan, null, null, _ct, journal.JobId).ConfigureAwait(false);
|
||||
journal.Replaced.Add(replace.Key);
|
||||
Save();
|
||||
}
|
||||
catch (Exception ex) when (ex is StaleDeletePlanException or IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Keep(replace, ex.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void Keep(MergeReplace replace, string why)
|
||||
{
|
||||
journal.KeptLosers.Add(replace.Key);
|
||||
Save();
|
||||
context.Log($"{replace.FileName}: {why}");
|
||||
}
|
||||
|
||||
/// <summary>Whether a usable frame of the kept sequence holds <paramref name="sha"/>, hashed again.</summary>
|
||||
private async Task<bool> WinnerInPlaceAsync(string sha)
|
||||
{
|
||||
foreach (var frame in await _db.Frames.FindBySha256Async(sha.ToLowerInvariant(), _ct).ConfigureAwait(false))
|
||||
{
|
||||
if (frame.SequenceId != journal.KeepId || (frame.Flags & CopySides.UnusableFlags) != 0) continue;
|
||||
if (string.Equals(await HashAsync(keepPaths.FromRelative(frame.RelativePath)).ConfigureAwait(false), sha, StringComparison.OrdinalIgnoreCase)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// ---- details ----
|
||||
|
||||
private async Task DetailsAsync()
|
||||
{
|
||||
if (journal.Payload is null || journal.DetailsDone) return;
|
||||
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Details)).ConfigureAwait(false);
|
||||
await new SequencePayloadWriter(_db, owner.Archives).ApplyAsync(journal.KeepId, journal.Payload, journal.KeepSettingsVersion, _ct).ConfigureAwait(false);
|
||||
journal.DetailsDone = true;
|
||||
Save();
|
||||
}
|
||||
|
||||
// ---- projects ----
|
||||
|
||||
private async Task ProjectsAsync()
|
||||
{
|
||||
if (journal.Projects.Count == 0) return;
|
||||
var from = new ArchiveProjectHome(_db, owner.Archives.Projects, journal.DropId);
|
||||
var to = new ArchiveProjectHome(_db, owner.Archives.Projects, journal.KeepId);
|
||||
var sources = (await _db.Projects.ListBySequenceAsync(journal.DropId, includeDeleted: false, _ct).ConfigureAwait(false)).ToDictionary(p => p.Id);
|
||||
var failed = 0;
|
||||
var n = 0;
|
||||
foreach (var step in journal.Projects)
|
||||
{
|
||||
n++;
|
||||
if (journal.ProjectsDone.Contains(step.SourceProjectId)) continue;
|
||||
_ct.ThrowIfCancellationRequested();
|
||||
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Projects, step.TargetName, n, journal.Projects.Count)).ConfigureAwait(false);
|
||||
if (!sources.TryGetValue(step.SourceProjectId, out var source))
|
||||
{
|
||||
// Deleted from the other since the plan: its own trash keeps it.
|
||||
journal.ProjectsDone.Add(step.SourceProjectId);
|
||||
Save();
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
var copied = await owner.Projects.CopyAsync(from, source, to, step.TargetName, journal.ProjectSteps.GetValueOrDefault(step.SourceProjectId),
|
||||
saved =>
|
||||
{
|
||||
journal.ProjectSteps[step.SourceProjectId] = saved;
|
||||
Save();
|
||||
return ValueTask.CompletedTask;
|
||||
}, _ct).ConfigureAwait(false);
|
||||
await RenamesRendersAsync(step.SourceProjectId, copied.Id).ConfigureAwait(false);
|
||||
journal.ProjectsDone.Add(step.SourceProjectId);
|
||||
Save();
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
failed++;
|
||||
context.Log($"{step.SourceName}: {ex.Message}");
|
||||
}
|
||||
}
|
||||
if (failed > 0) throw new InvalidOperationException(Loc.Plural("Errors.Copies.ProjectsKept", failed));
|
||||
}
|
||||
|
||||
/// <summary>The renders moved to the kept sequence name the project copy (and its versions) instead of the other's.</summary>
|
||||
private async Task RenamesRendersAsync(Guid source, Guid copy)
|
||||
{
|
||||
var access = new DbAccess(_db.Database);
|
||||
if (journal.ProjectSteps.GetValueOrDefault(source) is { } step)
|
||||
{
|
||||
foreach (var move in step.Moves)
|
||||
await access.ExecuteAsync("UPDATE renders SET project_id = $2, project_version_id = $3 WHERE sequence_id = $1 AND project_version_id = $4",
|
||||
[journal.KeepId, copy, move.Copy, move.Source], _ct).ConfigureAwait(false);
|
||||
}
|
||||
await access.ExecuteAsync("UPDATE renders SET project_id = $2 WHERE sequence_id = $1 AND project_id = $3", [journal.KeepId, copy, source], _ct)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// ---- links ----
|
||||
|
||||
private async Task LinksAsync()
|
||||
{
|
||||
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Process)).ConfigureAwait(false);
|
||||
var keep = await _db.Sequences.GetAsync(journal.KeepId, _ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
|
||||
var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false) ?? throw Changed(_dropTitle);
|
||||
var keys = journal.SessionKeys.ToHashSet(StringComparer.Ordinal);
|
||||
if (!journal.KeepKeysWritten)
|
||||
{
|
||||
if (keys.Count > 0 && File.Exists(keepPaths.Sequence(keep.Folder).ManifestPath))
|
||||
{
|
||||
await owner.Archives.Writer.UpdateAsync(keepPaths, keep.Id, keep.Folder, manifest =>
|
||||
keys.All(manifest.SessionKeys.Contains) ? null : manifest with { SessionKeys = [.. manifest.SessionKeys.Union(keys, StringComparer.Ordinal)] },
|
||||
_ct).ConfigureAwait(false);
|
||||
}
|
||||
journal.KeepKeysWritten = true;
|
||||
Save();
|
||||
}
|
||||
if (!journal.DropKeysRemoved)
|
||||
{
|
||||
if (keys.Count > 0 && File.Exists(_dropPaths.Sequence(drop.Folder).ManifestPath))
|
||||
{
|
||||
await owner.Archives.Writer.UpdateAsync(_dropPaths, drop.Id, drop.Folder, manifest =>
|
||||
!manifest.SessionKeys.Any(keys.Contains) ? null : manifest with { SessionKeys = [.. manifest.SessionKeys.Where(k => !keys.Contains(k))] },
|
||||
_ct).ConfigureAwait(false);
|
||||
}
|
||||
journal.DropKeysRemoved = true;
|
||||
Save();
|
||||
}
|
||||
if (!journal.LinksMoved)
|
||||
{
|
||||
var moved = await new FoundCopyStore(_db).MoveToKeptAsync(journal.DropId, journal.KeepId, keepArchive.Id, journal.SessionKeys, journal.CopyId, _ct)
|
||||
.ConfigureAwait(false);
|
||||
foreach (var key in moved.SkippedKeys) context.Log(key);
|
||||
journal.LinksMoved = true;
|
||||
Save();
|
||||
}
|
||||
}
|
||||
|
||||
// ---- trash ----
|
||||
|
||||
private async Task TrashAsync()
|
||||
{
|
||||
var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false);
|
||||
if (drop is null) return; // a run that moved it before its journal said so
|
||||
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Trash, _dropTitle)).ConfigureAwait(false);
|
||||
if (await owner.Workspace.HasUnsentAsync(journal.DropId, _ct).ConfigureAwait(false))
|
||||
throw new InvalidOperationException(Loc.L("Errors.Copies.Unsent", _dropTitle));
|
||||
var expected = journal.VerifiedDropFramesVersion ?? throw Changed(_dropTitle);
|
||||
if (drop.FramesVersion != expected) throw Changed(_dropTitle);
|
||||
await using (await owner.Archives.Writer.LockAsync(keepPaths, journal.KeepId, _ct).ConfigureAwait(false))
|
||||
{
|
||||
await EnsureVerifiedHeldAsync().ConfigureAwait(false);
|
||||
var plan = await owner.Archives.Trash.PlanAsync([journal.DropId], null, _ct, journal.JobId).ConfigureAwait(false);
|
||||
if (plan.Blocked.Count > 0)
|
||||
throw new InvalidOperationException(Loc.L("Errors.Copies.TrashBlocked", _dropTitle, Loc.L("Archive.Delete.Reason." + plan.Blocked[0].Reason)));
|
||||
try
|
||||
{
|
||||
await owner.Archives.Trash.DeleteAsync(plan, null, null, _ct, journal.JobId, new Dictionary<Guid, int> { [journal.DropId] = expected })
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (StaleDeletePlanException ex)
|
||||
{
|
||||
throw new InvalidOperationException(Loc.L("Errors.Copies.Changed", _dropTitle), ex);
|
||||
}
|
||||
}
|
||||
context.Log(Loc.L("Archive.Log.MergeTrashed", _dropTitle));
|
||||
}
|
||||
|
||||
/// <summary>Every photo the verify found in the kept sequence is still there and usable (read under its lease).</summary>
|
||||
private async Task EnsureVerifiedHeldAsync()
|
||||
{
|
||||
var frames = await ProcessingFrames.ListAsync(_db, journal.KeepId, _ct).ConfigureAwait(false);
|
||||
var byId = frames.Where(f => (f.Flags & CopySides.UnusableFlags) == 0).ToDictionary(f => f.Id);
|
||||
var hashes = byId.Values.Select(f => f.Sha256).OfType<string>().Select(s => s.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
|
||||
foreach (var verified in journal.Verified.Values)
|
||||
{
|
||||
if (byId.TryGetValue(verified.KeepFrameId, out var frame)
|
||||
&& (frame.Sha256 is null || string.Equals(frame.Sha256, verified.Sha256, StringComparison.OrdinalIgnoreCase)))
|
||||
continue;
|
||||
if (hashes.Contains(verified.Sha256)) continue;
|
||||
throw Changed(_keepTitle);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- done ----
|
||||
|
||||
private async Task FinishAsync()
|
||||
{
|
||||
await context.ReportAsync(Total, Total, new JobDetail(JobStep.FollowUps)).ConfigureAwait(false);
|
||||
await new FoundCopyStore(_db).DeleteAsync(journal.CopyId, CancellationToken.None).ConfigureAwait(false);
|
||||
if (await _db.Sequences.GetAsync(journal.KeepId, CancellationToken.None).ConfigureAwait(false) is { } keep)
|
||||
{
|
||||
foreach (var request in owner.Archives.FrameTrash.FollowUps(keepArchive, keep, (SequenceStatus)journal.KeepStatus))
|
||||
{
|
||||
try
|
||||
{
|
||||
if (request.Targets.Count > 0 && await _db.Jobs.IsQueuedAsync(request.Kind, request.Targets[0], CancellationToken.None).ConfigureAwait(false)) continue;
|
||||
await context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (DatabaseException ex)
|
||||
{
|
||||
Log.Warning("archive", $"{request.Label}: the passes the merged photos need were not queued; the next scan queues them", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
await TellAsync(journal.KeepId, KeepChange).ConfigureAwait(false);
|
||||
await TellAsync(journal.DropId, DropChange).ConfigureAwait(false);
|
||||
var kept = journal.KeptLosers.Count;
|
||||
try
|
||||
{
|
||||
MergeJournal.Delete(keepPaths, journal.JobId);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("archive", $"The journal of merge {journal.JobId} was not removed", ex);
|
||||
}
|
||||
if (kept > 0) throw new InvalidOperationException(Loc.Plural("Errors.Copies.Kept", kept));
|
||||
}
|
||||
|
||||
private async Task TellAsync(Guid id, string change)
|
||||
{
|
||||
if (owner.Archives.Writer.Options.SequenceChanged is not { } changed) return;
|
||||
try
|
||||
{
|
||||
await changed(id, change, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Warning("archive", $"The change of sequence {id} was not told", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.IO;
|
||||
using Chronos.Data.Copies;
|
||||
using Chronos.Data.Sync;
|
||||
|
||||
namespace Chronos.Data.Archiving;
|
||||
|
||||
/// <summary>The stages of a merge, in the order they run (design §5d).</summary>
|
||||
internal enum MergeStage
|
||||
{
|
||||
Copy,
|
||||
Renders,
|
||||
Verify,
|
||||
Replace,
|
||||
Details,
|
||||
Projects,
|
||||
Links,
|
||||
Trash,
|
||||
Done,
|
||||
}
|
||||
|
||||
/// <summary>A photo of the dropped sequence a merge copies into the kept one: its key, the drop's frame, name, size and recorded SHA-256.</summary>
|
||||
internal sealed record MergeCopy(string Key, PhotoAction Action, Guid DropFrameId, string FileName, long Bytes, string? Sha256);
|
||||
|
||||
/// <summary>A losing version of the kept sequence that goes to its photo trash once the winner (the drop's version) is in place.</summary>
|
||||
internal sealed record MergeReplace(string Key, Guid KeepFrameId, string FileName, string WinnerSha256);
|
||||
|
||||
/// <summary>A drop original the verify found in the kept sequence: its SHA-256 now and the keep frame that holds it.</summary>
|
||||
internal sealed record MergeVerified(string Sha256, Guid KeepFrameId);
|
||||
|
||||
/// <summary>
|
||||
/// The journal of a merge (owner review 6, #73; design §5d): <c><keep archive>/.chronos/journal/merge-<job:N>.json</c>,
|
||||
/// written atomically (temp + rename) after each step. It keeps what the confirmed plan decided (a resumed run never plans
|
||||
/// again: its own steps moved the versions) and how far each stage came, so every stage resumes and nothing is done twice.
|
||||
/// </summary>
|
||||
internal sealed class MergeJournal
|
||||
{
|
||||
public Guid JobId { get; set; }
|
||||
|
||||
public Guid CopyId { get; set; }
|
||||
|
||||
public Guid KeepId { get; set; }
|
||||
|
||||
public Guid DropId { get; set; }
|
||||
|
||||
public string Fingerprint { get; set; } = string.Empty;
|
||||
|
||||
public MergeStage Stage { get; set; }
|
||||
|
||||
/// <summary>The kept sequence's settings version the plan read (the details are written only at it).</summary>
|
||||
public int KeepSettingsVersion { get; set; }
|
||||
|
||||
/// <summary>The dropped sequence's frames version the plan read (checked before each photo and at the verify).</summary>
|
||||
public int DropFramesVersion { get; set; }
|
||||
|
||||
/// <summary>The kept sequence's statuses before the merge (what its follow-ups make again).</summary>
|
||||
public int KeepStatus { get; set; }
|
||||
|
||||
public List<MergeCopy> Copies { get; set; } = [];
|
||||
|
||||
public List<MergeReplace> Replaces { get; set; } = [];
|
||||
|
||||
/// <summary>Raw SHA-256 of drop originals the plan leaves out of the kept sequence (last copies, losers, tombstoned in keep, converted).</summary>
|
||||
public List<string> Absent { get; set; } = [];
|
||||
|
||||
public List<RenderStep> Renders { get; set; } = [];
|
||||
|
||||
public SyncPayload? Payload { get; set; }
|
||||
|
||||
public List<ProjectStep> Projects { get; set; } = [];
|
||||
|
||||
public List<string> SessionKeys { get; set; } = [];
|
||||
|
||||
/// <summary>Drop frame → the keep frame it was filed as (or that already held it).</summary>
|
||||
public Dictionary<Guid, Guid> Committed { get; set; } = [];
|
||||
|
||||
/// <summary>Render → its archive-relative path in the kept sequence, written before the copy.</summary>
|
||||
public Dictionary<Guid, string> RenderTargets { get; set; } = [];
|
||||
|
||||
public List<Guid> RendersMoved { get; set; } = [];
|
||||
|
||||
/// <summary>Drop frame → its SHA-256 as hashed by the verify and the keep frame that holds it.</summary>
|
||||
public Dictionary<Guid, MergeVerified> Verified { get; set; } = [];
|
||||
|
||||
/// <summary>Drop frames the verify found to be another photo than keep's (unhashed, same size and time) and filed as a second version.</summary>
|
||||
public List<Guid> SecondVersions { get; set; } = [];
|
||||
|
||||
/// <summary>The dropped sequence's frames version the verify read: the trash refuses another.</summary>
|
||||
public int? VerifiedDropFramesVersion { get; set; }
|
||||
|
||||
public List<string> Replaced { get; set; } = [];
|
||||
|
||||
/// <summary>Keys of losing versions a refused trash step kept (reported at the end).</summary>
|
||||
public List<string> KeptLosers { get; set; } = [];
|
||||
|
||||
public bool DetailsDone { get; set; }
|
||||
|
||||
public Dictionary<Guid, TransplantStep> ProjectSteps { get; set; } = [];
|
||||
|
||||
public List<Guid> ProjectsDone { get; set; } = [];
|
||||
|
||||
public bool KeepKeysWritten { get; set; }
|
||||
|
||||
public bool DropKeysRemoved { get; set; }
|
||||
|
||||
public bool LinksMoved { get; set; }
|
||||
|
||||
/// <summary>The journal of job <paramref name="jobId"/> in the archive at <paramref name="paths"/>.</summary>
|
||||
public static string PathFor(ArchivePaths paths, Guid jobId) => Path.Combine(paths.JournalDirectory, $"merge-{jobId:N}.json");
|
||||
|
||||
/// <summary>Where the job stages the files it copies.</summary>
|
||||
public static string StagingFor(ArchivePaths paths, Guid jobId) => Path.Combine(paths.StagingDirectory, $"merge-{jobId:N}");
|
||||
|
||||
/// <summary>The journal of job <paramref name="jobId"/>; null when there is none (a first run), or it cannot be read.</summary>
|
||||
/// <exception cref="InvalidDataException">The journal is damaged: the merge stops rather than start again over half a merge.</exception>
|
||||
public static MergeJournal? Read(ArchivePaths paths, Guid jobId)
|
||||
{
|
||||
var path = PathFor(paths, jobId);
|
||||
if (!File.Exists(path)) return null;
|
||||
try
|
||||
{
|
||||
return JsonSerializer.Deserialize(AtomicFile.ReadAllText(path), MergeJournalJson.Default.MergeJournal);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
throw new InvalidDataException($"The merge journal {path} is damaged.", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Writes the journal atomically.</summary>
|
||||
public void Save(ArchivePaths paths)
|
||||
{
|
||||
Directory.CreateDirectory(paths.JournalDirectory);
|
||||
AtomicFile.WriteAllText(PathFor(paths, JobId), JsonSerializer.Serialize(this, MergeJournalJson.Default.MergeJournal));
|
||||
}
|
||||
|
||||
/// <summary>Deletes the journal and the staging folder of job <paramref name="jobId"/>.</summary>
|
||||
public static void Delete(ArchivePaths paths, Guid jobId)
|
||||
{
|
||||
var staging = StagingFor(paths, jobId);
|
||||
if (Directory.Exists(staging)) Directory.Delete(staging, recursive: true);
|
||||
File.Delete(PathFor(paths, jobId));
|
||||
}
|
||||
}
|
||||
|
||||
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, UseStringEnumConverter = true,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
[JsonSerializable(typeof(MergeJournal))]
|
||||
internal sealed partial class MergeJournalJson : JsonSerializerContext;
|
||||
@@ -0,0 +1,439 @@
|
||||
using System.Globalization;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.Localization;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Data.Archiving.Jobs;
|
||||
using Chronos.Data.Copies;
|
||||
using Chronos.Data.Jobs;
|
||||
using Chronos.Data.Projects;
|
||||
using Chronos.Data.Repositories;
|
||||
using Chronos.Data.Sync;
|
||||
|
||||
namespace Chronos.Data.Archiving;
|
||||
|
||||
/// <summary>
|
||||
/// Merges two local copies of one time-lapse into the one kept (owner review 6, #73; design §5d; BRIEF §12.5 Merge): the
|
||||
/// dry run (<see cref="PlanAsync(Guid, Guid?, Guid, MergeChoices, CopyPlanContext, CancellationToken)"/>) says which photos
|
||||
/// of the other copy are copied into the kept one, which go to the trash with it and are held nowhere else (last copies),
|
||||
/// the renders and projects that move, the details written, the session keys and links that follow, and what blocks it;
|
||||
/// <see cref="EnqueueAsync"/> queues the <see cref="JobKind.MergeSequences"/> job that applies exactly that plan
|
||||
/// (<see cref="CreateHandler"/>), every photo hash-verified in the kept sequence before the other goes to its archive's trash.
|
||||
/// </summary>
|
||||
public sealed class SequenceMerger(ChronosDatabase db, ArchiveService archives, CopySides sides)
|
||||
{
|
||||
/// <summary>The free space a merge leaves on the kept sequence's volume beyond what it copies.</summary>
|
||||
public const long SpaceReserve = 512L << 20;
|
||||
|
||||
private readonly ChronosDatabase _db = db ?? throw new ArgumentNullException(nameof(db));
|
||||
private readonly ArchiveService _archives = archives ?? throw new ArgumentNullException(nameof(archives));
|
||||
private readonly CopySides _sides = sides ?? throw new ArgumentNullException(nameof(sides));
|
||||
private readonly DbAccess _access = new(db.Database);
|
||||
|
||||
internal ChronosDatabase Db => _db;
|
||||
|
||||
internal ArchiveService Archives => _archives;
|
||||
|
||||
/// <summary>The handler of <see cref="JobKind.MergeSequences"/> jobs; register it on the <see cref="JobEngine"/>.</summary>
|
||||
public static IJobHandler CreateHandler(ChronosDatabase db, SequenceMerger merger, ArchiveService archives, IProjectCopier projects,
|
||||
IWorkspaceGuard workspace) => new MergeJobHandler(db, merger, archives, projects, workspace);
|
||||
|
||||
/// <summary>
|
||||
/// The dry run of merging the pair <paramref name="copyId"/>: <paramref name="keepId"/> is kept and the other goes to the
|
||||
/// trash. With <paramref name="keepId"/> null the kept one is preselected only when exactly one of the two is linked to a
|
||||
/// server and lies in a server's originals archive; otherwise the plan is made keeping the one that is not
|
||||
/// <paramref name="otherId"/> and blocks with <see cref="CopyBlock.KeptUnchosen"/>.
|
||||
/// </summary>
|
||||
/// <exception cref="ArgumentException">The pair is not a local pair of these sequences, or a sequence is not in the index.</exception>
|
||||
public Task<MergePlan> PlanAsync(Guid copyId, Guid? keepId, Guid otherId, MergeChoices choices, CopyPlanContext context, CancellationToken ct) =>
|
||||
PlanAsync(copyId, keepId, otherId, choices, context, null, ct);
|
||||
|
||||
/// <summary>The dry run, with the jobs of <paramref name="exceptJob"/> (the merge job planning again) not counted as busy.</summary>
|
||||
internal async Task<MergePlan> PlanAsync(Guid copyId, Guid? keepId, Guid otherId, MergeChoices choices, CopyPlanContext context, Guid? exceptJob,
|
||||
CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(choices);
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
var copy = await new FoundCopyStore(_db).GetAsync(copyId, ct).ConfigureAwait(false)
|
||||
?? throw new ArgumentException($"Unknown pair {copyId}.", nameof(copyId));
|
||||
if (copy is not { Kind: CopyKind.LocalLocal, OtherId: { } second } || !copy.Names(otherId))
|
||||
throw new ArgumentException($"Pair {copyId} is not a local pair of sequence {otherId}.", nameof(otherId));
|
||||
var first = copy.LocalId;
|
||||
var blocks = new List<CopyBlock>();
|
||||
|
||||
Guid keep, drop;
|
||||
if (keepId is { } chosen)
|
||||
{
|
||||
if (!copy.Names(chosen)) throw new ArgumentException($"Sequence {chosen} is not one of pair {copyId}.", nameof(keepId));
|
||||
(keep, drop) = (chosen, chosen == first ? second : first);
|
||||
}
|
||||
else if (await PreselectAsync(first, second, context, ct).ConfigureAwait(false) is { } preselected)
|
||||
{
|
||||
(keep, drop) = (preselected, preselected == first ? second : first);
|
||||
}
|
||||
else
|
||||
{
|
||||
(keep, drop) = (otherId == first ? second : first, otherId);
|
||||
blocks.Add(CopyBlock.KeptUnchosen);
|
||||
}
|
||||
|
||||
var keepRow = await _db.Sequences.GetAsync(keep, ct).ConfigureAwait(false) ?? throw new ArgumentException($"Unknown sequence {keep}.", nameof(copyId));
|
||||
var dropRow = await _db.Sequences.GetAsync(drop, ct).ConfigureAwait(false) ?? throw new ArgumentException($"Unknown sequence {drop}.", nameof(copyId));
|
||||
var keepArchive = await _db.Archives.GetAsync(keepRow.ArchiveId, ct).ConfigureAwait(false)
|
||||
?? throw new ArgumentException($"Unknown archive {keepRow.ArchiveId}.", nameof(copyId));
|
||||
var dropArchive = await _db.Archives.GetAsync(dropRow.ArchiveId, ct).ConfigureAwait(false)
|
||||
?? throw new ArgumentException($"Unknown archive {dropRow.ArchiveId}.", nameof(copyId));
|
||||
var keepPaths = new ArchivePaths(keepArchive.RootPath);
|
||||
var dropPaths = new ArchivePaths(dropArchive.RootPath);
|
||||
|
||||
// Where the two are: online, in the folders the index has, not worked on by a job.
|
||||
var online = ArchiveVolume.Probe(keepPaths) == ArchiveAvailability.Online && ArchiveVolume.Probe(dropPaths) == ArchiveAvailability.Online;
|
||||
if (!online) blocks.Add(CopyBlock.ArchiveOffline);
|
||||
else if (Moved(keepPaths, keepRow) || Moved(dropPaths, dropRow)) blocks.Add(CopyBlock.FolderMoved);
|
||||
if (await BusyAsync([keep, drop, keepRow.ArchiveId, dropRow.ArchiveId], exceptJob, ct).ConfigureAwait(false)) blocks.Add(CopyBlock.Busy);
|
||||
if (context.UnsentWorkspaceSequences.Contains(drop)) blocks.Add(CopyBlock.UnsentWorkspace);
|
||||
|
||||
// Photos: keep is "here", drop is "there".
|
||||
var here = await _sides.LocalAsync(keep, ct).ConfigureAwait(false);
|
||||
var there = await _sides.LocalAsync(drop, ct).ConfigureAwait(false);
|
||||
var comparison = CopyComparer.Compare(here, there);
|
||||
var steps = new List<PhotoStep>();
|
||||
var excluded = new List<string>();
|
||||
foreach (var photo in comparison.Photos)
|
||||
{
|
||||
switch (photo.Class)
|
||||
{
|
||||
case PhotoClass.OnlyThere:
|
||||
// Every photo only the other holds is copied: it goes to the trash, nothing it alone holds stays behind.
|
||||
steps.Add(Step(photo.Key, PhotoAction.Fetch, photo.There!));
|
||||
break;
|
||||
case PhotoClass.Differs:
|
||||
var choice = choices.Photos switch
|
||||
{
|
||||
CopyPhotoPolicy.Union => PhotoChoice.KeepBoth,
|
||||
CopyPhotoPolicy.ThisPcWins => PhotoChoice.TakeHere,
|
||||
CopyPhotoPolicy.OtherWins => PhotoChoice.TakeThere,
|
||||
_ => choices.PerPhoto.GetValueOrDefault(photo.Key, PhotoChoice.Leave),
|
||||
};
|
||||
if (choice is PhotoChoice.KeepBoth or PhotoChoice.TakeThere) steps.Add(Step(photo.Key, PhotoAction.FetchVersion, photo.There!));
|
||||
if (choice == PhotoChoice.KeepBoth && photo.There!.Sha256 is { Length: > 0 } version) excluded.Add(version.ToLowerInvariant());
|
||||
if (choice == PhotoChoice.TakeThere) steps.Add(Step(photo.Key, PhotoAction.TrashHere, photo.Here!));
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Last copies: drop originals nothing will hold once the copies are made (losers, tombstoned in keep, converted), and
|
||||
// keep's losing versions nothing else holds.
|
||||
var fetched = steps.Where(s => s.Action is PhotoAction.Fetch or PhotoAction.FetchVersion).Select(s => s.Sha256).OfType<string>()
|
||||
.Select(s => s.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
|
||||
var losers = steps.Where(s => s.Action == PhotoAction.TrashHere).Select(s => s.FrameId!.Value).ToHashSet();
|
||||
var keepHolds = here.Photos.Where(p => !p.Unusable && p.Sha256 is { Length: > 0 } && !losers.Contains(p.FrameId))
|
||||
.Select(p => p.Sha256!.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
|
||||
var absent = new List<CopyPhotoInput>();
|
||||
foreach (var frame in there.Photos.Where(p => !p.Unusable && p.Sha256 is { Length: > 0 }))
|
||||
{
|
||||
var sha = frame.Sha256!.ToLowerInvariant();
|
||||
if (keepHolds.Contains(sha) || fetched.Contains(sha) || absent.Any(a => string.Equals(a.Sha256, sha, StringComparison.OrdinalIgnoreCase))) continue;
|
||||
absent.Add(frame with { Sha256 = sha });
|
||||
}
|
||||
var lastCopies = new List<LastCopy>();
|
||||
var heldElsewhere = await _sides.HeldElsewhereAsync([.. absent.Select(a => a.Sha256!)], [drop], ct).ConfigureAwait(false);
|
||||
foreach (var frame in absent.Where(a => !heldElsewhere.Contains(a.Sha256!)))
|
||||
lastCopies.Add(new LastCopy(frame.Sha256!, CopySide.Other, frame.FileName, frame.Size));
|
||||
var loserSteps = steps.Where(s => s.Action == PhotoAction.TrashHere && s.Sha256 is { Length: > 0 }).ToList();
|
||||
if (loserSteps.Count > 0)
|
||||
{
|
||||
var loserHeld = await _sides.HeldElsewhereAsync([.. loserSteps.Select(s => s.Sha256!)], [keep, drop], ct).ConfigureAwait(false);
|
||||
foreach (var step in loserSteps)
|
||||
{
|
||||
var sha = step.Sha256!.ToLowerInvariant();
|
||||
if (loserHeld.Contains(sha) || keepHolds.Contains(sha)) continue;
|
||||
lastCopies.Add(new LastCopy(sha, CopySide.ThisPc, step.FileName, step.Bytes));
|
||||
}
|
||||
}
|
||||
|
||||
// Renders the kept sequence lacks (by id and SHA-256) move to it.
|
||||
var keepRenders = await _db.Renders.ListBySequenceAsync(keep, ct).ConfigureAwait(false);
|
||||
var keepRenderHashes = keepRenders.Select(r => r.Sha256).OfType<string>().ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var renders = (await _db.Renders.ListBySequenceAsync(drop, ct).ConfigureAwait(false))
|
||||
.Where(r => keepRenders.All(k => k.Id != r.Id) && (r.Sha256 is null || !keepRenderHashes.Contains(r.Sha256)))
|
||||
.OrderBy(r => r.Created).ThenBy(r => r.Id)
|
||||
.Select(r => new RenderStep(r.Id, Path.GetFileName(r.OutputPath.Replace('\\', '/').TrimEnd('/')), r.Bytes, r.Sha256)).ToList();
|
||||
|
||||
// Details, line by line, applied to the kept sequence only.
|
||||
var keepRemovals = await _db.FrameRemovals.ListBySequenceAsync(keep, ct).ConfigureAwait(false);
|
||||
var dropRemovals = await _db.FrameRemovals.ListBySequenceAsync(drop, ct).ConfigureAwait(false);
|
||||
var keepPhotos = await SequencePayloadWriter.PhotosAsync(_db, keep, dropRemovals, ct).ConfigureAwait(false);
|
||||
var dropPhotos = await SequencePayloadWriter.PhotosAsync(_db, drop, keepRemovals, ct).ConfigureAwait(false);
|
||||
var lines = CopyPayload.Lines(SyncPayload.Of(keepRow), keepPhotos, SyncPayload.Of(dropRow), dropPhotos);
|
||||
SyncPayload? payload = null;
|
||||
var changes = 0;
|
||||
if (lines.Any(l => !choices.Details.ContainsKey(l.Path)))
|
||||
{
|
||||
blocks.Add(CopyBlock.DetailsUnchosen);
|
||||
}
|
||||
else
|
||||
{
|
||||
changes = lines.DistinctBy(l => l.Path, StringComparer.Ordinal).Count(l => choices.Details[l.Path] == CopySide.Other);
|
||||
if (changes > 0 || excluded.Count > 0)
|
||||
{
|
||||
var mine = CopyPayload.Portable(SyncPayload.Of(keepRow), keepPhotos);
|
||||
var merged = changes > 0 ? CopyPayload.Merge(mine, CopyPayload.Portable(SyncPayload.Of(dropRow), dropPhotos), choices.Details, lines) : mine;
|
||||
try
|
||||
{
|
||||
merged = CopyPayload.ExcludeVersions(merged, excluded);
|
||||
}
|
||||
catch (Exception ex) when (ex is JsonException or InvalidDataException)
|
||||
{
|
||||
// Settings that are not a project of this Chronos: the incoming versions stay included.
|
||||
}
|
||||
payload = merged.SameAs(mine) && changes == 0 ? null : merged;
|
||||
}
|
||||
}
|
||||
|
||||
// Projects are always copied across: the other's home goes to the trash.
|
||||
var projects = await ProjectStepsAsync(keep, drop, dropArchive.Name, ct).ConfigureAwait(false);
|
||||
|
||||
// Session keys and links follow the kept sequence.
|
||||
var (sessionKeys, skippedKeys) = await SessionKeysAsync(keep, drop, keepRow.ArchiveId, ct).ConfigureAwait(false);
|
||||
var links = await _access.ListAsync("SELECT server, local_id FROM sync_links WHERE local_id = $1 OR local_id = $2", [keep, drop],
|
||||
r => (Server: r.GetString(0), Local: r.GetGuid(1)), ct).ConfigureAwait(false);
|
||||
var moved = links.Where(l => l.Local == drop).Select(l => l.Server).Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal).ToList();
|
||||
if (links.GroupBy(l => l.Server, StringComparer.Ordinal).Any(g => g.Select(l => l.Local).Distinct().Count() > 1)) blocks.Add(CopyBlock.BothLinked);
|
||||
|
||||
// The other goes to the trash.
|
||||
var trash = await _archives.Trash.PlanAsync([drop], null, ct, exceptJob).ConfigureAwait(false);
|
||||
if (trash.Blocked.Any(b => b.Reason is not (DeleteBlock.Busy or DeleteBlock.Offline))) blocks.Add(CopyBlock.TrashBlocked);
|
||||
else if (trash.Blocked.Any(b => b.Reason == DeleteBlock.Busy) && !blocks.Contains(CopyBlock.Busy)) blocks.Add(CopyBlock.Busy);
|
||||
|
||||
// Space on the kept sequence's volume.
|
||||
var copyBytes = steps.Where(s => s.Action is PhotoAction.Fetch or PhotoAction.FetchVersion).Sum(s => s.Bytes) + renders.Sum(r => r.Bytes);
|
||||
var free = online ? FreeSpace(keepPaths) : null;
|
||||
if (free is { } bytes && copyBytes + SpaceReserve > bytes) blocks.Add(CopyBlock.NoSpace);
|
||||
|
||||
// Details are in the fingerprint as they read now: an edit that keeps the versions (a title) still moves it.
|
||||
var keepStamp = await StampAsync(keep, ct).ConfigureAwait(false) + ":" + PayloadHash(SyncPayload.Of(keepRow));
|
||||
var dropStamp = await StampAsync(drop, ct).ConfigureAwait(false) + ":" + PayloadHash(SyncPayload.Of(dropRow));
|
||||
var fingerprint = CopyRules.PlanFingerprint(FingerprintLines(keepRow, dropRow, keepStamp, dropStamp, choices, steps, lastCopies, renders, payload,
|
||||
projects, sessionKeys, skippedKeys));
|
||||
return new MergePlan(copyId, keep, drop, choices, steps, lastCopies, renders, payload, changes, projects, sessionKeys, skippedKeys, moved, trash,
|
||||
copyBytes, free ?? -1, [.. blocks.Distinct()], keepRow.FramesVersion, keepRow.SettingsVersion, dropRow.FramesVersion, dropRow.SettingsVersion,
|
||||
fingerprint);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queues the job that applies <paramref name="plan"/>: the pair goes to <see cref="CopyState.Integrating"/> (its state
|
||||
/// before kept as the previous one), the job targets the kept sequence, then the other (both held from other jobs).
|
||||
/// </summary>
|
||||
/// <param name="title">The kept sequence's title, for the job's label ("Merge into {title}").</param>
|
||||
/// <exception cref="InvalidOperationException">The plan is blocked, or the pair is already being merged or integrated.</exception>
|
||||
/// <exception cref="StaleCopyPlanException">The pair changed state since the plan.</exception>
|
||||
public async Task<Job> EnqueueAsync(JobEngine engine, FoundCopyStore store, MergePlan plan, string title, CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(engine);
|
||||
ArgumentNullException.ThrowIfNull(store);
|
||||
ArgumentNullException.ThrowIfNull(plan);
|
||||
if (!plan.CanApply) throw new InvalidOperationException($"The merge plan is blocked: {string.Join(", ", plan.Blocks)}.");
|
||||
var copy = await store.GetAsync(plan.CopyId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
|
||||
if (copy.State == CopyState.Integrating) throw new InvalidOperationException(Loc.L("Errors.Copies.Busy"));
|
||||
var id = Guid.CreateVersion7();
|
||||
if (!await store.SetStateAsync(plan.CopyId, copy.State, CopyState.Integrating, new CopyAnswer(null, null, id, null, DateTimeOffset.UtcNow), ct)
|
||||
.ConfigureAwait(false))
|
||||
throw new StaleCopyPlanException();
|
||||
var parameters = new MergeJobParameters(plan.CopyId, plan.KeepId, plan.DropId, plan.Choices, plan.Fingerprint);
|
||||
try
|
||||
{
|
||||
return await engine.EnqueueAsync(id, JobKind.MergeSequences, [plan.KeepId, plan.DropId], Loc.L("Jobs.Target.Merge", title), parameters.ToJson(),
|
||||
JobPriority.User, null, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
await store.SetStateAsync(plan.CopyId, CopyState.Integrating, copy.State, null, CancellationToken.None).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The sentence a job fails with for the first block of a plan it made again.</summary>
|
||||
internal async Task<string> BlockMessageAsync(MergePlan plan, CancellationToken ct)
|
||||
{
|
||||
var keep = await _db.Sequences.GetAsync(plan.KeepId, ct).ConfigureAwait(false);
|
||||
var drop = await _db.Sequences.GetAsync(plan.DropId, ct).ConfigureAwait(false);
|
||||
var (keepTitle, dropTitle) = (keep?.DisplayTitle ?? string.Empty, drop?.DisplayTitle ?? string.Empty);
|
||||
return plan.Blocks[0] switch
|
||||
{
|
||||
CopyBlock.Busy => Loc.L("Errors.Copies.Busy"),
|
||||
CopyBlock.ArchiveOffline => Loc.L("Errors.Copies.ArchiveOffline", dropTitle),
|
||||
CopyBlock.FolderMoved => Loc.L("Errors.Copies.FolderMoved", dropTitle),
|
||||
CopyBlock.NoSpace => Loc.L("Errors.Copies.NoSpace", keepTitle, Size(plan.CopyBytes + SpaceReserve)),
|
||||
CopyBlock.BothLinked => Loc.L("Errors.Copies.BothLinked", string.Empty),
|
||||
CopyBlock.UnsentWorkspace => Loc.L("Errors.Copies.Unsent", dropTitle),
|
||||
CopyBlock.TrashBlocked => Loc.L("Errors.Copies.TrashBlocked", dropTitle,
|
||||
plan.Trash.Blocked.Select(b => Loc.L("Archive.Delete.Reason." + b.Reason)).FirstOrDefault() ?? string.Empty),
|
||||
CopyBlock.DetailsUnchosen => Loc.Plural("Errors.Copies.DetailsUnchosen", 1),
|
||||
CopyBlock.KeptUnchosen => Loc.L("Errors.Copies.KeptUnchosen"),
|
||||
_ => Loc.L("Errors.Copies.StalePlan"),
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>Free bytes on the kept archive's volume (its root and its staging folder, the tighter); null when they cannot be measured.</summary>
|
||||
internal long? FreeSpace(ArchivePaths paths)
|
||||
{
|
||||
long? free = null;
|
||||
foreach (var folder in (string[])[paths.Root, paths.StagingDirectory])
|
||||
{
|
||||
if (!Directory.Exists(folder) || Measure(folder) is not { } bytes) continue;
|
||||
free = free is { } known ? Math.Min(known, bytes) : bytes;
|
||||
}
|
||||
return free;
|
||||
|
||||
long? Measure(string folder)
|
||||
{
|
||||
if (_archives.FreeSpace is { } measure) return measure(folder);
|
||||
try
|
||||
{
|
||||
return new DriveInfo(OperatingSystem.IsWindows() ? Path.GetPathRoot(folder)! : folder).AvailableFreeSpace;
|
||||
}
|
||||
catch (Exception ex) when (ex is ArgumentException or IOException or UnauthorizedAccessException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static string Size(long bytes) => bytes >= 1_000_000_000
|
||||
? Loc.L("Common.Size.GB", (bytes / 1e9).ToString("0.0", Loc.Culture))
|
||||
: Loc.L("Common.Size.MB", (bytes / 1e6).ToString("0", Loc.Culture));
|
||||
|
||||
private static PhotoStep Step(string key, PhotoAction action, CopyPhotoInput frame) =>
|
||||
new(key, action, frame.FrameId, frame.FileName, frame.Size, string.IsNullOrEmpty(frame.Sha256) ? null : frame.Sha256.ToLowerInvariant());
|
||||
|
||||
private static bool Moved(ArchivePaths paths, Sequence sequence) =>
|
||||
sequence.Folder.Length > 0 && !SequenceFolders.Holds(paths.Sequence(sequence.Folder), sequence.Id);
|
||||
|
||||
/// <summary>
|
||||
/// The kept sequence preselected: the one of the two that is linked to a server and lies in a server's originals archive,
|
||||
/// when exactly one is (owner review 5 homes: a server's originals stay in its archive); null otherwise.
|
||||
/// </summary>
|
||||
private async Task<Guid?> PreselectAsync(Guid a, Guid b, CopyPlanContext context, CancellationToken ct)
|
||||
{
|
||||
if (context.ServerOriginalsArchives.Count == 0) return null;
|
||||
var linked = await _access.ListAsync("SELECT DISTINCT local_id FROM sync_links WHERE local_id = $1 OR local_id = $2", [a, b], r => r.GetGuid(0), ct)
|
||||
.ConfigureAwait(false);
|
||||
var candidates = new List<Guid>();
|
||||
foreach (var id in linked)
|
||||
{
|
||||
if (await _db.Sequences.GetAsync(id, ct).ConfigureAwait(false) is { } row && context.ServerOriginalsArchives.Contains(row.ArchiveId)) candidates.Add(id);
|
||||
}
|
||||
return candidates.Count == 1 ? candidates[0] : null;
|
||||
}
|
||||
|
||||
/// <summary>Whether a queued, running or paused job (but <paramref name="exceptJob"/>) targets any of <paramref name="targets"/>.</summary>
|
||||
private async Task<bool> BusyAsync(IReadOnlyCollection<Guid> targets, Guid? exceptJob, CancellationToken ct)
|
||||
{
|
||||
var rows = await _access.ListAsync(
|
||||
$"SELECT id, target_ids FROM jobs WHERE state IN ({(int)JobState.Queued}, {(int)JobState.Running}, {(int)JobState.Paused})", [],
|
||||
r => (Id: r.GetGuid(0), Targets: r.GetString(1)), ct).ConfigureAwait(false);
|
||||
var wanted = targets.ToHashSet();
|
||||
foreach (var (id, ids) in rows)
|
||||
{
|
||||
if (id == exceptJob || ids.Length == 0) continue;
|
||||
foreach (var part in ids.Split(','))
|
||||
{
|
||||
if (Guid.TryParse(part, out var target) && wanted.Contains(target)) return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>The drop's non-Main projects, each with the name it gets at the kept sequence's home.</summary>
|
||||
private async Task<List<ProjectStep>> ProjectStepsAsync(Guid keep, Guid drop, string dropArchiveName, CancellationToken ct)
|
||||
{
|
||||
var taken = (await _db.Projects.ListBySequenceAsync(keep, includeDeleted: false, ct).ConfigureAwait(false)).Select(p => p.Name)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var steps = new List<ProjectStep>();
|
||||
foreach (var project in (await _db.Projects.ListBySequenceAsync(drop, includeDeleted: false, ct).ConfigureAwait(false))
|
||||
.Where(p => !p.IsMain).OrderBy(p => p.CreatedAt).ThenBy(p => p.Id))
|
||||
{
|
||||
var name = FreeName(project.Name, dropArchiveName, taken);
|
||||
taken.Add(name);
|
||||
steps.Add(new ProjectStep(CopySide.Other, project.Id, project.Name, name, project.HeadNumber, project.HeadVersionId ?? Guid.Empty));
|
||||
}
|
||||
return steps;
|
||||
}
|
||||
|
||||
/// <summary>The name itself when free, else "<name> (<archive>)", then " (2)"…, each within <see cref="ProjectService.MaxNameLength"/>.</summary>
|
||||
internal static string FreeName(string name, string archiveName, IReadOnlySet<string> taken)
|
||||
{
|
||||
if (!taken.Contains(name)) return name;
|
||||
var stem = Fit(name, $" ({archiveName})");
|
||||
if (!taken.Contains(stem)) return stem;
|
||||
for (var n = 2; ; n++)
|
||||
{
|
||||
var candidate = Fit(stem, string.Create(CultureInfo.InvariantCulture, $" ({n})"));
|
||||
if (!taken.Contains(candidate)) return candidate;
|
||||
}
|
||||
|
||||
static string Fit(string stem, string suffix)
|
||||
{
|
||||
const int max = ProjectService.MaxNameLength;
|
||||
if (suffix.Length >= max) suffix = suffix[..(max / 2)];
|
||||
return (stem.Length + suffix.Length > max ? stem[..(max - suffix.Length)].TrimEnd() : stem) + suffix;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The drop's session keys that move to the kept sequence, and those the kept archive maps to a third sequence already.</summary>
|
||||
private async Task<(List<string> Moved, List<string> Skipped)> SessionKeysAsync(Guid keep, Guid drop, Guid keepArchiveId, CancellationToken ct)
|
||||
{
|
||||
var keys = await _access.ListAsync("SELECT session_key FROM sequence_sessions WHERE sequence_id = $1 ORDER BY session_key", [drop],
|
||||
r => r.GetString(0), ct).ConfigureAwait(false);
|
||||
var (moved, skipped) = (new List<string>(), new List<string>());
|
||||
foreach (var key in keys.Distinct(StringComparer.Ordinal))
|
||||
{
|
||||
var holder = await _db.Sequences.FindBySessionKeyAsync(keepArchiveId, key, ct).ConfigureAwait(false);
|
||||
if (holder is null || holder == keep || holder == drop) moved.Add(key);
|
||||
else skipped.Add(key);
|
||||
}
|
||||
return (moved, skipped);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// What a sequence's photos are in the index, beyond its versions: how many, their bytes, how many are hashed, flagged
|
||||
/// missing, corrupt, read from the preview only or undecodable (a verify or a decode records those without another
|
||||
/// frames version), and its tombstones. Of this sequence alone: an import elsewhere in its archive leaves it as it was.
|
||||
/// </summary>
|
||||
private async Task<string> StampAsync(Guid sequenceId, CancellationToken ct)
|
||||
{
|
||||
var flags = string.Join(", ", new[] { FrameFlags.Missing, FrameFlags.Corrupt, FrameFlags.PreviewOnly, FrameFlags.Undecodable }
|
||||
.Select(flag => $"CAST(COALESCE(SUM(CASE WHEN (flags & {(int)flag}) <> 0 THEN 1 ELSE 0 END), 0) AS BIGINT)"));
|
||||
var photos = await _access.SingleAsync(
|
||||
$"SELECT CAST(count(*) AS BIGINT), CAST(COALESCE(SUM(size), 0) AS BIGINT), CAST(COALESCE(SUM(CASE WHEN sha256 IS NULL OR sha256 = '' THEN 0 ELSE 1 END), 0) AS BIGINT), {flags} FROM frames WHERE sequence_id = $1",
|
||||
[sequenceId], r => string.Join(':', Enumerable.Range(0, 7).Select(i => r.GetInt64(i).ToString(CultureInfo.InvariantCulture))), ct).ConfigureAwait(false);
|
||||
var removals = await _access.ScalarAsync<long>("SELECT count(*) FROM frame_removals WHERE sequence_id = $1", [sequenceId], ct).ConfigureAwait(false);
|
||||
return string.Create(CultureInfo.InvariantCulture, $"{photos}:{removals}");
|
||||
}
|
||||
|
||||
private static string PayloadHash(SyncPayload payload) =>
|
||||
Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(payload, SyncPayloadJson.Default.SyncPayload))));
|
||||
|
||||
private static IEnumerable<string> FingerprintLines(Sequence keep, Sequence drop, string keepStamp, string dropStamp, MergeChoices choices,
|
||||
List<PhotoStep> steps, List<LastCopy> lastCopies, List<RenderStep> renders, SyncPayload? payload, List<ProjectStep> projects,
|
||||
List<string> sessionKeys, List<string> skippedKeys)
|
||||
{
|
||||
var inv = CultureInfo.InvariantCulture;
|
||||
yield return string.Create(inv, $"keep:{keep.Id:N}:{keep.FramesVersion}:{keep.SettingsVersion}:{keepStamp}");
|
||||
yield return string.Create(inv, $"drop:{drop.Id:N}:{drop.FramesVersion}:{drop.SettingsVersion}:{dropStamp}");
|
||||
yield return $"policy:{choices.Photos}";
|
||||
foreach (var (key, choice) in choices.PerPhoto.OrderBy(p => p.Key, StringComparer.Ordinal)) yield return $"choice:{key}:{choice}";
|
||||
foreach (var (path, side) in choices.Details.OrderBy(p => p.Key, StringComparer.Ordinal)) yield return $"detail:{path}:{side}";
|
||||
foreach (var line in steps.Select(s => $"{s.Action}:{s.Key}").Order(StringComparer.Ordinal)) yield return line;
|
||||
foreach (var line in lastCopies.Select(l => $"last:{l.Side}:{l.Key}").Order(StringComparer.Ordinal)) yield return line;
|
||||
foreach (var render in renders) yield return string.Create(inv, $"render:{render.RenderId:N}:{render.Sha256}");
|
||||
if (payload is not null)
|
||||
yield return "payload:" + PayloadHash(payload);
|
||||
foreach (var project in projects) yield return string.Create(inv, $"{project.From}:{project.SourceProjectId:N}:{project.SourceHeadId:N}:{project.TargetName}");
|
||||
foreach (var key in sessionKeys) yield return "session:" + key;
|
||||
foreach (var key in skippedKeys) yield return "skipped:" + key;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
using System.Text.Json;
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.IO;
|
||||
using Chronos.Core.Localization;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Core.Project;
|
||||
using Chronos.Data.Processing;
|
||||
using Chronos.Data.Repositories;
|
||||
using Chronos.Data.Sync;
|
||||
|
||||
namespace Chronos.Data.Archiving;
|
||||
|
||||
/// <summary>
|
||||
/// Writes the details a merge settled (owner review 6, #73; design §5d step 6) into the kept sequence in one step: title,
|
||||
/// tags, rating, notes and settings, under the sequence's lease, only while its settings are still at the version the plan
|
||||
/// read. The payload comes in portable form (frame positions by photo); its positions are placed on the sequence's own
|
||||
/// frames as they are under the lease, then the index, <c>project.json</c> and the sidecar are written as
|
||||
/// <see cref="SyncService"/> applies a server's payload. A payload the sequence already holds writes nothing (a resumed run).
|
||||
/// </summary>
|
||||
internal sealed class SequencePayloadWriter(ChronosDatabase db, ArchiveService archives)
|
||||
{
|
||||
/// <summary>
|
||||
/// Applies <paramref name="portable"/> to <paramref name="sequenceId"/> at <paramref name="expectedSettingsVersion"/>;
|
||||
/// returns the sequence as it is then.
|
||||
/// </summary>
|
||||
/// <exception cref="SyncConflictChangedException">The settings moved since the plan (an edit, an analysis): nothing was written.</exception>
|
||||
/// <exception cref="SequenceFolderMovedException">The sequence is not in the folder the index has: nothing was written.</exception>
|
||||
/// <exception cref="ArgumentException">The sequence or its archive is not in the index.</exception>
|
||||
public async Task<Sequence> ApplyAsync(Guid sequenceId, SyncPayload portable, int expectedSettingsVersion, CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(portable);
|
||||
var sequence = await db.Sequences.GetAsync(sequenceId, ct).ConfigureAwait(false)
|
||||
?? throw new ArgumentException($"Unknown sequence {sequenceId}.", nameof(sequenceId));
|
||||
var archive = await db.Archives.GetAsync(sequence.ArchiveId, ct).ConfigureAwait(false)
|
||||
?? throw new ArgumentException($"Unknown archive {sequence.ArchiveId}.", nameof(sequenceId));
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
await using (await archives.Writer.LockAsync(paths, sequenceId, ct).ConfigureAwait(false))
|
||||
{
|
||||
var row = await db.Sequences.GetAsync(sequenceId, ct).ConfigureAwait(false)
|
||||
?? throw new ArgumentException($"Unknown sequence {sequenceId}.", nameof(sequenceId));
|
||||
var folder = paths.Sequence(row.Folder);
|
||||
if (row.Folder.Length > 0 && !SequenceFolders.Holds(folder, row.Id)) throw new SequenceFolderMovedException();
|
||||
// The frames cannot change under the lease: the positions are placed on them as they are now.
|
||||
var payload = portable with { SettingsJson = await PlaceAsync(row.Id, portable.SettingsJson, ct).ConfigureAwait(false) };
|
||||
if (row.SettingsVersion != expectedSettingsVersion)
|
||||
{
|
||||
// A resumed run that wrote the payload before its journal said so finds it in place.
|
||||
if (payload.SameAs(SyncPayload.Of(row))) return row;
|
||||
throw new SyncConflictChangedException(Loc.L("Errors.Copies.DetailsChanged"));
|
||||
}
|
||||
var metadata = !payload.SameMetadata(SyncPayload.Of(row));
|
||||
var settings = !SyncRules.SameSettings(payload.SettingsJson, row.SettingsJson);
|
||||
if (!metadata && !settings) return row;
|
||||
|
||||
string? project = null;
|
||||
Guid? hero = null;
|
||||
if (settings)
|
||||
{
|
||||
var edited = ProjectStore.Parse(SyncRules.NormalizeSettings(payload.SettingsJson));
|
||||
project = edited.ToJson();
|
||||
hero = await HeroFrame.EditAsync(db, row.Id, ProjectStore.Read(row, folder).HeroFrame, edited.HeroFrame, ct).ConfigureAwait(false);
|
||||
}
|
||||
var edit = new SequenceEdit
|
||||
{
|
||||
Title = metadata ? payload.Title?.Trim() ?? string.Empty : null,
|
||||
Tags = metadata ? payload.Tags : null,
|
||||
Rating = metadata ? payload.Rating : null,
|
||||
Notes = metadata ? payload.Notes ?? string.Empty : null,
|
||||
SettingsJson = project,
|
||||
HeroFrameId = hero,
|
||||
};
|
||||
// The index first, at the version read under the lease: a writer that skips the lease still cannot be overwritten.
|
||||
var result = await db.Sequences.EditAsync(row.Id, edit, expectedSettingsVersion, ct).ConfigureAwait(false);
|
||||
if (result is not { Outcome: SequenceEditOutcome.Updated, Sequence: { } updated })
|
||||
throw new SyncConflictChangedException(Loc.L("Errors.Copies.DetailsChanged"));
|
||||
if (row.Folder.Length == 0 && !File.Exists(folder.ManifestPath)) return updated;
|
||||
if (project is not null)
|
||||
{
|
||||
Directory.CreateDirectory(folder.MetaDirectory);
|
||||
AtomicFile.WriteAllText(folder.ProjectPath, project);
|
||||
}
|
||||
if (metadata)
|
||||
{
|
||||
var manifest = SequenceWriter.ReadLocked(folder, row.Id);
|
||||
// Asked before the sidecar is written: frames another process wrote there since this index last synced the
|
||||
// folder keep it changed for the next scan.
|
||||
var inStep = await archives.Indexer.InStepAsync(archive, folder, manifest, ct).ConfigureAwait(false);
|
||||
ManifestStore.WriteSequence(folder.ManifestPath, manifest with
|
||||
{
|
||||
Title = updated.Title,
|
||||
Tags = updated.Tags,
|
||||
Rating = updated.Rating,
|
||||
Notes = updated.Notes,
|
||||
Versions = manifest.Versions with { Settings = Math.Max(manifest.Versions.Settings, updated.SettingsVersion) },
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
});
|
||||
if (inStep) await archives.Indexer.RecordStampsAsync(archive, folder, row.Id, ct).ConfigureAwait(false);
|
||||
}
|
||||
return updated;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The photos of a sequence's frames by index, as a merge names them (raw SHA-256, else size and capture time), with the
|
||||
/// encodings its conversions joined (<see cref="PhotoAlias"/>), so a position parked on a converted photo finds its frame.
|
||||
/// </summary>
|
||||
internal static async Task<FramePhotos> PhotosAsync(ChronosDatabase db, Guid sequenceId, IEnumerable<FrameRemoval>? extraRemovals, CancellationToken ct)
|
||||
{
|
||||
var frames = await Processing.SequenceFrames.ListAsync(db, sequenceId, ct).ConfigureAwait(false);
|
||||
var removals = await db.FrameRemovals.ListBySequenceAsync(sequenceId, ct).ConfigureAwait(false);
|
||||
var aliases = removals.Concat(extraRemovals ?? [])
|
||||
.Where(r => r.Reason == RemovalReason.Replaced && !string.IsNullOrEmpty(r.ReplacementSha256))
|
||||
.Select(r => new PhotoAlias(r.Sha256, r.ReplacementSha256!));
|
||||
return new FramePhotos(frames.Select(f => (f.Index, new FramePhoto(string.IsNullOrEmpty(f.Sha256) ? null : f.Sha256.ToLowerInvariant(), f.Size, f.CapturedAt))),
|
||||
aliases);
|
||||
}
|
||||
|
||||
/// <summary>The portable settings placed on the sequence's frames; settings that are not a project stay as they are.</summary>
|
||||
private async Task<string?> PlaceAsync(Guid sequenceId, string? portableSettings, CancellationToken ct)
|
||||
{
|
||||
var normalized = SyncRules.NormalizeSettings(portableSettings);
|
||||
if (!ProjectSettings.NamesFrames(normalized)) return portableSettings;
|
||||
var photos = await PhotosAsync(db, sequenceId, null, ct).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
return ProjectContent.Place(normalized, photos);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return portableSettings;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,630 @@
|
||||
using System.Security.Cryptography;
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Data.Archiving;
|
||||
using Chronos.Data.Archiving.Jobs;
|
||||
using Chronos.Data.Copies;
|
||||
using Chronos.Data.Jobs;
|
||||
using Chronos.Data.Repositories;
|
||||
using Chronos.Data.Sync;
|
||||
using Chronos.Data.Tests.Infrastructure;
|
||||
using Chronos.Testing;
|
||||
using static Chronos.Data.Tests.Archiving.ArchiveFixture;
|
||||
|
||||
namespace Chronos.Data.Tests.Archiving;
|
||||
|
||||
/// <summary>
|
||||
/// Merging two local copies of one time-lapse (owner review 6, #73; design §5d, §11 Merge): the dry run (kept sequence,
|
||||
/// photos copied, last copies, session keys, links, blocks, fingerprint) and the job that applies it, verifies every photo
|
||||
/// of the other in the kept one and only then moves the other to its archive's trash.
|
||||
/// </summary>
|
||||
[Category("archive")]
|
||||
public abstract class SequenceMergerTests(TestContext context) : DatabaseTest(context)
|
||||
{
|
||||
private const string Nas = "nas.local";
|
||||
|
||||
private static readonly CopyPlanContext NoContext = PlanContext();
|
||||
|
||||
private async Task<ArchiveFixture> FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true)));
|
||||
|
||||
private static CopyPlanContext PlanContext(params Guid[] originals) =>
|
||||
new("PC", null, false, false, false, false, false, false, null, new HashSet<Guid>(), originals.ToHashSet());
|
||||
|
||||
private static SequenceMerger Merger(ArchiveFixture fx) => new(fx.Db, fx.Service, new CopySides(fx.Db));
|
||||
|
||||
/// <summary>A sequence uploaded photo by photo into <paramref name="archive"/>: (seed, capture offset in ms, file name).</summary>
|
||||
private static async Task<Sequence> SequenceAsync(ArchiveFixture fx, string session, IReadOnlyList<(int Seed, int Ms, string Name)> photos, Archive? archive = null)
|
||||
{
|
||||
Guid? id = null;
|
||||
var staging = Path.Combine(new ArchivePaths((archive ?? fx.Archive).RootPath).StagingDirectory, "uploads");
|
||||
Directory.CreateDirectory(staging);
|
||||
foreach (var (seed, ms, name) in photos)
|
||||
{
|
||||
var bytes = Photos.Jpeg(Start.AddMilliseconds(ms), seed);
|
||||
var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part");
|
||||
File.WriteAllBytes(path, bytes);
|
||||
var result = await fx.Service.Ingest.CommitAsync(new IngestRequest((archive ?? fx.Archive).Id, id, null, id is null ? session : null, path, name,
|
||||
bytes.Length, Sha(bytes), null, null), fx.Ct);
|
||||
id = result.SequenceId;
|
||||
}
|
||||
return (await fx.Db.Sequences.GetAsync(id!.Value))!;
|
||||
}
|
||||
|
||||
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes));
|
||||
|
||||
private static string Sha(int seed, int ms) => Sha(Photos.Jpeg(Start.AddMilliseconds(ms), seed));
|
||||
|
||||
private static async Task<FoundCopy> PairAsync(ArchiveFixture fx, Guid a, Guid b) =>
|
||||
await new FoundCopyStore(fx.Db).UpsertFoundAsync(new FoundCopy
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Kind = CopyKind.LocalLocal,
|
||||
LocalId = a,
|
||||
OtherId = b,
|
||||
Evidence = CopyEvidence.Hashes,
|
||||
Fingerprint = "f",
|
||||
}, fx.Ct);
|
||||
|
||||
/// <summary>Choices with every detail line on the kept side but those in <paramref name="other"/>.</summary>
|
||||
private static async Task<MergeChoices> ChoicesAsync(ArchiveFixture fx, Guid keep, Guid drop, CopyPhotoPolicy policy,
|
||||
IReadOnlyDictionary<string, PhotoChoice>? perPhoto = null, params string[] other)
|
||||
{
|
||||
var keepRow = (await fx.Db.Sequences.GetAsync(keep))!;
|
||||
var dropRow = (await fx.Db.Sequences.GetAsync(drop))!;
|
||||
var lines = CopyPayload.Lines(SyncPayload.Of(keepRow), await SequencePayloadWriter.PhotosAsync(fx.Db, keep, null, fx.Ct),
|
||||
SyncPayload.Of(dropRow), await SequencePayloadWriter.PhotosAsync(fx.Db, drop, null, fx.Ct));
|
||||
var details = lines.Select(l => l.Path).Distinct().ToDictionary(p => p, p => other.Contains(p) ? CopySide.Other : CopySide.ThisPc);
|
||||
return new MergeChoices(policy, perPhoto ?? new Dictionary<string, PhotoChoice>(), details);
|
||||
}
|
||||
|
||||
private static async Task<Sequence> TitledAsync(ArchiveFixture fx, Sequence sequence, string title)
|
||||
{
|
||||
var result = await fx.Db.Sequences.EditAsync(sequence.Id, new SequenceEdit { Title = title }, null, fx.Ct);
|
||||
return result.Sequence!;
|
||||
}
|
||||
|
||||
private static Task<Job> RunningAsync(ArchiveFixture fx, JobKind kind, params Guid[] targets) => EnqueuedAsync(fx, new Job
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
Kind = kind,
|
||||
TargetIds = targets,
|
||||
State = JobState.Running,
|
||||
NodeId = "test#0",
|
||||
Created = DateTimeOffset.UtcNow,
|
||||
Started = DateTimeOffset.UtcNow,
|
||||
});
|
||||
|
||||
private static async Task<Job> EnqueuedAsync(ArchiveFixture fx, Job job)
|
||||
{
|
||||
await fx.Db.Jobs.EnqueueAsync(job);
|
||||
return job;
|
||||
}
|
||||
|
||||
private (JobEngine Engine, MergeJobHandler Handler) Engine(ArchiveFixture fx, SequenceMerger merger, FakeProjects projects, FakeWorkspace workspace)
|
||||
{
|
||||
var handler = (MergeJobHandler)SequenceMerger.CreateHandler(fx.Db, merger, fx.Service, projects, workspace);
|
||||
var engine = Own(new JobEngine(fx.Db, [.. fx.Service.CreateJobHandlers(), handler],
|
||||
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) }));
|
||||
return (engine, handler);
|
||||
}
|
||||
|
||||
private static async Task<IReadOnlySet<string>> HashesAsync(ArchiveFixture fx, Guid sequenceId) =>
|
||||
(await fx.Db.Frames.ListBySequenceAsync(sequenceId)).Select(f => f.Sha256!).ToHashSet(StringComparer.Ordinal);
|
||||
|
||||
/// <summary>
|
||||
/// A pair with something in every stage: shared photo 1, photo 2 kept only, photo 3 only in the other, photo 4 different at
|
||||
/// one instant (same name), a render inside the other's folder, a project, a session key, a link and a title.
|
||||
/// </summary>
|
||||
private static async Task<Scenario> ScenarioAsync(ArchiveFixture fx, int n)
|
||||
{
|
||||
var b = 100 * n;
|
||||
var keep = await SequenceAsync(fx, $"keep-{n}", [(b + 1, 0, "IMG_0001.JPG"), (b + 2, 10_000, "IMG_0002.JPG"), (b + 4, 30_250, "IMG_0004.JPG")]);
|
||||
var drop = await SequenceAsync(fx, $"drop-{n}", [(b + 1, 0, "IMG_0001.JPG"), (b + 3, 20_000, "IMG_0003.JPG"), (b + 5, 30_250, "IMG_0004.JPG")]);
|
||||
drop = await TitledAsync(fx, drop, $"Harbour {n}");
|
||||
var exports = fx.Paths.Sequence(drop.Folder).Exports;
|
||||
Directory.CreateDirectory(exports);
|
||||
var clip = Path.Combine(exports, "clip.mp4");
|
||||
var bytes = RandomNumberGenerator.GetBytes(4096);
|
||||
File.WriteAllBytes(clip, bytes);
|
||||
var render = new Render
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
SequenceId = drop.Id,
|
||||
PresetName = "1080p",
|
||||
OutputPath = fx.Paths.ToRelative(clip),
|
||||
Bytes = bytes.Length,
|
||||
Sha256 = Sha(bytes),
|
||||
Created = DateTimeOffset.UtcNow,
|
||||
};
|
||||
await fx.Db.Renders.InsertAsync(render, fx.Ct);
|
||||
var project = await fx.Service.Projects.CreateAsync(drop.Id, "Night cut", cancellationToken: fx.Ct);
|
||||
await new SyncLinkStore(fx.Db).UpsertAsync(new SyncLink { Server = Nas, RemoteId = Guid.CreateVersion7(), LocalId = drop.Id, Title = drop.DisplayTitle }, fx.Ct);
|
||||
var copy = await PairAsync(fx, keep.Id, drop.Id);
|
||||
var differs = Sha(b + 4, 30_250);
|
||||
var choices = await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.OtherWins, null, SyncRules.MetadataPath + "title");
|
||||
return new Scenario(keep, drop, copy, render, project, choices, differs);
|
||||
}
|
||||
|
||||
private sealed record Scenario(Sequence Keep, Sequence Drop, FoundCopy Copy, Render Render, SequenceProject Project, MergeChoices Choices, string KeepLoser);
|
||||
|
||||
// ---- the dry run ----
|
||||
|
||||
[Test]
|
||||
public async Task TheKeptOneIsPreselectedOnlyWhenOneIsTheLinkedOriginalsCopy()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var a = await SequenceAsync(fx, "a", [(1, 0, "IMG_0001.JPG")]);
|
||||
var b = await SequenceAsync(fx, "b", [(1, 0, "IMG_0001.JPG")]);
|
||||
var copy = await PairAsync(fx, a.Id, b.Id);
|
||||
var merger = Merger(fx);
|
||||
var choices = await ChoicesAsync(fx, a.Id, b.Id, CopyPhotoPolicy.Union);
|
||||
|
||||
// Nothing linked: the user chooses; the plan keeps the one the dialog does not name.
|
||||
var plan = await merger.PlanAsync(copy.Id, null, b.Id, choices, PlanContext(fx.Archive.Id), Ct);
|
||||
Assert.Contains(CopyBlock.KeptUnchosen, plan.Blocks);
|
||||
Assert.Equal((a.Id, b.Id), (plan.KeepId, plan.DropId));
|
||||
|
||||
// One linked, in a server's originals archive: preselected (also when the dialog names it as the other).
|
||||
await new SyncLinkStore(fx.Db).UpsertAsync(new SyncLink { Server = Nas, RemoteId = Guid.CreateVersion7(), LocalId = b.Id, Title = "b" }, Ct);
|
||||
plan = await merger.PlanAsync(copy.Id, null, b.Id, choices, PlanContext(fx.Archive.Id), Ct);
|
||||
Assert.DoesNotContain(CopyBlock.KeptUnchosen, plan.Blocks);
|
||||
Assert.Equal((b.Id, a.Id), (plan.KeepId, plan.DropId));
|
||||
|
||||
// Linked but not in an originals archive: asked.
|
||||
plan = await merger.PlanAsync(copy.Id, null, b.Id, choices, NoContext, Ct);
|
||||
Assert.Contains(CopyBlock.KeptUnchosen, plan.Blocks);
|
||||
|
||||
// Both linked: asked again.
|
||||
await new SyncLinkStore(fx.Db).UpsertAsync(new SyncLink { Server = "other.local", RemoteId = Guid.CreateVersion7(), LocalId = a.Id, Title = "a" }, Ct);
|
||||
plan = await merger.PlanAsync(copy.Id, null, b.Id, choices, PlanContext(fx.Archive.Id), Ct);
|
||||
Assert.Contains(CopyBlock.KeptUnchosen, plan.Blocks);
|
||||
|
||||
// Chosen: never asked.
|
||||
plan = await merger.PlanAsync(copy.Id, a.Id, b.Id, choices, NoContext, Ct);
|
||||
Assert.Empty(plan.Blocks);
|
||||
Assert.Equal((a.Id, b.Id), (plan.KeepId, plan.DropId));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task OnlyThereIsAlwaysCopiedAndThePolicyDecidesDifferentPhotosOnly()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG"), (4, 30_250, "IMG_0004.JPG")]);
|
||||
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (3, 20_000, "IMG_0003.JPG"), (5, 30_250, "IMG_0004.JPG")]);
|
||||
var copy = await PairAsync(fx, keep.Id, drop.Id);
|
||||
var merger = Merger(fx);
|
||||
var (only, theirs, mine) = (Sha(3, 20_000), Sha(5, 30_250), Sha(4, 30_250));
|
||||
|
||||
// Choose each with nothing chosen: the only-there photo still comes; the other's different version is a last copy.
|
||||
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.ChoosePerPhoto), NoContext, Ct);
|
||||
Assert.Empty(plan.Blocks);
|
||||
Assert.SequenceEqual([(PhotoAction.Fetch, only)], plan.Photos.Select(p => (p.Action, p.Sha256!)));
|
||||
Assert.SequenceEqual([(CopySide.Other, theirs)], plan.LastCopies.Select(l => (l.Side, l.Key)));
|
||||
|
||||
// Choose each, Take the other's: its version comes, the kept one's goes (a last copy of this PC).
|
||||
var key = mine;
|
||||
plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id,
|
||||
await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.ChoosePerPhoto, new Dictionary<string, PhotoChoice> { [key] = PhotoChoice.TakeThere }), NoContext, Ct);
|
||||
Assert.SequenceEqual([(PhotoAction.Fetch, only), (PhotoAction.FetchVersion, theirs), (PhotoAction.TrashHere, mine)],
|
||||
plan.Photos.Select(p => (p.Action, p.Sha256!)).OrderBy(p => p.Action));
|
||||
Assert.SequenceEqual([(CopySide.ThisPc, mine)], plan.LastCopies.Select(l => (l.Side, l.Key)));
|
||||
|
||||
// Union keeps both versions: nothing is a last copy.
|
||||
plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
|
||||
Assert.SequenceEqual([(PhotoAction.Fetch, only), (PhotoAction.FetchVersion, theirs)], plan.Photos.Select(p => (p.Action, p.Sha256!)).OrderBy(p => p.Action));
|
||||
Assert.Empty(plan.LastCopies);
|
||||
Assert.Equal(plan.Photos.Sum(p => p.Bytes), plan.CopyBytes);
|
||||
|
||||
// The kept one wins: nothing of the different photo moves; the other's version is a last copy.
|
||||
plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.ThisPcWins), NoContext, Ct);
|
||||
Assert.SequenceEqual([(PhotoAction.Fetch, only)], plan.Photos.Select(p => (p.Action, p.Sha256!)));
|
||||
Assert.SequenceEqual([theirs], plan.LastCopies.Select(l => l.Key));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task APhotoTombstonedInTheKeptOneIsALastCopyUnlessAnotherSequenceHoldsIt()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
|
||||
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
|
||||
var gone = Sha(2, 10_000);
|
||||
await fx.Db.FrameRemovals.SyncSequenceAsync(keep.Id, [new FrameRemoval(keep.Id, gone, null, RemovalReason.Deleted, DateTimeOffset.UtcNow)], Ct);
|
||||
var copy = await PairAsync(fx, keep.Id, drop.Id);
|
||||
var merger = Merger(fx);
|
||||
var choices = await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union);
|
||||
|
||||
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
|
||||
Assert.Empty(plan.Photos);
|
||||
Assert.SequenceEqual([(CopySide.Other, gone)], plan.LastCopies.Select(l => (l.Side, l.Key)));
|
||||
|
||||
await SequenceAsync(fx, "third", [(2, 10_000, "IMG_0002.JPG")]);
|
||||
plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
|
||||
Assert.Empty(plan.LastCopies);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task SessionKeysFollowTheKeptOneButOneItsArchiveMapsElsewhereAndLinksArePlanned()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var second = await fx.Service.AdoptAsync(Context.CreateTempDirectory("second"), "Second", Ct);
|
||||
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
|
||||
var third = await SequenceAsync(fx, "card-7", [(9, 0, "IMG_0009.JPG")]);
|
||||
var drop = await SequenceAsync(fx, "card-7", [(1, 0, "IMG_0001.JPG")], second);
|
||||
await fx.Db.Sequences.MapSessionKeyAsync(second.Id, "card-8", drop.Id, Ct);
|
||||
await new SyncLinkStore(fx.Db).UpsertAsync(new SyncLink { Server = Nas, RemoteId = Guid.CreateVersion7(), LocalId = drop.Id, Title = "d" }, Ct);
|
||||
var copy = await PairAsync(fx, keep.Id, drop.Id);
|
||||
|
||||
var plan = await Merger(fx).PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
|
||||
Assert.Empty(plan.Blocks);
|
||||
Assert.SequenceEqual(["card-8"], plan.SessionKeys);
|
||||
Assert.SequenceEqual(["card-7"], plan.SkippedSessionKeys);
|
||||
Assert.SequenceEqual([Nas], plan.Links);
|
||||
Assert.Equal(third.Id, await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, "card-7", Ct));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnotherJobOnEitherSequenceAndBothLinkedToOneServerBlockThePlan()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
|
||||
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG")]);
|
||||
var copy = await PairAsync(fx, keep.Id, drop.Id);
|
||||
var merger = Merger(fx);
|
||||
var choices = await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union);
|
||||
|
||||
var render = await RunningAsync(fx, JobKind.Render, keep.Id);
|
||||
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
|
||||
Assert.Contains(CopyBlock.Busy, plan.Blocks);
|
||||
await fx.Db.Jobs.SetStateAsync(render.Id, JobState.Done, null, Ct);
|
||||
Assert.Empty((await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct)).Blocks);
|
||||
|
||||
var links = new SyncLinkStore(fx.Db);
|
||||
await links.UpsertAsync(new SyncLink { Server = Nas, RemoteId = Guid.CreateVersion7(), LocalId = keep.Id, Title = "k" }, Ct);
|
||||
await links.UpsertAsync(new SyncLink { Server = Nas, RemoteId = Guid.CreateVersion7(), LocalId = drop.Id, Title = "d" }, Ct);
|
||||
Assert.Contains(CopyBlock.BothLinked, (await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct)).Blocks);
|
||||
|
||||
var unsent = PlanContext() with { UnsentWorkspaceSequences = new HashSet<Guid> { drop.Id } };
|
||||
Assert.Contains(CopyBlock.UnsentWorkspace, (await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, unsent, Ct)).Blocks);
|
||||
// A detail that differs and is not chosen.
|
||||
await TitledAsync(fx, drop, "Other title");
|
||||
var unchosen = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
|
||||
Assert.Contains(CopyBlock.DetailsUnchosen, unchosen.Blocks);
|
||||
Assert.Null(unchosen.Payload);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TheFingerprintMovesWithADetailOfTheKeptOneNotWithAnImportElsewhere()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
|
||||
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
|
||||
var copy = await PairAsync(fx, keep.Id, drop.Id);
|
||||
var merger = Merger(fx);
|
||||
var choices = await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union);
|
||||
var first = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct);
|
||||
|
||||
await SequenceAsync(fx, "elsewhere", [(7, 0, "IMG_0007.JPG")]);
|
||||
Assert.Equal(first.Fingerprint, (await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct)).Fingerprint);
|
||||
|
||||
await fx.Db.Sequences.EditAsync(keep.Id, new SequenceEdit { Rating = 4 }, null, Ct);
|
||||
Assert.NotEqual(first.Fingerprint, (await merger.PlanAsync(copy.Id, keep.Id, drop.Id, choices, NoContext, Ct)).Fingerprint);
|
||||
}
|
||||
|
||||
// ---- the job ----
|
||||
|
||||
[Test]
|
||||
public async Task AMergeCopiesVerifiesAndMovesEverythingThenTrashesTheOther()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var s = await ScenarioAsync(fx, 0);
|
||||
var merger = Merger(fx);
|
||||
var projects = new FakeProjects();
|
||||
var (engine, _) = Engine(fx, merger, projects, new FakeWorkspace());
|
||||
await engine.StartAsync(Ct);
|
||||
|
||||
var plan = await merger.PlanAsync(s.Copy.Id, s.Keep.Id, s.Drop.Id, s.Choices, NoContext, Ct);
|
||||
Assert.Empty(plan.Blocks);
|
||||
Assert.SequenceEqual([(CopySide.ThisPc, s.KeepLoser)], plan.LastCopies.Select(l => (l.Side, l.Key)));
|
||||
Assert.SequenceEqual([s.Render.Id], plan.Renders.Select(r => r.RenderId));
|
||||
Assert.SequenceEqual([("Night cut", "Night cut")], plan.Projects.Select(p => (p.SourceName, p.TargetName)));
|
||||
Assert.Equal(1, plan.DetailChanges);
|
||||
Assert.Equal("Harbour 0", plan.Payload!.Title);
|
||||
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, s.Keep.DisplayTitle, Ct);
|
||||
Assert.Equal(CopyState.Integrating, (await new FoundCopyStore(fx.Db).GetAsync(s.Copy.Id, Ct))!.State);
|
||||
await fx.WaitForAsync(job.Id, JobState.Done);
|
||||
|
||||
// Photos: shared, kept only, only in the other and the other's winning version; the kept one's losing version went.
|
||||
Assert.True((await HashesAsync(fx, s.Keep.Id)).SetEquals([Sha(1, 0), Sha(2, 10_000), Sha(3, 20_000), Sha(5, 30_250)]));
|
||||
var keep = (await fx.Db.Sequences.GetAsync(s.Keep.Id))!;
|
||||
Assert.Equal("Harbour 0", keep.Title);
|
||||
// The render keeps its id, its file copied into the kept sequence.
|
||||
var render = (await fx.Db.Renders.GetAsync(s.Render.Id))!;
|
||||
Assert.Equal(s.Keep.Id, render.SequenceId);
|
||||
Assert.Equal(s.Render.Sha256, Sha(File.ReadAllBytes(fx.Paths.FromRelative(render.OutputPath))));
|
||||
Assert.Equal(1, keep.RenderedCount);
|
||||
Assert.SequenceEqual([("Night cut", s.Project.Id)], projects.Copied.Select(c => (c.Name, c.Source)));
|
||||
// Keys, links and the pair.
|
||||
Assert.Equal(s.Keep.Id, await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, "drop-0", Ct));
|
||||
Assert.Contains("drop-0", ManifestStore.ReadSequence(fx.Paths.Sequence(keep.Folder).ManifestPath).SessionKeys);
|
||||
Assert.Equal(s.Keep.Id, (await new SyncLinkStore(fx.Db).ListAsync(Nas, Ct)).Single().LocalId);
|
||||
Assert.Null(await new FoundCopyStore(fx.Db).GetAsync(s.Copy.Id, Ct));
|
||||
Assert.False(File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)));
|
||||
|
||||
// The other is in its archive's trash, restorable, without the keys it gave away.
|
||||
Assert.Null(await fx.Db.Sequences.GetAsync(s.Drop.Id));
|
||||
var entry = (await fx.Service.ListTrashAsync(fx.Archive.Id, Ct)).Single(e => e.SequenceId == s.Drop.Id && e.Kind == TrashKind.Sequence);
|
||||
var restored = await fx.Service.RestoreAsync(fx.Archive.Id, entry.Name, Ct);
|
||||
Assert.Equal(s.Drop.Id, restored.Id);
|
||||
Assert.Equal(3, restored.FrameCount);
|
||||
Assert.DoesNotContain("drop-0", ManifestStore.ReadSequence(fx.Paths.Sequence(restored.Folder).ManifestPath).SessionKeys);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ACrashAtEachStageResumesWhereItStopped()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var merger = Merger(fx);
|
||||
var projects = new FakeProjects();
|
||||
var (engine, handler) = Engine(fx, merger, projects, new FakeWorkspace());
|
||||
await engine.StartAsync(Ct);
|
||||
var stages = Enum.GetValues<MergeStage>().Where(s => s != MergeStage.Done).ToList();
|
||||
for (var i = 0; i < stages.Count; i++)
|
||||
{
|
||||
var s = await ScenarioAsync(fx, i + 1);
|
||||
var crashed = false;
|
||||
var at = stages[i];
|
||||
handler.BeforeStage = stage =>
|
||||
{
|
||||
if (stage != at || crashed) return Task.CompletedTask;
|
||||
crashed = true;
|
||||
throw new IOException($"crash before {at}");
|
||||
};
|
||||
var plan = await merger.PlanAsync(s.Copy.Id, s.Keep.Id, s.Drop.Id, s.Choices, NoContext, Ct);
|
||||
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, s.Keep.DisplayTitle, Ct);
|
||||
await fx.WaitForAsync(job.Id, JobState.Failed);
|
||||
Assert.NotNull(await fx.Db.Sequences.GetAsync(s.Drop.Id));
|
||||
Assert.Equal(CopyState.Candidate, (await new FoundCopyStore(fx.Db).GetAsync(s.Copy.Id, Ct))!.State);
|
||||
|
||||
await engine.ResumeAsync(job.Id, Ct);
|
||||
await fx.WaitForAsync(job.Id, JobState.Done);
|
||||
var b = 100 * (i + 1);
|
||||
Assert.True((await HashesAsync(fx, s.Keep.Id)).SetEquals([Sha(b + 1, 0), Sha(b + 2, 10_000), Sha(b + 3, 20_000), Sha(b + 5, 30_250)]), at.ToString());
|
||||
Assert.Null(await fx.Db.Sequences.GetAsync(s.Drop.Id));
|
||||
Assert.Equal(s.Keep.Id, (await fx.Db.Renders.GetAsync(s.Render.Id))!.SequenceId);
|
||||
}
|
||||
// Each project copied once, whatever stage the run stopped at.
|
||||
Assert.Equal(stages.Count, projects.Copied.Count);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task APhotoOfTheOtherThatChangedOnDiskFailsTheVerifyAndNothingGoesToTheTrash()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
|
||||
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
|
||||
var copy = await PairAsync(fx, keep.Id, drop.Id);
|
||||
var merger = Merger(fx);
|
||||
var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
|
||||
await engine.StartAsync(Ct);
|
||||
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
|
||||
var shared = (await fx.Db.Frames.ListBySequenceAsync(drop.Id)).First(f => f.Sha256 == Sha(1, 0));
|
||||
File.WriteAllBytes(fx.Paths.FromRelative(shared.RelativePath), Photos.Jpeg(Start, 77));
|
||||
|
||||
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
|
||||
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
|
||||
Assert.Contains(shared.FileName, failed.Error!);
|
||||
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
|
||||
Assert.Count(2, await fx.Db.Frames.ListBySequenceAsync(drop.Id));
|
||||
Assert.Equal(CopyState.Candidate, (await new FoundCopyStore(fx.Db).GetAsync(copy.Id, Ct))!.State);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnUnhashedPhotoThatIsAnotherPhotoAfterAllIsKeptAsASecondVersion()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
|
||||
var drop = await SequenceAsync(fx, "drop", [(2, 0, "IMG_0001.JPG")]);
|
||||
// Neither hashed, the same size and time: compared as the same photo.
|
||||
var access = new DbAccess(fx.Db.Database);
|
||||
await access.ExecuteAsync("UPDATE frames SET sha256 = NULL, size = 1000 WHERE sequence_id = $1 OR sequence_id = $2", [keep.Id, drop.Id], Ct);
|
||||
var copy = await PairAsync(fx, keep.Id, drop.Id);
|
||||
var merger = Merger(fx);
|
||||
var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
|
||||
await engine.StartAsync(Ct);
|
||||
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
|
||||
Assert.Empty(plan.Photos);
|
||||
|
||||
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
|
||||
await fx.WaitForAsync(job.Id, JobState.Done);
|
||||
var frames = await fx.Db.Frames.ListBySequenceAsync(keep.Id);
|
||||
Assert.Count(2, frames);
|
||||
Assert.Contains(Sha(2, 0), frames.Select(f => f.Sha256));
|
||||
Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ADamagedPhotoOfTheKeptOneIsRepairedFromTheOther()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
|
||||
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
|
||||
var damaged = (await fx.Db.Frames.ListBySequenceAsync(keep.Id)).First(f => f.Sha256 == Sha(2, 10_000));
|
||||
File.WriteAllBytes(fx.Paths.FromRelative(damaged.RelativePath), [1, 2, 3]);
|
||||
await fx.Db.Frames.UpdateFlagsAsync([damaged.Id], FrameFlags.Corrupt, FrameFlags.None, Ct);
|
||||
var copy = await PairAsync(fx, keep.Id, drop.Id);
|
||||
var merger = Merger(fx);
|
||||
var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
|
||||
await engine.StartAsync(Ct);
|
||||
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
|
||||
Assert.SequenceEqual([(PhotoAction.Fetch, Sha(2, 10_000))], plan.Photos.Select(p => (p.Action, p.Sha256!)));
|
||||
|
||||
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
|
||||
await fx.WaitForAsync(job.Id, JobState.Done);
|
||||
var frames = await fx.Db.Frames.ListBySequenceAsync(keep.Id);
|
||||
Assert.Count(2, frames);
|
||||
var repaired = frames.Single(f => f.Sha256 == Sha(2, 10_000));
|
||||
Assert.Equal(Sha(2, 10_000), Sha(File.ReadAllBytes(fx.Paths.FromRelative(repaired.RelativePath))));
|
||||
Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task APhotoAddedToTheOtherOrDeletedFromTheKeptOneMeanwhileKeepsTheOtherOutOfTheTrash()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var merger = Merger(fx);
|
||||
var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
|
||||
await engine.StartAsync(Ct);
|
||||
|
||||
// A photo added to the other after the verify.
|
||||
var keep = await SequenceAsync(fx, "keep-a", [(1, 0, "IMG_0001.JPG")]);
|
||||
var drop = await SequenceAsync(fx, "drop-a", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
|
||||
var copy = await PairAsync(fx, keep.Id, drop.Id);
|
||||
handler.BeforeStage = async stage =>
|
||||
{
|
||||
if (stage != MergeStage.Trash) return;
|
||||
var (path, size, sha) = fx.Stage(Start.AddSeconds(40), 3);
|
||||
await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, drop.Id, null, null, path, "IMG_0003.JPG", size, sha, null, null), fx.Ct);
|
||||
};
|
||||
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
|
||||
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
|
||||
await fx.WaitForAsync(job.Id, JobState.Failed);
|
||||
Assert.Count(3, await fx.Db.Frames.ListBySequenceAsync(drop.Id));
|
||||
|
||||
// A verified photo deleted from the kept one before the trash.
|
||||
keep = await SequenceAsync(fx, "keep-b", [(11, 0, "IMG_0001.JPG")]);
|
||||
drop = await SequenceAsync(fx, "drop-b", [(11, 0, "IMG_0001.JPG"), (12, 10_000, "IMG_0002.JPG")]);
|
||||
copy = await PairAsync(fx, keep.Id, drop.Id);
|
||||
var keepId = keep.Id;
|
||||
Guid? running = null;
|
||||
handler.BeforeStage = async stage =>
|
||||
{
|
||||
if (stage != MergeStage.Trash) return;
|
||||
// Another hand deletes it (the job's own hold does not count, so the deletion is not refused as busy).
|
||||
var gone = (await fx.Db.Frames.ListBySequenceAsync(keepId)).First(f => f.Sha256 == Sha(12, 10_000));
|
||||
var trash = await fx.Service.FrameTrash.PlanDeleteAsync(keepId, [gone.Id], null, fx.Ct, running);
|
||||
await fx.Service.FrameTrash.DeleteAsync(trash, null, null, fx.Ct, running);
|
||||
};
|
||||
plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
|
||||
job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
|
||||
running = job.Id;
|
||||
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
|
||||
Assert.Contains(keep.DisplayTitle, failed.Error!);
|
||||
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task UnsentWorkspaceWorkStopsTheTrashAndTheTrashStepIsNotBusyWithItsOwnJob()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
|
||||
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
|
||||
var copy = await PairAsync(fx, keep.Id, drop.Id);
|
||||
var merger = Merger(fx);
|
||||
var workspace = new FakeWorkspace();
|
||||
var (engine, handler) = Engine(fx, merger, new FakeProjects(), workspace);
|
||||
await engine.StartAsync(Ct);
|
||||
handler.BeforeStage = stage =>
|
||||
{
|
||||
if (stage == MergeStage.Trash) workspace.Unsent.Add(drop.Id);
|
||||
return Task.CompletedTask;
|
||||
};
|
||||
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
|
||||
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
|
||||
await fx.WaitForAsync(job.Id, JobState.Failed);
|
||||
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
|
||||
// The photos are in the kept one already.
|
||||
Assert.True((await HashesAsync(fx, keep.Id)).SetEquals([Sha(1, 0), Sha(2, 10_000)]));
|
||||
|
||||
handler.BeforeStage = null;
|
||||
workspace.Unsent.Clear();
|
||||
await engine.ResumeAsync(job.Id, Ct);
|
||||
await fx.WaitForAsync(job.Id, JobState.Done);
|
||||
Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ACancelReturnsThePairToItsStateAndLeavesTheOtherAlone()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
|
||||
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
|
||||
var copy = await PairAsync(fx, keep.Id, drop.Id);
|
||||
var merger = Merger(fx);
|
||||
var (engine, handler) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
|
||||
await engine.StartAsync(Ct);
|
||||
Guid? id = null;
|
||||
handler.BeforeStage = async stage =>
|
||||
{
|
||||
if (stage == MergeStage.Verify) await engine.CancelAsync(id!.Value, CancellationToken.None);
|
||||
};
|
||||
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
|
||||
var store = new FoundCopyStore(fx.Db);
|
||||
var job = await merger.EnqueueAsync(engine, store, plan, keep.DisplayTitle, Ct);
|
||||
id = job.Id;
|
||||
await fx.WaitForAsync(job.Id, JobState.Cancelled);
|
||||
var until = DateTime.UtcNow.AddSeconds(10);
|
||||
while (File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)) && DateTime.UtcNow < until) await Task.Delay(20, Ct);
|
||||
Assert.False(File.Exists(MergeJournal.PathFor(fx.Paths, job.Id)));
|
||||
var pair = (await store.GetAsync(copy.Id, Ct))!;
|
||||
Assert.Equal(CopyState.Candidate, pair.State);
|
||||
Assert.NotNull(await fx.Db.Sequences.GetAsync(drop.Id));
|
||||
Assert.Count(2, await fx.Db.Frames.ListBySequenceAsync(drop.Id));
|
||||
|
||||
// Planned and merged again from the start.
|
||||
plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
|
||||
handler.BeforeStage = null;
|
||||
job = await merger.EnqueueAsync(engine, store, plan, keep.DisplayTitle, Ct);
|
||||
await fx.WaitForAsync(job.Id, JobState.Done);
|
||||
Assert.Null(await fx.Db.Sequences.GetAsync(drop.Id));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AJobWhosePlanMovedSinceTheDialogRefusesAndChangesNothing()
|
||||
{
|
||||
var fx = await FixtureAsync();
|
||||
var keep = await SequenceAsync(fx, "keep", [(1, 0, "IMG_0001.JPG")]);
|
||||
var drop = await SequenceAsync(fx, "drop", [(1, 0, "IMG_0001.JPG"), (2, 10_000, "IMG_0002.JPG")]);
|
||||
var copy = await PairAsync(fx, keep.Id, drop.Id);
|
||||
var merger = Merger(fx);
|
||||
var (engine, _) = Engine(fx, merger, new FakeProjects(), new FakeWorkspace());
|
||||
var plan = await merger.PlanAsync(copy.Id, keep.Id, drop.Id, await ChoicesAsync(fx, keep.Id, drop.Id, CopyPhotoPolicy.Union), NoContext, Ct);
|
||||
var job = await merger.EnqueueAsync(engine, new FoundCopyStore(fx.Db), plan, keep.DisplayTitle, Ct);
|
||||
await SequenceAsync(fx, "drop", [(3, 20_000, "IMG_0003.JPG")]); // the session key files it into the other
|
||||
await engine.StartAsync(Ct);
|
||||
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
|
||||
Assert.Equal(Chronos.Core.Localization.Loc.L("Errors.Copies.StalePlan"), failed.Error);
|
||||
Assert.Count(1, await fx.Db.Frames.ListBySequenceAsync(keep.Id));
|
||||
Assert.Count(3, await fx.Db.Frames.ListBySequenceAsync(drop.Id));
|
||||
}
|
||||
|
||||
private sealed class FakeProjects : IProjectCopier
|
||||
{
|
||||
public List<(Guid Source, string Name)> Copied { get; } = [];
|
||||
|
||||
public async Task<SequenceProject> CopyAsync(Chronos.Data.Projects.IProjectHome from, SequenceProject source, Chronos.Data.Projects.IProjectHome to,
|
||||
string name, TransplantStep? journal, Func<TransplantStep, ValueTask> save, CancellationToken ct)
|
||||
{
|
||||
await save(new TransplantStep(name, DateTimeOffset.UtcNow, null, null, 0, []));
|
||||
lock (Copied) Copied.Add((source.Id, name));
|
||||
return source with { Id = Guid.CreateVersion7(), Name = name };
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeWorkspace : IWorkspaceGuard
|
||||
{
|
||||
public HashSet<Guid> Unsent { get; } = [];
|
||||
|
||||
public Task<bool> HasUnsentAsync(Guid sequenceId, CancellationToken ct) => Task.FromResult(Unsent.Contains(sequenceId));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class SqliteSequenceMergerTests(TestContext context) : SequenceMergerTests(context)
|
||||
{
|
||||
protected override Backend Backend => Backend.SqliteFile;
|
||||
}
|
||||
|
||||
[RequiresEnvironment(PgTestServer.Variable)]
|
||||
public sealed class PostgresSequenceMergerTests(TestContext context) : SequenceMergerTests(context)
|
||||
{
|
||||
protected override Backend Backend => Backend.Postgres;
|
||||
}
|
||||
Reference in new issue
Block a user