The fourth verification round of the matches work found 18 defects in the merge, Integrate and Matches surfaces. This commit fixes them: - An Integrate job now carries the planning account's id, so its re-plan decides ownership the way the dialog did. - A merge skips only a project the kept sequence holds a real copy of (name, offset head and history content) and re-points its renders to that copy. - A merge runs its trash checks before the Links stage and records an undo, so an abandoned merge gives the other sequence back its link, session keys and pairs. - The metered hold adopts automatic transfers that were paused before a restart. - Apply in the Compare dialog re-plans when the plan is pending or stale. - The Library's Two versions... lists the second versions a Union merge leaves, each with Drop... (dry run, confirmation, verified partner). A merge job's Open shows that list. - The merge's typed confirmation is the invariant count, and the prompt names it in all 38 languages. - Italian text uses Principale and Gemello. The Main project is named through WorkspaceText in the Projects part and in the twin tooltip. - The session keys tooltip is split into two keys, and To trash... and Delete on server... say why they are disabled. - The capture restores the editor account before the Matches scenes. - Upload again follows SyncHost.CanSendSync (Upload with Edit). - Matches rows trim archive names in the middle on a rebalanced grid; the Projects part rows trim, with the full text in the tooltip. - The sample project targets use ProjectTransplant.FreeName. DECISIONS, CHANGELOG, STATUS 8.7, the request doc and the 16 changed capture baselines are updated. Refs #73 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
987 lines
55 KiB
C#
987 lines
55 KiB
C#
using Chronos.Core.Archiving;
|
|
using Chronos.Core.Diagnostics;
|
|
using Chronos.Core.Import;
|
|
using Chronos.Core.Localization;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Data.Copies;
|
|
using Chronos.Data.Jobs;
|
|
using Chronos.Data.Projects;
|
|
using Chronos.Data.Repositories;
|
|
using Chronos.Data.Sync;
|
|
using ProcessingFrames = Chronos.Data.Processing.SequenceFrames;
|
|
|
|
namespace Chronos.Data.Archiving.Jobs;
|
|
|
|
/// <summary>
|
|
/// <see cref="JobKind.MergeSequences"/> (owner review 6, #73; design §5d): applies the merge plan the user confirmed
|
|
/// (<see cref="MergeJobParameters"/>). Its first run plans again and refuses a plan whose fingerprint moved; the journal
|
|
/// (<see cref="MergeJournal"/>) then keeps what that plan decided and how far each stage came, so a crash, a pause or a
|
|
/// retry resumes where it stopped. Stages: copy the other's photos into the kept sequence (each hash-checked, filed with
|
|
/// tombstones honoured), move the renders, verify every original of the other is held by the kept one (hashed again on
|
|
/// both sides) or listed as a last copy, trash the kept sequence's losing versions, write the details, copy the projects,
|
|
/// move session keys, links and pairs, then move the other to its archive's trash, only while it is still at the frames
|
|
/// version the verify read and the kept photos verified are still there. Every check that can still abandon the merge runs
|
|
/// before the links stage changes anything, and again at the trash; a merge abandoned after the links stage (a check that
|
|
/// fails at the trash, a cancel) gives the other back its keys, links and pairs first (<see cref="MergeLinksUndo"/>). A
|
|
/// cancel leaves the photos copied so far in the kept sequence and drops the journal.
|
|
/// </summary>
|
|
internal sealed class MergeJobHandler(ChronosDatabase db, SequenceMerger merger, ArchiveService archives, IProjectCopier projects,
|
|
IWorkspaceGuard workspace) : ICancelledJobHandler
|
|
{
|
|
/// <summary>The change told about the kept sequence.</summary>
|
|
public const string KeepChange = "frames";
|
|
|
|
/// <summary>The change told about the sequence that went to the trash.</summary>
|
|
public const string DropChange = "deleted";
|
|
|
|
private static readonly CopyPlanContext JobContextPlan = new(string.Empty, null, false, false, false, false, false, false, null,
|
|
new HashSet<Guid>(), new HashSet<Guid>());
|
|
|
|
public JobKind Kind => JobKind.MergeSequences;
|
|
|
|
private ChronosDatabase Db => db;
|
|
|
|
private ArchiveService Archives => archives;
|
|
|
|
private IProjectCopier Projects => projects;
|
|
|
|
private IWorkspaceGuard Workspace => workspace;
|
|
|
|
private SequenceMerger Merger => merger;
|
|
|
|
/// <summary>Called before each stage runs (tests: a crash, a change made between stages).</summary>
|
|
internal Func<MergeStage, Task>? BeforeStage { get; set; }
|
|
|
|
public async Task RunAsync(JobContext context)
|
|
{
|
|
var job = context.Job;
|
|
var ct = context.Cancellation;
|
|
var parameters = MergeJobParameters.Read(job.ParametersJson) ?? throw new InvalidOperationException($"Merge job {job.Id} has no plan.");
|
|
var keepRow = await db.Sequences.GetAsync(parameters.KeepId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
|
|
var keepArchive = await db.Archives.GetAsync(keepRow.ArchiveId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
|
|
var keepPaths = new ArchivePaths(keepArchive.RootPath);
|
|
try
|
|
{
|
|
var journal = MergeJournal.Read(keepPaths, job.Id);
|
|
if (journal is null) journal = await StartAsync(context, parameters, keepRow, keepPaths).ConfigureAwait(false);
|
|
else await ResumeAsync(job, parameters, keepPaths, ct).ConfigureAwait(false);
|
|
var staging = MergeJournal.StagingFor(keepPaths, job.Id);
|
|
if (Directory.Exists(staging)) Directory.Delete(staging, recursive: true);
|
|
await new Run(this, context, journal, keepArchive, keepPaths).RunAsync().ConfigureAwait(false);
|
|
}
|
|
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
|
{
|
|
if (await db.Jobs.GetAsync(job.Id, CancellationToken.None).ConfigureAwait(false) is { State: JobState.Cancelled })
|
|
await ForgetAsync(job, parameters, keepPaths).ConfigureAwait(false);
|
|
throw;
|
|
}
|
|
}
|
|
|
|
public async Task CancelledAsync(Job job, CancellationToken cancellationToken)
|
|
{
|
|
try
|
|
{
|
|
if (MergeJobParameters.Read(job.ParametersJson) is not { } parameters) return;
|
|
if (await db.Sequences.GetAsync(parameters.KeepId, cancellationToken).ConfigureAwait(false) is not { } keep) return;
|
|
if (await db.Archives.GetAsync(keep.ArchiveId, cancellationToken).ConfigureAwait(false) is not { } archive) return;
|
|
await ForgetAsync(job, parameters, new ArchivePaths(archive.RootPath)).ConfigureAwait(false);
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
|
{
|
|
Log.Warning("archive", $"The cancelled merge {job.Id} left its journal; the archive's next maintenance removes it", ex);
|
|
}
|
|
}
|
|
|
|
/// <summary>A cancelled merge: its journal and staging go, and its pair returns to the state it had (when it is still this job's).</summary>
|
|
private async Task ForgetAsync(Job job, MergeJobParameters parameters, ArchivePaths keepPaths)
|
|
{
|
|
await MoveBackAsync(keepPaths, job.Id).ConfigureAwait(false);
|
|
try
|
|
{
|
|
MergeJournal.Delete(keepPaths, job.Id);
|
|
}
|
|
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
|
{
|
|
Log.Warning("archive", $"The journal of the cancelled merge {job.Id} was not removed", ex);
|
|
}
|
|
// By the stored row: the engine marked the job Cancelled first, so the row already reads as its state before.
|
|
await new FoundCopyStore(db).ReleaseAsync(parameters.CopyId, job.Id, null, keepJob: false, CancellationToken.None).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <summary>Gives back what the links stage of job <paramref name="jobId"/>'s journal moved (when it has one and the stage began).</summary>
|
|
private async Task MoveBackAsync(ArchivePaths keepPaths, Guid jobId)
|
|
{
|
|
MergeJournal? journal;
|
|
try
|
|
{
|
|
journal = MergeJournal.Read(keepPaths, jobId);
|
|
}
|
|
catch (InvalidDataException ex)
|
|
{
|
|
Log.Warning("archive", $"The journal of merge {jobId} could not be read: what its links stage moved stays with the kept sequence", ex);
|
|
return;
|
|
}
|
|
if (journal is not null) await MoveBackAsync(journal).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gives the other sequence back what the links stage took (<see cref="MergeJournal.LinksUndo"/>): its rows (session keys,
|
|
/// links, pairs), then the session keys of both manifests; the journal is set back to before the links stage (not saved).
|
|
/// Nothing when the stage had not begun, or the other is gone (the trash ran). False when it could not be done.
|
|
/// </summary>
|
|
internal async Task<bool> MoveBackAsync(MergeJournal journal)
|
|
{
|
|
if (journal.LinksUndo is not { } undo) return true;
|
|
var ct = CancellationToken.None;
|
|
try
|
|
{
|
|
var drop = await db.Sequences.GetAsync(journal.DropId, ct).ConfigureAwait(false);
|
|
var keep = await db.Sequences.GetAsync(journal.KeepId, ct).ConfigureAwait(false);
|
|
if (drop is null || keep is null) return true;
|
|
await new FoundCopyStore(db).MoveBackAsync(journal.DropId, journal.KeepId, keep.ArchiveId, undo.DropArchiveId, undo.Moved, ct).ConfigureAwait(false);
|
|
var keys = journal.SessionKeys.ToHashSet(StringComparer.Ordinal);
|
|
var added = keys.Where(k => !undo.KeepKeysBefore.Contains(k, StringComparer.Ordinal)).ToHashSet(StringComparer.Ordinal);
|
|
var removed = keys.Where(k => undo.DropKeysBefore.Contains(k, StringComparer.Ordinal)).ToList();
|
|
if (added.Count > 0 && await db.Archives.GetAsync(keep.ArchiveId, ct).ConfigureAwait(false) is { } keepArchive)
|
|
{
|
|
var paths = new ArchivePaths(keepArchive.RootPath);
|
|
if (File.Exists(paths.Sequence(keep.Folder).ManifestPath))
|
|
await archives.Writer.UpdateAsync(paths, keep.Id, keep.Folder, manifest =>
|
|
!manifest.SessionKeys.Any(added.Contains) ? null : manifest with { SessionKeys = [.. manifest.SessionKeys.Where(k => !added.Contains(k))] },
|
|
ct).ConfigureAwait(false);
|
|
}
|
|
if (removed.Count > 0 && await db.Archives.GetAsync(drop.ArchiveId, ct).ConfigureAwait(false) is { } dropArchive)
|
|
{
|
|
var paths = new ArchivePaths(dropArchive.RootPath);
|
|
if (File.Exists(paths.Sequence(drop.Folder).ManifestPath))
|
|
await archives.Writer.UpdateAsync(paths, drop.Id, drop.Folder, manifest =>
|
|
removed.All(manifest.SessionKeys.Contains) ? null : manifest with { SessionKeys = [.. manifest.SessionKeys.Union(removed, StringComparer.Ordinal)] },
|
|
ct).ConfigureAwait(false);
|
|
}
|
|
journal.LinksUndo = null;
|
|
journal.KeepKeysWritten = false;
|
|
journal.DropKeysRemoved = false;
|
|
journal.LinksMoved = false;
|
|
if (journal.Stage > MergeStage.Links) journal.Stage = MergeStage.Links;
|
|
return true;
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
|
{
|
|
Log.Warning("archive", $"Merge {journal.JobId}: what its links stage moved could not be given back to the other sequence", ex);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// A resumed run (a pause, a crash, or a retry after a failure) goes on only while the pair is still this job's: it reads
|
|
/// Integrating with this job (a failure keeps both; a Keep apart, an Ask again or another merge since clears them), or
|
|
/// it is gone with the other sequence (the trash stage ran before the journal said so). Otherwise the journal is stale:
|
|
/// it and its staging go and nothing more is applied; the pair, no longer this job's, is left as it is.
|
|
/// </summary>
|
|
private async Task ResumeAsync(Job job, MergeJobParameters parameters, ArchivePaths keepPaths, CancellationToken ct)
|
|
{
|
|
var copy = await new FoundCopyStore(db).GetAsync(parameters.CopyId, ct).ConfigureAwait(false);
|
|
if (copy is { State: CopyState.Integrating } && copy.JobId == job.Id) return;
|
|
if (copy is null && await db.Sequences.GetAsync(parameters.DropId, ct).ConfigureAwait(false) is null) return;
|
|
await MoveBackAsync(keepPaths, job.Id).ConfigureAwait(false);
|
|
try
|
|
{
|
|
MergeJournal.Delete(keepPaths, job.Id);
|
|
}
|
|
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
|
{
|
|
Log.Warning("archive", $"The stale journal of merge {job.Id} was not removed", ex);
|
|
}
|
|
throw new StaleCopyPlanException();
|
|
}
|
|
|
|
/// <summary>The first run: plans again, refuses a plan that moved or is blocked, and journals it.</summary>
|
|
private async Task<MergeJournal> StartAsync(JobContext context, MergeJobParameters parameters, Sequence keepRow, ArchivePaths keepPaths)
|
|
{
|
|
var ct = context.Cancellation;
|
|
var copy = await new FoundCopyStore(db).GetAsync(parameters.CopyId, ct).ConfigureAwait(false);
|
|
if (copy is null || copy.JobId != context.Job.Id) throw new StaleCopyPlanException();
|
|
if (await workspace.HasUnsentAsync(parameters.DropId, ct).ConfigureAwait(false))
|
|
throw new InvalidOperationException(Loc.L("Errors.Copies.Unsent", (await db.Sequences.GetAsync(parameters.DropId, ct).ConfigureAwait(false))?.DisplayTitle ?? string.Empty));
|
|
var plan = await merger.PlanAsync(parameters.CopyId, parameters.KeepId, parameters.DropId, parameters.Choices, JobContextPlan, context.Job.Id, ct)
|
|
.ConfigureAwait(false);
|
|
if (plan.KeepId != parameters.KeepId || plan.DropId != parameters.DropId) throw new StaleCopyPlanException();
|
|
if (!plan.CanApply) throw new InvalidOperationException(await merger.BlockMessageAsync(plan, ct).ConfigureAwait(false));
|
|
if (!string.Equals(plan.Fingerprint, parameters.Fingerprint, StringComparison.Ordinal)) throw new StaleCopyPlanException();
|
|
|
|
var winners = plan.Photos.Where(p => p.Action == PhotoAction.FetchVersion).ToDictionary(p => p.Key, p => p.Sha256, StringComparer.Ordinal);
|
|
var journal = new MergeJournal
|
|
{
|
|
JobId = context.Job.Id,
|
|
CopyId = plan.CopyId,
|
|
KeepId = plan.KeepId,
|
|
DropId = plan.DropId,
|
|
Fingerprint = plan.Fingerprint,
|
|
Stage = MergeStage.Copy,
|
|
KeepSettingsVersion = plan.KeepSettingsVersion,
|
|
DropFramesVersion = plan.DropFramesVersion,
|
|
DropSettingsVersion = plan.DropSettingsVersion,
|
|
KeepStatus = (int)keepRow.Status,
|
|
Copies = [.. plan.Photos.Where(p => p.Action is PhotoAction.Fetch or PhotoAction.FetchVersion && p.FrameId is not null)
|
|
.Select(p => new MergeCopy(p.Key, p.Action, p.FrameId!.Value, p.FileName, p.Bytes, p.Sha256))],
|
|
Replaces = [.. plan.Photos.Where(p => p.Action == PhotoAction.TrashHere && p.FrameId is not null && winners.GetValueOrDefault(p.Key) is { Length: > 0 })
|
|
.Select(p => new MergeReplace(p.Key, p.FrameId!.Value, p.FileName, winners[p.Key]!))],
|
|
Absent = [.. plan.LastCopies.Where(l => l.Side == CopySide.Other).Select(l => l.Key.ToLowerInvariant())],
|
|
Renders = [.. plan.Renders],
|
|
Payload = plan.Payload,
|
|
Projects = [.. plan.Projects],
|
|
SessionKeys = [.. plan.SessionKeys],
|
|
};
|
|
journal.Save(keepPaths);
|
|
return journal;
|
|
}
|
|
|
|
/// <summary>One run of a merge, from the stage its journal reached.</summary>
|
|
private sealed class Run(MergeJobHandler owner, JobContext context, MergeJournal journal, Archive keepArchive, ArchivePaths keepPaths)
|
|
{
|
|
private readonly ChronosDatabase _db = owner.Db;
|
|
private readonly CancellationToken _ct = context.Cancellation;
|
|
private readonly string _staging = MergeJournal.StagingFor(keepPaths, journal.JobId);
|
|
private Archive _dropArchive = null!;
|
|
private ArchivePaths _dropPaths = null!;
|
|
private string _keepTitle = string.Empty;
|
|
private string _dropTitle = string.Empty;
|
|
private bool _forget;
|
|
|
|
private long Total => journal.Copies.Count + journal.Renders.Count + journal.Replaces.Count + journal.Projects.Count + 4;
|
|
|
|
public async Task RunAsync()
|
|
{
|
|
var keep = await _db.Sequences.GetAsync(journal.KeepId, _ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
|
|
_keepTitle = keep.DisplayTitle;
|
|
var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false);
|
|
if (drop is null && journal.Stage < MergeStage.Trash) throw new InvalidOperationException(Loc.L("Errors.Copies.StalePlan"));
|
|
if (drop is not null)
|
|
{
|
|
_dropTitle = drop.DisplayTitle;
|
|
_dropArchive = await _db.Archives.GetAsync(drop.ArchiveId, _ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
|
|
_dropPaths = new ArchivePaths(_dropArchive.RootPath);
|
|
}
|
|
|
|
while (journal.Stage != MergeStage.Done)
|
|
{
|
|
_ct.ThrowIfCancellationRequested();
|
|
if (owner.BeforeStage is { } hook) await hook(journal.Stage).ConfigureAwait(false);
|
|
try
|
|
{
|
|
switch (journal.Stage)
|
|
{
|
|
case MergeStage.Copy: await CopyAsync().ConfigureAwait(false); break;
|
|
case MergeStage.Renders: await RendersAsync().ConfigureAwait(false); break;
|
|
case MergeStage.Verify: await VerifyAsync().ConfigureAwait(false); break;
|
|
case MergeStage.Replace: await ReplaceAsync().ConfigureAwait(false); break;
|
|
case MergeStage.Details: await DetailsAsync().ConfigureAwait(false); break;
|
|
case MergeStage.Projects: await ProjectsAsync().ConfigureAwait(false); break;
|
|
case MergeStage.Links: await LinksAsync().ConfigureAwait(false); break;
|
|
case MergeStage.Trash: await TrashAsync().ConfigureAwait(false); break;
|
|
}
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
|
{
|
|
await AbandonAsync().ConfigureAwait(false);
|
|
throw;
|
|
}
|
|
journal.Stage++;
|
|
Save();
|
|
await context.CheckpointAsync($"merge:{journal.Stage}", Done(), Total).ConfigureAwait(false);
|
|
}
|
|
await FinishAsync().ConfigureAwait(false);
|
|
}
|
|
|
|
private long Done() => journal.Committed.Count + journal.RendersMoved.Count + journal.Replaced.Count + journal.KeptLosers.Count
|
|
+ journal.ProjectsDone.Count + Math.Min(4, Math.Max(0, journal.Stage - MergeStage.Verify));
|
|
|
|
private void Save() => journal.Save(keepPaths);
|
|
|
|
private InvalidOperationException Changed(string title) => new(Loc.L("Errors.Copies.Changed", title));
|
|
|
|
/// <summary>Asks for this run's journal and staging to go as it fails (<see cref="AbandonAsync"/>): what it planned no longer holds.</summary>
|
|
private void ForgetJournal() => _forget = true;
|
|
|
|
/// <summary>
|
|
/// A stage failed: what the links stage took from the other goes back to it first (a retry runs that stage again, after
|
|
/// its checks), then the journal goes when <see cref="ForgetJournal"/> asked for it, else it is saved as set back. When
|
|
/// nothing could be given back, the journal stays (at the stage it reached) so a retry or a cancel tries again.
|
|
/// </summary>
|
|
private async Task AbandonAsync()
|
|
{
|
|
var linksBegun = journal.LinksUndo is not null;
|
|
if (!await owner.MoveBackAsync(journal).ConfigureAwait(false)) return;
|
|
if (_forget)
|
|
{
|
|
try
|
|
{
|
|
MergeJournal.Delete(keepPaths, journal.JobId);
|
|
}
|
|
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
|
{
|
|
Log.Warning("archive", $"The stale journal of merge {journal.JobId} was not removed", ex);
|
|
}
|
|
}
|
|
else if (linksBegun)
|
|
{
|
|
Save();
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// The other sequence's files are where the index says, on a volume that answers: a verify or a trash never reads an
|
|
/// absent drive or a moved folder as "nothing there to lose".
|
|
/// </summary>
|
|
private void EnsureDropReadable(Sequence drop)
|
|
{
|
|
if (ArchiveVolume.Probe(_dropPaths) != ArchiveAvailability.Online)
|
|
throw new InvalidOperationException(Loc.L("Errors.Copies.ArchiveOffline", _dropTitle));
|
|
if (drop.Folder.Length > 0 && !SequenceFolders.Holds(_dropPaths.Sequence(drop.Folder), drop.Id))
|
|
throw new InvalidOperationException(Loc.L("Errors.Copies.FolderMoved", _dropTitle));
|
|
}
|
|
|
|
private async Task EnsureDropUnchangedAsync(int expected)
|
|
{
|
|
var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false);
|
|
if (drop is null || drop.FramesVersion != expected) throw Changed(_dropTitle);
|
|
}
|
|
|
|
// ---- copy ----
|
|
|
|
private async Task CopyAsync()
|
|
{
|
|
var n = 0;
|
|
foreach (var copy in journal.Copies)
|
|
{
|
|
n++;
|
|
if (journal.Committed.ContainsKey(copy.DropFrameId)) continue;
|
|
_ct.ThrowIfCancellationRequested();
|
|
await EnsureDropUnchangedAsync(journal.DropFramesVersion).ConfigureAwait(false);
|
|
var frame = await _db.Frames.GetAsync(copy.DropFrameId, _ct).ConfigureAwait(false) ?? throw Changed(_dropTitle);
|
|
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.CopyPhoto, copy.FileName, n, journal.Copies.Count)).ConfigureAwait(false);
|
|
var result = await FileAsync(frame, copy.Sha256, planned: copy.Sha256 is { Length: > 0 }).ConfigureAwait(false);
|
|
journal.Committed[copy.DropFrameId] = result.FrameId;
|
|
Save();
|
|
context.Log(Loc.L("Archive.Log.MergeCopied", copy.FileName));
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Copies a frame of the other into staging (hash-checked against <paramref name="expected"/>) and files it in the
|
|
/// kept sequence. <paramref name="planned"/>: the plan knew the photo's content (else a tombstone of the kept sequence
|
|
/// may refuse it although the plan listed no last copy: the plan is stale, never this job's to retry).
|
|
/// </summary>
|
|
private async Task<IngestResult> FileAsync(Frame frame, string? expected, bool planned)
|
|
{
|
|
Directory.CreateDirectory(_staging);
|
|
var staged = Path.Combine(_staging, frame.Id.ToString("N") + Path.GetExtension(frame.FileName));
|
|
if (File.Exists(staged)) File.Delete(staged);
|
|
string sha;
|
|
try
|
|
{
|
|
sha = await VerifiedCopy.CopyAsync(_dropPaths.FromRelative(frame.RelativePath), staged, verify: true, null, _ct).ConfigureAwait(false);
|
|
}
|
|
catch (Exception ex) when (ex is FileNotFoundException or DirectoryNotFoundException)
|
|
{
|
|
throw new InvalidOperationException(Loc.L("Errors.Copies.ChangedFile", frame.FileName), ex);
|
|
}
|
|
if (expected is { Length: > 0 } && !string.Equals(sha, expected, StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
File.Delete(staged);
|
|
throw new InvalidOperationException(Loc.L("Errors.Copies.ChangedFile", frame.FileName));
|
|
}
|
|
var size = new FileInfo(staged).Length;
|
|
IngestResult result;
|
|
try
|
|
{
|
|
result = await owner.Archives.Ingest.CommitAsync(new IngestRequest(keepArchive.Id, journal.KeepId, null, null, staged, frame.FileName, size,
|
|
sha.ToLowerInvariant(), frame.CapturedAt, null, RefuseRemoved: true), _ct).ConfigureAwait(false);
|
|
}
|
|
catch (PhotoRemovedException)
|
|
{
|
|
File.Delete(staged);
|
|
if (planned) throw Changed(_keepTitle); // deleted from the kept sequence since the plan: a last copy the plan did not list
|
|
// A photo the plan could not compare by content (unhashed) that the kept sequence deleted: a last copy the
|
|
// plan did not list. Retrying this journal would fail the same way: it goes, and the user plans again.
|
|
ForgetJournal();
|
|
throw new StaleCopyPlanException();
|
|
}
|
|
if (File.Exists(staged)) File.Delete(staged);
|
|
return result;
|
|
}
|
|
|
|
// ---- renders ----
|
|
|
|
private async Task RendersAsync()
|
|
{
|
|
var keep = await _db.Sequences.GetAsync(journal.KeepId, _ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
|
|
var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false) ?? throw Changed(_dropTitle);
|
|
var n = 0;
|
|
foreach (var step in journal.Renders)
|
|
{
|
|
n++;
|
|
if (journal.RendersMoved.Contains(step.RenderId)) continue;
|
|
_ct.ThrowIfCancellationRequested();
|
|
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.CopyVideo, step.FileName, n, journal.Renders.Count)).ConfigureAwait(false);
|
|
var render = await _db.Renders.GetAsync(step.RenderId, _ct).ConfigureAwait(false);
|
|
if (render is null || render.SequenceId == journal.KeepId)
|
|
{
|
|
journal.RendersMoved.Add(step.RenderId);
|
|
Save();
|
|
continue;
|
|
}
|
|
var source = SyncService.RenderFile(_dropArchive, render);
|
|
var relative = render.OutputPath.Replace('\\', '/');
|
|
var inside = !Path.IsPathRooted(render.OutputPath) && drop.Folder.Length > 0
|
|
&& relative.StartsWith(drop.Folder.TrimEnd('/') + "/", StringComparison.Ordinal);
|
|
string output;
|
|
if (inside && (File.Exists(source) || Directory.Exists(source)))
|
|
{
|
|
// Inside the folder that goes to the trash: copied (verified) into the kept sequence's exports first.
|
|
if (!journal.RenderTargets.TryGetValue(step.RenderId, out var target))
|
|
{
|
|
target = keepPaths.ToRelative(Free(keepPaths.Sequence(keep.Folder).Exports, Path.GetFileName(source)));
|
|
journal.RenderTargets[step.RenderId] = target;
|
|
Save();
|
|
}
|
|
var destination = keepPaths.FromRelative(target);
|
|
if (Directory.Exists(source)) await CopyFolderAsync(source, destination).ConfigureAwait(false);
|
|
else await CopyRenderAsync(source, destination, render.Sha256, step.FileName).ConfigureAwait(false);
|
|
output = target;
|
|
}
|
|
else
|
|
{
|
|
// Outside it (or gone): only the row moves, naming the same file wherever the kept sequence's archive is.
|
|
output = Path.IsPathRooted(render.OutputPath) || keepArchive.Id == _dropArchive.Id ? render.OutputPath : source;
|
|
}
|
|
var access = new DbAccess(_db.Database);
|
|
await access.InTransactionAsync(async session =>
|
|
{
|
|
var inner = new DbAccess(session);
|
|
await inner.ExecuteAsync("UPDATE renders SET sequence_id = $2, output_path = $3 WHERE id = $1", [step.RenderId, journal.KeepId, output], _ct)
|
|
.ConfigureAwait(false);
|
|
foreach (var id in (Guid[])[journal.KeepId, journal.DropId])
|
|
await inner.ExecuteAsync("UPDATE sequences SET rendered_count = (SELECT count(*) FROM renders WHERE sequence_id = $1) WHERE id = $1", [id], _ct)
|
|
.ConfigureAwait(false);
|
|
return 0;
|
|
}, _ct).ConfigureAwait(false);
|
|
journal.RendersMoved.Add(step.RenderId);
|
|
Save();
|
|
}
|
|
}
|
|
|
|
private async Task CopyRenderAsync(string source, string destination, string? sha, string name)
|
|
{
|
|
if (File.Exists(destination))
|
|
{
|
|
// A run that copied it before its row moved: kept when it is the same file.
|
|
if (sha is { Length: > 0 } && string.Equals(await ContentHash.Sha256Async(destination, null, _ct).ConfigureAwait(false), sha, StringComparison.OrdinalIgnoreCase))
|
|
return;
|
|
File.Delete(destination);
|
|
}
|
|
var copied = await VerifiedCopy.CopyAsync(source, destination, verify: true, null, _ct).ConfigureAwait(false);
|
|
if (sha is { Length: > 0 } && !string.Equals(copied, sha, StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
File.Delete(destination);
|
|
throw new InvalidOperationException(Loc.L("Errors.Copies.ChangedFile", name));
|
|
}
|
|
}
|
|
|
|
private async Task CopyFolderAsync(string source, string destination)
|
|
{
|
|
foreach (var file in Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories))
|
|
{
|
|
var target = Path.Combine(destination, Path.GetRelativePath(source, file));
|
|
if (File.Exists(target) && new FileInfo(target).Length == new FileInfo(file).Length
|
|
&& string.Equals(await ContentHash.Sha256Async(target, null, _ct).ConfigureAwait(false),
|
|
await ContentHash.Sha256Async(file, null, _ct).ConfigureAwait(false), StringComparison.Ordinal))
|
|
continue;
|
|
if (File.Exists(target)) File.Delete(target);
|
|
await VerifiedCopy.CopyAsync(file, target, verify: true, null, _ct).ConfigureAwait(false);
|
|
}
|
|
}
|
|
|
|
private static string Free(string folder, string name)
|
|
{
|
|
var stem = Path.GetFileNameWithoutExtension(name);
|
|
var extension = Path.GetExtension(name);
|
|
var candidate = Path.Combine(folder, name);
|
|
for (var n = 2; File.Exists(candidate) || Directory.Exists(candidate); n++) candidate = Path.Combine(folder, $"{stem} ({n}){extension}");
|
|
return candidate;
|
|
}
|
|
|
|
// ---- verify ----
|
|
|
|
private async Task VerifyAsync()
|
|
{
|
|
var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false) ?? throw Changed(_dropTitle);
|
|
if (drop.FramesVersion != journal.DropFramesVersion) throw Changed(_dropTitle);
|
|
EnsureDropReadable(drop);
|
|
var keepFrames = await ProcessingFrames.ListAsync(_db, journal.KeepId, _ct).ConfigureAwait(false);
|
|
var dropFrames = (await ProcessingFrames.ListAsync(_db, journal.DropId, _ct).ConfigureAwait(false))
|
|
.Where(f => (f.Flags & CopySides.UnusableFlags) == 0).ToList();
|
|
var total = keepFrames.Count + dropFrames.Count;
|
|
var done = 0;
|
|
|
|
// What the kept sequence holds now, hashed again.
|
|
var held = new Dictionary<string, Guid>(StringComparer.Ordinal);
|
|
foreach (var frame in keepFrames.Where(f => (f.Flags & CopySides.UnusableFlags) == 0))
|
|
{
|
|
_ct.ThrowIfCancellationRequested();
|
|
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Verify, frame.FileName, ++done, total)).ConfigureAwait(false);
|
|
if (await HashAsync(keepPaths.FromRelative(frame.RelativePath)).ConfigureAwait(false) is { } sha) held.TryAdd(sha, frame.Id);
|
|
}
|
|
|
|
var keepRemoved = (await _db.FrameRemovals.ListBySequenceAsync(journal.KeepId, _ct).ConfigureAwait(false))
|
|
.Select(r => r.Sha256.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
|
|
var failed = new List<(Guid FrameId, string Sha256)>();
|
|
foreach (var frame in dropFrames)
|
|
{
|
|
_ct.ThrowIfCancellationRequested();
|
|
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Verify, frame.FileName, ++done, total)).ConfigureAwait(false);
|
|
if (journal.Verified.ContainsKey(frame.Id) || journal.Settled.Contains(frame.Id)) continue;
|
|
// A usable frame (not flagged Missing) whose file is absent stops the merge (design §5d step 4): it is not
|
|
// "nothing to lose" until a verify says so.
|
|
var sha = await HashAsync(_dropPaths.FromRelative(frame.RelativePath)).ConfigureAwait(false)
|
|
?? throw new InvalidOperationException(Loc.L("Errors.Copies.ChangedFile", frame.FileName));
|
|
if (frame.Sha256 is { Length: > 0 } recorded && !string.Equals(recorded, sha, StringComparison.OrdinalIgnoreCase))
|
|
throw new InvalidOperationException(Loc.L("Errors.Copies.ChangedFile", frame.FileName));
|
|
if (held.TryGetValue(sha, out var keepFrame))
|
|
{
|
|
journal.Verified[frame.Id] = new MergeVerified(sha, keepFrame);
|
|
continue;
|
|
}
|
|
if (journal.Absent.Contains(sha))
|
|
{
|
|
// A last copy the plan listed (or left out by the user's choice).
|
|
journal.Settled.Add(frame.Id);
|
|
continue;
|
|
}
|
|
if (frame.Sha256 is not { Length: > 0 } && !keepRemoved.Contains(sha))
|
|
{
|
|
// Compared by size and time only, and another photo after all: filed in the kept sequence as a second version.
|
|
var filed = await FileAsync(frame, sha, planned: false).ConfigureAwait(false);
|
|
journal.SecondVersions.Add(frame.Id);
|
|
journal.Verified[frame.Id] = new MergeVerified(sha, filed.FrameId);
|
|
held.TryAdd(sha, filed.FrameId);
|
|
Save();
|
|
continue;
|
|
}
|
|
failed.Add((frame.Id, sha));
|
|
}
|
|
if (failed.Count > 0)
|
|
{
|
|
// A photo held by another sequence of the index is not lost with the other; any other is. The kept
|
|
// sequence's index rows prove nothing: its files were hashed again above and did not hold it.
|
|
var others = await new CopySides(_db).HeldElsewhereAsync([.. failed.Select(f => f.Sha256)], [journal.DropId, journal.KeepId], _ct)
|
|
.ConfigureAwait(false);
|
|
var lost = failed.Where(f => !others.Contains(f.Sha256)).Select(f => f.Sha256).Distinct(StringComparer.Ordinal).Count();
|
|
if (lost > 0) throw new InvalidOperationException(Loc.Plural("Errors.Copies.VerifyFailed", lost));
|
|
journal.Settled.AddRange(failed.Select(f => f.FrameId));
|
|
// Held by a third sequence now: asked again before the trash, which that sequence may not wait for.
|
|
foreach (var sha in failed.Select(f => f.Sha256).Distinct(StringComparer.Ordinal))
|
|
if (!journal.SettledElsewhere.Contains(sha)) journal.SettledElsewhere.Add(sha);
|
|
}
|
|
journal.VerifiedDropFramesVersion = drop.FramesVersion;
|
|
Save();
|
|
}
|
|
|
|
private async Task<string?> HashAsync(string path)
|
|
{
|
|
try
|
|
{
|
|
return (await ContentHash.Sha256Async(path, null, _ct).ConfigureAwait(false)).ToLowerInvariant();
|
|
}
|
|
catch (Exception ex) when (ex is FileNotFoundException or DirectoryNotFoundException)
|
|
{
|
|
return null;
|
|
}
|
|
}
|
|
|
|
// ---- replace ----
|
|
|
|
private async Task ReplaceAsync()
|
|
{
|
|
var n = 0;
|
|
foreach (var replace in journal.Replaces)
|
|
{
|
|
n++;
|
|
if (journal.Replaced.Contains(replace.Key) || journal.KeptLosers.Contains(replace.Key)) continue;
|
|
_ct.ThrowIfCancellationRequested();
|
|
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Replace, replace.FileName, n, journal.Replaces.Count)).ConfigureAwait(false);
|
|
var loser = await _db.Frames.GetAsync(replace.KeepFrameId, _ct).ConfigureAwait(false);
|
|
if (loser is null || loser.SequenceId != journal.KeepId)
|
|
{
|
|
journal.Replaced.Add(replace.Key);
|
|
Save();
|
|
continue;
|
|
}
|
|
if (!await WinnerInPlaceAsync(replace.WinnerSha256).ConfigureAwait(false))
|
|
{
|
|
Keep(replace, Loc.L("Archive.Log.KeptReason.NotInPlace"));
|
|
continue;
|
|
}
|
|
try
|
|
{
|
|
var plan = await owner.Archives.FrameTrash.PlanDeleteAsync(journal.KeepId, [replace.KeepFrameId], null, _ct, journal.JobId).ConfigureAwait(false);
|
|
if (!plan.CanDelete)
|
|
{
|
|
Keep(replace, CopyLog.Reasons(plan.Blocked));
|
|
continue;
|
|
}
|
|
await owner.Archives.FrameTrash.DeleteAsync(plan, null, null, _ct, journal.JobId).ConfigureAwait(false);
|
|
journal.Replaced.Add(replace.Key);
|
|
Save();
|
|
context.Log(Loc.L("Archive.Log.CopyReplaced", replace.FileName));
|
|
}
|
|
catch (Exception ex) when (ex is StaleDeletePlanException or IOException or UnauthorizedAccessException)
|
|
{
|
|
Keep(replace, CopyLog.Reason(ex));
|
|
}
|
|
}
|
|
}
|
|
|
|
private void Keep(MergeReplace replace, string why)
|
|
{
|
|
journal.KeptLosers.Add(replace.Key);
|
|
Save();
|
|
context.Log(Loc.L("Archive.Log.CopyKept", replace.FileName, why));
|
|
}
|
|
|
|
/// <summary>Whether a usable frame of the kept sequence holds <paramref name="sha"/>, hashed again.</summary>
|
|
private async Task<bool> WinnerInPlaceAsync(string sha)
|
|
{
|
|
foreach (var frame in await _db.Frames.FindBySha256Async(sha.ToLowerInvariant(), _ct).ConfigureAwait(false))
|
|
{
|
|
if (frame.SequenceId != journal.KeepId || (frame.Flags & CopySides.UnusableFlags) != 0) continue;
|
|
if (string.Equals(await HashAsync(keepPaths.FromRelative(frame.RelativePath)).ConfigureAwait(false), sha, StringComparison.OrdinalIgnoreCase)) return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
// ---- details ----
|
|
|
|
private async Task DetailsAsync()
|
|
{
|
|
if (journal.Payload is null || journal.DetailsDone) return;
|
|
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Details)).ConfigureAwait(false);
|
|
await new SequencePayloadWriter(_db, owner.Archives).ApplyAsync(journal.KeepId, journal.Payload, journal.KeepSettingsVersion, _ct).ConfigureAwait(false);
|
|
journal.DetailsDone = true;
|
|
Save();
|
|
}
|
|
|
|
// ---- projects ----
|
|
|
|
private async Task ProjectsAsync()
|
|
{
|
|
await RenamesHeldRendersAsync().ConfigureAwait(false);
|
|
if (journal.Projects.Count == 0) return;
|
|
var from = new ArchiveProjectHome(_db, owner.Archives.Projects, journal.DropId);
|
|
var to = new ArchiveProjectHome(_db, owner.Archives.Projects, journal.KeepId);
|
|
var sources = (await _db.Projects.ListBySequenceAsync(journal.DropId, includeDeleted: false, _ct).ConfigureAwait(false)).ToDictionary(p => p.Id);
|
|
var failed = 0;
|
|
var n = 0;
|
|
foreach (var step in journal.Projects)
|
|
{
|
|
n++;
|
|
if (journal.ProjectsDone.Contains(step.SourceProjectId)) continue;
|
|
_ct.ThrowIfCancellationRequested();
|
|
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Projects, step.TargetName, n, journal.Projects.Count)).ConfigureAwait(false);
|
|
if (!sources.TryGetValue(step.SourceProjectId, out var source))
|
|
{
|
|
// Deleted from the other since the plan: its own trash keeps it.
|
|
journal.ProjectsDone.Add(step.SourceProjectId);
|
|
Save();
|
|
continue;
|
|
}
|
|
try
|
|
{
|
|
var copied = await owner.Projects.CopyAsync(from, source, to, step.TargetName, journal.ProjectSteps.GetValueOrDefault(step.SourceProjectId),
|
|
saved =>
|
|
{
|
|
journal.ProjectSteps[step.SourceProjectId] = saved;
|
|
Save();
|
|
return ValueTask.CompletedTask;
|
|
}, _ct).ConfigureAwait(false);
|
|
await RenamesRendersAsync(step.SourceProjectId, copied.Id).ConfigureAwait(false);
|
|
journal.ProjectsDone.Add(step.SourceProjectId);
|
|
Save();
|
|
context.Log(Loc.L("Archive.Log.CopyProject", step.SourceName, step.TargetName));
|
|
}
|
|
catch (Exception ex) when (ex is not OperationCanceledException)
|
|
{
|
|
failed++;
|
|
context.Log(Loc.L("Archive.Log.CopyProjectKept", step.SourceName, ex.Message));
|
|
}
|
|
}
|
|
if (failed > 0) throw new InvalidOperationException(Loc.Plural("Errors.Copies.ProjectsKept", failed));
|
|
}
|
|
|
|
/// <summary>
|
|
/// The renders moved to the kept sequence that name a project of the other the plan left because the kept sequence
|
|
/// holds its copy already name that copy (and its versions), as those of a copied project do.
|
|
/// </summary>
|
|
private async Task RenamesHeldRendersAsync()
|
|
{
|
|
var planned = journal.Projects.Select(p => p.SourceProjectId).ToHashSet();
|
|
IReadOnlyList<SequenceProject>? keepProjects = null;
|
|
foreach (var project in await _db.Projects.ListBySequenceAsync(journal.DropId, includeDeleted: false, _ct).ConfigureAwait(false))
|
|
{
|
|
if (project.IsMain || planned.Contains(project.Id)) continue;
|
|
keepProjects ??= await _db.Projects.ListBySequenceAsync(journal.KeepId, includeDeleted: false, _ct).ConfigureAwait(false);
|
|
if (await owner.Merger.HeldAsync(project, keepProjects, _dropArchive.Name, _ct).ConfigureAwait(false) is { } held)
|
|
await RenamesRendersAsync(project.Id, held.KeptId, held.Versions.Select(v => new TransplantMove(v.Key, v.Value))).ConfigureAwait(false);
|
|
}
|
|
}
|
|
|
|
/// <summary>The renders moved to the kept sequence name the project copy (and its versions) instead of the other's.</summary>
|
|
private Task RenamesRendersAsync(Guid source, Guid copy) =>
|
|
RenamesRendersAsync(source, copy, journal.ProjectSteps.GetValueOrDefault(source)?.Moves ?? []);
|
|
|
|
private async Task RenamesRendersAsync(Guid source, Guid copy, IEnumerable<TransplantMove> moves)
|
|
{
|
|
var access = new DbAccess(_db.Database);
|
|
foreach (var move in moves)
|
|
await access.ExecuteAsync("UPDATE renders SET project_id = $2, project_version_id = $3 WHERE sequence_id = $1 AND project_version_id = $4",
|
|
[journal.KeepId, copy, move.Copy, move.Source], _ct).ConfigureAwait(false);
|
|
await access.ExecuteAsync("UPDATE renders SET project_id = $2 WHERE sequence_id = $1 AND project_id = $3", [journal.KeepId, copy, source], _ct)
|
|
.ConfigureAwait(false);
|
|
}
|
|
|
|
// ---- links ----
|
|
|
|
private async Task LinksAsync()
|
|
{
|
|
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Process)).ConfigureAwait(false);
|
|
var keep = await _db.Sequences.GetAsync(journal.KeepId, _ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
|
|
var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false) ?? throw Changed(_dropTitle);
|
|
if (journal.LinksUndo is null)
|
|
{
|
|
// Nothing of the other is touched while a check before the trash can still abandon the merge.
|
|
await CheckTrashAsync(drop).ConfigureAwait(false);
|
|
await using (await owner.Archives.Writer.LockAsync(keepPaths, journal.KeepId, _ct).ConfigureAwait(false))
|
|
await PlanTrashAsync().ConfigureAwait(false);
|
|
journal.LinksUndo = new MergeLinksUndo(KeysOf(keepPaths, keep), KeysOf(_dropPaths, drop), _dropArchive.Id,
|
|
await new FoundCopyStore(_db).UndoOfMoveAsync(journal.DropId, journal.KeepId, keepArchive.Id, journal.SessionKeys, journal.CopyId, _ct)
|
|
.ConfigureAwait(false));
|
|
Save();
|
|
}
|
|
var keys = journal.SessionKeys.ToHashSet(StringComparer.Ordinal);
|
|
if (!journal.KeepKeysWritten)
|
|
{
|
|
if (keys.Count > 0 && File.Exists(keepPaths.Sequence(keep.Folder).ManifestPath))
|
|
{
|
|
await owner.Archives.Writer.UpdateAsync(keepPaths, keep.Id, keep.Folder, manifest =>
|
|
keys.All(manifest.SessionKeys.Contains) ? null : manifest with { SessionKeys = [.. manifest.SessionKeys.Union(keys, StringComparer.Ordinal)] },
|
|
_ct).ConfigureAwait(false);
|
|
}
|
|
journal.KeepKeysWritten = true;
|
|
Save();
|
|
}
|
|
if (!journal.DropKeysRemoved)
|
|
{
|
|
if (keys.Count > 0 && File.Exists(_dropPaths.Sequence(drop.Folder).ManifestPath))
|
|
{
|
|
await owner.Archives.Writer.UpdateAsync(_dropPaths, drop.Id, drop.Folder, manifest =>
|
|
!manifest.SessionKeys.Any(keys.Contains) ? null : manifest with { SessionKeys = [.. manifest.SessionKeys.Where(k => !keys.Contains(k))] },
|
|
_ct).ConfigureAwait(false);
|
|
}
|
|
journal.DropKeysRemoved = true;
|
|
Save();
|
|
}
|
|
if (!journal.LinksMoved)
|
|
{
|
|
MoveResult moved;
|
|
try
|
|
{
|
|
moved = await new FoundCopyStore(_db).MoveToKeptAsync(journal.DropId, journal.KeepId, keepArchive.Id, journal.SessionKeys, journal.CopyId, _ct)
|
|
.ConfigureAwait(false);
|
|
}
|
|
catch (CopyLinkTakenException ex) when (ex.Server is { } key)
|
|
{
|
|
throw CopyLinkTakenException.BothLinked(key, owner.Merger.ServerName(key));
|
|
}
|
|
foreach (var key in moved.SkippedKeys) context.Log(key);
|
|
journal.LinksMoved = true;
|
|
Save();
|
|
}
|
|
}
|
|
|
|
/// <summary>The session keys of the manifest of <paramref name="sequence"/> (none when it has no manifest).</summary>
|
|
private static List<string> KeysOf(ArchivePaths paths, Sequence sequence)
|
|
{
|
|
var manifest = paths.Sequence(sequence.Folder).ManifestPath;
|
|
return sequence.Folder.Length > 0 && File.Exists(manifest) ? [.. ManifestStore.ReadSequence(manifest).SessionKeys] : [];
|
|
}
|
|
|
|
// ---- trash ----
|
|
|
|
/// <summary>
|
|
/// The checks before the trash that may abandon the merge, run before the links stage and again at the trash: no
|
|
/// unsent workspace work, no work saved into the other since the plan, the other at the frames version the verify read
|
|
/// and readable. Returns that frames version.
|
|
/// </summary>
|
|
private async Task<int> CheckTrashAsync(Sequence drop)
|
|
{
|
|
if (await owner.Workspace.HasUnsentAsync(journal.DropId, _ct).ConfigureAwait(false))
|
|
throw new InvalidOperationException(Loc.L("Errors.Copies.Unsent", _dropTitle));
|
|
if (await WorkSinceThePlanAsync(drop).ConfigureAwait(false))
|
|
{
|
|
// Work saved into the other since the plan (sent home, rendered) never reached the kept one: planned again.
|
|
ForgetJournal();
|
|
throw Changed(_dropTitle);
|
|
}
|
|
var expected = journal.VerifiedDropFramesVersion ?? throw Changed(_dropTitle);
|
|
if (drop.FramesVersion != expected) throw Changed(_dropTitle);
|
|
EnsureDropReadable(drop);
|
|
return expected;
|
|
}
|
|
|
|
/// <summary>Under the kept sequence's lease: every verified photo is still held, and the trash's dry run of the other is not blocked.</summary>
|
|
private async Task<DeletePlan> PlanTrashAsync()
|
|
{
|
|
await EnsureVerifiedHeldAsync().ConfigureAwait(false);
|
|
var plan = await owner.Archives.Trash.PlanAsync([journal.DropId], null, _ct, journal.JobId).ConfigureAwait(false);
|
|
if (plan.Blocked.Count > 0)
|
|
throw new InvalidOperationException(Loc.L("Errors.Copies.TrashBlocked", _dropTitle, Loc.L("Archive.Delete.Reason." + plan.Blocked[0].Reason)));
|
|
return plan;
|
|
}
|
|
|
|
private async Task TrashAsync()
|
|
{
|
|
var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false);
|
|
if (drop is null) return; // a run that moved it before its journal said so
|
|
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Trash, _dropTitle)).ConfigureAwait(false);
|
|
var expected = await CheckTrashAsync(drop).ConfigureAwait(false);
|
|
await using (await owner.Archives.Writer.LockAsync(keepPaths, journal.KeepId, _ct).ConfigureAwait(false))
|
|
{
|
|
var plan = await PlanTrashAsync().ConfigureAwait(false);
|
|
try
|
|
{
|
|
await owner.Archives.Trash.DeleteAsync(plan, null, null, _ct, journal.JobId, new Dictionary<Guid, int> { [journal.DropId] = expected })
|
|
.ConfigureAwait(false);
|
|
}
|
|
catch (StaleDeletePlanException ex)
|
|
{
|
|
throw new InvalidOperationException(Loc.L("Errors.Copies.Changed", _dropTitle), ex);
|
|
}
|
|
}
|
|
context.Log(Loc.L("Archive.Log.MergeTrashed", _dropTitle));
|
|
}
|
|
|
|
/// <summary>
|
|
/// Whether the other holds work the plan did not see and nothing copied: details or Main settings saved since (its
|
|
/// settings version moved), a project made or a version added since (missing from the plan, or another head than the
|
|
/// one it copied), or a render the kept sequence does not hold.
|
|
/// </summary>
|
|
private async Task<bool> WorkSinceThePlanAsync(Sequence drop)
|
|
{
|
|
if (journal.DropSettingsVersion is { } settings && drop.SettingsVersion != settings) return true;
|
|
var planned = journal.Projects.ToDictionary(p => p.SourceProjectId);
|
|
IReadOnlyList<SequenceProject>? keepProjects = null;
|
|
foreach (var project in await _db.Projects.ListBySequenceAsync(journal.DropId, includeDeleted: false, _ct).ConfigureAwait(false))
|
|
{
|
|
if (project.IsMain) continue;
|
|
if (planned.TryGetValue(project.Id, out var step) && (project.HeadVersionId ?? Guid.Empty) == step.SourceHeadId) continue;
|
|
// Not planned (or moved on since) is fine only when the kept sequence holds a copy of it already, as the plan saw it.
|
|
keepProjects ??= await _db.Projects.ListBySequenceAsync(journal.KeepId, includeDeleted: false, _ct).ConfigureAwait(false);
|
|
if (await owner.Merger.HeldAsync(project, keepProjects, _dropArchive.Name, _ct).ConfigureAwait(false) is null) return true;
|
|
}
|
|
var renders = await _db.Renders.ListBySequenceAsync(journal.DropId, _ct).ConfigureAwait(false);
|
|
if (renders.Count == 0) return false;
|
|
// A render the plan left because the kept sequence holds the same file stays with the other, as planned.
|
|
var keepHashes = (await _db.Renders.ListBySequenceAsync(journal.KeepId, _ct).ConfigureAwait(false)).Select(r => r.Sha256).OfType<string>()
|
|
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
|
return renders.Any(r => !journal.RendersMoved.Contains(r.Id) && (r.Sha256 is null || !keepHashes.Contains(r.Sha256)));
|
|
}
|
|
|
|
/// <summary>
|
|
/// Every usable photo of the other was verified (held by the kept sequence, or a last copy the plan listed, or held by
|
|
/// a third sequence that still holds it now), and every photo the verify found in the kept sequence is still there and
|
|
/// usable (read under its lease).
|
|
/// </summary>
|
|
private async Task EnsureVerifiedHeldAsync()
|
|
{
|
|
// Every usable photo of the other was verified in this journal: a verify cut short (an absent drive, a moved
|
|
// folder) never lets the rest through unread.
|
|
var dropFrames = await ProcessingFrames.ListAsync(_db, journal.DropId, _ct).ConfigureAwait(false);
|
|
if (dropFrames.Any(f => (f.Flags & CopySides.UnusableFlags) == 0 && !journal.Verified.ContainsKey(f.Id) && !journal.Settled.Contains(f.Id)))
|
|
throw Changed(_dropTitle);
|
|
|
|
// The photos a third sequence held at the verify: one it no longer holds would leave only with the other, never
|
|
// listed as a last copy. Planned again, the plan lists it.
|
|
if (journal.SettledElsewhere.Count > 0)
|
|
{
|
|
var still = await new CopySides(_db).HeldElsewhereAsync(journal.SettledElsewhere, [journal.DropId, journal.KeepId], _ct).ConfigureAwait(false);
|
|
if (journal.SettledElsewhere.Any(sha => !still.Contains(sha)))
|
|
{
|
|
ForgetJournal();
|
|
throw Changed(_dropTitle);
|
|
}
|
|
}
|
|
|
|
var frames = await ProcessingFrames.ListAsync(_db, journal.KeepId, _ct).ConfigureAwait(false);
|
|
var byId = frames.Where(f => (f.Flags & CopySides.UnusableFlags) == 0).ToDictionary(f => f.Id);
|
|
var hashes = byId.Values.Select(f => f.Sha256).OfType<string>().Select(s => s.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
|
|
foreach (var verified in journal.Verified.Values)
|
|
{
|
|
if (byId.TryGetValue(verified.KeepFrameId, out var frame)
|
|
&& (frame.Sha256 is null || string.Equals(frame.Sha256, verified.Sha256, StringComparison.OrdinalIgnoreCase)))
|
|
continue;
|
|
if (hashes.Contains(verified.Sha256)) continue;
|
|
throw Changed(_keepTitle);
|
|
}
|
|
}
|
|
|
|
// ---- done ----
|
|
|
|
private async Task FinishAsync()
|
|
{
|
|
await context.ReportAsync(Total, Total, new JobDetail(JobStep.FollowUps)).ConfigureAwait(false);
|
|
await new FoundCopyStore(_db).DeleteAsync(journal.CopyId, CancellationToken.None).ConfigureAwait(false);
|
|
if (await _db.Sequences.GetAsync(journal.KeepId, CancellationToken.None).ConfigureAwait(false) is { } keep)
|
|
{
|
|
foreach (var request in owner.Archives.FrameTrash.FollowUps(keepArchive, keep, (SequenceStatus)journal.KeepStatus))
|
|
{
|
|
try
|
|
{
|
|
if (request.Targets.Count > 0 && await _db.Jobs.IsQueuedAsync(request.Kind, request.Targets[0], CancellationToken.None).ConfigureAwait(false)) continue;
|
|
await context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, CancellationToken.None)
|
|
.ConfigureAwait(false);
|
|
}
|
|
catch (DatabaseException ex)
|
|
{
|
|
Log.Warning("archive", $"{request.Label}: the passes the merged photos need were not queued; the next scan queues them", ex);
|
|
}
|
|
}
|
|
}
|
|
await TellAsync(journal.KeepId, KeepChange).ConfigureAwait(false);
|
|
await TellAsync(journal.DropId, DropChange).ConfigureAwait(false);
|
|
var kept = journal.KeptLosers.Count;
|
|
try
|
|
{
|
|
MergeJournal.Delete(keepPaths, journal.JobId);
|
|
}
|
|
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
|
{
|
|
Log.Warning("archive", $"The journal of merge {journal.JobId} was not removed", ex);
|
|
}
|
|
if (kept > 0) throw new InvalidOperationException(Loc.Plural("Errors.Copies.Kept", kept));
|
|
}
|
|
|
|
private async Task TellAsync(Guid id, string change)
|
|
{
|
|
if (owner.Archives.Writer.Options.SequenceChanged is not { } changed) return;
|
|
try
|
|
{
|
|
await changed(id, change, CancellationToken.None).ConfigureAwait(false);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.Warning("archive", $"The change of sequence {id} was not told", ex);
|
|
}
|
|
}
|
|
}
|
|
}
|