feat(archive): merge two local sequences into the kept one (owner review 6, P4)
SequenceMerger plans the merge of a local pair (kept sequence, photos copied by policy, last copies, renders, details, projects, session keys, links, blocks and a fingerprint) and queues the MergeSequences job. MergeJobHandler applies the confirmed plan through a journal, so every stage resumes: copy and file the other's photos, move renders, hash-verify every original of the other in the kept one (or list it as a last copy), trash the kept one's losing versions, write the details (SequencePayloadWriter), copy projects, move keys and links, then move the other to its archive's trash only at the verified frames version. A cancel drops the journal and returns the pair to its previous state. Refs #73 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9912baf78a
commit
b4070ddfd4
6 files changed
+2062
No files matched your search
@@ -54,6 +54,9 @@ public sealed class ArchiveService : IJobHandlerProvider
|
||||
|
||||
public ArchiveIndexer Indexer { get; }
|
||||
|
||||
/// <summary>The sidecar writer this service's writers share (its in-process gates and leases): a merge holds sequences with it.</summary>
|
||||
internal SequenceWriter Writer => _writer;
|
||||
|
||||
/// <summary>Commits staged files into sequences (server uploads, client card imports).</summary>
|
||||
public ArchiveIngest Ingest { get; }
|
||||
|
||||
|
||||
@@ -0,0 +1,705 @@
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.Diagnostics;
|
||||
using Chronos.Core.Import;
|
||||
using Chronos.Core.Localization;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Data.Copies;
|
||||
using Chronos.Data.Jobs;
|
||||
using Chronos.Data.Projects;
|
||||
using Chronos.Data.Repositories;
|
||||
using Chronos.Data.Sync;
|
||||
using ProcessingFrames = Chronos.Data.Processing.SequenceFrames;
|
||||
|
||||
namespace Chronos.Data.Archiving.Jobs;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.MergeSequences"/> (owner review 6, #73; design §5d): applies the merge plan the user confirmed
|
||||
/// (<see cref="MergeJobParameters"/>). Its first run plans again and refuses a plan whose fingerprint moved; the journal
|
||||
/// (<see cref="MergeJournal"/>) then keeps what that plan decided and how far each stage came, so a crash, a pause or a
|
||||
/// retry resumes where it stopped. Stages: copy the other's photos into the kept sequence (each hash-checked, filed with
|
||||
/// tombstones honoured), move the renders, verify every original of the other is held by the kept one (hashed again on
|
||||
/// both sides) or listed as a last copy, trash the kept sequence's losing versions, write the details, copy the projects,
|
||||
/// move session keys, links and pairs, then move the other to its archive's trash, only while it is still at the frames
|
||||
/// version the verify read and the kept photos verified are still there. A cancel leaves the photos copied so far in the
|
||||
/// kept sequence (nothing of the other is touched before the trash stage) and drops the journal.
|
||||
/// </summary>
|
||||
internal sealed class MergeJobHandler(ChronosDatabase db, SequenceMerger merger, ArchiveService archives, IProjectCopier projects,
|
||||
IWorkspaceGuard workspace) : ICancelledJobHandler
|
||||
{
|
||||
/// <summary>The change told about the kept sequence.</summary>
|
||||
public const string KeepChange = "frames";
|
||||
|
||||
/// <summary>The change told about the sequence that went to the trash.</summary>
|
||||
public const string DropChange = "deleted";
|
||||
|
||||
private static readonly CopyPlanContext JobContextPlan = new(string.Empty, null, false, false, false, false, false, false, null,
|
||||
new HashSet<Guid>(), new HashSet<Guid>());
|
||||
|
||||
public JobKind Kind => JobKind.MergeSequences;
|
||||
|
||||
private ChronosDatabase Db => db;
|
||||
|
||||
private ArchiveService Archives => archives;
|
||||
|
||||
private IProjectCopier Projects => projects;
|
||||
|
||||
private IWorkspaceGuard Workspace => workspace;
|
||||
|
||||
/// <summary>Called before each stage runs (tests: a crash, a change made between stages).</summary>
|
||||
internal Func<MergeStage, Task>? BeforeStage { get; set; }
|
||||
|
||||
public async Task RunAsync(JobContext context)
|
||||
{
|
||||
var job = context.Job;
|
||||
var ct = context.Cancellation;
|
||||
var parameters = MergeJobParameters.Read(job.ParametersJson) ?? throw new InvalidOperationException($"Merge job {job.Id} has no plan.");
|
||||
var keepRow = await db.Sequences.GetAsync(parameters.KeepId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
|
||||
var keepArchive = await db.Archives.GetAsync(keepRow.ArchiveId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
|
||||
var keepPaths = new ArchivePaths(keepArchive.RootPath);
|
||||
try
|
||||
{
|
||||
var journal = MergeJournal.Read(keepPaths, job.Id) ?? await StartAsync(context, parameters, keepRow, keepPaths).ConfigureAwait(false);
|
||||
var staging = MergeJournal.StagingFor(keepPaths, job.Id);
|
||||
if (Directory.Exists(staging)) Directory.Delete(staging, recursive: true);
|
||||
await new Run(this, context, journal, keepArchive, keepPaths).RunAsync().ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
if (await db.Jobs.GetAsync(job.Id, CancellationToken.None).ConfigureAwait(false) is { State: JobState.Cancelled })
|
||||
await ForgetAsync(job, parameters, keepPaths).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task CancelledAsync(Job job, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (MergeJobParameters.Read(job.ParametersJson) is not { } parameters) return;
|
||||
if (await db.Sequences.GetAsync(parameters.KeepId, cancellationToken).ConfigureAwait(false) is not { } keep) return;
|
||||
if (await db.Archives.GetAsync(keep.ArchiveId, cancellationToken).ConfigureAwait(false) is not { } archive) return;
|
||||
await ForgetAsync(job, parameters, new ArchivePaths(archive.RootPath)).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
Log.Warning("archive", $"The cancelled merge {job.Id} left its journal; the archive's next maintenance removes it", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A cancelled merge: its journal and staging go, and its pair returns to the state it had (when it is still this job's).</summary>
|
||||
private async Task ForgetAsync(Job job, MergeJobParameters parameters, ArchivePaths keepPaths)
|
||||
{
|
||||
try
|
||||
{
|
||||
MergeJournal.Delete(keepPaths, job.Id);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("archive", $"The journal of the cancelled merge {job.Id} was not removed", ex);
|
||||
}
|
||||
var store = new FoundCopyStore(db);
|
||||
if (await store.GetAsync(parameters.CopyId, CancellationToken.None).ConfigureAwait(false) is { JobId: { } owner } copy && owner == job.Id
|
||||
&& copy.State != CopyState.Integrating)
|
||||
await store.SetStateAsync(copy.Id, copy.State, copy.State, null, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>The first run: plans again, refuses a plan that moved or is blocked, and journals it.</summary>
|
||||
private async Task<MergeJournal> StartAsync(JobContext context, MergeJobParameters parameters, Sequence keepRow, ArchivePaths keepPaths)
|
||||
{
|
||||
var ct = context.Cancellation;
|
||||
var copy = await new FoundCopyStore(db).GetAsync(parameters.CopyId, ct).ConfigureAwait(false);
|
||||
if (copy is null || copy.JobId != context.Job.Id) throw new StaleCopyPlanException();
|
||||
if (await workspace.HasUnsentAsync(parameters.DropId, ct).ConfigureAwait(false))
|
||||
throw new InvalidOperationException(Loc.L("Errors.Copies.Unsent", (await db.Sequences.GetAsync(parameters.DropId, ct).ConfigureAwait(false))?.DisplayTitle ?? string.Empty));
|
||||
var plan = await merger.PlanAsync(parameters.CopyId, parameters.KeepId, parameters.DropId, parameters.Choices, JobContextPlan, context.Job.Id, ct)
|
||||
.ConfigureAwait(false);
|
||||
if (plan.KeepId != parameters.KeepId || plan.DropId != parameters.DropId) throw new StaleCopyPlanException();
|
||||
if (!plan.CanApply) throw new InvalidOperationException(await merger.BlockMessageAsync(plan, ct).ConfigureAwait(false));
|
||||
if (!string.Equals(plan.Fingerprint, parameters.Fingerprint, StringComparison.Ordinal)) throw new StaleCopyPlanException();
|
||||
|
||||
var winners = plan.Photos.Where(p => p.Action == PhotoAction.FetchVersion).ToDictionary(p => p.Key, p => p.Sha256, StringComparer.Ordinal);
|
||||
var journal = new MergeJournal
|
||||
{
|
||||
JobId = context.Job.Id,
|
||||
CopyId = plan.CopyId,
|
||||
KeepId = plan.KeepId,
|
||||
DropId = plan.DropId,
|
||||
Fingerprint = plan.Fingerprint,
|
||||
Stage = MergeStage.Copy,
|
||||
KeepSettingsVersion = plan.KeepSettingsVersion,
|
||||
DropFramesVersion = plan.DropFramesVersion,
|
||||
KeepStatus = (int)keepRow.Status,
|
||||
Copies = [.. plan.Photos.Where(p => p.Action is PhotoAction.Fetch or PhotoAction.FetchVersion && p.FrameId is not null)
|
||||
.Select(p => new MergeCopy(p.Key, p.Action, p.FrameId!.Value, p.FileName, p.Bytes, p.Sha256))],
|
||||
Replaces = [.. plan.Photos.Where(p => p.Action == PhotoAction.TrashHere && p.FrameId is not null && winners.GetValueOrDefault(p.Key) is { Length: > 0 })
|
||||
.Select(p => new MergeReplace(p.Key, p.FrameId!.Value, p.FileName, winners[p.Key]!))],
|
||||
Absent = [.. plan.LastCopies.Where(l => l.Side == CopySide.Other).Select(l => l.Key.ToLowerInvariant())],
|
||||
Renders = [.. plan.Renders],
|
||||
Payload = plan.Payload,
|
||||
Projects = [.. plan.Projects],
|
||||
SessionKeys = [.. plan.SessionKeys],
|
||||
};
|
||||
journal.Save(keepPaths);
|
||||
return journal;
|
||||
}
|
||||
|
||||
/// <summary>One run of a merge, from the stage its journal reached.</summary>
|
||||
private sealed class Run(MergeJobHandler owner, JobContext context, MergeJournal journal, Archive keepArchive, ArchivePaths keepPaths)
|
||||
{
|
||||
private readonly ChronosDatabase _db = owner.Db;
|
||||
private readonly CancellationToken _ct = context.Cancellation;
|
||||
private readonly string _staging = MergeJournal.StagingFor(keepPaths, journal.JobId);
|
||||
private Archive _dropArchive = null!;
|
||||
private ArchivePaths _dropPaths = null!;
|
||||
private string _keepTitle = string.Empty;
|
||||
private string _dropTitle = string.Empty;
|
||||
|
||||
private long Total => journal.Copies.Count + journal.Renders.Count + journal.Replaces.Count + journal.Projects.Count + 4;
|
||||
|
||||
public async Task RunAsync()
|
||||
{
|
||||
var keep = await _db.Sequences.GetAsync(journal.KeepId, _ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
|
||||
_keepTitle = keep.DisplayTitle;
|
||||
var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false);
|
||||
if (drop is null && journal.Stage < MergeStage.Trash) throw new InvalidOperationException(Loc.L("Errors.Copies.StalePlan"));
|
||||
if (drop is not null)
|
||||
{
|
||||
_dropTitle = drop.DisplayTitle;
|
||||
_dropArchive = await _db.Archives.GetAsync(drop.ArchiveId, _ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
|
||||
_dropPaths = new ArchivePaths(_dropArchive.RootPath);
|
||||
}
|
||||
|
||||
while (journal.Stage != MergeStage.Done)
|
||||
{
|
||||
_ct.ThrowIfCancellationRequested();
|
||||
if (owner.BeforeStage is { } hook) await hook(journal.Stage).ConfigureAwait(false);
|
||||
switch (journal.Stage)
|
||||
{
|
||||
case MergeStage.Copy: await CopyAsync().ConfigureAwait(false); break;
|
||||
case MergeStage.Renders: await RendersAsync().ConfigureAwait(false); break;
|
||||
case MergeStage.Verify: await VerifyAsync().ConfigureAwait(false); break;
|
||||
case MergeStage.Replace: await ReplaceAsync().ConfigureAwait(false); break;
|
||||
case MergeStage.Details: await DetailsAsync().ConfigureAwait(false); break;
|
||||
case MergeStage.Projects: await ProjectsAsync().ConfigureAwait(false); break;
|
||||
case MergeStage.Links: await LinksAsync().ConfigureAwait(false); break;
|
||||
case MergeStage.Trash: await TrashAsync().ConfigureAwait(false); break;
|
||||
}
|
||||
journal.Stage++;
|
||||
Save();
|
||||
await context.CheckpointAsync($"merge:{journal.Stage}", Done(), Total).ConfigureAwait(false);
|
||||
}
|
||||
await FinishAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private long Done() => journal.Committed.Count + journal.RendersMoved.Count + journal.Replaced.Count + journal.KeptLosers.Count
|
||||
+ journal.ProjectsDone.Count + Math.Min(4, Math.Max(0, journal.Stage - MergeStage.Verify));
|
||||
|
||||
private void Save() => journal.Save(keepPaths);
|
||||
|
||||
private InvalidOperationException Changed(string title) => new(Loc.L("Errors.Copies.Changed", title));
|
||||
|
||||
private async Task EnsureDropUnchangedAsync(int expected)
|
||||
{
|
||||
var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false);
|
||||
if (drop is null || drop.FramesVersion != expected) throw Changed(_dropTitle);
|
||||
}
|
||||
|
||||
// ---- copy ----
|
||||
|
||||
private async Task CopyAsync()
|
||||
{
|
||||
var n = 0;
|
||||
foreach (var copy in journal.Copies)
|
||||
{
|
||||
n++;
|
||||
if (journal.Committed.ContainsKey(copy.DropFrameId)) continue;
|
||||
_ct.ThrowIfCancellationRequested();
|
||||
await EnsureDropUnchangedAsync(journal.DropFramesVersion).ConfigureAwait(false);
|
||||
var frame = await _db.Frames.GetAsync(copy.DropFrameId, _ct).ConfigureAwait(false) ?? throw Changed(_dropTitle);
|
||||
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.CopyPhoto, copy.FileName, n, journal.Copies.Count)).ConfigureAwait(false);
|
||||
var result = await FileAsync(frame, copy.Sha256).ConfigureAwait(false);
|
||||
journal.Committed[copy.DropFrameId] = result.FrameId;
|
||||
Save();
|
||||
context.Log(Loc.L("Archive.Log.MergeCopied", copy.FileName));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Copies a frame of the other into staging (hash-checked against <paramref name="expected"/>) and files it in the kept sequence.</summary>
|
||||
private async Task<IngestResult> FileAsync(Frame frame, string? expected)
|
||||
{
|
||||
Directory.CreateDirectory(_staging);
|
||||
var staged = Path.Combine(_staging, frame.Id.ToString("N") + Path.GetExtension(frame.FileName));
|
||||
if (File.Exists(staged)) File.Delete(staged);
|
||||
string sha;
|
||||
try
|
||||
{
|
||||
sha = await VerifiedCopy.CopyAsync(_dropPaths.FromRelative(frame.RelativePath), staged, verify: true, null, _ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is FileNotFoundException or DirectoryNotFoundException)
|
||||
{
|
||||
throw new InvalidOperationException(Loc.L("Errors.Copies.ChangedFile", frame.FileName), ex);
|
||||
}
|
||||
if (expected is { Length: > 0 } && !string.Equals(sha, expected, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
File.Delete(staged);
|
||||
throw new InvalidOperationException(Loc.L("Errors.Copies.ChangedFile", frame.FileName));
|
||||
}
|
||||
var size = new FileInfo(staged).Length;
|
||||
IngestResult result;
|
||||
try
|
||||
{
|
||||
result = await owner.Archives.Ingest.CommitAsync(new IngestRequest(keepArchive.Id, journal.KeepId, null, null, staged, frame.FileName, size,
|
||||
sha.ToLowerInvariant(), frame.CapturedAt, null, RefuseRemoved: true), _ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (PhotoRemovedException)
|
||||
{
|
||||
// Deleted from the kept sequence since the plan: it would be a last copy the plan did not list.
|
||||
File.Delete(staged);
|
||||
throw Changed(_keepTitle);
|
||||
}
|
||||
if (File.Exists(staged)) File.Delete(staged);
|
||||
return result;
|
||||
}
|
||||
|
||||
// ---- renders ----
|
||||
|
||||
private async Task RendersAsync()
|
||||
{
|
||||
var keep = await _db.Sequences.GetAsync(journal.KeepId, _ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
|
||||
var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false) ?? throw Changed(_dropTitle);
|
||||
var n = 0;
|
||||
foreach (var step in journal.Renders)
|
||||
{
|
||||
n++;
|
||||
if (journal.RendersMoved.Contains(step.RenderId)) continue;
|
||||
_ct.ThrowIfCancellationRequested();
|
||||
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.CopyVideo, step.FileName, n, journal.Renders.Count)).ConfigureAwait(false);
|
||||
var render = await _db.Renders.GetAsync(step.RenderId, _ct).ConfigureAwait(false);
|
||||
if (render is null || render.SequenceId == journal.KeepId)
|
||||
{
|
||||
journal.RendersMoved.Add(step.RenderId);
|
||||
Save();
|
||||
continue;
|
||||
}
|
||||
var source = SyncService.RenderFile(_dropArchive, render);
|
||||
var relative = render.OutputPath.Replace('\\', '/');
|
||||
var inside = !Path.IsPathRooted(render.OutputPath) && drop.Folder.Length > 0
|
||||
&& relative.StartsWith(drop.Folder.TrimEnd('/') + "/", StringComparison.Ordinal);
|
||||
string output;
|
||||
if (inside && (File.Exists(source) || Directory.Exists(source)))
|
||||
{
|
||||
// Inside the folder that goes to the trash: copied (verified) into the kept sequence's exports first.
|
||||
if (!journal.RenderTargets.TryGetValue(step.RenderId, out var target))
|
||||
{
|
||||
target = keepPaths.ToRelative(Free(keepPaths.Sequence(keep.Folder).Exports, Path.GetFileName(source)));
|
||||
journal.RenderTargets[step.RenderId] = target;
|
||||
Save();
|
||||
}
|
||||
var destination = keepPaths.FromRelative(target);
|
||||
if (Directory.Exists(source)) await CopyFolderAsync(source, destination).ConfigureAwait(false);
|
||||
else await CopyRenderAsync(source, destination, render.Sha256, step.FileName).ConfigureAwait(false);
|
||||
output = target;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Outside it (or gone): only the row moves, naming the same file wherever the kept sequence's archive is.
|
||||
output = Path.IsPathRooted(render.OutputPath) || keepArchive.Id == _dropArchive.Id ? render.OutputPath : source;
|
||||
}
|
||||
var access = new DbAccess(_db.Database);
|
||||
await access.InTransactionAsync(async session =>
|
||||
{
|
||||
var inner = new DbAccess(session);
|
||||
await inner.ExecuteAsync("UPDATE renders SET sequence_id = $2, output_path = $3 WHERE id = $1", [step.RenderId, journal.KeepId, output], _ct)
|
||||
.ConfigureAwait(false);
|
||||
foreach (var id in (Guid[])[journal.KeepId, journal.DropId])
|
||||
await inner.ExecuteAsync("UPDATE sequences SET rendered_count = (SELECT count(*) FROM renders WHERE sequence_id = $1) WHERE id = $1", [id], _ct)
|
||||
.ConfigureAwait(false);
|
||||
return 0;
|
||||
}, _ct).ConfigureAwait(false);
|
||||
journal.RendersMoved.Add(step.RenderId);
|
||||
Save();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task CopyRenderAsync(string source, string destination, string? sha, string name)
|
||||
{
|
||||
if (File.Exists(destination))
|
||||
{
|
||||
// A run that copied it before its row moved: kept when it is the same file.
|
||||
if (sha is { Length: > 0 } && string.Equals(await ContentHash.Sha256Async(destination, null, _ct).ConfigureAwait(false), sha, StringComparison.OrdinalIgnoreCase))
|
||||
return;
|
||||
File.Delete(destination);
|
||||
}
|
||||
var copied = await VerifiedCopy.CopyAsync(source, destination, verify: true, null, _ct).ConfigureAwait(false);
|
||||
if (sha is { Length: > 0 } && !string.Equals(copied, sha, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
File.Delete(destination);
|
||||
throw new InvalidOperationException(Loc.L("Errors.Copies.ChangedFile", name));
|
||||
}
|
||||
}
|
||||
|
||||
private async Task CopyFolderAsync(string source, string destination)
|
||||
{
|
||||
foreach (var file in Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories))
|
||||
{
|
||||
var target = Path.Combine(destination, Path.GetRelativePath(source, file));
|
||||
if (File.Exists(target) && new FileInfo(target).Length == new FileInfo(file).Length
|
||||
&& string.Equals(await ContentHash.Sha256Async(target, null, _ct).ConfigureAwait(false),
|
||||
await ContentHash.Sha256Async(file, null, _ct).ConfigureAwait(false), StringComparison.Ordinal))
|
||||
continue;
|
||||
if (File.Exists(target)) File.Delete(target);
|
||||
await VerifiedCopy.CopyAsync(file, target, verify: true, null, _ct).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private static string Free(string folder, string name)
|
||||
{
|
||||
var stem = Path.GetFileNameWithoutExtension(name);
|
||||
var extension = Path.GetExtension(name);
|
||||
var candidate = Path.Combine(folder, name);
|
||||
for (var n = 2; File.Exists(candidate) || Directory.Exists(candidate); n++) candidate = Path.Combine(folder, $"{stem} ({n}){extension}");
|
||||
return candidate;
|
||||
}
|
||||
|
||||
// ---- verify ----
|
||||
|
||||
private async Task VerifyAsync()
|
||||
{
|
||||
var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false) ?? throw Changed(_dropTitle);
|
||||
if (drop.FramesVersion != journal.DropFramesVersion) throw Changed(_dropTitle);
|
||||
var keepFrames = await ProcessingFrames.ListAsync(_db, journal.KeepId, _ct).ConfigureAwait(false);
|
||||
var dropFrames = (await ProcessingFrames.ListAsync(_db, journal.DropId, _ct).ConfigureAwait(false))
|
||||
.Where(f => (f.Flags & CopySides.UnusableFlags) == 0).ToList();
|
||||
var total = keepFrames.Count + dropFrames.Count;
|
||||
var done = 0;
|
||||
|
||||
// What the kept sequence holds now, hashed again.
|
||||
var held = new Dictionary<string, Guid>(StringComparer.Ordinal);
|
||||
foreach (var frame in keepFrames.Where(f => (f.Flags & CopySides.UnusableFlags) == 0))
|
||||
{
|
||||
_ct.ThrowIfCancellationRequested();
|
||||
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Verify, frame.FileName, ++done, total)).ConfigureAwait(false);
|
||||
if (await HashAsync(keepPaths.FromRelative(frame.RelativePath)).ConfigureAwait(false) is { } sha) held.TryAdd(sha, frame.Id);
|
||||
}
|
||||
|
||||
var failed = new List<string>();
|
||||
foreach (var frame in dropFrames)
|
||||
{
|
||||
_ct.ThrowIfCancellationRequested();
|
||||
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Verify, frame.FileName, ++done, total)).ConfigureAwait(false);
|
||||
if (journal.Verified.ContainsKey(frame.Id)) continue;
|
||||
var sha = await HashAsync(_dropPaths.FromRelative(frame.RelativePath)).ConfigureAwait(false);
|
||||
if (sha is null)
|
||||
{
|
||||
// Nothing on disk: nothing of it can be lost.
|
||||
continue;
|
||||
}
|
||||
if (frame.Sha256 is { Length: > 0 } recorded && !string.Equals(recorded, sha, StringComparison.OrdinalIgnoreCase))
|
||||
throw new InvalidOperationException(Loc.L("Errors.Copies.ChangedFile", frame.FileName));
|
||||
if (held.TryGetValue(sha, out var keepFrame))
|
||||
{
|
||||
journal.Verified[frame.Id] = new MergeVerified(sha, keepFrame);
|
||||
continue;
|
||||
}
|
||||
if (frame.Sha256 is not { Length: > 0 })
|
||||
{
|
||||
// Compared by size and time only, and another photo after all: filed in the kept sequence as a second version.
|
||||
var filed = await FileAsync(frame, sha).ConfigureAwait(false);
|
||||
journal.SecondVersions.Add(frame.Id);
|
||||
journal.Verified[frame.Id] = new MergeVerified(sha, filed.FrameId);
|
||||
held.TryAdd(sha, filed.FrameId);
|
||||
Save();
|
||||
continue;
|
||||
}
|
||||
if (journal.Absent.Contains(sha)) continue;
|
||||
failed.Add(sha);
|
||||
}
|
||||
if (failed.Count > 0)
|
||||
{
|
||||
// A photo held by another sequence of the index is not lost with the other; any other is.
|
||||
var others = await new CopySides(_db).HeldElsewhereAsync(failed, [journal.DropId], _ct).ConfigureAwait(false);
|
||||
var lost = failed.Count(s => !others.Contains(s));
|
||||
if (lost > 0) throw new InvalidOperationException(Loc.Plural("Errors.Copies.VerifyFailed", lost));
|
||||
}
|
||||
journal.VerifiedDropFramesVersion = drop.FramesVersion;
|
||||
Save();
|
||||
}
|
||||
|
||||
private async Task<string?> HashAsync(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
return (await ContentHash.Sha256Async(path, null, _ct).ConfigureAwait(false)).ToLowerInvariant();
|
||||
}
|
||||
catch (Exception ex) when (ex is FileNotFoundException or DirectoryNotFoundException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ---- replace ----
|
||||
|
||||
private async Task ReplaceAsync()
|
||||
{
|
||||
var n = 0;
|
||||
foreach (var replace in journal.Replaces)
|
||||
{
|
||||
n++;
|
||||
if (journal.Replaced.Contains(replace.Key) || journal.KeptLosers.Contains(replace.Key)) continue;
|
||||
_ct.ThrowIfCancellationRequested();
|
||||
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Replace, replace.FileName, n, journal.Replaces.Count)).ConfigureAwait(false);
|
||||
var loser = await _db.Frames.GetAsync(replace.KeepFrameId, _ct).ConfigureAwait(false);
|
||||
if (loser is null || loser.SequenceId != journal.KeepId)
|
||||
{
|
||||
journal.Replaced.Add(replace.Key);
|
||||
Save();
|
||||
continue;
|
||||
}
|
||||
if (!await WinnerInPlaceAsync(replace.WinnerSha256).ConfigureAwait(false))
|
||||
{
|
||||
Keep(replace, "the version that replaces it is not in place");
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
var plan = await owner.Archives.FrameTrash.PlanDeleteAsync(journal.KeepId, [replace.KeepFrameId], null, _ct, journal.JobId).ConfigureAwait(false);
|
||||
if (!plan.CanDelete)
|
||||
{
|
||||
Keep(replace, string.Join(", ", plan.Blocked));
|
||||
continue;
|
||||
}
|
||||
await owner.Archives.FrameTrash.DeleteAsync(plan, null, null, _ct, journal.JobId).ConfigureAwait(false);
|
||||
journal.Replaced.Add(replace.Key);
|
||||
Save();
|
||||
}
|
||||
catch (Exception ex) when (ex is StaleDeletePlanException or IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Keep(replace, ex.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void Keep(MergeReplace replace, string why)
|
||||
{
|
||||
journal.KeptLosers.Add(replace.Key);
|
||||
Save();
|
||||
context.Log($"{replace.FileName}: {why}");
|
||||
}
|
||||
|
||||
/// <summary>Whether a usable frame of the kept sequence holds <paramref name="sha"/>, hashed again.</summary>
|
||||
private async Task<bool> WinnerInPlaceAsync(string sha)
|
||||
{
|
||||
foreach (var frame in await _db.Frames.FindBySha256Async(sha.ToLowerInvariant(), _ct).ConfigureAwait(false))
|
||||
{
|
||||
if (frame.SequenceId != journal.KeepId || (frame.Flags & CopySides.UnusableFlags) != 0) continue;
|
||||
if (string.Equals(await HashAsync(keepPaths.FromRelative(frame.RelativePath)).ConfigureAwait(false), sha, StringComparison.OrdinalIgnoreCase)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// ---- details ----
|
||||
|
||||
private async Task DetailsAsync()
|
||||
{
|
||||
if (journal.Payload is null || journal.DetailsDone) return;
|
||||
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Details)).ConfigureAwait(false);
|
||||
await new SequencePayloadWriter(_db, owner.Archives).ApplyAsync(journal.KeepId, journal.Payload, journal.KeepSettingsVersion, _ct).ConfigureAwait(false);
|
||||
journal.DetailsDone = true;
|
||||
Save();
|
||||
}
|
||||
|
||||
// ---- projects ----
|
||||
|
||||
private async Task ProjectsAsync()
|
||||
{
|
||||
if (journal.Projects.Count == 0) return;
|
||||
var from = new ArchiveProjectHome(_db, owner.Archives.Projects, journal.DropId);
|
||||
var to = new ArchiveProjectHome(_db, owner.Archives.Projects, journal.KeepId);
|
||||
var sources = (await _db.Projects.ListBySequenceAsync(journal.DropId, includeDeleted: false, _ct).ConfigureAwait(false)).ToDictionary(p => p.Id);
|
||||
var failed = 0;
|
||||
var n = 0;
|
||||
foreach (var step in journal.Projects)
|
||||
{
|
||||
n++;
|
||||
if (journal.ProjectsDone.Contains(step.SourceProjectId)) continue;
|
||||
_ct.ThrowIfCancellationRequested();
|
||||
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Projects, step.TargetName, n, journal.Projects.Count)).ConfigureAwait(false);
|
||||
if (!sources.TryGetValue(step.SourceProjectId, out var source))
|
||||
{
|
||||
// Deleted from the other since the plan: its own trash keeps it.
|
||||
journal.ProjectsDone.Add(step.SourceProjectId);
|
||||
Save();
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
var copied = await owner.Projects.CopyAsync(from, source, to, step.TargetName, journal.ProjectSteps.GetValueOrDefault(step.SourceProjectId),
|
||||
saved =>
|
||||
{
|
||||
journal.ProjectSteps[step.SourceProjectId] = saved;
|
||||
Save();
|
||||
return ValueTask.CompletedTask;
|
||||
}, _ct).ConfigureAwait(false);
|
||||
await RenamesRendersAsync(step.SourceProjectId, copied.Id).ConfigureAwait(false);
|
||||
journal.ProjectsDone.Add(step.SourceProjectId);
|
||||
Save();
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
failed++;
|
||||
context.Log($"{step.SourceName}: {ex.Message}");
|
||||
}
|
||||
}
|
||||
if (failed > 0) throw new InvalidOperationException(Loc.Plural("Errors.Copies.ProjectsKept", failed));
|
||||
}
|
||||
|
||||
/// <summary>The renders moved to the kept sequence name the project copy (and its versions) instead of the other's.</summary>
|
||||
private async Task RenamesRendersAsync(Guid source, Guid copy)
|
||||
{
|
||||
var access = new DbAccess(_db.Database);
|
||||
if (journal.ProjectSteps.GetValueOrDefault(source) is { } step)
|
||||
{
|
||||
foreach (var move in step.Moves)
|
||||
await access.ExecuteAsync("UPDATE renders SET project_id = $2, project_version_id = $3 WHERE sequence_id = $1 AND project_version_id = $4",
|
||||
[journal.KeepId, copy, move.Copy, move.Source], _ct).ConfigureAwait(false);
|
||||
}
|
||||
await access.ExecuteAsync("UPDATE renders SET project_id = $2 WHERE sequence_id = $1 AND project_id = $3", [journal.KeepId, copy, source], _ct)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// ---- links ----
|
||||
|
||||
private async Task LinksAsync()
|
||||
{
|
||||
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Process)).ConfigureAwait(false);
|
||||
var keep = await _db.Sequences.GetAsync(journal.KeepId, _ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
|
||||
var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false) ?? throw Changed(_dropTitle);
|
||||
var keys = journal.SessionKeys.ToHashSet(StringComparer.Ordinal);
|
||||
if (!journal.KeepKeysWritten)
|
||||
{
|
||||
if (keys.Count > 0 && File.Exists(keepPaths.Sequence(keep.Folder).ManifestPath))
|
||||
{
|
||||
await owner.Archives.Writer.UpdateAsync(keepPaths, keep.Id, keep.Folder, manifest =>
|
||||
keys.All(manifest.SessionKeys.Contains) ? null : manifest with { SessionKeys = [.. manifest.SessionKeys.Union(keys, StringComparer.Ordinal)] },
|
||||
_ct).ConfigureAwait(false);
|
||||
}
|
||||
journal.KeepKeysWritten = true;
|
||||
Save();
|
||||
}
|
||||
if (!journal.DropKeysRemoved)
|
||||
{
|
||||
if (keys.Count > 0 && File.Exists(_dropPaths.Sequence(drop.Folder).ManifestPath))
|
||||
{
|
||||
await owner.Archives.Writer.UpdateAsync(_dropPaths, drop.Id, drop.Folder, manifest =>
|
||||
!manifest.SessionKeys.Any(keys.Contains) ? null : manifest with { SessionKeys = [.. manifest.SessionKeys.Where(k => !keys.Contains(k))] },
|
||||
_ct).ConfigureAwait(false);
|
||||
}
|
||||
journal.DropKeysRemoved = true;
|
||||
Save();
|
||||
}
|
||||
if (!journal.LinksMoved)
|
||||
{
|
||||
var moved = await new FoundCopyStore(_db).MoveToKeptAsync(journal.DropId, journal.KeepId, keepArchive.Id, journal.SessionKeys, journal.CopyId, _ct)
|
||||
.ConfigureAwait(false);
|
||||
foreach (var key in moved.SkippedKeys) context.Log(key);
|
||||
journal.LinksMoved = true;
|
||||
Save();
|
||||
}
|
||||
}
|
||||
|
||||
// ---- trash ----
|
||||
|
||||
private async Task TrashAsync()
|
||||
{
|
||||
var drop = await _db.Sequences.GetAsync(journal.DropId, _ct).ConfigureAwait(false);
|
||||
if (drop is null) return; // a run that moved it before its journal said so
|
||||
await context.ReportAsync(Done(), Total, new JobDetail(JobStep.Trash, _dropTitle)).ConfigureAwait(false);
|
||||
if (await owner.Workspace.HasUnsentAsync(journal.DropId, _ct).ConfigureAwait(false))
|
||||
throw new InvalidOperationException(Loc.L("Errors.Copies.Unsent", _dropTitle));
|
||||
var expected = journal.VerifiedDropFramesVersion ?? throw Changed(_dropTitle);
|
||||
if (drop.FramesVersion != expected) throw Changed(_dropTitle);
|
||||
await using (await owner.Archives.Writer.LockAsync(keepPaths, journal.KeepId, _ct).ConfigureAwait(false))
|
||||
{
|
||||
await EnsureVerifiedHeldAsync().ConfigureAwait(false);
|
||||
var plan = await owner.Archives.Trash.PlanAsync([journal.DropId], null, _ct, journal.JobId).ConfigureAwait(false);
|
||||
if (plan.Blocked.Count > 0)
|
||||
throw new InvalidOperationException(Loc.L("Errors.Copies.TrashBlocked", _dropTitle, Loc.L("Archive.Delete.Reason." + plan.Blocked[0].Reason)));
|
||||
try
|
||||
{
|
||||
await owner.Archives.Trash.DeleteAsync(plan, null, null, _ct, journal.JobId, new Dictionary<Guid, int> { [journal.DropId] = expected })
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (StaleDeletePlanException ex)
|
||||
{
|
||||
throw new InvalidOperationException(Loc.L("Errors.Copies.Changed", _dropTitle), ex);
|
||||
}
|
||||
}
|
||||
context.Log(Loc.L("Archive.Log.MergeTrashed", _dropTitle));
|
||||
}
|
||||
|
||||
/// <summary>Every photo the verify found in the kept sequence is still there and usable (read under its lease).</summary>
|
||||
private async Task EnsureVerifiedHeldAsync()
|
||||
{
|
||||
var frames = await ProcessingFrames.ListAsync(_db, journal.KeepId, _ct).ConfigureAwait(false);
|
||||
var byId = frames.Where(f => (f.Flags & CopySides.UnusableFlags) == 0).ToDictionary(f => f.Id);
|
||||
var hashes = byId.Values.Select(f => f.Sha256).OfType<string>().Select(s => s.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
|
||||
foreach (var verified in journal.Verified.Values)
|
||||
{
|
||||
if (byId.TryGetValue(verified.KeepFrameId, out var frame)
|
||||
&& (frame.Sha256 is null || string.Equals(frame.Sha256, verified.Sha256, StringComparison.OrdinalIgnoreCase)))
|
||||
continue;
|
||||
if (hashes.Contains(verified.Sha256)) continue;
|
||||
throw Changed(_keepTitle);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- done ----
|
||||
|
||||
private async Task FinishAsync()
|
||||
{
|
||||
await context.ReportAsync(Total, Total, new JobDetail(JobStep.FollowUps)).ConfigureAwait(false);
|
||||
await new FoundCopyStore(_db).DeleteAsync(journal.CopyId, CancellationToken.None).ConfigureAwait(false);
|
||||
if (await _db.Sequences.GetAsync(journal.KeepId, CancellationToken.None).ConfigureAwait(false) is { } keep)
|
||||
{
|
||||
foreach (var request in owner.Archives.FrameTrash.FollowUps(keepArchive, keep, (SequenceStatus)journal.KeepStatus))
|
||||
{
|
||||
try
|
||||
{
|
||||
if (request.Targets.Count > 0 && await _db.Jobs.IsQueuedAsync(request.Kind, request.Targets[0], CancellationToken.None).ConfigureAwait(false)) continue;
|
||||
await context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (DatabaseException ex)
|
||||
{
|
||||
Log.Warning("archive", $"{request.Label}: the passes the merged photos need were not queued; the next scan queues them", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
await TellAsync(journal.KeepId, KeepChange).ConfigureAwait(false);
|
||||
await TellAsync(journal.DropId, DropChange).ConfigureAwait(false);
|
||||
var kept = journal.KeptLosers.Count;
|
||||
try
|
||||
{
|
||||
MergeJournal.Delete(keepPaths, journal.JobId);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("archive", $"The journal of merge {journal.JobId} was not removed", ex);
|
||||
}
|
||||
if (kept > 0) throw new InvalidOperationException(Loc.Plural("Errors.Copies.Kept", kept));
|
||||
}
|
||||
|
||||
private async Task TellAsync(Guid id, string change)
|
||||
{
|
||||
if (owner.Archives.Writer.Options.SequenceChanged is not { } changed) return;
|
||||
try
|
||||
{
|
||||
await changed(id, change, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Warning("archive", $"The change of sequence {id} was not told", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.IO;
|
||||
using Chronos.Data.Copies;
|
||||
using Chronos.Data.Sync;
|
||||
|
||||
namespace Chronos.Data.Archiving;
|
||||
|
||||
/// <summary>The stages of a merge, in the order they run (design §5d).</summary>
|
||||
internal enum MergeStage
|
||||
{
|
||||
Copy,
|
||||
Renders,
|
||||
Verify,
|
||||
Replace,
|
||||
Details,
|
||||
Projects,
|
||||
Links,
|
||||
Trash,
|
||||
Done,
|
||||
}
|
||||
|
||||
/// <summary>A photo of the dropped sequence a merge copies into the kept one: its key, the drop's frame, name, size and recorded SHA-256.</summary>
|
||||
internal sealed record MergeCopy(string Key, PhotoAction Action, Guid DropFrameId, string FileName, long Bytes, string? Sha256);
|
||||
|
||||
/// <summary>A losing version of the kept sequence that goes to its photo trash once the winner (the drop's version) is in place.</summary>
|
||||
internal sealed record MergeReplace(string Key, Guid KeepFrameId, string FileName, string WinnerSha256);
|
||||
|
||||
/// <summary>A drop original the verify found in the kept sequence: its SHA-256 now and the keep frame that holds it.</summary>
|
||||
internal sealed record MergeVerified(string Sha256, Guid KeepFrameId);
|
||||
|
||||
/// <summary>
|
||||
/// The journal of a merge (owner review 6, #73; design §5d): <c><keep archive>/.chronos/journal/merge-<job:N>.json</c>,
|
||||
/// written atomically (temp + rename) after each step. It keeps what the confirmed plan decided (a resumed run never plans
|
||||
/// again: its own steps moved the versions) and how far each stage came, so every stage resumes and nothing is done twice.
|
||||
/// </summary>
|
||||
internal sealed class MergeJournal
|
||||
{
|
||||
public Guid JobId { get; set; }
|
||||
|
||||
public Guid CopyId { get; set; }
|
||||
|
||||
public Guid KeepId { get; set; }
|
||||
|
||||
public Guid DropId { get; set; }
|
||||
|
||||
public string Fingerprint { get; set; } = string.Empty;
|
||||
|
||||
public MergeStage Stage { get; set; }
|
||||
|
||||
/// <summary>The kept sequence's settings version the plan read (the details are written only at it).</summary>
|
||||
public int KeepSettingsVersion { get; set; }
|
||||
|
||||
/// <summary>The dropped sequence's frames version the plan read (checked before each photo and at the verify).</summary>
|
||||
public int DropFramesVersion { get; set; }
|
||||
|
||||
/// <summary>The kept sequence's statuses before the merge (what its follow-ups make again).</summary>
|
||||
public int KeepStatus { get; set; }
|
||||
|
||||
public List<MergeCopy> Copies { get; set; } = [];
|
||||
|
||||
public List<MergeReplace> Replaces { get; set; } = [];
|
||||
|
||||
/// <summary>Raw SHA-256 of drop originals the plan leaves out of the kept sequence (last copies, losers, tombstoned in keep, converted).</summary>
|
||||
public List<string> Absent { get; set; } = [];
|
||||
|
||||
public List<RenderStep> Renders { get; set; } = [];
|
||||
|
||||
public SyncPayload? Payload { get; set; }
|
||||
|
||||
public List<ProjectStep> Projects { get; set; } = [];
|
||||
|
||||
public List<string> SessionKeys { get; set; } = [];
|
||||
|
||||
/// <summary>Drop frame → the keep frame it was filed as (or that already held it).</summary>
|
||||
public Dictionary<Guid, Guid> Committed { get; set; } = [];
|
||||
|
||||
/// <summary>Render → its archive-relative path in the kept sequence, written before the copy.</summary>
|
||||
public Dictionary<Guid, string> RenderTargets { get; set; } = [];
|
||||
|
||||
public List<Guid> RendersMoved { get; set; } = [];
|
||||
|
||||
/// <summary>Drop frame → its SHA-256 as hashed by the verify and the keep frame that holds it.</summary>
|
||||
public Dictionary<Guid, MergeVerified> Verified { get; set; } = [];
|
||||
|
||||
/// <summary>Drop frames the verify found to be another photo than keep's (unhashed, same size and time) and filed as a second version.</summary>
|
||||
public List<Guid> SecondVersions { get; set; } = [];
|
||||
|
||||
/// <summary>The dropped sequence's frames version the verify read: the trash refuses another.</summary>
|
||||
public int? VerifiedDropFramesVersion { get; set; }
|
||||
|
||||
public List<string> Replaced { get; set; } = [];
|
||||
|
||||
/// <summary>Keys of losing versions a refused trash step kept (reported at the end).</summary>
|
||||
public List<string> KeptLosers { get; set; } = [];
|
||||
|
||||
public bool DetailsDone { get; set; }
|
||||
|
||||
public Dictionary<Guid, TransplantStep> ProjectSteps { get; set; } = [];
|
||||
|
||||
public List<Guid> ProjectsDone { get; set; } = [];
|
||||
|
||||
public bool KeepKeysWritten { get; set; }
|
||||
|
||||
public bool DropKeysRemoved { get; set; }
|
||||
|
||||
public bool LinksMoved { get; set; }
|
||||
|
||||
/// <summary>The journal of job <paramref name="jobId"/> in the archive at <paramref name="paths"/>.</summary>
|
||||
public static string PathFor(ArchivePaths paths, Guid jobId) => Path.Combine(paths.JournalDirectory, $"merge-{jobId:N}.json");
|
||||
|
||||
/// <summary>Where the job stages the files it copies.</summary>
|
||||
public static string StagingFor(ArchivePaths paths, Guid jobId) => Path.Combine(paths.StagingDirectory, $"merge-{jobId:N}");
|
||||
|
||||
/// <summary>The journal of job <paramref name="jobId"/>; null when there is none (a first run), or it cannot be read.</summary>
|
||||
/// <exception cref="InvalidDataException">The journal is damaged: the merge stops rather than start again over half a merge.</exception>
|
||||
public static MergeJournal? Read(ArchivePaths paths, Guid jobId)
|
||||
{
|
||||
var path = PathFor(paths, jobId);
|
||||
if (!File.Exists(path)) return null;
|
||||
try
|
||||
{
|
||||
return JsonSerializer.Deserialize(AtomicFile.ReadAllText(path), MergeJournalJson.Default.MergeJournal);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
throw new InvalidDataException($"The merge journal {path} is damaged.", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Writes the journal atomically.</summary>
|
||||
public void Save(ArchivePaths paths)
|
||||
{
|
||||
Directory.CreateDirectory(paths.JournalDirectory);
|
||||
AtomicFile.WriteAllText(PathFor(paths, JobId), JsonSerializer.Serialize(this, MergeJournalJson.Default.MergeJournal));
|
||||
}
|
||||
|
||||
/// <summary>Deletes the journal and the staging folder of job <paramref name="jobId"/>.</summary>
|
||||
public static void Delete(ArchivePaths paths, Guid jobId)
|
||||
{
|
||||
var staging = StagingFor(paths, jobId);
|
||||
if (Directory.Exists(staging)) Directory.Delete(staging, recursive: true);
|
||||
File.Delete(PathFor(paths, jobId));
|
||||
}
|
||||
}
|
||||
|
||||
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, UseStringEnumConverter = true,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
[JsonSerializable(typeof(MergeJournal))]
|
||||
internal sealed partial class MergeJournalJson : JsonSerializerContext;
|
||||
@@ -0,0 +1,439 @@
|
||||
using System.Globalization;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.Localization;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Data.Archiving.Jobs;
|
||||
using Chronos.Data.Copies;
|
||||
using Chronos.Data.Jobs;
|
||||
using Chronos.Data.Projects;
|
||||
using Chronos.Data.Repositories;
|
||||
using Chronos.Data.Sync;
|
||||
|
||||
namespace Chronos.Data.Archiving;
|
||||
|
||||
/// <summary>
|
||||
/// Merges two local copies of one time-lapse into the one kept (owner review 6, #73; design §5d; BRIEF §12.5 Merge): the
|
||||
/// dry run (<see cref="PlanAsync(Guid, Guid?, Guid, MergeChoices, CopyPlanContext, CancellationToken)"/>) says which photos
|
||||
/// of the other copy are copied into the kept one, which go to the trash with it and are held nowhere else (last copies),
|
||||
/// the renders and projects that move, the details written, the session keys and links that follow, and what blocks it;
|
||||
/// <see cref="EnqueueAsync"/> queues the <see cref="JobKind.MergeSequences"/> job that applies exactly that plan
|
||||
/// (<see cref="CreateHandler"/>), every photo hash-verified in the kept sequence before the other goes to its archive's trash.
|
||||
/// </summary>
|
||||
public sealed class SequenceMerger(ChronosDatabase db, ArchiveService archives, CopySides sides)
|
||||
{
|
||||
/// <summary>The free space a merge leaves on the kept sequence's volume beyond what it copies.</summary>
|
||||
public const long SpaceReserve = 512L << 20;
|
||||
|
||||
private readonly ChronosDatabase _db = db ?? throw new ArgumentNullException(nameof(db));
|
||||
private readonly ArchiveService _archives = archives ?? throw new ArgumentNullException(nameof(archives));
|
||||
private readonly CopySides _sides = sides ?? throw new ArgumentNullException(nameof(sides));
|
||||
private readonly DbAccess _access = new(db.Database);
|
||||
|
||||
internal ChronosDatabase Db => _db;
|
||||
|
||||
internal ArchiveService Archives => _archives;
|
||||
|
||||
/// <summary>The handler of <see cref="JobKind.MergeSequences"/> jobs; register it on the <see cref="JobEngine"/>.</summary>
|
||||
public static IJobHandler CreateHandler(ChronosDatabase db, SequenceMerger merger, ArchiveService archives, IProjectCopier projects,
|
||||
IWorkspaceGuard workspace) => new MergeJobHandler(db, merger, archives, projects, workspace);
|
||||
|
||||
/// <summary>
|
||||
/// The dry run of merging the pair <paramref name="copyId"/>: <paramref name="keepId"/> is kept and the other goes to the
|
||||
/// trash. With <paramref name="keepId"/> null the kept one is preselected only when exactly one of the two is linked to a
|
||||
/// server and lies in a server's originals archive; otherwise the plan is made keeping the one that is not
|
||||
/// <paramref name="otherId"/> and blocks with <see cref="CopyBlock.KeptUnchosen"/>.
|
||||
/// </summary>
|
||||
/// <exception cref="ArgumentException">The pair is not a local pair of these sequences, or a sequence is not in the index.</exception>
|
||||
public Task<MergePlan> PlanAsync(Guid copyId, Guid? keepId, Guid otherId, MergeChoices choices, CopyPlanContext context, CancellationToken ct) =>
|
||||
PlanAsync(copyId, keepId, otherId, choices, context, null, ct);
|
||||
|
||||
/// <summary>The dry run, with the jobs of <paramref name="exceptJob"/> (the merge job planning again) not counted as busy.</summary>
|
||||
internal async Task<MergePlan> PlanAsync(Guid copyId, Guid? keepId, Guid otherId, MergeChoices choices, CopyPlanContext context, Guid? exceptJob,
|
||||
CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(choices);
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
var copy = await new FoundCopyStore(_db).GetAsync(copyId, ct).ConfigureAwait(false)
|
||||
?? throw new ArgumentException($"Unknown pair {copyId}.", nameof(copyId));
|
||||
if (copy is not { Kind: CopyKind.LocalLocal, OtherId: { } second } || !copy.Names(otherId))
|
||||
throw new ArgumentException($"Pair {copyId} is not a local pair of sequence {otherId}.", nameof(otherId));
|
||||
var first = copy.LocalId;
|
||||
var blocks = new List<CopyBlock>();
|
||||
|
||||
Guid keep, drop;
|
||||
if (keepId is { } chosen)
|
||||
{
|
||||
if (!copy.Names(chosen)) throw new ArgumentException($"Sequence {chosen} is not one of pair {copyId}.", nameof(keepId));
|
||||
(keep, drop) = (chosen, chosen == first ? second : first);
|
||||
}
|
||||
else if (await PreselectAsync(first, second, context, ct).ConfigureAwait(false) is { } preselected)
|
||||
{
|
||||
(keep, drop) = (preselected, preselected == first ? second : first);
|
||||
}
|
||||
else
|
||||
{
|
||||
(keep, drop) = (otherId == first ? second : first, otherId);
|
||||
blocks.Add(CopyBlock.KeptUnchosen);
|
||||
}
|
||||
|
||||
var keepRow = await _db.Sequences.GetAsync(keep, ct).ConfigureAwait(false) ?? throw new ArgumentException($"Unknown sequence {keep}.", nameof(copyId));
|
||||
var dropRow = await _db.Sequences.GetAsync(drop, ct).ConfigureAwait(false) ?? throw new ArgumentException($"Unknown sequence {drop}.", nameof(copyId));
|
||||
var keepArchive = await _db.Archives.GetAsync(keepRow.ArchiveId, ct).ConfigureAwait(false)
|
||||
?? throw new ArgumentException($"Unknown archive {keepRow.ArchiveId}.", nameof(copyId));
|
||||
var dropArchive = await _db.Archives.GetAsync(dropRow.ArchiveId, ct).ConfigureAwait(false)
|
||||
?? throw new ArgumentException($"Unknown archive {dropRow.ArchiveId}.", nameof(copyId));
|
||||
var keepPaths = new ArchivePaths(keepArchive.RootPath);
|
||||
var dropPaths = new ArchivePaths(dropArchive.RootPath);
|
||||
|
||||
// Where the two are: online, in the folders the index has, not worked on by a job.
|
||||
var online = ArchiveVolume.Probe(keepPaths) == ArchiveAvailability.Online && ArchiveVolume.Probe(dropPaths) == ArchiveAvailability.Online;
|
||||
if (!online) blocks.Add(CopyBlock.ArchiveOffline);
|
||||
else if (Moved(keepPaths, keepRow) || Moved(dropPaths, dropRow)) blocks.Add(CopyBlock.FolderMoved);
|
||||
if (await BusyAsync([keep, drop, keepRow.ArchiveId, dropRow.ArchiveId], exceptJob, ct).ConfigureAwait(false)) blocks.Add(CopyBlock.Busy);
|
||||
if (context.UnsentWorkspaceSequences.Contains(drop)) blocks.Add(CopyBlock.UnsentWorkspace);
|
||||
|
||||
// Photos: keep is "here", drop is "there".
|
||||
var here = await _sides.LocalAsync(keep, ct).ConfigureAwait(false);
|
||||
var there = await _sides.LocalAsync(drop, ct).ConfigureAwait(false);
|
||||
var comparison = CopyComparer.Compare(here, there);
|
||||
var steps = new List<PhotoStep>();
|
||||
var excluded = new List<string>();
|
||||
foreach (var photo in comparison.Photos)
|
||||
{
|
||||
switch (photo.Class)
|
||||
{
|
||||
case PhotoClass.OnlyThere:
|
||||
// Every photo only the other holds is copied: it goes to the trash, nothing it alone holds stays behind.
|
||||
steps.Add(Step(photo.Key, PhotoAction.Fetch, photo.There!));
|
||||
break;
|
||||
case PhotoClass.Differs:
|
||||
var choice = choices.Photos switch
|
||||
{
|
||||
CopyPhotoPolicy.Union => PhotoChoice.KeepBoth,
|
||||
CopyPhotoPolicy.ThisPcWins => PhotoChoice.TakeHere,
|
||||
CopyPhotoPolicy.OtherWins => PhotoChoice.TakeThere,
|
||||
_ => choices.PerPhoto.GetValueOrDefault(photo.Key, PhotoChoice.Leave),
|
||||
};
|
||||
if (choice is PhotoChoice.KeepBoth or PhotoChoice.TakeThere) steps.Add(Step(photo.Key, PhotoAction.FetchVersion, photo.There!));
|
||||
if (choice == PhotoChoice.KeepBoth && photo.There!.Sha256 is { Length: > 0 } version) excluded.Add(version.ToLowerInvariant());
|
||||
if (choice == PhotoChoice.TakeThere) steps.Add(Step(photo.Key, PhotoAction.TrashHere, photo.Here!));
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Last copies: drop originals nothing will hold once the copies are made (losers, tombstoned in keep, converted), and
|
||||
// keep's losing versions nothing else holds.
|
||||
var fetched = steps.Where(s => s.Action is PhotoAction.Fetch or PhotoAction.FetchVersion).Select(s => s.Sha256).OfType<string>()
|
||||
.Select(s => s.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
|
||||
var losers = steps.Where(s => s.Action == PhotoAction.TrashHere).Select(s => s.FrameId!.Value).ToHashSet();
|
||||
var keepHolds = here.Photos.Where(p => !p.Unusable && p.Sha256 is { Length: > 0 } && !losers.Contains(p.FrameId))
|
||||
.Select(p => p.Sha256!.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
|
||||
var absent = new List<CopyPhotoInput>();
|
||||
foreach (var frame in there.Photos.Where(p => !p.Unusable && p.Sha256 is { Length: > 0 }))
|
||||
{
|
||||
var sha = frame.Sha256!.ToLowerInvariant();
|
||||
if (keepHolds.Contains(sha) || fetched.Contains(sha) || absent.Any(a => string.Equals(a.Sha256, sha, StringComparison.OrdinalIgnoreCase))) continue;
|
||||
absent.Add(frame with { Sha256 = sha });
|
||||
}
|
||||
var lastCopies = new List<LastCopy>();
|
||||
var heldElsewhere = await _sides.HeldElsewhereAsync([.. absent.Select(a => a.Sha256!)], [drop], ct).ConfigureAwait(false);
|
||||
foreach (var frame in absent.Where(a => !heldElsewhere.Contains(a.Sha256!)))
|
||||
lastCopies.Add(new LastCopy(frame.Sha256!, CopySide.Other, frame.FileName, frame.Size));
|
||||
var loserSteps = steps.Where(s => s.Action == PhotoAction.TrashHere && s.Sha256 is { Length: > 0 }).ToList();
|
||||
if (loserSteps.Count > 0)
|
||||
{
|
||||
var loserHeld = await _sides.HeldElsewhereAsync([.. loserSteps.Select(s => s.Sha256!)], [keep, drop], ct).ConfigureAwait(false);
|
||||
foreach (var step in loserSteps)
|
||||
{
|
||||
var sha = step.Sha256!.ToLowerInvariant();
|
||||
if (loserHeld.Contains(sha) || keepHolds.Contains(sha)) continue;
|
||||
lastCopies.Add(new LastCopy(sha, CopySide.ThisPc, step.FileName, step.Bytes));
|
||||
}
|
||||
}
|
||||
|
||||
// Renders the kept sequence lacks (by id and SHA-256) move to it.
|
||||
var keepRenders = await _db.Renders.ListBySequenceAsync(keep, ct).ConfigureAwait(false);
|
||||
var keepRenderHashes = keepRenders.Select(r => r.Sha256).OfType<string>().ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var renders = (await _db.Renders.ListBySequenceAsync(drop, ct).ConfigureAwait(false))
|
||||
.Where(r => keepRenders.All(k => k.Id != r.Id) && (r.Sha256 is null || !keepRenderHashes.Contains(r.Sha256)))
|
||||
.OrderBy(r => r.Created).ThenBy(r => r.Id)
|
||||
.Select(r => new RenderStep(r.Id, Path.GetFileName(r.OutputPath.Replace('\\', '/').TrimEnd('/')), r.Bytes, r.Sha256)).ToList();
|
||||
|
||||
// Details, line by line, applied to the kept sequence only.
|
||||
var keepRemovals = await _db.FrameRemovals.ListBySequenceAsync(keep, ct).ConfigureAwait(false);
|
||||
var dropRemovals = await _db.FrameRemovals.ListBySequenceAsync(drop, ct).ConfigureAwait(false);
|
||||
var keepPhotos = await SequencePayloadWriter.PhotosAsync(_db, keep, dropRemovals, ct).ConfigureAwait(false);
|
||||
var dropPhotos = await SequencePayloadWriter.PhotosAsync(_db, drop, keepRemovals, ct).ConfigureAwait(false);
|
||||
var lines = CopyPayload.Lines(SyncPayload.Of(keepRow), keepPhotos, SyncPayload.Of(dropRow), dropPhotos);
|
||||
SyncPayload? payload = null;
|
||||
var changes = 0;
|
||||
if (lines.Any(l => !choices.Details.ContainsKey(l.Path)))
|
||||
{
|
||||
blocks.Add(CopyBlock.DetailsUnchosen);
|
||||
}
|
||||
else
|
||||
{
|
||||
changes = lines.DistinctBy(l => l.Path, StringComparer.Ordinal).Count(l => choices.Details[l.Path] == CopySide.Other);
|
||||
if (changes > 0 || excluded.Count > 0)
|
||||
{
|
||||
var mine = CopyPayload.Portable(SyncPayload.Of(keepRow), keepPhotos);
|
||||
var merged = changes > 0 ? CopyPayload.Merge(mine, CopyPayload.Portable(SyncPayload.Of(dropRow), dropPhotos), choices.Details, lines) : mine;
|
||||
try
|
||||
{
|
||||
merged = CopyPayload.ExcludeVersions(merged, excluded);
|
||||
}
|
||||
catch (Exception ex) when (ex is JsonException or InvalidDataException)
|
||||
{
|
||||
// Settings that are not a project of this Chronos: the incoming versions stay included.
|
||||
}
|
||||
payload = merged.SameAs(mine) && changes == 0 ? null : merged;
|
||||
}
|
||||
}
|
||||
|
||||
// Projects are always copied across: the other's home goes to the trash.
|
||||
var projects = await ProjectStepsAsync(keep, drop, dropArchive.Name, ct).ConfigureAwait(false);
|
||||
|
||||
// Session keys and links follow the kept sequence.
|
||||
var (sessionKeys, skippedKeys) = await SessionKeysAsync(keep, drop, keepRow.ArchiveId, ct).ConfigureAwait(false);
|
||||
var links = await _access.ListAsync("SELECT server, local_id FROM sync_links WHERE local_id = $1 OR local_id = $2", [keep, drop],
|
||||
r => (Server: r.GetString(0), Local: r.GetGuid(1)), ct).ConfigureAwait(false);
|
||||
var moved = links.Where(l => l.Local == drop).Select(l => l.Server).Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal).ToList();
|
||||
if (links.GroupBy(l => l.Server, StringComparer.Ordinal).Any(g => g.Select(l => l.Local).Distinct().Count() > 1)) blocks.Add(CopyBlock.BothLinked);
|
||||
|
||||
// The other goes to the trash.
|
||||
var trash = await _archives.Trash.PlanAsync([drop], null, ct, exceptJob).ConfigureAwait(false);
|
||||
if (trash.Blocked.Any(b => b.Reason is not (DeleteBlock.Busy or DeleteBlock.Offline))) blocks.Add(CopyBlock.TrashBlocked);
|
||||
else if (trash.Blocked.Any(b => b.Reason == DeleteBlock.Busy) && !blocks.Contains(CopyBlock.Busy)) blocks.Add(CopyBlock.Busy);
|
||||
|
||||
// Space on the kept sequence's volume.
|
||||
var copyBytes = steps.Where(s => s.Action is PhotoAction.Fetch or PhotoAction.FetchVersion).Sum(s => s.Bytes) + renders.Sum(r => r.Bytes);
|
||||
var free = online ? FreeSpace(keepPaths) : null;
|
||||
if (free is { } bytes && copyBytes + SpaceReserve > bytes) blocks.Add(CopyBlock.NoSpace);
|
||||
|
||||
// Details are in the fingerprint as they read now: an edit that keeps the versions (a title) still moves it.
|
||||
var keepStamp = await StampAsync(keep, ct).ConfigureAwait(false) + ":" + PayloadHash(SyncPayload.Of(keepRow));
|
||||
var dropStamp = await StampAsync(drop, ct).ConfigureAwait(false) + ":" + PayloadHash(SyncPayload.Of(dropRow));
|
||||
var fingerprint = CopyRules.PlanFingerprint(FingerprintLines(keepRow, dropRow, keepStamp, dropStamp, choices, steps, lastCopies, renders, payload,
|
||||
projects, sessionKeys, skippedKeys));
|
||||
return new MergePlan(copyId, keep, drop, choices, steps, lastCopies, renders, payload, changes, projects, sessionKeys, skippedKeys, moved, trash,
|
||||
copyBytes, free ?? -1, [.. blocks.Distinct()], keepRow.FramesVersion, keepRow.SettingsVersion, dropRow.FramesVersion, dropRow.SettingsVersion,
|
||||
fingerprint);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queues the job that applies <paramref name="plan"/>: the pair goes to <see cref="CopyState.Integrating"/> (its state
|
||||
/// before kept as the previous one), the job targets the kept sequence, then the other (both held from other jobs).
|
||||
/// </summary>
|
||||
/// <param name="title">The kept sequence's title, for the job's label ("Merge into {title}").</param>
|
||||
/// <exception cref="InvalidOperationException">The plan is blocked, or the pair is already being merged or integrated.</exception>
|
||||
/// <exception cref="StaleCopyPlanException">The pair changed state since the plan.</exception>
|
||||
public async Task<Job> EnqueueAsync(JobEngine engine, FoundCopyStore store, MergePlan plan, string title, CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(engine);
|
||||
ArgumentNullException.ThrowIfNull(store);
|
||||
ArgumentNullException.ThrowIfNull(plan);
|
||||
if (!plan.CanApply) throw new InvalidOperationException($"The merge plan is blocked: {string.Join(", ", plan.Blocks)}.");
|
||||
var copy = await store.GetAsync(plan.CopyId, ct).ConfigureAwait(false) ?? throw new StaleCopyPlanException();
|
||||
if (copy.State == CopyState.Integrating) throw new InvalidOperationException(Loc.L("Errors.Copies.Busy"));
|
||||
var id = Guid.CreateVersion7();
|
||||
if (!await store.SetStateAsync(plan.CopyId, copy.State, CopyState.Integrating, new CopyAnswer(null, null, id, null, DateTimeOffset.UtcNow), ct)
|
||||
.ConfigureAwait(false))
|
||||
throw new StaleCopyPlanException();
|
||||
var parameters = new MergeJobParameters(plan.CopyId, plan.KeepId, plan.DropId, plan.Choices, plan.Fingerprint);
|
||||
try
|
||||
{
|
||||
return await engine.EnqueueAsync(id, JobKind.MergeSequences, [plan.KeepId, plan.DropId], Loc.L("Jobs.Target.Merge", title), parameters.ToJson(),
|
||||
JobPriority.User, null, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
await store.SetStateAsync(plan.CopyId, CopyState.Integrating, copy.State, null, CancellationToken.None).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The sentence a job fails with for the first block of a plan it made again.</summary>
|
||||
internal async Task<string> BlockMessageAsync(MergePlan plan, CancellationToken ct)
|
||||
{
|
||||
var keep = await _db.Sequences.GetAsync(plan.KeepId, ct).ConfigureAwait(false);
|
||||
var drop = await _db.Sequences.GetAsync(plan.DropId, ct).ConfigureAwait(false);
|
||||
var (keepTitle, dropTitle) = (keep?.DisplayTitle ?? string.Empty, drop?.DisplayTitle ?? string.Empty);
|
||||
return plan.Blocks[0] switch
|
||||
{
|
||||
CopyBlock.Busy => Loc.L("Errors.Copies.Busy"),
|
||||
CopyBlock.ArchiveOffline => Loc.L("Errors.Copies.ArchiveOffline", dropTitle),
|
||||
CopyBlock.FolderMoved => Loc.L("Errors.Copies.FolderMoved", dropTitle),
|
||||
CopyBlock.NoSpace => Loc.L("Errors.Copies.NoSpace", keepTitle, Size(plan.CopyBytes + SpaceReserve)),
|
||||
CopyBlock.BothLinked => Loc.L("Errors.Copies.BothLinked", string.Empty),
|
||||
CopyBlock.UnsentWorkspace => Loc.L("Errors.Copies.Unsent", dropTitle),
|
||||
CopyBlock.TrashBlocked => Loc.L("Errors.Copies.TrashBlocked", dropTitle,
|
||||
plan.Trash.Blocked.Select(b => Loc.L("Archive.Delete.Reason." + b.Reason)).FirstOrDefault() ?? string.Empty),
|
||||
CopyBlock.DetailsUnchosen => Loc.Plural("Errors.Copies.DetailsUnchosen", 1),
|
||||
CopyBlock.KeptUnchosen => Loc.L("Errors.Copies.KeptUnchosen"),
|
||||
_ => Loc.L("Errors.Copies.StalePlan"),
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>Free bytes on the kept archive's volume (its root and its staging folder, the tighter); null when they cannot be measured.</summary>
|
||||
internal long? FreeSpace(ArchivePaths paths)
|
||||
{
|
||||
long? free = null;
|
||||
foreach (var folder in (string[])[paths.Root, paths.StagingDirectory])
|
||||
{
|
||||
if (!Directory.Exists(folder) || Measure(folder) is not { } bytes) continue;
|
||||
free = free is { } known ? Math.Min(known, bytes) : bytes;
|
||||
}
|
||||
return free;
|
||||
|
||||
long? Measure(string folder)
|
||||
{
|
||||
if (_archives.FreeSpace is { } measure) return measure(folder);
|
||||
try
|
||||
{
|
||||
return new DriveInfo(OperatingSystem.IsWindows() ? Path.GetPathRoot(folder)! : folder).AvailableFreeSpace;
|
||||
}
|
||||
catch (Exception ex) when (ex is ArgumentException or IOException or UnauthorizedAccessException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static string Size(long bytes) => bytes >= 1_000_000_000
|
||||
? Loc.L("Common.Size.GB", (bytes / 1e9).ToString("0.0", Loc.Culture))
|
||||
: Loc.L("Common.Size.MB", (bytes / 1e6).ToString("0", Loc.Culture));
|
||||
|
||||
private static PhotoStep Step(string key, PhotoAction action, CopyPhotoInput frame) =>
|
||||
new(key, action, frame.FrameId, frame.FileName, frame.Size, string.IsNullOrEmpty(frame.Sha256) ? null : frame.Sha256.ToLowerInvariant());
|
||||
|
||||
private static bool Moved(ArchivePaths paths, Sequence sequence) =>
|
||||
sequence.Folder.Length > 0 && !SequenceFolders.Holds(paths.Sequence(sequence.Folder), sequence.Id);
|
||||
|
||||
/// <summary>
|
||||
/// The kept sequence preselected: the one of the two that is linked to a server and lies in a server's originals archive,
|
||||
/// when exactly one is (owner review 5 homes: a server's originals stay in its archive); null otherwise.
|
||||
/// </summary>
|
||||
private async Task<Guid?> PreselectAsync(Guid a, Guid b, CopyPlanContext context, CancellationToken ct)
|
||||
{
|
||||
if (context.ServerOriginalsArchives.Count == 0) return null;
|
||||
var linked = await _access.ListAsync("SELECT DISTINCT local_id FROM sync_links WHERE local_id = $1 OR local_id = $2", [a, b], r => r.GetGuid(0), ct)
|
||||
.ConfigureAwait(false);
|
||||
var candidates = new List<Guid>();
|
||||
foreach (var id in linked)
|
||||
{
|
||||
if (await _db.Sequences.GetAsync(id, ct).ConfigureAwait(false) is { } row && context.ServerOriginalsArchives.Contains(row.ArchiveId)) candidates.Add(id);
|
||||
}
|
||||
return candidates.Count == 1 ? candidates[0] : null;
|
||||
}
|
||||
|
||||
/// <summary>Whether a queued, running or paused job (but <paramref name="exceptJob"/>) targets any of <paramref name="targets"/>.</summary>
|
||||
private async Task<bool> BusyAsync(IReadOnlyCollection<Guid> targets, Guid? exceptJob, CancellationToken ct)
|
||||
{
|
||||
var rows = await _access.ListAsync(
|
||||
$"SELECT id, target_ids FROM jobs WHERE state IN ({(int)JobState.Queued}, {(int)JobState.Running}, {(int)JobState.Paused})", [],
|
||||
r => (Id: r.GetGuid(0), Targets: r.GetString(1)), ct).ConfigureAwait(false);
|
||||
var wanted = targets.ToHashSet();
|
||||
foreach (var (id, ids) in rows)
|
||||
{
|
||||
if (id == exceptJob || ids.Length == 0) continue;
|
||||
foreach (var part in ids.Split(','))
|
||||
{
|
||||
if (Guid.TryParse(part, out var target) && wanted.Contains(target)) return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>The drop's non-Main projects, each with the name it gets at the kept sequence's home.</summary>
|
||||
private async Task<List<ProjectStep>> ProjectStepsAsync(Guid keep, Guid drop, string dropArchiveName, CancellationToken ct)
|
||||
{
|
||||
var taken = (await _db.Projects.ListBySequenceAsync(keep, includeDeleted: false, ct).ConfigureAwait(false)).Select(p => p.Name)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var steps = new List<ProjectStep>();
|
||||
foreach (var project in (await _db.Projects.ListBySequenceAsync(drop, includeDeleted: false, ct).ConfigureAwait(false))
|
||||
.Where(p => !p.IsMain).OrderBy(p => p.CreatedAt).ThenBy(p => p.Id))
|
||||
{
|
||||
var name = FreeName(project.Name, dropArchiveName, taken);
|
||||
taken.Add(name);
|
||||
steps.Add(new ProjectStep(CopySide.Other, project.Id, project.Name, name, project.HeadNumber, project.HeadVersionId ?? Guid.Empty));
|
||||
}
|
||||
return steps;
|
||||
}
|
||||
|
||||
/// <summary>The name itself when free, else "<name> (<archive>)", then " (2)"…, each within <see cref="ProjectService.MaxNameLength"/>.</summary>
|
||||
internal static string FreeName(string name, string archiveName, IReadOnlySet<string> taken)
|
||||
{
|
||||
if (!taken.Contains(name)) return name;
|
||||
var stem = Fit(name, $" ({archiveName})");
|
||||
if (!taken.Contains(stem)) return stem;
|
||||
for (var n = 2; ; n++)
|
||||
{
|
||||
var candidate = Fit(stem, string.Create(CultureInfo.InvariantCulture, $" ({n})"));
|
||||
if (!taken.Contains(candidate)) return candidate;
|
||||
}
|
||||
|
||||
static string Fit(string stem, string suffix)
|
||||
{
|
||||
const int max = ProjectService.MaxNameLength;
|
||||
if (suffix.Length >= max) suffix = suffix[..(max / 2)];
|
||||
return (stem.Length + suffix.Length > max ? stem[..(max - suffix.Length)].TrimEnd() : stem) + suffix;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The drop's session keys that move to the kept sequence, and those the kept archive maps to a third sequence already.</summary>
|
||||
private async Task<(List<string> Moved, List<string> Skipped)> SessionKeysAsync(Guid keep, Guid drop, Guid keepArchiveId, CancellationToken ct)
|
||||
{
|
||||
var keys = await _access.ListAsync("SELECT session_key FROM sequence_sessions WHERE sequence_id = $1 ORDER BY session_key", [drop],
|
||||
r => r.GetString(0), ct).ConfigureAwait(false);
|
||||
var (moved, skipped) = (new List<string>(), new List<string>());
|
||||
foreach (var key in keys.Distinct(StringComparer.Ordinal))
|
||||
{
|
||||
var holder = await _db.Sequences.FindBySessionKeyAsync(keepArchiveId, key, ct).ConfigureAwait(false);
|
||||
if (holder is null || holder == keep || holder == drop) moved.Add(key);
|
||||
else skipped.Add(key);
|
||||
}
|
||||
return (moved, skipped);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// What a sequence's photos are in the index, beyond its versions: how many, their bytes, how many are hashed, flagged
|
||||
/// missing, corrupt, read from the preview only or undecodable (a verify or a decode records those without another
|
||||
/// frames version), and its tombstones. Of this sequence alone: an import elsewhere in its archive leaves it as it was.
|
||||
/// </summary>
|
||||
private async Task<string> StampAsync(Guid sequenceId, CancellationToken ct)
|
||||
{
|
||||
var flags = string.Join(", ", new[] { FrameFlags.Missing, FrameFlags.Corrupt, FrameFlags.PreviewOnly, FrameFlags.Undecodable }
|
||||
.Select(flag => $"CAST(COALESCE(SUM(CASE WHEN (flags & {(int)flag}) <> 0 THEN 1 ELSE 0 END), 0) AS BIGINT)"));
|
||||
var photos = await _access.SingleAsync(
|
||||
$"SELECT CAST(count(*) AS BIGINT), CAST(COALESCE(SUM(size), 0) AS BIGINT), CAST(COALESCE(SUM(CASE WHEN sha256 IS NULL OR sha256 = '' THEN 0 ELSE 1 END), 0) AS BIGINT), {flags} FROM frames WHERE sequence_id = $1",
|
||||
[sequenceId], r => string.Join(':', Enumerable.Range(0, 7).Select(i => r.GetInt64(i).ToString(CultureInfo.InvariantCulture))), ct).ConfigureAwait(false);
|
||||
var removals = await _access.ScalarAsync<long>("SELECT count(*) FROM frame_removals WHERE sequence_id = $1", [sequenceId], ct).ConfigureAwait(false);
|
||||
return string.Create(CultureInfo.InvariantCulture, $"{photos}:{removals}");
|
||||
}
|
||||
|
||||
private static string PayloadHash(SyncPayload payload) =>
|
||||
Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(payload, SyncPayloadJson.Default.SyncPayload))));
|
||||
|
||||
private static IEnumerable<string> FingerprintLines(Sequence keep, Sequence drop, string keepStamp, string dropStamp, MergeChoices choices,
|
||||
List<PhotoStep> steps, List<LastCopy> lastCopies, List<RenderStep> renders, SyncPayload? payload, List<ProjectStep> projects,
|
||||
List<string> sessionKeys, List<string> skippedKeys)
|
||||
{
|
||||
var inv = CultureInfo.InvariantCulture;
|
||||
yield return string.Create(inv, $"keep:{keep.Id:N}:{keep.FramesVersion}:{keep.SettingsVersion}:{keepStamp}");
|
||||
yield return string.Create(inv, $"drop:{drop.Id:N}:{drop.FramesVersion}:{drop.SettingsVersion}:{dropStamp}");
|
||||
yield return $"policy:{choices.Photos}";
|
||||
foreach (var (key, choice) in choices.PerPhoto.OrderBy(p => p.Key, StringComparer.Ordinal)) yield return $"choice:{key}:{choice}";
|
||||
foreach (var (path, side) in choices.Details.OrderBy(p => p.Key, StringComparer.Ordinal)) yield return $"detail:{path}:{side}";
|
||||
foreach (var line in steps.Select(s => $"{s.Action}:{s.Key}").Order(StringComparer.Ordinal)) yield return line;
|
||||
foreach (var line in lastCopies.Select(l => $"last:{l.Side}:{l.Key}").Order(StringComparer.Ordinal)) yield return line;
|
||||
foreach (var render in renders) yield return string.Create(inv, $"render:{render.RenderId:N}:{render.Sha256}");
|
||||
if (payload is not null)
|
||||
yield return "payload:" + PayloadHash(payload);
|
||||
foreach (var project in projects) yield return string.Create(inv, $"{project.From}:{project.SourceProjectId:N}:{project.SourceHeadId:N}:{project.TargetName}");
|
||||
foreach (var key in sessionKeys) yield return "session:" + key;
|
||||
foreach (var key in skippedKeys) yield return "skipped:" + key;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
using System.Text.Json;
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.IO;
|
||||
using Chronos.Core.Localization;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Core.Project;
|
||||
using Chronos.Data.Processing;
|
||||
using Chronos.Data.Repositories;
|
||||
using Chronos.Data.Sync;
|
||||
|
||||
namespace Chronos.Data.Archiving;
|
||||
|
||||
/// <summary>
|
||||
/// Writes the details a merge settled (owner review 6, #73; design §5d step 6) into the kept sequence in one step: title,
|
||||
/// tags, rating, notes and settings, under the sequence's lease, only while its settings are still at the version the plan
|
||||
/// read. The payload comes in portable form (frame positions by photo); its positions are placed on the sequence's own
|
||||
/// frames as they are under the lease, then the index, <c>project.json</c> and the sidecar are written as
|
||||
/// <see cref="SyncService"/> applies a server's payload. A payload the sequence already holds writes nothing (a resumed run).
|
||||
/// </summary>
|
||||
internal sealed class SequencePayloadWriter(ChronosDatabase db, ArchiveService archives)
|
||||
{
|
||||
/// <summary>
|
||||
/// Applies <paramref name="portable"/> to <paramref name="sequenceId"/> at <paramref name="expectedSettingsVersion"/>;
|
||||
/// returns the sequence as it is then.
|
||||
/// </summary>
|
||||
/// <exception cref="SyncConflictChangedException">The settings moved since the plan (an edit, an analysis): nothing was written.</exception>
|
||||
/// <exception cref="SequenceFolderMovedException">The sequence is not in the folder the index has: nothing was written.</exception>
|
||||
/// <exception cref="ArgumentException">The sequence or its archive is not in the index.</exception>
|
||||
public async Task<Sequence> ApplyAsync(Guid sequenceId, SyncPayload portable, int expectedSettingsVersion, CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(portable);
|
||||
var sequence = await db.Sequences.GetAsync(sequenceId, ct).ConfigureAwait(false)
|
||||
?? throw new ArgumentException($"Unknown sequence {sequenceId}.", nameof(sequenceId));
|
||||
var archive = await db.Archives.GetAsync(sequence.ArchiveId, ct).ConfigureAwait(false)
|
||||
?? throw new ArgumentException($"Unknown archive {sequence.ArchiveId}.", nameof(sequenceId));
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
await using (await archives.Writer.LockAsync(paths, sequenceId, ct).ConfigureAwait(false))
|
||||
{
|
||||
var row = await db.Sequences.GetAsync(sequenceId, ct).ConfigureAwait(false)
|
||||
?? throw new ArgumentException($"Unknown sequence {sequenceId}.", nameof(sequenceId));
|
||||
var folder = paths.Sequence(row.Folder);
|
||||
if (row.Folder.Length > 0 && !SequenceFolders.Holds(folder, row.Id)) throw new SequenceFolderMovedException();
|
||||
// The frames cannot change under the lease: the positions are placed on them as they are now.
|
||||
var payload = portable with { SettingsJson = await PlaceAsync(row.Id, portable.SettingsJson, ct).ConfigureAwait(false) };
|
||||
if (row.SettingsVersion != expectedSettingsVersion)
|
||||
{
|
||||
// A resumed run that wrote the payload before its journal said so finds it in place.
|
||||
if (payload.SameAs(SyncPayload.Of(row))) return row;
|
||||
throw new SyncConflictChangedException(Loc.L("Errors.Copies.DetailsChanged"));
|
||||
}
|
||||
var metadata = !payload.SameMetadata(SyncPayload.Of(row));
|
||||
var settings = !SyncRules.SameSettings(payload.SettingsJson, row.SettingsJson);
|
||||
if (!metadata && !settings) return row;
|
||||
|
||||
string? project = null;
|
||||
Guid? hero = null;
|
||||
if (settings)
|
||||
{
|
||||
var edited = ProjectStore.Parse(SyncRules.NormalizeSettings(payload.SettingsJson));
|
||||
project = edited.ToJson();
|
||||
hero = await HeroFrame.EditAsync(db, row.Id, ProjectStore.Read(row, folder).HeroFrame, edited.HeroFrame, ct).ConfigureAwait(false);
|
||||
}
|
||||
var edit = new SequenceEdit
|
||||
{
|
||||
Title = metadata ? payload.Title?.Trim() ?? string.Empty : null,
|
||||
Tags = metadata ? payload.Tags : null,
|
||||
Rating = metadata ? payload.Rating : null,
|
||||
Notes = metadata ? payload.Notes ?? string.Empty : null,
|
||||
SettingsJson = project,
|
||||
HeroFrameId = hero,
|
||||
};
|
||||
// The index first, at the version read under the lease: a writer that skips the lease still cannot be overwritten.
|
||||
var result = await db.Sequences.EditAsync(row.Id, edit, expectedSettingsVersion, ct).ConfigureAwait(false);
|
||||
if (result is not { Outcome: SequenceEditOutcome.Updated, Sequence: { } updated })
|
||||
throw new SyncConflictChangedException(Loc.L("Errors.Copies.DetailsChanged"));
|
||||
if (row.Folder.Length == 0 && !File.Exists(folder.ManifestPath)) return updated;
|
||||
if (project is not null)
|
||||
{
|
||||
Directory.CreateDirectory(folder.MetaDirectory);
|
||||
AtomicFile.WriteAllText(folder.ProjectPath, project);
|
||||
}
|
||||
if (metadata)
|
||||
{
|
||||
var manifest = SequenceWriter.ReadLocked(folder, row.Id);
|
||||
// Asked before the sidecar is written: frames another process wrote there since this index last synced the
|
||||
// folder keep it changed for the next scan.
|
||||
var inStep = await archives.Indexer.InStepAsync(archive, folder, manifest, ct).ConfigureAwait(false);
|
||||
ManifestStore.WriteSequence(folder.ManifestPath, manifest with
|
||||
{
|
||||
Title = updated.Title,
|
||||
Tags = updated.Tags,
|
||||
Rating = updated.Rating,
|
||||
Notes = updated.Notes,
|
||||
Versions = manifest.Versions with { Settings = Math.Max(manifest.Versions.Settings, updated.SettingsVersion) },
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
});
|
||||
if (inStep) await archives.Indexer.RecordStampsAsync(archive, folder, row.Id, ct).ConfigureAwait(false);
|
||||
}
|
||||
return updated;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The photos of a sequence's frames by index, as a merge names them (raw SHA-256, else size and capture time), with the
|
||||
/// encodings its conversions joined (<see cref="PhotoAlias"/>), so a position parked on a converted photo finds its frame.
|
||||
/// </summary>
|
||||
internal static async Task<FramePhotos> PhotosAsync(ChronosDatabase db, Guid sequenceId, IEnumerable<FrameRemoval>? extraRemovals, CancellationToken ct)
|
||||
{
|
||||
var frames = await Processing.SequenceFrames.ListAsync(db, sequenceId, ct).ConfigureAwait(false);
|
||||
var removals = await db.FrameRemovals.ListBySequenceAsync(sequenceId, ct).ConfigureAwait(false);
|
||||
var aliases = removals.Concat(extraRemovals ?? [])
|
||||
.Where(r => r.Reason == RemovalReason.Replaced && !string.IsNullOrEmpty(r.ReplacementSha256))
|
||||
.Select(r => new PhotoAlias(r.Sha256, r.ReplacementSha256!));
|
||||
return new FramePhotos(frames.Select(f => (f.Index, new FramePhoto(string.IsNullOrEmpty(f.Sha256) ? null : f.Sha256.ToLowerInvariant(), f.Size, f.CapturedAt))),
|
||||
aliases);
|
||||
}
|
||||
|
||||
/// <summary>The portable settings placed on the sequence's frames; settings that are not a project stay as they are.</summary>
|
||||
private async Task<string?> PlaceAsync(Guid sequenceId, string? portableSettings, CancellationToken ct)
|
||||
{
|
||||
var normalized = SyncRules.NormalizeSettings(portableSettings);
|
||||
if (!ProjectSettings.NamesFrames(normalized)) return portableSettings;
|
||||
var photos = await PhotosAsync(db, sequenceId, null, ct).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
return ProjectContent.Place(normalized, photos);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return portableSettings;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user