feat: owner review 2 phase 4a: Reorganize sequences after import (#12)
CI / Build and test (push) Successful in 9m15s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 6s
Release / Build, test and publish (push) Successful in 15m35s
CI / PostgreSQL 16 integration (push) Successful in 7m26s
CI / PostgreSQL 17 integration (push) Successful in 6m47s
CI / Docker image (push) Successful in 59s
Release / Server image (push) Successful in 1m24s
CI / Build and test (push) Successful in 9m15s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 6s
Release / Build, test and publish (push) Successful in 15m35s
CI / PostgreSQL 16 integration (push) Successful in 7m26s
CI / PostgreSQL 17 integration (push) Successful in 6m47s
CI / Docker image (push) Successful in 59s
Release / Server image (push) Successful in 1m24s
- Reorganize… on the web panel and the desktop: folder, file and title templates applied to existing sequences, with a dry run and an explicit Apply that runs as a job. - One naming rule (FrameNaming) for import, upload and Reorganize. - Crash-safe plan and file journals, held plans, stale-plan refusal; exports move with their folder. - Writers refuse a folder another program moved or gave to another sequence. - Server: POST /api/v1/archive/reorganize with a bounded plan cache; template rules. - Phase 3 docs and translations merged; strings in all 38 languages. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
This commit was merged in pull request #12.
This commit is contained in:
1 parent
9aa818ed47
commit
7e4a3b481d
416 files changed
+15304
-1572
No files matched your search
@@ -25,7 +25,8 @@ namespace Chronos.Data.Sync;
|
||||
/// original the server has not hashed yet, its size, and hash it here) and commit them through the archive ingest,
|
||||
/// which names them and writes the sidecar; the first original creates the local
|
||||
/// sequence, which then takes the server's settings. Frames of the local copy not hashed yet are hashed first (so the
|
||||
/// server's copies of them are recognised), and originals flagged missing here are restored to their recorded place.
|
||||
/// server's copies of them are recognised), and originals flagged missing here are restored to their recorded place in
|
||||
/// the folder the sequence is in (never one it left).
|
||||
/// Proxy downloads fill the proxy cache. Renders travel on request: this PC's to the server (hash-verified chunked
|
||||
/// upload) and the server's into the local copy's exports. A resumed job skips what is already on the other side, so
|
||||
/// pausing or a crash loses at most the files in flight.
|
||||
@@ -244,14 +245,14 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
await progress.StepAsync().ConfigureAwait(false);
|
||||
}
|
||||
var sequenceId = local?.Id;
|
||||
int restored = 0, refused = 0;
|
||||
var refused = 0;
|
||||
var restores = new ConcurrentBag<(string Staged, Guid FrameId)>();
|
||||
await Parallel.ForEachAsync(rest, Workers(sync.Options.Downloads, ct), async (frame, token) =>
|
||||
{
|
||||
var (staged, sha) = await FetchOriginalAsync(remote, frame, staging, token).ConfigureAwait(false);
|
||||
if (restore.TryGetValue(sha, out var place))
|
||||
if (restore.TryGetValue(sha, out var frameId))
|
||||
{
|
||||
Restore(staged, place);
|
||||
Interlocked.Increment(ref restored);
|
||||
restores.Add((staged, frameId));
|
||||
}
|
||||
else if (frame.Sha256.Length == 0 && (tombstones.DeletedLocally(sha) || have.Contains(tombstones.Canonical(sha)!)))
|
||||
{
|
||||
@@ -267,26 +268,30 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
}
|
||||
await progress.StepAsync().ConfigureAwait(false);
|
||||
}).ConfigureAwait(false);
|
||||
TryDeleteDirectory(staging);
|
||||
// Null when the sequence is not in the folder the index has: the copies stay staged for the next attempt.
|
||||
var restored = restores.IsEmpty ? 0 : await RestoreAsync(local!, paths, restores, ct).ConfigureAwait(false);
|
||||
if (restored is not null) TryDeleteDirectory(staging);
|
||||
// Restored originals are checked against their records, which clears their missing flag.
|
||||
if (restored > 0) await VerifyAsync(archive, local!.Id, ct).ConfigureAwait(false);
|
||||
|
||||
if (created) await sync.TakeAllAsync(remote, remoteId, ct).ConfigureAwait(false);
|
||||
await sync.RecompareAsync(remote, remoteId, ct).ConfigureAwait(false);
|
||||
if (sequenceId is { } downloaded && wanted.Count > restored + refused)
|
||||
if (sequenceId is { } downloaded && wanted.Count > restores.Count + refused)
|
||||
{
|
||||
var label = local!.DisplayTitle;
|
||||
await EnqueueFollowUpAsync(context, ArchiveJobs.Thumbnails([downloaded], label)).ConfigureAwait(false);
|
||||
await EnqueueFollowUpAsync(context, ProcessingJobs.LightAnalysis([downloaded], label)).ConfigureAwait(false);
|
||||
}
|
||||
if (restored is null) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The originals a local copy holds (lower-case SHA-256, and their photos for server frames not hashed yet) and where
|
||||
/// each original flagged missing belongs. Frames not hashed yet (an adopted folder never verified) are hashed first
|
||||
/// through the archive's verify, so the server's copies of them are recognised instead of committed again as duplicates.
|
||||
/// The originals a local copy holds (lower-case SHA-256, and their photos for server frames not hashed yet) and the
|
||||
/// frame each original flagged missing restores. Frames not hashed yet (an adopted folder never verified) are hashed
|
||||
/// first through the archive's verify, so the server's copies of them are recognised instead of committed again as
|
||||
/// duplicates.
|
||||
/// </summary>
|
||||
private async Task<(HashSet<string> Have, Dictionary<string, string> Restore, FramePhotos Photos)> LocalContentAsync(Archive archive, Sequence local,
|
||||
private async Task<(HashSet<string> Have, Dictionary<string, Guid> Restore, FramePhotos Photos)> LocalContentAsync(Archive archive, Sequence local,
|
||||
CancellationToken ct)
|
||||
{
|
||||
var frames = await SequenceFrames.ListAsync(sync.Database, local.Id, ct).ConfigureAwait(false);
|
||||
@@ -297,12 +302,11 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
}
|
||||
var have = frames.Where(f => (f.Flags & FrameFlags.Missing) == 0 && f.Sha256 is not null).Select(f => f.Sha256!.ToLowerInvariant())
|
||||
.ToHashSet(StringComparer.Ordinal);
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
var restore = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
var restore = new Dictionary<string, Guid>(StringComparer.Ordinal);
|
||||
foreach (var frame in frames.Where(f => (f.Flags & FrameFlags.Missing) != 0 && f.Sha256 is not null))
|
||||
{
|
||||
var sha = frame.Sha256!.ToLowerInvariant();
|
||||
if (!have.Contains(sha)) restore.TryAdd(sha, paths.FromRelative(frame.RelativePath));
|
||||
if (!have.Contains(sha)) restore.TryAdd(sha, frame.Id);
|
||||
}
|
||||
var photos = SyncRules.Photos([.. frames.Where(f => (f.Flags & FrameFlags.Missing) == 0)]);
|
||||
return (have, restore, photos);
|
||||
@@ -311,6 +315,35 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
private Task VerifyAsync(Archive archive, Guid sequenceId, CancellationToken ct) =>
|
||||
sync.Archives.Verifier.VerifyAsync(archive, [sequenceId], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: ct);
|
||||
|
||||
/// <summary>
|
||||
/// Puts the verified copies of originals flagged missing (<paramref name="staged"/>, each with the frame it restores)
|
||||
/// back where their records say, under the sequence's lease in its folder as the index has it then: a Reorganize moves
|
||||
/// the folder and the frames' paths under that lease. Returns how many went back; a file that reappeared meanwhile is
|
||||
/// kept, and the copy of a frame that left the index is dropped. Null, with nothing restored and every copy kept in
|
||||
/// staging for the next attempt, when the sequence is not in that folder (another index moved it, and maybe another
|
||||
/// sequence into it; this index's next scan finds them): the folder it left is not created again, nor given an
|
||||
/// original of this sequence when it holds another one now.
|
||||
/// </summary>
|
||||
private Task<int?> RestoreAsync(Sequence local, ArchivePaths paths, IReadOnlyCollection<(string Staged, Guid FrameId)> staged, CancellationToken ct) =>
|
||||
sync.Archives.UnderLeaseAsync(local, async row =>
|
||||
{
|
||||
if (row is null) throw new InvalidOperationException(Loc.L("Errors.Sync.LocalMissing"));
|
||||
if (!SequenceFolders.Holds(paths.Sequence(row.Folder), row.Id)) return (int?)null;
|
||||
var restored = 0;
|
||||
foreach (var (copy, frameId) in staged)
|
||||
{
|
||||
// Read under the lease, as the folder: where the photo belongs in the folder the sequence is in now.
|
||||
if (await sync.Database.Frames.GetAsync(frameId, ct).ConfigureAwait(false) is not { } frame || frame.SequenceId != row.Id)
|
||||
{
|
||||
File.Delete(copy);
|
||||
continue;
|
||||
}
|
||||
Restore(copy, paths.FromRelative(frame.RelativePath));
|
||||
restored++;
|
||||
}
|
||||
return restored;
|
||||
}, ct);
|
||||
|
||||
/// <summary>Puts a verified copy of an original flagged missing back where its record says; a file that reappeared meanwhile is kept.</summary>
|
||||
private static void Restore(string staged, string place)
|
||||
{
|
||||
@@ -370,31 +403,44 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
EnsureSpace(paths.Root, wanted.Sum(r => r.Bytes));
|
||||
|
||||
var staging = Path.Combine(paths.StagingDirectory, "exports-" + remoteId.ToString("N"));
|
||||
var exports = paths.Sequence(local.Folder).Exports;
|
||||
Directory.CreateDirectory(staging);
|
||||
Directory.CreateDirectory(exports);
|
||||
var progress = new TransferProgress(context, wanted.Count, 0);
|
||||
await progress.ReportAsync().ConfigureAwait(false);
|
||||
foreach (var render in wanted)
|
||||
{
|
||||
var (part, name) = await FetchRenderAsync(remote, render, staging, ct).ConfigureAwait(false);
|
||||
var target = Unique(exports, name);
|
||||
File.Move(part, target);
|
||||
await db.Renders.InsertAsync(new Render
|
||||
// Filed under the sequence's lease into its folder as the index has it then, as a server files a render: a
|
||||
// Reorganize may have moved it since the sequence was read, and moves the row's path along when it moves it later.
|
||||
// One another index moved is found by this index's next scan; until then nothing is filed (the download stays
|
||||
// staged for the next attempt), so the folder it left is not created again, nor filed into when it holds another
|
||||
// sequence now.
|
||||
var filed = await sync.Archives.UnderLeaseAsync(local, async row =>
|
||||
{
|
||||
Id = render.Id,
|
||||
SequenceId = local.Id,
|
||||
PresetName = render.PresetName,
|
||||
OutputPath = paths.ToRelative(target),
|
||||
Codec = render.Codec,
|
||||
Width = render.Width,
|
||||
Height = render.Height,
|
||||
Fps = render.Fps,
|
||||
DurationSeconds = render.DurationSeconds,
|
||||
Bytes = new FileInfo(target).Length,
|
||||
Created = render.Created,
|
||||
ProducedBy = render.ProducedBy,
|
||||
if (row is null) throw new InvalidOperationException(Loc.L("Errors.Sync.LocalMissing"));
|
||||
var folder = paths.Sequence(row.Folder);
|
||||
if (!SequenceFolders.Holds(folder, row.Id)) return false;
|
||||
Directory.CreateDirectory(folder.Exports);
|
||||
var target = Unique(folder.Exports, name);
|
||||
File.Move(part, target);
|
||||
// Recorded even if the job is being cancelled: a filed export is never left without its row.
|
||||
await db.Renders.InsertAsync(new Render
|
||||
{
|
||||
Id = render.Id,
|
||||
SequenceId = row.Id,
|
||||
PresetName = render.PresetName,
|
||||
OutputPath = paths.ToRelative(target),
|
||||
Codec = render.Codec,
|
||||
Width = render.Width,
|
||||
Height = render.Height,
|
||||
Fps = render.Fps,
|
||||
DurationSeconds = render.DurationSeconds,
|
||||
Bytes = new FileInfo(target).Length,
|
||||
Created = render.Created,
|
||||
ProducedBy = render.ProducedBy,
|
||||
}, CancellationToken.None).ConfigureAwait(false);
|
||||
return true;
|
||||
}, ct).ConfigureAwait(false);
|
||||
if (!filed) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
|
||||
await progress.StepAsync().ConfigureAwait(false);
|
||||
}
|
||||
TryDeleteDirectory(staging);
|
||||
|
||||
Reference in new issue
Block a user