feat: owner review 2 phase 4b: mass conversion after import (#13)
CI / Build and test (push) Failing after 3m1s
CI / PostgreSQL 16 integration (push) Skipped
CI / PostgreSQL 17 integration (push) Skipped
CI / Docker image (push) Skipped
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 10s
Release / Build, test and publish (push) Successful in 15m31s
Release / Server image (push) Successful in 1m13s
CI / Build and test (push) Failing after 3m1s
CI / PostgreSQL 16 integration (push) Skipped
CI / PostgreSQL 17 integration (push) Skipped
CI / Docker image (push) Skipped
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 10s
Release / Build, test and publish (push) Successful in 15m31s
Release / Server image (push) Successful in 1m13s
Convert photos already in the archive to JPEG, TIFF-16 or Linear DNG, from the panel's Library and the desktop's Archive and Library. Each run asks Replace (same frames, originals to the trash, recoverable) or New sequence. A dry run lists what converts, what is skipped and why, size before and after, and free space; the job converts exactly the confirmed photos, resumes, and commits all or nothing per sequence. A conversion never changes what Chronos reads from a photo. Replaced originals are erased only on the archive's own word that their conversion is gone. Desktop TIFFs that WIC showed black or as noise now go to the managed reader; the Edit picker no longer spins a core. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
This commit was merged in pull request #13.
This commit is contained in:
1 parent
7e4a3b481d
commit
efa87f68ce
354 files changed
+24530
-843
No files matched your search
@@ -26,7 +26,8 @@ namespace Chronos.Data.Sync;
|
||||
/// which names them and writes the sidecar; the first original creates the local
|
||||
/// sequence, which then takes the server's settings. Frames of the local copy not hashed yet are hashed first (so the
|
||||
/// server's copies of them are recognised), and originals flagged missing here are restored to their recorded place in
|
||||
/// the folder the sequence is in (never one it left).
|
||||
/// the folder the sequence is in (never one it left; one the server holds in another encoding, a conversion on either
|
||||
/// side, takes the frame's place under its id instead).
|
||||
/// Proxy downloads fill the proxy cache. Renders travel on request: this PC's to the server (hash-verified chunked
|
||||
/// upload) and the server's into the local copy's exports. A resumed job skips what is already on the other side, so
|
||||
/// pausing or a crash loses at most the files in flight.
|
||||
@@ -111,10 +112,11 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
/// <summary>
|
||||
/// The proxies the server asks for (<see cref="ManifestFrame.WantsProxy"/>) that this PC's decoders can make from its
|
||||
/// originals (by their archived SHA-256, or the one <paramref name="hashes"/> holds for an original the index has none
|
||||
/// for; by size and capture time for a frame the server has not hashed, <see cref="SyncRules.ProxyOriginal"/>), made as
|
||||
/// the server's proxy policy says with the server's capture data, and sent. One the server refuses is
|
||||
/// counted in the job log; a server that makes no proxies, or an account that may not send them, ends the pass; a
|
||||
/// server busy or out of reach fails the job like an original would.
|
||||
/// for, compared as the photo either side's conversions make it, <see cref="SyncTombstones"/>: a frame this PC converted
|
||||
/// makes the proxy of the server's original; by size and capture time for a frame the server has not hashed,
|
||||
/// <see cref="SyncRules.ProxyOriginal"/>), made as the server's proxy policy says with the server's capture data, and
|
||||
/// sent. One the server refuses is counted in the job log; a server that makes no proxies, or an account that may not
|
||||
/// send them, ends the pass; a server busy or out of reach fails the job like an original would.
|
||||
/// </summary>
|
||||
private async Task SendProxiesAsync(JobContext context, ISyncRemote remote, Guid remoteId, Sequence local, ArchivePaths paths, IReadOnlyList<Frame> frames,
|
||||
IReadOnlyDictionary<Guid, string> hashes)
|
||||
@@ -123,10 +125,11 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false);
|
||||
var wanted = manifest.Frames.Where(f => f.WantsProxy).ToList();
|
||||
if (manifest.Proxies is not { } spec || wanted.Count == 0) return;
|
||||
var tombstones = new SyncTombstones(await sync.Database.FrameRemovals.ListBySequenceAsync(local.Id, ct).ConfigureAwait(false), manifest.Removed);
|
||||
var originals = new Dictionary<string, Frame>(StringComparer.Ordinal);
|
||||
foreach (var frame in frames)
|
||||
{
|
||||
if ((frame.Sha256 ?? hashes.GetValueOrDefault(frame.Id)) is { } sha) originals.TryAdd(sha.ToLowerInvariant(), frame);
|
||||
if ((frame.Sha256 ?? hashes.GetValueOrDefault(frame.Id)) is { } sha) originals.TryAdd(tombstones.Canonical(sha)!, frame);
|
||||
}
|
||||
var records = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id);
|
||||
var photos = SyncRules.Photos(frames);
|
||||
@@ -144,7 +147,7 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
// A frame the server has not hashed yet is this PC's original of its size and capture time.
|
||||
var original = frame.Sha256.Length == 0
|
||||
? SyncRules.ProxyOriginal(frame, record?.CapturedAt, frames, photos)
|
||||
: originals.GetValueOrDefault(frame.Sha256.ToLowerInvariant());
|
||||
: originals.GetValueOrDefault(tombstones.Canonical(frame.Sha256)!);
|
||||
var jpeg = original is not null && record is not null
|
||||
? await Task.Run(() => MakeProxy(decoder, paths.FromRelative(original.RelativePath), RenderNodeClient.ToFrame(record), camera, spec, frame.FileName,
|
||||
token), token).ConfigureAwait(false)
|
||||
@@ -211,11 +214,22 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
|
||||
var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false);
|
||||
var captured = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt);
|
||||
var (have, restore, photos) = local is null ? ([], [], new FramePhotos([])) : await LocalContentAsync(archive, local, ct).ConfigureAwait(false);
|
||||
var (have, restore, photos, missing) = local is null ? ([], [], new FramePhotos([]), []) : await LocalContentAsync(archive, local, ct).ConfigureAwait(false);
|
||||
// A photo this PC deleted is not fetched again; one replaced here by a converted file is here as its replacement (owner review 2, R6).
|
||||
var tombstones = local is null ? SyncTombstones.None
|
||||
: new SyncTombstones(await db.FrameRemovals.ListBySequenceAsync(local.Id, ct).ConfigureAwait(false), manifest.Removed);
|
||||
if (!tombstones.IsEmpty) have = [.. have.Select(h => tombstones.Canonical(h)!)];
|
||||
// A frame flagged missing here whose photo the server holds in another encoding (a conversion on either side, R7): the
|
||||
// copy fetched takes that frame's place under its id, rather than coming in as a frame of its own beside it.
|
||||
// One whose own encoding the server holds too is restored from that one instead, and one whose photo another frame
|
||||
// here holds intact stays as it is, as a restore leaves it.
|
||||
var byAlias = new ConcurrentDictionary<string, Frame>(StringComparer.Ordinal);
|
||||
if (!tombstones.IsEmpty)
|
||||
{
|
||||
var served = manifest.Frames.Where(f => f.Sha256.Length > 0).Select(f => f.Sha256.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
|
||||
foreach (var frame in missing.Where(f => !served.Contains(f.Sha256!.ToLowerInvariant()) && !have.Contains(tombstones.Canonical(f.Sha256)!)))
|
||||
byAlias.TryAdd(tombstones.Canonical(frame.Sha256)!, frame);
|
||||
}
|
||||
// An original the server has not hashed yet (adopted there, not verified) is here when an original of its size and
|
||||
// capture time is; each one is fetched on its own and hashed here.
|
||||
var wanted = manifest.Frames
|
||||
@@ -247,6 +261,7 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
var sequenceId = local?.Id;
|
||||
var refused = 0;
|
||||
var restores = new ConcurrentBag<(string Staged, Guid FrameId)>();
|
||||
var aliases = new ConcurrentBag<AliasCopy>();
|
||||
await Parallel.ForEachAsync(rest, Workers(sync.Options.Downloads, ct), async (frame, token) =>
|
||||
{
|
||||
var (staged, sha) = await FetchOriginalAsync(remote, frame, staging, token).ConfigureAwait(false);
|
||||
@@ -254,13 +269,18 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
{
|
||||
restores.Add((staged, frameId));
|
||||
}
|
||||
else if (byAlias.TryRemove(tombstones.Canonical(sha)!, out var lost))
|
||||
{
|
||||
// This encoding takes the place of the one missing here, after the transfers as the restores do.
|
||||
aliases.Add(new AliasCopy(staged, sha, lost, frame.FileName));
|
||||
}
|
||||
else if (frame.Sha256.Length == 0 && (tombstones.DeletedLocally(sha) || have.Contains(tombstones.Canonical(sha)!)))
|
||||
{
|
||||
// An original the server has not hashed is known by content only now: one this PC deleted (or holds as
|
||||
// the converted file that replaced it) is not filed again, which would also drop its tombstone.
|
||||
File.Delete(staged);
|
||||
Interlocked.Increment(ref refused);
|
||||
if (tombstones.DeletedLocally(sha)) context.Log($"{frame.FileName}: deleted on this PC, not downloaded");
|
||||
if (tombstones.DeletedLocally(sha)) context.Log(Loc.L("Archive.Log.SyncDeletedHere", frame.FileName));
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -269,30 +289,35 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
await progress.StepAsync().ConfigureAwait(false);
|
||||
}).ConfigureAwait(false);
|
||||
// Null when the sequence is not in the folder the index has: the copies stay staged for the next attempt.
|
||||
var restored = restores.IsEmpty ? 0 : await RestoreAsync(local!, paths, restores, ct).ConfigureAwait(false);
|
||||
if (restored is not null) TryDeleteDirectory(staging);
|
||||
// Restored originals are checked against their records, which clears their missing flag.
|
||||
if (restored > 0) await VerifyAsync(archive, local!.Id, ct).ConfigureAwait(false);
|
||||
var restored = restores.IsEmpty && aliases.IsEmpty ? 0 : await RestoreAsync(local!, paths, restores, ct).ConfigureAwait(false);
|
||||
var (swapped, moved) = restored is null || aliases.IsEmpty ? (0, restored is null)
|
||||
: await SwapAliasesAsync(context, archive, local!, paths, aliases, ct).ConfigureAwait(false);
|
||||
if (!moved) TryDeleteDirectory(staging);
|
||||
// Restored originals are checked against their records, which clears their missing flag (and reads the tier of a
|
||||
// frame that took another encoding).
|
||||
if (restored + swapped > 0) await VerifyAsync(archive, local!.Id, ct).ConfigureAwait(false);
|
||||
|
||||
if (created) await sync.TakeAllAsync(remote, remoteId, ct).ConfigureAwait(false);
|
||||
await sync.RecompareAsync(remote, remoteId, ct).ConfigureAwait(false);
|
||||
if (sequenceId is { } downloaded && wanted.Count > restores.Count + refused)
|
||||
// Originals filed into the sequence (not restores, nor frames swapped to another encoding, whose passes are queued as
|
||||
// they swap) need their thumbnails and light analysis.
|
||||
if (sequenceId is { } downloaded && wanted.Count > restores.Count + aliases.Count + refused)
|
||||
{
|
||||
var label = local!.DisplayTitle;
|
||||
await EnqueueFollowUpAsync(context, ArchiveJobs.Thumbnails([downloaded], label)).ConfigureAwait(false);
|
||||
await EnqueueFollowUpAsync(context, ProcessingJobs.LightAnalysis([downloaded], label)).ConfigureAwait(false);
|
||||
await QueuePassesAsync(context, downloaded, [ArchiveJobs.Thumbnails([downloaded], label), ProcessingJobs.LightAnalysis([downloaded], label)])
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
if (restored is null) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
|
||||
if (moved) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The originals a local copy holds (lower-case SHA-256, and their photos for server frames not hashed yet) and the
|
||||
/// frame each original flagged missing restores. Frames not hashed yet (an adopted folder never verified) are hashed
|
||||
/// first through the archive's verify, so the server's copies of them are recognised instead of committed again as
|
||||
/// duplicates.
|
||||
/// The originals a local copy holds (lower-case SHA-256, and their photos for server frames not hashed yet), the frame
|
||||
/// each original flagged missing restores, and the frames flagged missing whose content no intact frame holds. Frames
|
||||
/// not hashed yet (an adopted folder never verified) are hashed first through the archive's verify, so the server's
|
||||
/// copies of them are recognised instead of committed again as duplicates.
|
||||
/// </summary>
|
||||
private async Task<(HashSet<string> Have, Dictionary<string, Guid> Restore, FramePhotos Photos)> LocalContentAsync(Archive archive, Sequence local,
|
||||
CancellationToken ct)
|
||||
private async Task<(HashSet<string> Have, Dictionary<string, Guid> Restore, FramePhotos Photos, IReadOnlyList<Frame> Missing)> LocalContentAsync(
|
||||
Archive archive, Sequence local, CancellationToken ct)
|
||||
{
|
||||
var frames = await SequenceFrames.ListAsync(sync.Database, local.Id, ct).ConfigureAwait(false);
|
||||
if (frames.Any(f => f.Sha256 is null && (f.Flags & FrameFlags.Missing) == 0))
|
||||
@@ -303,13 +328,16 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
var have = frames.Where(f => (f.Flags & FrameFlags.Missing) == 0 && f.Sha256 is not null).Select(f => f.Sha256!.ToLowerInvariant())
|
||||
.ToHashSet(StringComparer.Ordinal);
|
||||
var restore = new Dictionary<string, Guid>(StringComparer.Ordinal);
|
||||
var missing = new List<Frame>();
|
||||
foreach (var frame in frames.Where(f => (f.Flags & FrameFlags.Missing) != 0 && f.Sha256 is not null))
|
||||
{
|
||||
var sha = frame.Sha256!.ToLowerInvariant();
|
||||
if (!have.Contains(sha)) restore.TryAdd(sha, frame.Id);
|
||||
if (have.Contains(sha)) continue;
|
||||
restore.TryAdd(sha, frame.Id);
|
||||
missing.Add(frame);
|
||||
}
|
||||
var photos = SyncRules.Photos([.. frames.Where(f => (f.Flags & FrameFlags.Missing) == 0)]);
|
||||
return (have, restore, photos);
|
||||
return (have, restore, photos, missing);
|
||||
}
|
||||
|
||||
private Task VerifyAsync(Archive archive, Guid sequenceId, CancellationToken ct) =>
|
||||
@@ -344,6 +372,69 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
return restored;
|
||||
}, ct);
|
||||
|
||||
/// <summary>
|
||||
/// Gives each frame flagged missing here whose photo the server holds only in another encoding (a conversion on either
|
||||
/// side, owner review 2, R7) the verified copy fetched (<see cref="FrameTrash.ReplaceMissingAsync"/>: under the sequence's
|
||||
/// lease, in its folder as the index has it then, whose sidecar must be the sequence's, <see cref="SequenceWriter.ReadLocked"/>).
|
||||
/// Run after the transfers, once <see cref="RestoreAsync"/> found the sequence in that folder. A swap clears what covered
|
||||
/// the sequence, as any replacement does, so the passes it had (<see cref="FrameTrash.FollowUps"/>: the light analysis, a
|
||||
/// background deep analysis, the thumbnails and proxies) are queued after each one, whatever stops the download later.
|
||||
/// Returns how many took their frame's place; the copy of a frame that changed meanwhile, or whose file came back, is
|
||||
/// dropped (logged). Moved, with the copies not swapped yet kept in staging for the next attempt, when the sequence left
|
||||
/// that folder meanwhile.
|
||||
/// </summary>
|
||||
private async Task<(int Swapped, bool Moved)> SwapAliasesAsync(JobContext context, Archive archive, Sequence local, ArchivePaths paths,
|
||||
IEnumerable<AliasCopy> aliases, CancellationToken ct)
|
||||
{
|
||||
var swapped = 0;
|
||||
// What the sequence had before the first swap cleared it: the passes that make it again.
|
||||
var had = SequenceStatus.None;
|
||||
foreach (var alias in aliases)
|
||||
{
|
||||
ReplaceResult? result;
|
||||
try
|
||||
{
|
||||
result = await sync.Archives.FrameTrash.ReplaceMissingAsync(archive, local.Id, alias.Lost.Id, alias.Lost.Sha256!, alias.Staged, alias.Sha256,
|
||||
alias.FileName, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is InvalidDataException or DirectoryNotFoundException or FileNotFoundException)
|
||||
{
|
||||
// Another index moved the sequence away, maybe another one into the folder it left: nothing is filed there.
|
||||
if (await sync.Database.Sequences.GetAsync(local.Id, ct).ConfigureAwait(false) is { } row && SequenceFolders.Holds(paths.Sequence(row.Folder), row.Id))
|
||||
throw;
|
||||
return (swapped, true);
|
||||
}
|
||||
if (result is null)
|
||||
{
|
||||
File.Delete(alias.Staged);
|
||||
context.Log(Loc.L("Archive.Log.SyncFrameChanged", alias.FileName));
|
||||
continue;
|
||||
}
|
||||
swapped++;
|
||||
had |= result.Had;
|
||||
await QueuePassesAsync(context, local.Id, sync.Archives.FrameTrash.FollowUps(archive, result.Sequence, had)).ConfigureAwait(false);
|
||||
}
|
||||
return (swapped, false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queues <paramref name="requests"/>, passes over the sequence <paramref name="sequenceId"/>, as follow-ups of this job,
|
||||
/// whatever stops it meanwhile, as a restore from the trash queues its own: the files they cover have changed already, and
|
||||
/// the next download, which finds nothing missing, would not queue them again. One already queued for the sequence (not
|
||||
/// started yet, so it covers what changed) is left out.
|
||||
/// </summary>
|
||||
private async Task QueuePassesAsync(JobContext context, Guid sequenceId, IEnumerable<ArchiveJobRequest> requests)
|
||||
{
|
||||
foreach (var request in requests)
|
||||
{
|
||||
if (await sync.Database.Jobs.IsQueuedAsync(request.Kind, sequenceId, CancellationToken.None).ConfigureAwait(false)) continue;
|
||||
await EnqueueFollowUpAsync(context, request, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A verified copy fetched in another encoding of a frame flagged missing here (<see cref="SwapAliasesAsync"/>).</summary>
|
||||
private sealed record AliasCopy(string Staged, string Sha256, Frame Lost, string FileName);
|
||||
|
||||
/// <summary>Puts a verified copy of an original flagged missing back where its record says; a file that reappeared meanwhile is kept.</summary>
|
||||
private static void Restore(string staged, string place)
|
||||
{
|
||||
@@ -513,7 +604,7 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
var cached = files.Count(f => f is not null);
|
||||
var standIns = Enumerable.Range(0, frames.Count).Count(i => kinds[i] != ProxyCache.Proxy && !NeverProxied(frames[i]));
|
||||
await sync.SetProxiesAsync(remote.Server, dto, cached, standIns, manifest.FramesVersion, ct).ConfigureAwait(false);
|
||||
if (cached < frames.Count) context.Log($"{frames.Count - cached} frame(s) have neither proxy nor thumbnail on the server");
|
||||
if (cached < frames.Count) context.Log(Loc.Plural("Archive.Log.SyncNoPreview", frames.Count - cached));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -563,7 +654,7 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
{
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
context.Log($"{frame.FileName}: missing, not uploaded");
|
||||
context.Log(Loc.L("Archive.Log.SyncMissing", frame.FileName));
|
||||
return null;
|
||||
}
|
||||
var sha = await ChronosClient.HashFileAsync(path, cancellationToken).ConfigureAwait(false);
|
||||
@@ -573,13 +664,13 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
hashes[frame.Id] = sha;
|
||||
if (tombstones.DeletedRemotely(sha))
|
||||
{
|
||||
context.Log($"{frame.FileName}: deleted on the server, not uploaded");
|
||||
context.Log(Loc.L("Archive.Log.SyncDeletedThere", frame.FileName));
|
||||
return notSent;
|
||||
}
|
||||
}
|
||||
if (frame.Sha256 is { } archived && !archived.Equals(sha, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
context.Log($"{frame.FileName}: changed since it was archived, not uploaded");
|
||||
context.Log(Loc.L("Archive.Log.SyncChanged", frame.FileName));
|
||||
return null;
|
||||
}
|
||||
return await remote.UploadFileAsync(path, sequenceId, sessionKey, sha, frame.CapturedAt, cancellationToken).ConfigureAwait(false) ?? notSent;
|
||||
@@ -717,8 +808,8 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
};
|
||||
|
||||
/// <summary>Queues a follow-up on this job's engine for the same user; skipped when the engine has no handler for it.</summary>
|
||||
private static Task<Job?> EnqueueFollowUpAsync(JobContext context, ArchiveJobRequest request) =>
|
||||
context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority);
|
||||
private static Task<Job?> EnqueueFollowUpAsync(JobContext context, ArchiveJobRequest request, CancellationToken cancellationToken) =>
|
||||
context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, cancellationToken);
|
||||
|
||||
private static void EnsureSpace(string root, long bytes)
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user