feat(archive): photo delete, tombstones, trash kinds and emptying, library API
Owner review 2, R6: the web panel manages the library without editing it, so the engine and the server must delete single photos safely, keep them out of sync afterwards, and let an administrator restore or finally erase the trash. - Migration 0008: frame_removals (tombstones by sequence and SHA-256, reason, replacement); the sidecar gains "removed" (omitted when empty) and frames a reserved "source"; the indexer writes the tombstones on every sync, so a rebuild keeps them, and drops a hero frame that is gone. - FrameTrash: dry run (busy, frames changed, offline, would empty), delete under the sequence lease with a write-ahead journal that the next writer undoes after a crash, files to trash/photos/<seq>-<time>/originals with a record, sidecar without the frames plus tombstones, index sync clearing analyses, thumbnails and proxies, proxies renamed to follow their frames, follow-up jobs. Restore re-hashes every file, keeps mismatches in the trash, puts the same frame records back and drops their tombstones, or files the photos into a new sequence through the ingest when theirs is gone. - Project positions on vanished photos are translated by photo and wait as pending frames, so they come back with a restore. - Trash entries have a kind (sequence, photos, replaced); emptying is a dry run with a fingerprint and a new EmptyTrash job (18) that erases exactly the planned entries, refuses a stale plan and erases replaced originals only while their replacements still hash right. - Sync honours tombstones: nothing one side deleted is offered back, local files are never deleted, replaced photos count as their replacement; the server answers an upload of a tombstoned SHA-256 as already there. - Server: sequence thumbnail, photo delete, bulk delete/retag/jobs, trash list/restore/empty endpoints with their error messages; a verify of sequences now targets their archive (POST /jobs failed before). - Desktop trash dialog lists photo entries and empties the trash after a typed confirmation through the same job. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> (cherry picked from commit 35bb56690a1d69eae57153d036e1409201cd73f7)
This commit is contained in:
1 parent
4b9ba04fe4
commit
51782a3ada
63 files changed
+3303
-60
No files matched your search
@@ -24,6 +24,9 @@ internal sealed class SyncPositions(ChronosDatabase db, ISyncRemote remote, Guid
|
||||
/// <summary>The manifest's frames by hash, and with their capture times once fetched.</summary>
|
||||
private FramePhotos? _hashed, _timed;
|
||||
|
||||
/// <summary>The photos either copy removed: a replaced photo is translated as its replacement (owner review 2, R6).</summary>
|
||||
private SyncTombstones? _tombstones;
|
||||
|
||||
/// <summary>The server's frames version the translation to the server's frames is made against.</summary>
|
||||
public int ServerFrames => manifest.FramesVersion;
|
||||
|
||||
@@ -73,7 +76,10 @@ internal sealed class SyncPositions(ChronosDatabase db, ISyncRemote remote, Guid
|
||||
var frames = await SequenceFrames.ListAsync(db, localId, cancellationToken).ConfigureAwait(false);
|
||||
if (await db.Sequences.GetAsync(localId, cancellationToken).ConfigureAwait(false) is not { } after) return null;
|
||||
if (after.FramesVersion == before.FramesVersion)
|
||||
return new LocalState(after, new FramePhotos(frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, f.CapturedAt)))));
|
||||
{
|
||||
var tombstones = await TombstonesAsync(cancellationToken).ConfigureAwait(false);
|
||||
return new LocalState(after, new FramePhotos(frames.Select(f => (f.Index, tombstones.Canonical(new FramePhoto(f.Sha256, f.Size, f.CapturedAt))))));
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -87,20 +93,25 @@ internal sealed class SyncPositions(ChronosDatabase db, ISyncRemote remote, Guid
|
||||
/// </summary>
|
||||
private async Task<FramePhotos?> ServerAsync(FramePhotos local, CancellationToken cancellationToken)
|
||||
{
|
||||
var tombstones = await TombstonesAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (_hashed is null)
|
||||
{
|
||||
if (!OneList(manifest.Frames)) return null;
|
||||
_hashed = new FramePhotos(manifest.Frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, null))));
|
||||
_hashed = new FramePhotos(manifest.Frames.Select(f => (f.Index, tombstones.Canonical(new FramePhoto(f.Sha256, f.Size, null)))));
|
||||
}
|
||||
if (_hashed.SameAs(local)) return _hashed;
|
||||
if (_timed is null)
|
||||
{
|
||||
var captured = (await remote.ListFramesAsync(manifest.SequenceId, cancellationToken).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt);
|
||||
_timed = new FramePhotos(manifest.Frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, captured.GetValueOrDefault(f.Id)))));
|
||||
_timed = new FramePhotos(manifest.Frames.Select(f => (f.Index, tombstones.Canonical(new FramePhoto(f.Sha256, f.Size, captured.GetValueOrDefault(f.Id))))));
|
||||
}
|
||||
return _timed;
|
||||
}
|
||||
|
||||
/// <summary>This copy's tombstones and the server's, read once.</summary>
|
||||
private async Task<SyncTombstones> TombstonesAsync(CancellationToken cancellationToken) =>
|
||||
_tombstones ??= new SyncTombstones(await db.FrameRemovals.ListBySequenceAsync(localId, cancellationToken).ConfigureAwait(false), manifest.Removed);
|
||||
|
||||
/// <summary>Each frame once, at the indices 0…n-1 a sequence's frames always take.</summary>
|
||||
private static bool OneList(IReadOnlyList<ManifestFrame> frames)
|
||||
{
|
||||
|
||||
@@ -164,6 +164,22 @@ public static class SyncRules
|
||||
];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="CompareOriginals(IReadOnlyCollection{FramePhoto}, IReadOnlyCollection{FramePhoto})"/> honouring the photos
|
||||
/// either side removed (owner review 2, R6): a photo the server deleted is not missing there, one this PC deleted is not
|
||||
/// missing here, and a photo replaced by a converted file on either side is compared as its replacement.
|
||||
/// </summary>
|
||||
public static (int MissingRemote, int MissingLocal) CompareOriginals(IReadOnlyCollection<FramePhoto> local, IReadOnlyCollection<FramePhoto> remote,
|
||||
SyncTombstones tombstones)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(local);
|
||||
ArgumentNullException.ThrowIfNull(remote);
|
||||
ArgumentNullException.ThrowIfNull(tombstones);
|
||||
if (tombstones.IsEmpty) return CompareOriginals(local, remote);
|
||||
return CompareOriginals([.. local.Where(p => !tombstones.DeletedRemotely(p.Sha256)).Select(tombstones.Canonical)],
|
||||
[.. remote.Where(p => !tombstones.DeletedLocally(p.Sha256)).Select(tombstones.Canonical)]);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Frames whose proxy the server asks for (it cannot decode their originals, BRIEF §2.3) that this PC holds and its
|
||||
/// <paramref name="decoder"/> reads: an upload makes and sends them. A local original not hashed yet (a folder adopted
|
||||
|
||||
@@ -528,8 +528,10 @@ public sealed class SyncService
|
||||
var captured = manifest.Frames.Any(f => f.Sha256.Length == 0) || present.Any(f => f.Sha256 is null)
|
||||
? (await remote.ListFramesAsync(link.RemoteId, cancellationToken).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt)
|
||||
: null;
|
||||
// Photos either side removed are not offered back; one replaced by a converted file counts as its replacement (owner review 2, R6).
|
||||
var tombstones = new SyncTombstones(await _db.FrameRemovals.ListBySequenceAsync(local.Id, cancellationToken).ConfigureAwait(false), manifest.Removed);
|
||||
var (missingRemote, missingLocal) = SyncRules.CompareOriginals([.. present.Select(f => new FramePhoto(f.Sha256, f.Size, f.CapturedAt))],
|
||||
[.. manifest.Frames.Select(f => new FramePhoto(f.Sha256, f.Size, captured?.GetValueOrDefault(f.Id)))]);
|
||||
[.. manifest.Frames.Select(f => new FramePhoto(f.Sha256, f.Size, captured?.GetValueOrDefault(f.Id)))], tombstones);
|
||||
// Proxies the server asks for that this PC can make from its originals travel with an upload too (BRIEF §2.3).
|
||||
var proxies = policy.CanEdit && manifest.Proxies is not null ? SyncRules.ProxiesToSend(manifest, present, Options.Archive.Decoder, captured) : 0;
|
||||
// Those change without the server's frame counters moving: while some may be made here, the server's next
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Core.Project;
|
||||
using Chronos.Protocol;
|
||||
|
||||
namespace Chronos.Data.Sync;
|
||||
|
||||
/// <summary>
|
||||
/// The photos each copy of a linked pair removed (owner review 2, R6; BRIEF §14.5): a photo one side deleted is not
|
||||
/// offered to it again (no upload of what the server deleted, no download of what this PC deleted; local files are never
|
||||
/// deleted by sync), and a photo one side replaced by a converted file is the same photo as its replacement on either side
|
||||
/// (an alias: both are compared, matched and translated as the replacement). No I/O.
|
||||
/// </summary>
|
||||
public sealed class SyncTombstones
|
||||
{
|
||||
/// <summary>Links a replacement chain may have at most (a conversion of a conversion…); longer ones stop there.</summary>
|
||||
private const int MaxAliasSteps = 8;
|
||||
|
||||
private readonly HashSet<string> _deletedLocally;
|
||||
private readonly HashSet<string> _deletedRemotely;
|
||||
private readonly Dictionary<string, string> _aliases;
|
||||
|
||||
public SyncTombstones(IEnumerable<FrameRemoval> local, IEnumerable<RemovedPhotoDto>? remote)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(local);
|
||||
_deletedLocally = new HashSet<string>(StringComparer.Ordinal);
|
||||
_deletedRemotely = new HashSet<string>(StringComparer.Ordinal);
|
||||
_aliases = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
foreach (var removal in local) Add(removal.Sha256, removal.ReplacementSha256, removal.Reason == RemovalReason.Replaced, _deletedLocally);
|
||||
foreach (var removal in remote ?? [])
|
||||
Add(removal.Sha256, removal.Replacement, string.Equals(removal.Reason, Wire(RemovalReason.Replaced), StringComparison.OrdinalIgnoreCase), _deletedRemotely);
|
||||
}
|
||||
|
||||
/// <summary>No photo removed on either side.</summary>
|
||||
public static SyncTombstones None { get; } = new([], null);
|
||||
|
||||
/// <summary>Whether either side removed or replaced anything.</summary>
|
||||
public bool IsEmpty => _deletedLocally.Count == 0 && _deletedRemotely.Count == 0 && _aliases.Count == 0;
|
||||
|
||||
/// <summary>The wire name of a removal reason (<c>deleted</c>, <c>replaced</c>).</summary>
|
||||
public static string Wire(RemovalReason reason) => reason.ToString().ToLowerInvariant();
|
||||
|
||||
/// <summary>The tombstones of a sequence as the manifest carries them.</summary>
|
||||
public static IReadOnlyList<RemovedPhotoDto> ToDto(IEnumerable<FrameRemoval> removals) =>
|
||||
[.. removals.Select(r => new RemovedPhotoDto(r.Sha256, r.ReplacementSha256, Wire(r.Reason), r.RemovedAt))];
|
||||
|
||||
/// <summary>
|
||||
/// The SHA-256 a photo is compared as: the end of its replacement chain on either side, lower-case; null or empty
|
||||
/// (not hashed yet) stays as it is.
|
||||
/// </summary>
|
||||
public string? Canonical(string? sha256)
|
||||
{
|
||||
if (string.IsNullOrEmpty(sha256)) return sha256;
|
||||
var current = sha256.ToLowerInvariant();
|
||||
for (var step = 0; step < MaxAliasSteps && _aliases.TryGetValue(current, out var next); step++) current = next;
|
||||
return current;
|
||||
}
|
||||
|
||||
/// <summary>The photo as it is compared (<see cref="Canonical(string?)"/>).</summary>
|
||||
public FramePhoto Canonical(FramePhoto photo)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(photo);
|
||||
return photo.IsHashed && _aliases.Count > 0 ? photo with { Sha256 = Canonical(photo.Sha256) } : photo;
|
||||
}
|
||||
|
||||
/// <summary>This PC deleted the photo (or what replaced it): the server's copy is not downloaded.</summary>
|
||||
public bool DeletedLocally(string? sha256) => Deleted(sha256, _deletedLocally);
|
||||
|
||||
/// <summary>The server deleted the photo (or what replaced it): this PC's copy is not uploaded.</summary>
|
||||
public bool DeletedRemotely(string? sha256) => Deleted(sha256, _deletedRemotely);
|
||||
|
||||
private bool Deleted(string? sha256, HashSet<string> deleted) =>
|
||||
!string.IsNullOrEmpty(sha256) && deleted.Count > 0 && (deleted.Contains(sha256.ToLowerInvariant()) || deleted.Contains(Canonical(sha256)!));
|
||||
|
||||
private void Add(string sha256, string? replacement, bool replaced, HashSet<string> deleted)
|
||||
{
|
||||
if (string.IsNullOrEmpty(sha256)) return;
|
||||
var sha = sha256.ToLowerInvariant();
|
||||
if (replaced && !string.IsNullOrEmpty(replacement)) _aliases[sha] = replacement.ToLowerInvariant();
|
||||
else deleted.Add(sha);
|
||||
}
|
||||
}
|
||||
@@ -48,13 +48,15 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
|
||||
var frames = (await SequenceFrames.ListAsync(db, local.Id, ct).ConfigureAwait(false)).Where(f => (f.Flags & FrameFlags.Missing) == 0).ToList();
|
||||
var manifest = remoteId is { } known ? await remote.GetManifestAsync(known, ct).ConfigureAwait(false) : null;
|
||||
var present = manifest?.Frames.Where(f => f.Sha256.Length > 0).Select(f => f.Sha256.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal) ?? [];
|
||||
// A photo the server deleted is not sent back; one it replaced by a converted file is there as its replacement (owner review 2, R6).
|
||||
var tombstones = new SyncTombstones(await db.FrameRemovals.ListBySequenceAsync(local.Id, ct).ConfigureAwait(false), manifest?.Removed);
|
||||
var present = manifest?.Frames.Where(f => f.Sha256.Length > 0).Select(f => tombstones.Canonical(f.Sha256)!).ToHashSet(StringComparer.Ordinal) ?? [];
|
||||
// Originals the server holds without a hash (adopted there, not verified) are the ones of their size and capture
|
||||
// time: the server could not recognise them by content, so they are not sent again.
|
||||
var adopted = manifest is not null && manifest.Frames.Any(f => f.Sha256.Length == 0)
|
||||
? await UnhashedAsync(remote, manifest, ct).ConfigureAwait(false)
|
||||
: new FramePhotos([]);
|
||||
var pending = new Queue<Frame>(frames.Where(f => (f.Sha256 is null || !present.Contains(f.Sha256.ToLowerInvariant()))
|
||||
var pending = new Queue<Frame>(frames.Where(f => (f.Sha256 is null || (!present.Contains(tombstones.Canonical(f.Sha256)!) && !tombstones.DeletedRemotely(f.Sha256)))
|
||||
&& adopted.IndexOf(new FramePhoto(f.Sha256, f.Size, f.CapturedAt)) is null));
|
||||
var progress = new TransferProgress(context, frames.Count, frames.Count - pending.Count);
|
||||
await progress.ReportAsync().ConfigureAwait(false);
|
||||
@@ -205,12 +207,16 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false);
|
||||
var captured = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt);
|
||||
var (have, restore, photos) = local is null ? ([], [], new FramePhotos([])) : await LocalContentAsync(archive, local, ct).ConfigureAwait(false);
|
||||
// A photo this PC deleted is not fetched again; one replaced here by a converted file is here as its replacement (owner review 2, R6).
|
||||
var tombstones = local is null ? SyncTombstones.None
|
||||
: new SyncTombstones(await db.FrameRemovals.ListBySequenceAsync(local.Id, ct).ConfigureAwait(false), manifest.Removed);
|
||||
if (!tombstones.IsEmpty) have = [.. have.Select(h => tombstones.Canonical(h)!)];
|
||||
// An original the server has not hashed yet (adopted there, not verified) is here when an original of its size and
|
||||
// capture time is; each one is fetched on its own and hashed here.
|
||||
var wanted = manifest.Frames
|
||||
.Where(f => f.Sha256.Length == 0
|
||||
? photos.IndexOf(new FramePhoto(null, f.Size, captured.GetValueOrDefault(f.Id))) is null
|
||||
: !have.Contains(f.Sha256.ToLowerInvariant()))
|
||||
: !have.Contains(tombstones.Canonical(f.Sha256)!) && !tombstones.DeletedLocally(f.Sha256))
|
||||
.DistinctBy(f => f.Sha256.Length == 0 ? f.Id.ToString("N") : f.Sha256.ToLowerInvariant()).ToList();
|
||||
EnsureSpace(paths.Root, wanted.Sum(f => f.Size));
|
||||
|
||||
|
||||
Reference in new issue
Block a user