fix(sync,upload): keep tombstones honest and let an explicit upload bring a photo back

Sync:
- A server frame that has not been hashed is known by content only once it is
  downloaded: one this PC deleted (or holds as its replacement) is now deleted
  from staging instead of filed again, which also dropped its tombstone.
- An unhashed local original that turns out to be a photo the server deleted
  is not uploaded, and the archive verify then writes the hashes into the
  index, so the row settles instead of offering Upload for good.
- A tombstone leaves a photo out of the comparison only when its side no longer
  holds that content, so a tombstone left beside a held copy does not keep the
  other side's copy missing forever.

Uploads:
- StartUploadRequest.Restore (optional, never sent by sync or older clients)
  lets an explicit user upload file a photo deleted from the sequence again,
  also after the trash was emptied; the completion repeats it with
  ?restore=true and the ingest drops the tombstone.
- The completion checks the tombstone again: a photo deleted while its bytes
  were on the way is not filed (409 photo_removed, upload cancelled); the
  protocol client reports that as not sent.
- A tombstone of content a frame of the target still holds refuses no repair.
- Hidden dot names (AppleDouble "._" files) are refused, as the archive
  scanner leaves them out.

Web upload dialog:
- Upload again on rows marked Deleted before; the tooltip no longer promises
  a trash restore that may be impossible.
- Sends a capture time: the Exif DateTimeOriginal of a JPEG or TIFF-based file
  (minimal in-house reader, capture-time.js), else the file's time, so photos
  without Exif keep their order.
- Esc is stopped at keydown during a run (Chromium allows one prevented cancel
  per activation); hidden dot files are left out; a completion refused while
  another one files the upload waits for it; the same photo picked under two
  names waits for its first copy.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit d051bcfc351f5a08500ca864a6508576a95492a4)
This commit is contained in:
Alby96 committed 2026-10-27 12:45:33 +01:00
1 parent eb607d47aa
commit 316c182b19
18 files changed
+753 -63

No files matched your search

+9 -3
View File
@@ -167,7 +167,9 @@ public static class SyncRules
/// <summary>
/// <see cref="CompareOriginals(IReadOnlyCollection{FramePhoto}, IReadOnlyCollection{FramePhoto})"/> honouring the photos
/// either side removed (owner review 2, R6): a photo the server deleted is not missing there, one this PC deleted is not
/// missing here, and a photo replaced by a converted file on either side is compared as its replacement.
/// missing here, and a photo replaced by a converted file on either side is compared as its replacement. A tombstone
/// leaves out only a photo its side no longer holds: one left beside a frame of the same content (the other copy of a
/// duplicate, a file put back by hand) would otherwise keep the other side's copy missing for good.
/// </summary>
public static (int MissingRemote, int MissingLocal) CompareOriginals(IReadOnlyCollection<FramePhoto> local, IReadOnlyCollection<FramePhoto> remote,
SyncTombstones tombstones)
@@ -176,8 +178,12 @@ public static class SyncRules
ArgumentNullException.ThrowIfNull(remote);
ArgumentNullException.ThrowIfNull(tombstones);
if (tombstones.IsEmpty) return CompareOriginals(local, remote);
return CompareOriginals([.. local.Where(p => !tombstones.DeletedRemotely(p.Sha256)).Select(tombstones.Canonical)],
[.. remote.Where(p => !tombstones.DeletedLocally(p.Sha256)).Select(tombstones.Canonical)]);
var mine = local.Select(p => (p.Sha256, Photo: tombstones.Canonical(p))).ToList();
var theirs = remote.Select(p => (p.Sha256, Photo: tombstones.Canonical(p))).ToList();
var minePhotos = new FramePhotos(mine.Select((p, i) => (i, p.Photo)));
var theirPhotos = new FramePhotos(theirs.Select((p, i) => (i, p.Photo)));
return CompareOriginals([.. mine.Where(p => !tombstones.DeletedRemotely(p.Sha256) || theirPhotos.IndexOf(p.Photo) is not null).Select(p => p.Photo)],
[.. theirs.Where(p => !tombstones.DeletedLocally(p.Sha256) || minePhotos.IndexOf(p.Photo) is not null).Select(p => p.Photo)]);
}
/// <summary>
+32 -11
View File
@@ -70,8 +70,8 @@ internal sealed class SyncTransfers(SyncService sync)
// server has filed one, which creates the sequence (an original it already holds elsewhere creates nothing).
while (remoteId is null && pending.TryDequeue(out var first))
{
var sent = await UploadFrameAsync(context, remote, null, SyncService.SessionKey(local.Id), paths.FromRelative(first.RelativePath), first, hashes, ct)
.ConfigureAwait(false);
var sent = await UploadFrameAsync(context, remote, null, SyncService.SessionKey(local.Id), paths.FromRelative(first.RelativePath), first, tombstones,
hashes, ct).ConfigureAwait(false);
if (sent is null) skipped++;
else if (sent.SequenceId != Guid.Empty)
{
@@ -85,11 +85,15 @@ internal sealed class SyncTransfers(SyncService sync)
await Parallel.ForEachAsync(pending, Workers(sync.Options.Uploads, ct), async (frame, token) =>
{
if (await UploadFrameAsync(context, remote, target, null, paths.FromRelative(frame.RelativePath), frame, hashes, token).ConfigureAwait(false) is null)
Interlocked.Increment(ref skipped);
var sent = await UploadFrameAsync(context, remote, target, null, paths.FromRelative(frame.RelativePath), frame, tombstones, hashes, token)
.ConfigureAwait(false);
if (sent is null) Interlocked.Increment(ref skipped);
await progress.StepAsync().ConfigureAwait(false);
}).ConfigureAwait(false);
await SendProxiesAsync(context, remote, target, local, paths, frames, hashes).ConfigureAwait(false);
// An original the index has no hash for turned out to be a photo the server deleted: the verify gives the index
// the hashes (as before a download), so the comparison leaves it out instead of offering its upload again and again.
if (hashes.Values.Any(tombstones.DeletedRemotely)) await VerifyAsync(archive, local.Id, ct).ConfigureAwait(false);
if (created) await sync.PushAllAsync(remote, target, ct).ConfigureAwait(false);
await sync.RecompareAsync(remote, target, ct).ConfigureAwait(false);
@@ -240,7 +244,7 @@ internal sealed class SyncTransfers(SyncService sync)
await progress.StepAsync().ConfigureAwait(false);
}
var sequenceId = local?.Id;
var restored = 0;
int restored = 0, refused = 0;
await Parallel.ForEachAsync(rest, Workers(sync.Options.Downloads, ct), async (frame, token) =>
{
var (staged, sha) = await FetchOriginalAsync(remote, frame, staging, token).ConfigureAwait(false);
@@ -249,6 +253,14 @@ internal sealed class SyncTransfers(SyncService sync)
Restore(staged, place);
Interlocked.Increment(ref restored);
}
else if (frame.Sha256.Length == 0 && (tombstones.DeletedLocally(sha) || have.Contains(tombstones.Canonical(sha)!)))
{
// An original the server has not hashed is known by content only now: one this PC deleted (or holds as
// the converted file that replaced it) is not filed again, which would also drop its tombstone.
File.Delete(staged);
Interlocked.Increment(ref refused);
if (tombstones.DeletedLocally(sha)) context.Log($"{frame.FileName}: deleted on this PC, not downloaded");
}
else
{
await CommitAsync(remoteId, archive.Id, sequenceId, frame, staged, sha, captured, token).ConfigureAwait(false);
@@ -261,7 +273,7 @@ internal sealed class SyncTransfers(SyncService sync)
if (created) await sync.TakeAllAsync(remote, remoteId, ct).ConfigureAwait(false);
await sync.RecompareAsync(remote, remoteId, ct).ConfigureAwait(false);
if (sequenceId is { } downloaded && wanted.Count > restored)
if (sequenceId is { } downloaded && wanted.Count > restored + refused)
{
var label = local!.DisplayTitle;
await EnqueueFollowUpAsync(context, ArchiveJobs.Thumbnails([downloaded], label)).ConfigureAwait(false);
@@ -497,10 +509,11 @@ internal sealed class SyncTransfers(SyncService sync)
/// <summary>
/// Hashes the original again and sends it into <paramref name="sequenceId"/> (or the sequence of
/// <paramref name="sessionKey"/>); null (logged) when it is gone or no longer the archived file. The hash of an
/// original the index has none for goes into <paramref name="hashes"/>.
/// original the index has none for goes into <paramref name="hashes"/>; known by content only now, such an original
/// the server deleted (<paramref name="tombstones"/>) is not sent back.
/// </summary>
private static async Task<CompleteUploadResponse?> UploadFrameAsync(JobContext context, ISyncRemote remote, Guid? sequenceId, string? sessionKey, string path,
Frame frame, ConcurrentDictionary<Guid, string> hashes, CancellationToken cancellationToken)
Frame frame, SyncTombstones tombstones, ConcurrentDictionary<Guid, string> hashes, CancellationToken cancellationToken)
{
if (!File.Exists(path))
{
@@ -508,14 +521,22 @@ internal sealed class SyncTransfers(SyncService sync)
return null;
}
var sha = await ChronosClient.HashFileAsync(path, cancellationToken).ConfigureAwait(false);
if (frame.Sha256 is null) hashes[frame.Id] = sha;
var notSent = new CompleteUploadResponse(Guid.Empty, sequenceId ?? Guid.Empty, -1);
if (frame.Sha256 is null)
{
hashes[frame.Id] = sha;
if (tombstones.DeletedRemotely(sha))
{
context.Log($"{frame.FileName}: deleted on the server, not uploaded");
return notSent;
}
}
if (frame.Sha256 is { } archived && !archived.Equals(sha, StringComparison.OrdinalIgnoreCase))
{
context.Log($"{frame.FileName}: changed since it was archived, not uploaded");
return null;
}
return await remote.UploadFileAsync(path, sequenceId, sessionKey, sha, frame.CapturedAt, cancellationToken).ConfigureAwait(false)
?? new CompleteUploadResponse(Guid.Empty, sequenceId ?? Guid.Empty, -1);
return await remote.UploadFileAsync(path, sequenceId, sessionKey, sha, frame.CapturedAt, cancellationToken).ConfigureAwait(false) ?? notSent;
}
/// <summary>Commits one original from staging into the local sequence (created from the server id when null).</summary>