fix(sync,upload): keep tombstones honest and let an explicit upload bring a photo back
Sync: - A server frame that has not been hashed is known by content only once it is downloaded: one this PC deleted (or holds as its replacement) is now deleted from staging instead of filed again, which also dropped its tombstone. - An unhashed local original that turns out to be a photo the server deleted is not uploaded, and the archive verify then writes the hashes into the index, so the row settles instead of offering Upload for good. - A tombstone leaves a photo out of the comparison only when its side no longer holds that content, so a tombstone left beside a held copy does not keep the other side's copy missing forever. Uploads: - StartUploadRequest.Restore (optional, never sent by sync or older clients) lets an explicit user upload file a photo deleted from the sequence again, also after the trash was emptied; the completion repeats it with ?restore=true and the ingest drops the tombstone. - The completion checks the tombstone again: a photo deleted while its bytes were on the way is not filed (409 photo_removed, upload cancelled); the protocol client reports that as not sent. - A tombstone of content a frame of the target still holds refuses no repair. - Hidden dot names (AppleDouble "._" files) are refused, as the archive scanner leaves them out. Web upload dialog: - Upload again on rows marked Deleted before; the tooltip no longer promises a trash restore that may be impossible. - Sends a capture time: the Exif DateTimeOriginal of a JPEG or TIFF-based file (minimal in-house reader, capture-time.js), else the file's time, so photos without Exif keep their order. - Esc is stopped at keydown during a run (Chromium allows one prevented cancel per activation); hidden dot files are left out; a completion refused while another one files the upload waits for it; the same photo picked under two names waits for its first copy. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> (cherry picked from commit d051bcfc351f5a08500ca864a6508576a95492a4)
This commit is contained in:
1 parent
eb607d47aa
commit
316c182b19
18 files changed
+753
-63
No files matched your search
@@ -167,7 +167,9 @@ public static class SyncRules
|
||||
/// <summary>
|
||||
/// <see cref="CompareOriginals(IReadOnlyCollection{FramePhoto}, IReadOnlyCollection{FramePhoto})"/> honouring the photos
|
||||
/// either side removed (owner review 2, R6): a photo the server deleted is not missing there, one this PC deleted is not
|
||||
/// missing here, and a photo replaced by a converted file on either side is compared as its replacement.
|
||||
/// missing here, and a photo replaced by a converted file on either side is compared as its replacement. A tombstone
|
||||
/// leaves out only a photo its side no longer holds: one left beside a frame of the same content (the other copy of a
|
||||
/// duplicate, a file put back by hand) would otherwise keep the other side's copy missing for good.
|
||||
/// </summary>
|
||||
public static (int MissingRemote, int MissingLocal) CompareOriginals(IReadOnlyCollection<FramePhoto> local, IReadOnlyCollection<FramePhoto> remote,
|
||||
SyncTombstones tombstones)
|
||||
@@ -176,8 +178,12 @@ public static class SyncRules
|
||||
ArgumentNullException.ThrowIfNull(remote);
|
||||
ArgumentNullException.ThrowIfNull(tombstones);
|
||||
if (tombstones.IsEmpty) return CompareOriginals(local, remote);
|
||||
return CompareOriginals([.. local.Where(p => !tombstones.DeletedRemotely(p.Sha256)).Select(tombstones.Canonical)],
|
||||
[.. remote.Where(p => !tombstones.DeletedLocally(p.Sha256)).Select(tombstones.Canonical)]);
|
||||
var mine = local.Select(p => (p.Sha256, Photo: tombstones.Canonical(p))).ToList();
|
||||
var theirs = remote.Select(p => (p.Sha256, Photo: tombstones.Canonical(p))).ToList();
|
||||
var minePhotos = new FramePhotos(mine.Select((p, i) => (i, p.Photo)));
|
||||
var theirPhotos = new FramePhotos(theirs.Select((p, i) => (i, p.Photo)));
|
||||
return CompareOriginals([.. mine.Where(p => !tombstones.DeletedRemotely(p.Sha256) || theirPhotos.IndexOf(p.Photo) is not null).Select(p => p.Photo)],
|
||||
[.. theirs.Where(p => !tombstones.DeletedLocally(p.Sha256) || minePhotos.IndexOf(p.Photo) is not null).Select(p => p.Photo)]);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -70,8 +70,8 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
// server has filed one, which creates the sequence (an original it already holds elsewhere creates nothing).
|
||||
while (remoteId is null && pending.TryDequeue(out var first))
|
||||
{
|
||||
var sent = await UploadFrameAsync(context, remote, null, SyncService.SessionKey(local.Id), paths.FromRelative(first.RelativePath), first, hashes, ct)
|
||||
.ConfigureAwait(false);
|
||||
var sent = await UploadFrameAsync(context, remote, null, SyncService.SessionKey(local.Id), paths.FromRelative(first.RelativePath), first, tombstones,
|
||||
hashes, ct).ConfigureAwait(false);
|
||||
if (sent is null) skipped++;
|
||||
else if (sent.SequenceId != Guid.Empty)
|
||||
{
|
||||
@@ -85,11 +85,15 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
|
||||
await Parallel.ForEachAsync(pending, Workers(sync.Options.Uploads, ct), async (frame, token) =>
|
||||
{
|
||||
if (await UploadFrameAsync(context, remote, target, null, paths.FromRelative(frame.RelativePath), frame, hashes, token).ConfigureAwait(false) is null)
|
||||
Interlocked.Increment(ref skipped);
|
||||
var sent = await UploadFrameAsync(context, remote, target, null, paths.FromRelative(frame.RelativePath), frame, tombstones, hashes, token)
|
||||
.ConfigureAwait(false);
|
||||
if (sent is null) Interlocked.Increment(ref skipped);
|
||||
await progress.StepAsync().ConfigureAwait(false);
|
||||
}).ConfigureAwait(false);
|
||||
await SendProxiesAsync(context, remote, target, local, paths, frames, hashes).ConfigureAwait(false);
|
||||
// An original the index has no hash for turned out to be a photo the server deleted: the verify gives the index
|
||||
// the hashes (as before a download), so the comparison leaves it out instead of offering its upload again and again.
|
||||
if (hashes.Values.Any(tombstones.DeletedRemotely)) await VerifyAsync(archive, local.Id, ct).ConfigureAwait(false);
|
||||
|
||||
if (created) await sync.PushAllAsync(remote, target, ct).ConfigureAwait(false);
|
||||
await sync.RecompareAsync(remote, target, ct).ConfigureAwait(false);
|
||||
@@ -240,7 +244,7 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
await progress.StepAsync().ConfigureAwait(false);
|
||||
}
|
||||
var sequenceId = local?.Id;
|
||||
var restored = 0;
|
||||
int restored = 0, refused = 0;
|
||||
await Parallel.ForEachAsync(rest, Workers(sync.Options.Downloads, ct), async (frame, token) =>
|
||||
{
|
||||
var (staged, sha) = await FetchOriginalAsync(remote, frame, staging, token).ConfigureAwait(false);
|
||||
@@ -249,6 +253,14 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
Restore(staged, place);
|
||||
Interlocked.Increment(ref restored);
|
||||
}
|
||||
else if (frame.Sha256.Length == 0 && (tombstones.DeletedLocally(sha) || have.Contains(tombstones.Canonical(sha)!)))
|
||||
{
|
||||
// An original the server has not hashed is known by content only now: one this PC deleted (or holds as
|
||||
// the converted file that replaced it) is not filed again, which would also drop its tombstone.
|
||||
File.Delete(staged);
|
||||
Interlocked.Increment(ref refused);
|
||||
if (tombstones.DeletedLocally(sha)) context.Log($"{frame.FileName}: deleted on this PC, not downloaded");
|
||||
}
|
||||
else
|
||||
{
|
||||
await CommitAsync(remoteId, archive.Id, sequenceId, frame, staged, sha, captured, token).ConfigureAwait(false);
|
||||
@@ -261,7 +273,7 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
|
||||
if (created) await sync.TakeAllAsync(remote, remoteId, ct).ConfigureAwait(false);
|
||||
await sync.RecompareAsync(remote, remoteId, ct).ConfigureAwait(false);
|
||||
if (sequenceId is { } downloaded && wanted.Count > restored)
|
||||
if (sequenceId is { } downloaded && wanted.Count > restored + refused)
|
||||
{
|
||||
var label = local!.DisplayTitle;
|
||||
await EnqueueFollowUpAsync(context, ArchiveJobs.Thumbnails([downloaded], label)).ConfigureAwait(false);
|
||||
@@ -497,10 +509,11 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
/// <summary>
|
||||
/// Hashes the original again and sends it into <paramref name="sequenceId"/> (or the sequence of
|
||||
/// <paramref name="sessionKey"/>); null (logged) when it is gone or no longer the archived file. The hash of an
|
||||
/// original the index has none for goes into <paramref name="hashes"/>.
|
||||
/// original the index has none for goes into <paramref name="hashes"/>; known by content only now, such an original
|
||||
/// the server deleted (<paramref name="tombstones"/>) is not sent back.
|
||||
/// </summary>
|
||||
private static async Task<CompleteUploadResponse?> UploadFrameAsync(JobContext context, ISyncRemote remote, Guid? sequenceId, string? sessionKey, string path,
|
||||
Frame frame, ConcurrentDictionary<Guid, string> hashes, CancellationToken cancellationToken)
|
||||
Frame frame, SyncTombstones tombstones, ConcurrentDictionary<Guid, string> hashes, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
@@ -508,14 +521,22 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
return null;
|
||||
}
|
||||
var sha = await ChronosClient.HashFileAsync(path, cancellationToken).ConfigureAwait(false);
|
||||
if (frame.Sha256 is null) hashes[frame.Id] = sha;
|
||||
var notSent = new CompleteUploadResponse(Guid.Empty, sequenceId ?? Guid.Empty, -1);
|
||||
if (frame.Sha256 is null)
|
||||
{
|
||||
hashes[frame.Id] = sha;
|
||||
if (tombstones.DeletedRemotely(sha))
|
||||
{
|
||||
context.Log($"{frame.FileName}: deleted on the server, not uploaded");
|
||||
return notSent;
|
||||
}
|
||||
}
|
||||
if (frame.Sha256 is { } archived && !archived.Equals(sha, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
context.Log($"{frame.FileName}: changed since it was archived, not uploaded");
|
||||
return null;
|
||||
}
|
||||
return await remote.UploadFileAsync(path, sequenceId, sessionKey, sha, frame.CapturedAt, cancellationToken).ConfigureAwait(false)
|
||||
?? new CompleteUploadResponse(Guid.Empty, sequenceId ?? Guid.Empty, -1);
|
||||
return await remote.UploadFileAsync(path, sequenceId, sessionKey, sha, frame.CapturedAt, cancellationToken).ConfigureAwait(false) ?? notSent;
|
||||
}
|
||||
|
||||
/// <summary>Commits one original from staging into the local sequence (created from the server id when null).</summary>
|
||||
|
||||
Reference in new issue
Block a user