fix(sync,upload): keep tombstones honest and let an explicit upload bring a photo back

Sync:
- A server frame that has not been hashed is known by content only once it is
  downloaded: one this PC deleted (or holds as its replacement) is now deleted
  from staging instead of filed again, which also dropped its tombstone.
- An unhashed local original that turns out to be a photo the server deleted
  is not uploaded, and the archive verify then writes the hashes into the
  index, so the row settles instead of offering Upload for good.
- A tombstone leaves a photo out of the comparison only when its side no longer
  holds that content, so a tombstone left beside a held copy does not keep the
  other side's copy missing forever.

Uploads:
- StartUploadRequest.Restore (optional, never sent by sync or older clients)
  lets an explicit user upload file a photo deleted from the sequence again,
  also after the trash was emptied; the completion repeats it with
  ?restore=true and the ingest drops the tombstone.
- The completion checks the tombstone again: a photo deleted while its bytes
  were on the way is not filed (409 photo_removed, upload cancelled); the
  protocol client reports that as not sent.
- A tombstone of content a frame of the target still holds refuses no repair.
- Hidden dot names (AppleDouble "._" files) are refused, as the archive
  scanner leaves them out.

Web upload dialog:
- Upload again on rows marked Deleted before; the tooltip no longer promises
  a trash restore that may be impossible.
- Sends a capture time: the Exif DateTimeOriginal of a JPEG or TIFF-based file
  (minimal in-house reader, capture-time.js), else the file's time, so photos
  without Exif keep their order.
- Esc is stopped at keydown during a run (Chromium allows one prevented cancel
  per activation); hidden dot files are left out; a completion refused while
  another one files the upload waits for it; the same photo picked under two
  names waits for its first copy.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit d051bcfc351f5a08500ca864a6508576a95492a4)
This commit is contained in:
Alby96 committed 2026-10-27 12:45:33 +01:00
1 parent eb607d47aa
commit 316c182b19
18 files changed
+753 -63

No files matched your search

@@ -97,6 +97,27 @@ public sealed class SyncRulesTests
new FrameRemoval(localId, "bb", "b2", RemovalReason.Replaced, at)]).Select(r => r.Reason));
}
[Test]
public void ATombstoneBesideAPhotoItsSideStillHoldsHidesNothing()
{
var time = new DateTime(2026, 9, 28, 18, 0, 0);
var at = new DateTimeOffset(2026, 10, 27, 10, 0, 0, TimeSpan.Zero);
FramePhoto Photo(string sha, int second = 0) => new(sha, 100 + second, time.AddSeconds(second));
// The server tombstoned "xx" and still holds it (it deleted one of two copies, or the file came back by hand): this
// PC's copy is not missing there, and the server's is not missing here.
var server = new SyncTombstones([], [new RemovedPhotoDto("xx", null, "deleted", at)]);
Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa"), Photo("xx", 1)], [Photo("aa"), Photo("XX", 1)], server));
// Also when the server has not hashed its copy: it holds it by size and capture time.
Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa"), Photo("xx", 1)], [Photo("aa"), Photo("", 1)], server));
// The same on this side.
var here = new SyncTombstones([new FrameRemoval(Guid.CreateVersion7(), "xx", null, RemovalReason.Deleted, at)], null);
Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa"), Photo("xx", 1)], [Photo("aa"), Photo("xx", 1)], here));
// A tombstone of a photo its side no longer holds still leaves the other side's copy out.
Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa"), Photo("xx", 1)], [Photo("aa")], server));
Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa")], [Photo("aa"), Photo("xx", 1)], here));
Assert.Equal((1, 0), SyncRules.CompareOriginals([Photo("aa"), Photo("xx", 1)], [Photo("aa")], SyncTombstones.None));
}
[Test]
public void FramesEitherSideHasNotHashedAreMatchedBySizeAndCaptureTime()
{
@@ -151,6 +151,90 @@ public sealed class SyncTransferTests(TestContext context)
Assert.SequenceEqual(photos[1..].Select(Sha), kept.Select(k => k.Sha256!));
}
[Test]
public async Task APhotoDeletedHereIsNotFetchedAgainFromAServerThatHasNotHashedIt()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(3);
// A folder the server adopted and never verified (its manifest lists no SHA-256), downloaded here.
var remote = f.Remote.Add("Adopted on the server", photos);
remote.Unhashed = true;
await f.RefreshAsync();
var engine = await f.StartEngineAsync();
var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, f.Archive.Archive.Id);
await f.Archive.WaitForAsync(first.Id, JobState.Done);
var local = Assert.NotNull((await f.EntryAsync("Adopted on the server")).Local);
// This PC deletes a photo the server still lists, unhashed: only its content tells it is the one deleted here.
await UntilIdleAsync(f);
var gone = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Single(fr => fr.Sha256 == Sha(photos[0]));
var sequence = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
var plan = await f.Archive.Service.FrameTrash.PlanDeleteAsync(local.Id, [gone.Id], sequence.FramesVersion, f.Ct);
await f.Archive.Service.FrameTrash.DeleteAsync(plan, null, null, f.Ct);
await f.RefreshAsync();
var again = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, null);
var done = await f.Archive.WaitForAsync(again.Id, JobState.Done);
var kept = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
Assert.True(kept.Select(k => k.Sha256).ToHashSet().SetEquals(photos[1..].Select(Sha)), "the photo deleted here stays deleted");
Assert.True(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(photos[0])), "and keeps its tombstone");
Assert.Contains("deleted on this PC, not downloaded", done.LogTail);
Assert.False(Directory.Exists(Path.Combine(f.Archive.Paths.StagingDirectory, "sync-" + remote.Id.ToString("N"))), "staging cleaned up");
}
[Test]
public async Task AnUnverifiedCopyDoesNotSendBackAPhotoTheServerDeleted()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(4);
// An adopted folder (never verified, so no hashes here) that still holds a photo the server has deleted since.
var folder = Path.Combine(f.Archive.Paths.Root, "Adopted");
Directory.CreateDirectory(folder);
for (var i = 0; i < 4; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"DSC_{i + 1:0000}.JPG"), photos[i], f.Ct);
await f.Archive.ScanAsync();
var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Chronos.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted");
var remote = f.Remote.Add("Adopted", photos[..3]);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[3]), null, "deleted", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
// Without its hash the photo cannot be told from one the server lacks: the upload finds out.
Assert.Equal(1, (await f.EntryAsync(local.DisplayTitle)).Link!.MissingRemote);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(3, remote.Frames.Count);
Assert.Contains("deleted on the server, not uploaded", done.LogTail);
// Its hash is in the index now: the comparison leaves it out, so the row settles instead of offering Upload for good.
Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is not null), "hashed by the verify");
var entry = await f.EntryAsync(local.DisplayTitle);
Assert.Equal((SyncStatus.InSync, 0, 0), (entry.Status, entry.Link!.MissingRemote, entry.Link.MissingLocal));
var calls = remote.Frames.Count;
var next = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
var settled = await f.Archive.WaitForAsync(next.Id, JobState.Done);
Assert.Equal((4L, 4L, calls), (settled.ProgressDone, settled.ProgressTotal, remote.Frames.Count));
}
[Test]
public async Task ATombstoneBesideAPhotoTheServerStillHoldsLeavesTheCopiesInSync()
{
await using var f = await SyncFixture.CreateAsync(context);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Comet", photos);
// The server holds both photos and a tombstone of the second: it deleted one of two copies of it, or the file came
// back by hand. Neither side lacks anything.
var remote = f.Remote.Add("Comet", photos);
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[1]), null, "deleted", DateTimeOffset.UtcNow));
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await f.RefreshAsync();
var entry = await f.EntryAsync("Comet");
Assert.Equal((0, 0), (entry.Link!.MissingRemote, entry.Link.MissingLocal));
Assert.Equal(SyncStatus.InSync, entry.Status);
}
[Test]
public async Task AResumedUploadSendsOnlyWhatTheServerLacks()
{
@@ -634,6 +718,20 @@ public sealed class SyncTransferTests(TestContext context)
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(bytes));
/// <summary>Waits until no job is waiting or running (the thumbnails and analyses a download queues after it).</summary>
private static async Task UntilIdleAsync(SyncFixture f)
{
var until = DateTime.UtcNow + TimeSpan.FromSeconds(TestTiming.Budget(30));
foreach (var state in new[] { JobState.Queued, JobState.Running, JobState.Queued, JobState.Running })
{
while ((await f.Db.Jobs.ListAsync(state, cancellationToken: f.Ct)).Items.Count > 0)
{
Assert.True(DateTime.UtcNow < until, $"jobs still {state}");
await Task.Delay(50, f.Ct);
}
}
}
/// <summary>The photos (SHA-256) the local copy's project excludes, shows as hero, keys the camera on, and waits for.</summary>
private static async Task<(string[] Excluded, string? Hero, string[] Keyframes, string[] Pending)> NamedAsync(SyncFixture f, Guid localId)
{