fix(sync,upload): keep tombstones honest and let an explicit upload bring a photo back
Sync: - A server frame that has not been hashed is known by content only once it is downloaded: one this PC deleted (or holds as its replacement) is now deleted from staging instead of filed again, which also dropped its tombstone. - An unhashed local original that turns out to be a photo the server deleted is not uploaded, and the archive verify then writes the hashes into the index, so the row settles instead of offering Upload for good. - A tombstone leaves a photo out of the comparison only when its side no longer holds that content, so a tombstone left beside a held copy does not keep the other side's copy missing forever. Uploads: - StartUploadRequest.Restore (optional, never sent by sync or older clients) lets an explicit user upload file a photo deleted from the sequence again, also after the trash was emptied; the completion repeats it with ?restore=true and the ingest drops the tombstone. - The completion checks the tombstone again: a photo deleted while its bytes were on the way is not filed (409 photo_removed, upload cancelled); the protocol client reports that as not sent. - A tombstone of content a frame of the target still holds refuses no repair. - Hidden dot names (AppleDouble "._" files) are refused, as the archive scanner leaves them out. Web upload dialog: - Upload again on rows marked Deleted before; the tooltip no longer promises a trash restore that may be impossible. - Sends a capture time: the Exif DateTimeOriginal of a JPEG or TIFF-based file (minimal in-house reader, capture-time.js), else the file's time, so photos without Exif keep their order. - Esc is stopped at keydown during a run (Chromium allows one prevented cancel per activation); hidden dot files are left out; a completion refused while another one files the upload waits for it; the same photo picked under two names waits for its first copy. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> (cherry picked from commit d051bcfc351f5a08500ca864a6508576a95492a4)
This commit is contained in:
1 parent
eb607d47aa
commit
316c182b19
18 files changed
+753
-63
No files matched your search
@@ -97,6 +97,27 @@ public sealed class SyncRulesTests
|
||||
new FrameRemoval(localId, "bb", "b2", RemovalReason.Replaced, at)]).Select(r => r.Reason));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ATombstoneBesideAPhotoItsSideStillHoldsHidesNothing()
|
||||
{
|
||||
var time = new DateTime(2026, 9, 28, 18, 0, 0);
|
||||
var at = new DateTimeOffset(2026, 10, 27, 10, 0, 0, TimeSpan.Zero);
|
||||
FramePhoto Photo(string sha, int second = 0) => new(sha, 100 + second, time.AddSeconds(second));
|
||||
// The server tombstoned "xx" and still holds it (it deleted one of two copies, or the file came back by hand): this
|
||||
// PC's copy is not missing there, and the server's is not missing here.
|
||||
var server = new SyncTombstones([], [new RemovedPhotoDto("xx", null, "deleted", at)]);
|
||||
Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa"), Photo("xx", 1)], [Photo("aa"), Photo("XX", 1)], server));
|
||||
// Also when the server has not hashed its copy: it holds it by size and capture time.
|
||||
Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa"), Photo("xx", 1)], [Photo("aa"), Photo("", 1)], server));
|
||||
// The same on this side.
|
||||
var here = new SyncTombstones([new FrameRemoval(Guid.CreateVersion7(), "xx", null, RemovalReason.Deleted, at)], null);
|
||||
Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa"), Photo("xx", 1)], [Photo("aa"), Photo("xx", 1)], here));
|
||||
// A tombstone of a photo its side no longer holds still leaves the other side's copy out.
|
||||
Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa"), Photo("xx", 1)], [Photo("aa")], server));
|
||||
Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa")], [Photo("aa"), Photo("xx", 1)], here));
|
||||
Assert.Equal((1, 0), SyncRules.CompareOriginals([Photo("aa"), Photo("xx", 1)], [Photo("aa")], SyncTombstones.None));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void FramesEitherSideHasNotHashedAreMatchedBySizeAndCaptureTime()
|
||||
{
|
||||
|
||||
@@ -151,6 +151,90 @@ public sealed class SyncTransferTests(TestContext context)
|
||||
Assert.SequenceEqual(photos[1..].Select(Sha), kept.Select(k => k.Sha256!));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task APhotoDeletedHereIsNotFetchedAgainFromAServerThatHasNotHashedIt()
|
||||
{
|
||||
await using var f = await SyncFixture.CreateAsync(context);
|
||||
var photos = f.NewPhotos(3);
|
||||
// A folder the server adopted and never verified (its manifest lists no SHA-256), downloaded here.
|
||||
var remote = f.Remote.Add("Adopted on the server", photos);
|
||||
remote.Unhashed = true;
|
||||
await f.RefreshAsync();
|
||||
var engine = await f.StartEngineAsync();
|
||||
var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, f.Archive.Archive.Id);
|
||||
await f.Archive.WaitForAsync(first.Id, JobState.Done);
|
||||
var local = Assert.NotNull((await f.EntryAsync("Adopted on the server")).Local);
|
||||
|
||||
// This PC deletes a photo the server still lists, unhashed: only its content tells it is the one deleted here.
|
||||
await UntilIdleAsync(f);
|
||||
var gone = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Single(fr => fr.Sha256 == Sha(photos[0]));
|
||||
var sequence = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!;
|
||||
var plan = await f.Archive.Service.FrameTrash.PlanDeleteAsync(local.Id, [gone.Id], sequence.FramesVersion, f.Ct);
|
||||
await f.Archive.Service.FrameTrash.DeleteAsync(plan, null, null, f.Ct);
|
||||
await f.RefreshAsync();
|
||||
|
||||
var again = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, null);
|
||||
var done = await f.Archive.WaitForAsync(again.Id, JobState.Done);
|
||||
|
||||
var kept = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
|
||||
Assert.True(kept.Select(k => k.Sha256).ToHashSet().SetEquals(photos[1..].Select(Sha)), "the photo deleted here stays deleted");
|
||||
Assert.True(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(photos[0])), "and keeps its tombstone");
|
||||
Assert.Contains("deleted on this PC, not downloaded", done.LogTail);
|
||||
Assert.False(Directory.Exists(Path.Combine(f.Archive.Paths.StagingDirectory, "sync-" + remote.Id.ToString("N"))), "staging cleaned up");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnUnverifiedCopyDoesNotSendBackAPhotoTheServerDeleted()
|
||||
{
|
||||
await using var f = await SyncFixture.CreateAsync(context);
|
||||
var photos = f.NewPhotos(4);
|
||||
// An adopted folder (never verified, so no hashes here) that still holds a photo the server has deleted since.
|
||||
var folder = Path.Combine(f.Archive.Paths.Root, "Adopted");
|
||||
Directory.CreateDirectory(folder);
|
||||
for (var i = 0; i < 4; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"DSC_{i + 1:0000}.JPG"), photos[i], f.Ct);
|
||||
await f.Archive.ScanAsync();
|
||||
var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Chronos.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted");
|
||||
var remote = f.Remote.Add("Adopted", photos[..3]);
|
||||
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[3]), null, "deleted", DateTimeOffset.UtcNow));
|
||||
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
||||
await f.RefreshAsync();
|
||||
// Without its hash the photo cannot be told from one the server lacks: the upload finds out.
|
||||
Assert.Equal(1, (await f.EntryAsync(local.DisplayTitle)).Link!.MissingRemote);
|
||||
var engine = await f.StartEngineAsync();
|
||||
|
||||
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
||||
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
||||
|
||||
Assert.Equal(3, remote.Frames.Count);
|
||||
Assert.Contains("deleted on the server, not uploaded", done.LogTail);
|
||||
// Its hash is in the index now: the comparison leaves it out, so the row settles instead of offering Upload for good.
|
||||
Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is not null), "hashed by the verify");
|
||||
var entry = await f.EntryAsync(local.DisplayTitle);
|
||||
Assert.Equal((SyncStatus.InSync, 0, 0), (entry.Status, entry.Link!.MissingRemote, entry.Link.MissingLocal));
|
||||
var calls = remote.Frames.Count;
|
||||
var next = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
|
||||
var settled = await f.Archive.WaitForAsync(next.Id, JobState.Done);
|
||||
Assert.Equal((4L, 4L, calls), (settled.ProgressDone, settled.ProgressTotal, remote.Frames.Count));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ATombstoneBesideAPhotoTheServerStillHoldsLeavesTheCopiesInSync()
|
||||
{
|
||||
await using var f = await SyncFixture.CreateAsync(context);
|
||||
var photos = f.NewPhotos(2);
|
||||
var local = await f.LocalAsync("Comet", photos);
|
||||
// The server holds both photos and a tombstone of the second: it deleted one of two copies of it, or the file came
|
||||
// back by hand. Neither side lacks anything.
|
||||
var remote = f.Remote.Add("Comet", photos);
|
||||
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[1]), null, "deleted", DateTimeOffset.UtcNow));
|
||||
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
||||
await f.RefreshAsync();
|
||||
|
||||
var entry = await f.EntryAsync("Comet");
|
||||
Assert.Equal((0, 0), (entry.Link!.MissingRemote, entry.Link.MissingLocal));
|
||||
Assert.Equal(SyncStatus.InSync, entry.Status);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AResumedUploadSendsOnlyWhatTheServerLacks()
|
||||
{
|
||||
@@ -634,6 +718,20 @@ public sealed class SyncTransferTests(TestContext context)
|
||||
|
||||
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(bytes));
|
||||
|
||||
/// <summary>Waits until no job is waiting or running (the thumbnails and analyses a download queues after it).</summary>
|
||||
private static async Task UntilIdleAsync(SyncFixture f)
|
||||
{
|
||||
var until = DateTime.UtcNow + TimeSpan.FromSeconds(TestTiming.Budget(30));
|
||||
foreach (var state in new[] { JobState.Queued, JobState.Running, JobState.Queued, JobState.Running })
|
||||
{
|
||||
while ((await f.Db.Jobs.ListAsync(state, cancellationToken: f.Ct)).Items.Count > 0)
|
||||
{
|
||||
Assert.True(DateTime.UtcNow < until, $"jobs still {state}");
|
||||
await Task.Delay(50, f.Ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The photos (SHA-256) the local copy's project excludes, shows as hero, keys the camera on, and waits for.</summary>
|
||||
private static async Task<(string[] Excluded, string? Hero, string[] Keyframes, string[] Pending)> NamedAsync(SyncFixture f, Guid localId)
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user