Files
Chronos/tests/Chronos.Data.Tests/Sync/SyncRulesTests.cs
T
Alby96 316c182b19 fix(sync,upload): keep tombstones honest and let an explicit upload bring a photo back
Sync:
- A server frame that has not been hashed is known by content only once it is
  downloaded: one this PC deleted (or holds as its replacement) is now deleted
  from staging instead of filed again, which also dropped its tombstone.
- An unhashed local original that turns out to be a photo the server deleted
  is not uploaded, and the archive verify then writes the hashes into the
  index, so the row settles instead of offering Upload for good.
- A tombstone leaves a photo out of the comparison only when its side no longer
  holds that content, so a tombstone left beside a held copy does not keep the
  other side's copy missing forever.

Uploads:
- StartUploadRequest.Restore (optional, never sent by sync or older clients)
  lets an explicit user upload file a photo deleted from the sequence again,
  also after the trash was emptied; the completion repeats it with
  ?restore=true and the ingest drops the tombstone.
- The completion checks the tombstone again: a photo deleted while its bytes
  were on the way is not filed (409 photo_removed, upload cancelled); the
  protocol client reports that as not sent.
- A tombstone of content a frame of the target still holds refuses no repair.
- Hidden dot names (AppleDouble "._" files) are refused, as the archive
  scanner leaves them out.

Web upload dialog:
- Upload again on rows marked Deleted before; the tooltip no longer promises
  a trash restore that may be impossible.
- Sends a capture time: the Exif DateTimeOriginal of a JPEG or TIFF-based file
  (minimal in-house reader, capture-time.js), else the file's time, so photos
  without Exif keep their order.
- Esc is stopped at keydown during a run (Chromium allows one prevented cancel
  per activation); hidden dot files are left out; a completion refused while
  another one files the upload waits for it; the same photo picked under two
  names waits for its first copy.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit d051bcfc351f5a08500ca864a6508576a95492a4)
2026-10-27 12:45:33 +01:00

248 lines
15 KiB
C#

using Chronos.Core.Imaging;
using Chronos.Core.Imaging.Decoders;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Data.Sync;
using Chronos.Protocol;
using Chronos.Testing;
namespace Chronos.Data.Tests.Sync;
public sealed class SyncRulesTests
{
private static readonly Sequence Local = new() { Id = Guid.CreateVersion7(), ArchiveId = Guid.CreateVersion7(), Folder = "2026/a", SettingsVersion = 4 };
private static readonly SyncLink Agreed = new()
{
Server = "https://nas",
RemoteId = Guid.CreateVersion7(),
Title = "Harbour",
RemoteSettingsVersion = 7,
BaseLocalSettings = 4,
BaseRemoteSettings = 7,
};
[Test]
public void StateFollowsLinksOriginalsAndSettingsVersions()
{
Assert.Equal(SyncStatus.LocalOnly, SyncRules.Status(Local, null));
Assert.Equal(SyncStatus.RemoteOnly, SyncRules.Status(null, Agreed));
Assert.Equal(SyncStatus.ProxiesOnly, SyncRules.Status(null, Agreed with { ProxiesCached = 3 }));
Assert.Equal(SyncStatus.InSync, SyncRules.Status(Local, Agreed));
Assert.Equal(SyncStatus.LocalAhead, SyncRules.Status(Local, Agreed with { MissingRemote = 2 }));
Assert.Equal(SyncStatus.LocalAhead, SyncRules.Status(Local with { SettingsVersion = 5 }, Agreed));
Assert.Equal(SyncStatus.ServerAhead, SyncRules.Status(Local, Agreed with { MissingLocal = 1 }));
Assert.Equal(SyncStatus.ServerAhead, SyncRules.Status(Local, Agreed with { RemoteSettingsVersion = 8 }));
// Originals never conflict: what this PC lacks is downloaded after what the server lacks is uploaded.
Assert.Equal(SyncStatus.LocalAhead, SyncRules.Status(Local, Agreed with { MissingRemote = 1, MissingLocal = 1 }));
Assert.Equal(SyncStatus.Conflict, SyncRules.Status(Local with { SettingsVersion = 5 }, Agreed with { RemoteSettingsVersion = 8 }));
// A link nothing was agreed on yet: both sides count as changed until the content is compared.
Assert.Equal(SyncStatus.Conflict, SyncRules.Status(Local, Agreed with { BaseLocalSettings = null, BaseRemoteSettings = null }));
}
[Test]
public void TheLibraryStoresOnlyStatesOfLocalCopies()
{
Assert.Equal(SyncState.InSync, SyncRules.ToSequenceState(SyncStatus.InSync));
Assert.Equal(SyncState.Conflict, SyncRules.ToSequenceState(SyncStatus.Conflict));
Assert.Equal(SyncState.LocalAhead, SyncRules.ToSequenceState(SyncStatus.LocalAhead));
Assert.Equal(SyncState.ServerAhead, SyncRules.ToSequenceState(SyncStatus.ServerAhead));
Assert.Equal(SyncState.LocalOnly, SyncRules.ToSequenceState(SyncStatus.LocalOnly));
Assert.Null(SyncRules.ToSequenceState(SyncStatus.Transferring));
Assert.Null(SyncRules.ToSequenceState(SyncStatus.RemoteOnly));
}
[Test]
public void OriginalsAreComparedByContent()
{
Assert.Equal((0, 0), SyncRules.CompareOriginals(["AA", "bb"], ["aa", "BB"]));
Assert.Equal((1, 0), SyncRules.CompareOriginals(["aa", "bb", "cc"], ["aa", "bb"]));
Assert.Equal((0, 2), SyncRules.CompareOriginals(["aa"], ["aa", "bb", "cc"]));
Assert.Equal((1, 1), SyncRules.CompareOriginals(["aa", "dd"], ["aa", "ee"]));
// Frames not hashed yet (adopted, not verified) stand for server frames no hash matched.
Assert.Equal((0, 0), SyncRules.CompareOriginals(["aa", null, null], ["aa", "bb", "cc"]));
Assert.Equal((0, 1), SyncRules.CompareOriginals([null], ["bb", "cc"]));
Assert.Equal((1, 0), SyncRules.CompareOriginals([null, null], ["bb"]));
}
[Test]
public void PhotosEitherSideRemovedAreNotOfferedBack()
{
static FramePhoto Photo(string sha) => new(sha, 100, null);
var at = new DateTimeOffset(2026, 10, 27, 10, 0, 0, TimeSpan.Zero);
var localId = Guid.CreateVersion7();
// The server deleted "cc", which this PC still holds; this PC deleted "dd", which the server still holds.
var tombstones = new SyncTombstones([new FrameRemoval(localId, "DD", null, RemovalReason.Deleted, at)],
[new RemovedPhotoDto("cc", null, "deleted", at)]);
Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa"), Photo("bb"), Photo("cc")], [Photo("aa"), Photo("bb"), Photo("dd")], tombstones));
Assert.Equal((1, 1), SyncRules.CompareOriginals([Photo("aa"), Photo("cc")], [Photo("aa"), Photo("dd")], SyncTombstones.None));
Assert.True(tombstones.DeletedRemotely("CC"));
Assert.True(tombstones.DeletedLocally("dd"));
Assert.False(tombstones.DeletedLocally("cc"));
// A photo replaced by a converted file on either side is the same photo as its replacement, and so is a chain of them.
var replaced = new SyncTombstones([], [new RemovedPhotoDto("aa", "a2", "replaced", at), new RemovedPhotoDto("a2", "a3", "replaced", at)]);
Assert.Equal("a3", replaced.Canonical("AA"));
Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa"), Photo("bb")], [Photo("a3"), Photo("bb")], replaced));
Assert.False(replaced.DeletedRemotely("aa"), "a replaced photo is there as its replacement");
var replacedHere = new SyncTombstones([new FrameRemoval(localId, "bb", "b2", RemovalReason.Replaced, at)], null);
Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("b2")], [Photo("bb")], replacedHere));
// A replacement deleted since: neither the photo nor its replacement is offered back.
var gone = new SyncTombstones([], [new RemovedPhotoDto("aa", "a2", "replaced", at), new RemovedPhotoDto("a2", null, "deleted", at)]);
Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa")], [], gone));
Assert.True(SyncTombstones.None.IsEmpty);
Assert.SequenceEqual(["deleted", "replaced"], SyncTombstones.ToDto([new FrameRemoval(localId, "aa", null, RemovalReason.Deleted, at),
new FrameRemoval(localId, "bb", "b2", RemovalReason.Replaced, at)]).Select(r => r.Reason));
}
[Test]
public void ATombstoneBesideAPhotoItsSideStillHoldsHidesNothing()
{
var time = new DateTime(2026, 9, 28, 18, 0, 0);
var at = new DateTimeOffset(2026, 10, 27, 10, 0, 0, TimeSpan.Zero);
FramePhoto Photo(string sha, int second = 0) => new(sha, 100 + second, time.AddSeconds(second));
// The server tombstoned "xx" and still holds it (it deleted one of two copies, or the file came back by hand): this
// PC's copy is not missing there, and the server's is not missing here.
var server = new SyncTombstones([], [new RemovedPhotoDto("xx", null, "deleted", at)]);
Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa"), Photo("xx", 1)], [Photo("aa"), Photo("XX", 1)], server));
// Also when the server has not hashed its copy: it holds it by size and capture time.
Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa"), Photo("xx", 1)], [Photo("aa"), Photo("", 1)], server));
// The same on this side.
var here = new SyncTombstones([new FrameRemoval(Guid.CreateVersion7(), "xx", null, RemovalReason.Deleted, at)], null);
Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa"), Photo("xx", 1)], [Photo("aa"), Photo("xx", 1)], here));
// A tombstone of a photo its side no longer holds still leaves the other side's copy out.
Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa"), Photo("xx", 1)], [Photo("aa")], server));
Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa")], [Photo("aa"), Photo("xx", 1)], here));
Assert.Equal((1, 0), SyncRules.CompareOriginals([Photo("aa"), Photo("xx", 1)], [Photo("aa")], SyncTombstones.None));
}
[Test]
public void FramesEitherSideHasNotHashedAreMatchedBySizeAndCaptureTime()
{
var time = new DateTime(2026, 9, 28, 18, 0, 0);
FramePhoto Photo(string? sha, long size, int second) => new(sha, size, time.AddSeconds(second));
// A folder the server adopted and never verified: each of its frames counts on its own, matched by size and time.
Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa", 100, 0), Photo("bb", 200, 10)], [Photo("", 100, 0), Photo("", 200, 10)]));
Assert.Equal((1, 2), SyncRules.CompareOriginals([Photo("aa", 100, 0), Photo("bb", 200, 10)], [Photo("", 100, 0), Photo("", 300, 20), Photo("", 400, 30)]));
Assert.Equal((0, 3), SyncRules.CompareOriginals(Array.Empty<string?>(), ["", "", ""]));
// Hashed on both sides, the hash decides even when size and time agree.
Assert.Equal((1, 1), SyncRules.CompareOriginals([Photo("aa", 100, 0)], [Photo("cc", 100, 0)]));
// A photo the adopted folder holds twice ('IMG_0001.JPG' and 'IMG_0001 (1).JPG') is here in this PC's one copy,
// as the download finds it: nothing is left to fetch.
Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa", 100, 0)], [Photo("", 100, 0), Photo("", 100, 0)]));
Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo(null, 100, 0)], [Photo("", 100, 0), Photo("", 100, 0)]));
Assert.Equal((0, 1), SyncRules.CompareOriginals([Photo("aa", 100, 0)], [Photo("", 100, 0), Photo("", 100, 0), Photo("", 100, 10)]));
}
[Test]
public void ProxiesOfFramesTheServerHasNotHashedAreCountedByCaptureTime()
{
var heif = new ImageDecoder([new NativeHeif()]);
var time = new DateTime(2026, 9, 28, 18, 0, 0);
List<Frame> local =
[
Original("aa", 100, ImageFileFormat.Heif) with { CapturedAt = time },
Original("bb", 200, ImageFileFormat.Heif) with { CapturedAt = time.AddSeconds(10) },
];
var first = Wanted(string.Empty, 100, "IMG_0001.HEIC");
var second = Wanted(string.Empty, 200, "IMG_0002.HEIC");
var manifest = new SequenceManifest(Guid.CreateVersion7(), 1, 1, null, [], [first, second], [], new ProxySpec(1920, 90));
var captured = new Dictionary<Guid, DateTime?> { [first.Id] = time, [second.Id] = time.AddSeconds(11) };
Assert.Equal(1, SyncRules.ProxiesToSend(manifest, local, heif, captured));
Assert.Equal(0, SyncRules.ProxiesToSend(manifest, local, heif));
Assert.Same(local[0], SyncRules.ProxyOriginal(first, time, local, SyncRules.Photos(local)));
Assert.Null(SyncRules.ProxyOriginal(second, time.AddSeconds(11), local, SyncRules.Photos(local)));
}
[Test]
public void ProxiesTheServerAsksForAreCountedForOriginalsThisPcReads()
{
var heif = new ImageDecoder([new NativeHeif()]);
List<Frame> local =
[
Original("aa", 100, ImageFileFormat.Heif),
Original(null, 200, ImageFileFormat.Heif),
Original(null, 300, ImageFileFormat.Heif),
Original("dd", 400, ImageFileFormat.Avif),
];
var manifest = new SequenceManifest(Guid.CreateVersion7(), 1, 1, null, [],
[
Wanted("AA", 100, "2026-10-01_0001.heic"),
// Not hashed here: matched by size, whatever name the server gave its copy.
Wanted("bb", 200, "2026-10-01_0002.heic"),
Wanted("cc", 999, "IMG_0003.HEIC"),
// The server has no hash of it yet: nothing tells which original it is.
Wanted(string.Empty, 300, "IMG_0004.HEIC"),
// This PC cannot decode AVIF.
Wanted("dd", 400, "IMG_0005.AVIF"),
Wanted("ee", 200, "IMG_0006.HEIC") with { WantsProxy = false },
], [], new ProxySpec(1920, 90));
Assert.Equal(2, SyncRules.ProxiesToSend(manifest, local, heif));
Assert.Equal(0, SyncRules.ProxiesToSend(manifest, local, DecoderFactory.CreateServer()));
Assert.True(SyncRules.MakesProxiesFor(local, heif), "HEIF the server cannot decode and this PC can");
Assert.False(SyncRules.MakesProxiesFor(local, DecoderFactory.CreateServer()), "a PC that reads no more than the server");
Assert.False(SyncRules.MakesProxiesFor([Original("aa", 100, ImageFileFormat.Jpeg)], heif), "the server decodes JPEG itself");
}
[Test]
public void SettingsAreComparedAsProjectsNotAsText()
{
Assert.True(SyncRules.SameSettings(null, "{}"), "no settings is the default project");
Assert.True(SyncRules.SameSettings(SyncFixture.Window41, "{ \"exposure\": { \"deflickerWindow\": { \"manual\": 41, \"isAuto\": false } } }"));
Assert.False(SyncRules.SameSettings(SyncFixture.Window41, SyncFixture.Window33));
var difference = Assert.Contains(SyncRules.Differences(SyncFixture.Window41, SyncFixture.Window33), d => d.Path.StartsWith("exposure.deflickerWindow", StringComparison.Ordinal));
Assert.Equal("exposure.deflickerWindow.manual", difference.Path);
Assert.Equal("41", difference.Mine);
Assert.Equal("33", difference.Theirs);
Assert.Count(1, SyncRules.Differences("""{"exposure":{"exposureOffsetEv":1},"stack":{"length":3}}""", "{}", max: 1));
Assert.Empty(SyncRules.Differences(SyncFixture.Window33, SyncFixture.Window33));
}
[Test]
public void PayloadDifferencesListTheSequenceFieldsBeforeTheSettings()
{
var mine = new SyncPayload("Harbour", ["sunset", "Harbour"], 3, "mine", SyncFixture.Window41);
var theirs = new SyncPayload("Harbour at dusk", ["harbour", "SUNSET"], 3, " mine ", SyncFixture.Window33);
var differences = SyncRules.Differences(mine, theirs, 10);
Assert.SequenceEqual(["sequence.title", "exposure.deflickerWindow.manual"], differences.Select(d => d.Path));
Assert.Equal(("Harbour", "Harbour at dusk"), (differences[0].Mine, differences[0].Theirs));
Assert.Count(1, SyncRules.Differences(mine, theirs, 1));
Assert.True(mine.SameMetadata(mine with { Tags = ["HARBOUR", "sunset"], Notes = "mine " }), "tags ignore case and order, blanks are trimmed");
Assert.Empty(SyncRules.Differences(mine, mine with { Tags = ["harbour", "SUNSET"] }));
}
private static Frame Original(string? sha, long size, ImageFileFormat format) => new()
{
Id = Guid.CreateVersion7(),
SequenceId = Local.Id,
Index = (int)(size / 100),
RelativePath = $"2026/a/{size}",
FileName = $"{size}",
Size = size,
Sha256 = sha,
Format = format,
};
private static ManifestFrame Wanted(string sha, long size, string name) =>
new(Guid.CreateVersion7(), (int)(size / 100), name, size, sha, HasProxy: false, HasThumb: true, WantsProxy: true);
/// <summary>A provider that decodes HEIF natively (WIC with the HEIF extension on a desktop).</summary>
private sealed class NativeHeif : IImageDecoderProvider
{
public string Name => "native-heif";
public DecodeTier? TierFor(ImageFileFormat format) => format == ImageFileFormat.Heif ? DecodeTier.Native : null;
public DecodedImage? Decode(DecodeRequest request, CancellationToken cancellationToken = default) => null;
}
}