Files
Chronos/tests/Chronos.Data.Tests/Sync/SyncTransferCopyTests.cs
T
Alby96andClaude Opus 5.5 6db4c2e80c fix(copies): close the eleventh verification round of owner review 6
- A merge's Verify stage journals a second version, with the hash it is
  filed as, before it files it. A run that stops between filing it and
  leaving it out of Main (a drive dropping out, project.json held, the
  app closing) now leaves it out on resume, instead of reading it as the
  kept sequence's own photo and keeping both versions in Main for good.
- Upload all local ignores a Kept apart pair whose server sequence is
  confirmed deleted only when the answer came before the first 404. A
  Keep here, answered after the deletion was shown, skips the sequence
  again instead of uploading it and erasing the answer.
- An upload's checkpoint records whether the job joined a server
  sequence this PC already knew. A resumed or retried upload no longer
  takes such a sequence for one it made and overwrites its details.

Refs #73

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 14:01:45 +01:00

556 lines
31 KiB
C#

using System.Security.Cryptography;
using Chronos.Core.Model;
using Chronos.Data.Copies;
using Chronos.Data.Jobs;
using Chronos.Data.Repositories;
using Chronos.Data.Sync;
using Chronos.Protocol;
using Chronos.Testing;
namespace Chronos.Data.Tests.Sync;
/// <summary>
/// The filtered transfers an Integrate runs (owner review 6, #73; design §5c): only the photos asked for, each once even
/// when resumed, a server sequence gone marks the pair, and a linked upload files into its target when the server can.
/// </summary>
[Category("sync")]
public sealed class SyncTransferCopyTests(TestContext context)
{
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes));
private static async Task<JobContext> ContextAsync(SyncFixture f)
{
var engine = new JobEngine(f.Db, [], new JobEngineOptions { WorkerId = "t" });
await f.Db.Jobs.EnqueueAsync(new Job { Id = Guid.CreateVersion7(), Kind = JobKind.Verify, State = JobState.Queued, Created = DateTimeOffset.UtcNow }, f.Ct);
var claimed = Assert.NotNull(await f.Db.Jobs.ClaimNextAsync("t", [JobKind.Verify], f.Ct));
return new JobContext(engine, claimed, f.Ct);
}
private static async Task<HashSet<string>> LocalHashesAsync(SyncFixture f, Guid id) =>
(await f.Db.Frames.ListBySequenceAsync(id, cancellationToken: f.Ct)).Select(fr => fr.Sha256!).ToHashSet(StringComparer.Ordinal);
private static SyncFixture Capable(SyncFixture f)
{
f.Remote.Capabilities.Add(ServerCapabilities.UploadInto);
f.Remote.Capabilities.Add(ServerCapabilities.PhotoLookup);
return f;
}
/// <summary>Waits until no pass a download queued works on <paramref name="sequenceId"/>.</summary>
private static async Task WaitIdleAsync(SyncFixture f, Guid sequenceId)
{
var until = DateTime.UtcNow.AddSeconds(30);
while (DateTime.UtcNow < until)
{
var held = await f.Db.Jobs.ListPendingTargetsAsync(JobKind.Thumbs, f.Ct);
held.UnionWith(await f.Db.Jobs.ListPendingTargetsAsync(JobKind.LightAnalyse, f.Ct));
if (!held.Contains(sequenceId)) return;
await Task.Delay(20, f.Ct);
}
}
[Test]
public async Task OnlyThePhotosAskedForGoEachWay()
{
await using var f = Capable(await SyncFixture.CreateAsync(context));
var photos = f.NewPhotos(5);
var local = await f.LocalAsync("Comet", [photos[0], photos[1], photos[2]]);
var remote = f.Remote.Add("Comet", [photos[0], photos[3], photos[4]]);
var job = await ContextAsync(f);
var transfers = new SyncTransfers(f.Sync);
Assert.Equal(1, await transfers.SendPhotosAsync(job, f.Remote, local.Id, remote.Id, [Sha(photos[1])]));
Assert.Equal(1, await transfers.FetchPhotosAsync(job, f.Remote, remote.Id, local.Id, [Sha(photos[3])]));
Assert.True(remote.Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals([Sha(photos[0]), Sha(photos[1]), Sha(photos[3]), Sha(photos[4])]));
Assert.True((await LocalHashesAsync(f, local.Id)).SetEquals([Sha(photos[0]), Sha(photos[1]), Sha(photos[2]), Sha(photos[3])]));
Assert.Equal(1, f.Remote.IntoUploads);
}
[Test]
public async Task ARunAgainSendsAndFetchesNothingTwice()
{
await using var f = Capable(await SyncFixture.CreateAsync(context));
var photos = f.NewPhotos(4);
var local = await f.LocalAsync("Comet", [photos[0], photos[1]]);
var remote = f.Remote.Add("Comet", [photos[2], photos[3]]);
var job = await ContextAsync(f);
var transfers = new SyncTransfers(f.Sync);
var sent = new List<string>();
var fetched = new List<string>();
string[] send = [Sha(photos[0]), Sha(photos[1])], fetch = [Sha(photos[2]), Sha(photos[3])];
Assert.Equal(2, await transfers.SendPhotosAsync(job, f.Remote, local.Id, remote.Id, send, k => { sent.Add(k); return ValueTask.CompletedTask; }));
Assert.Equal(2, await transfers.FetchPhotosAsync(job, f.Remote, remote.Id, local.Id, fetch, k => { fetched.Add(k); return ValueTask.CompletedTask; }));
var uploads = f.Remote.IntoUploads;
Assert.Equal(0, await transfers.SendPhotosAsync(job, f.Remote, local.Id, remote.Id, send, k => { sent.Add(k); return ValueTask.CompletedTask; }));
Assert.Equal(0, await transfers.FetchPhotosAsync(job, f.Remote, remote.Id, local.Id, fetch, k => { fetched.Add(k); return ValueTask.CompletedTask; }));
Assert.Equal(uploads, f.Remote.IntoUploads);
Assert.Count(4, remote.Frames);
Assert.Count(4, await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct));
// Every key is reported done, the second time without moving a file.
Assert.SequenceEqual([.. send, .. send], sent);
Assert.SequenceEqual([.. fetch, .. fetch], fetched);
}
[Test]
public async Task AServerSequenceGoneMarksThePairAndSendsNowhereElse()
{
await using var f = await SyncFixture.CreateAsync(context);
// A server known by its id: a 404 there is that server's word (an address may name another server tomorrow).
var nas = new FakeSyncRemote { Server = ServerIdentity.IdKey(Guid.NewGuid()) };
nas.Capabilities.Add(ServerCapabilities.UploadInto);
nas.Capabilities.Add(ServerCapabilities.PhotoLookup);
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Comet", photos);
var remote = nas.Add("Comet", [photos[0]]);
var pair = await PairAsync(f, nas.Server, local.Id, remote.Id);
nas.Sequences.Remove(remote.Id);
var job = await ContextAsync(f);
var transfers = new SyncTransfers(f.Sync);
await Assert.ThrowsAsync<RemoteGoneException>(() => transfers.SendPhotosAsync(job, nas, local.Id, remote.Id, [Sha(photos[1])]));
await Assert.ThrowsAsync<RemoteGoneException>(() => transfers.FetchPhotosAsync(job, nas, remote.Id, local.Id, [Sha(photos[0])]));
Assert.Empty(nas.Sequences);
Assert.Equal(0, nas.IntoUploads);
Assert.Equal(1, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.RemoteGoneHits);
// A refresh right after asks again but does not count a second 404 within the recheck interval.
await f.Sync.RefreshAsync(nas, new SyncPolicy(), f.Ct);
Assert.Equal(1, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.RemoteGoneHits);
}
[Test]
public async Task AGoneAnswerFromAServerKnownByItsAddressMarksNothing()
{
await using var f = Capable(await SyncFixture.CreateAsync(context));
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Comet", photos);
var remote = f.Remote.Add("Comet", [photos[0]]);
var pair = await PairAsync(f, f.Server, local.Id, remote.Id);
f.Remote.Sequences.Remove(remote.Id);
var job = await ContextAsync(f);
await Assert.ThrowsAsync<RemoteGoneException>(() => new SyncTransfers(f.Sync).SendPhotosAsync(job, f.Remote, local.Id, remote.Id, [Sha(photos[1])]));
Assert.Equal(0, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.RemoteGoneHits);
}
[Test]
public async Task AFirstUploadWhoseFirstPhotoTheServerHoldsElsewhereEndsWithEveryPhotoInTheNewSequence()
{
await using var f = Capable(await SyncFixture.CreateAsync(context));
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", photos);
// The first photo is already on the server, in another sequence.
f.Remote.Add("Elsewhere", [photos[0]]);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var link = Assert.NotNull(await f.Sync.Links.FindByLocalAsync(f.Server, local.Id, f.Ct));
var target = f.Remote.Sequences[link.RemoteId];
Assert.True(target.Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)));
}
[Test]
public async Task AnUploadOfASequenceKeptApartMakesANewServerSequenceAndLeavesTheOtherAlone()
{
await using var f = Capable(await SyncFixture.CreateAsync(context));
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", [photos[0], photos[1]]);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var first = Assert.NotNull(await f.Sync.Links.FindByLocalAsync(f.Server, local.Id, f.Ct)).RemoteId;
// Unlinked ("Stop syncing these sequences"): the pair is kept apart.
await f.Sync.UnlinkAsync(f.Server, first, local.Id, f.Ct);
var pair = (await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(c => c.RemoteId == first);
Assert.Equal(CopyState.KeptApart, pair.State);
// A new photo, then a plain Upload.
await f.AddLocalAsync(local.Id, photos[2]);
job = await f.Sync.EnqueueUploadAsync(engine, f.Server, (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
// The sequence kept apart gets nothing, and stays apart and unlinked; a new one holds every photo.
Assert.True(f.Remote.Sequences[first].Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals([Sha(photos[0]), Sha(photos[1])]));
Assert.Equal(CopyState.KeptApart, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
Assert.Null((await f.Sync.Links.GetAsync(f.Server, first, f.Ct))!.LocalId);
var link = Assert.NotNull(await f.Sync.Links.FindByLocalAsync(f.Server, local.Id, f.Ct));
Assert.NotEqual(first, link.RemoteId);
Assert.True(f.Remote.Sequences[link.RemoteId].Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)));
}
[Test]
public async Task ARetriedUploadNeverLinksAgainTheServerSequenceItsFirstAttemptMadeAndTheUserKeptApart()
{
await using var f = Capable(await SyncFixture.CreateAsync(context));
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", [photos[0], photos[1]]);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var started = Assert.NotNull(await f.Sync.Links.FindByLocalAsync(f.Server, local.Id, f.Ct)).RemoteId;
// The user keeps the pair apart while a run of the upload that made it is still to be retried.
await f.Sync.UnlinkAsync(f.Server, started, local.Id, f.Ct);
var pair = (await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(c => c.RemoteId == started);
await f.AddLocalAsync(local.Id, photos[2]);
// The retry resumes from the checkpoint naming the sequence that run made.
var row = new Job
{
Id = Guid.CreateVersion7(),
Kind = JobKind.UploadCommit,
TargetIds = [local.Id],
State = JobState.Running,
NodeId = "t#0",
Created = DateTimeOffset.UtcNow,
Started = DateTimeOffset.UtcNow,
Checkpoint = started.ToString(),
};
await f.Db.Jobs.EnqueueAsync(row, f.Ct);
var retry = new JobContext(new JobEngine(f.Db, [], new JobEngineOptions { WorkerId = "t" }), row, f.Ct);
await new SyncTransfers(f.Sync).UploadAsync(retry, f.Remote, local.Id);
// The sequence kept apart stays apart, unlinked and as it was; a new one holds every photo.
Assert.Equal(CopyState.KeptApart, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
Assert.Null((await f.Sync.Links.GetAsync(f.Server, started, f.Ct))!.LocalId);
Assert.True(f.Remote.Sequences[started].Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals([Sha(photos[0]), Sha(photos[1])]));
var link = Assert.NotNull(await f.Sync.Links.FindByLocalAsync(f.Server, local.Id, f.Ct));
Assert.NotEqual(started, link.RemoteId);
Assert.True(f.Remote.Sequences[link.RemoteId].Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)));
}
[Test]
public async Task DownloadingAgainAServerSequenceUnlinkedFromItsDownloadMakesANewLocalSequence()
{
await using var f = Capable(await SyncFixture.CreateAsync(context));
var photos = f.NewPhotos(3);
var remote = f.Remote.Add("Comet", photos);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Comet", SyncDownloadMode.Originals, f.Archive.Archive.Id);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var local = Assert.NotNull(Assert.NotNull(await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct)).LocalId);
await WaitIdleAsync(f, local);
// Here: one photo deleted, the title edited; then the pair is unlinked (kept apart).
var sequence = (await f.Db.Sequences.GetAsync(local, f.Ct))!;
var gone = (await f.Db.Frames.ListBySequenceAsync(local, cancellationToken: f.Ct))[0];
var delete = await f.Archive.Service.FrameTrash.PlanDeleteAsync(local, [gone.Id], sequence.FramesVersion, f.Ct);
await f.Archive.Service.FrameTrash.DeleteAsync(delete, cancellationToken: f.Ct);
await f.Db.Sequences.EditAsync(local, new SequenceEdit { Title = "Mine" }, null, f.Ct);
await f.Sync.UnlinkAsync(f.Server, remote.Id, local, f.Ct);
var pair = (await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(c => c.RemoteId == remote.Id && c.LocalId == local);
Assert.Equal(CopyState.KeptApart, pair.State);
// Download originals again: a separate local copy.
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Comet", SyncDownloadMode.Originals, f.Archive.Archive.Id);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
// The sequence kept apart is as the user left it: its photos, its deletion, its title, its pair.
var after = (await f.Db.Sequences.GetAsync(local, f.Ct))!;
Assert.Equal("Mine", after.Title);
Assert.Count(2, await f.Db.Frames.ListBySequenceAsync(local, cancellationToken: f.Ct));
Assert.Contains(gone.Sha256, (await f.Db.FrameRemovals.ListBySequenceAsync(local, f.Ct)).Select(r => r.Sha256));
Assert.Equal(CopyState.KeptApart, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
var copy = Assert.NotNull(Assert.NotNull(await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct)).LocalId);
Assert.NotEqual(local, copy);
Assert.True((await LocalHashesAsync(f, copy)).SetEquals(photos.Select(Sha)));
}
[Test]
public async Task DownloadingAgainAServerSequenceWhoseLinkA404ForgotRefillsAndLinksItsOwnLocalCopy()
{
await using var f = Capable(await SyncFixture.CreateAsync(context));
var photos = f.NewPhotos(3);
var remote = f.Remote.Add("Comet", photos);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Comet", SyncDownloadMode.Originals, f.Archive.Archive.Id);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var local = Assert.NotNull(Assert.NotNull(await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct)).LocalId);
await WaitIdleAsync(f, local);
// A refresh's 404 forgot the link (the archive was briefly offline); listed again, the pair is asked about again.
await f.Sync.Links.DeleteAsync(f.Server, remote.Id, f.Ct);
var pair = (await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(c => c.RemoteId == remote.Id && c.LocalId == local);
Assert.True(await f.Sync.Copies.SetStateAsync(pair.Id, pair.State, CopyState.Candidate, null, f.Ct));
// Download originals of the server-only sequence: it goes back into its own local copy, linked again.
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Comet", SyncDownloadMode.Originals, f.Archive.Archive.Id);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.Equal(local, Assert.NotNull(await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct)).LocalId);
Assert.Count(3, await f.Db.Frames.ListBySequenceAsync(local, cancellationToken: f.Ct));
Assert.True((await LocalHashesAsync(f, local)).SetEquals(photos.Select(Sha)));
var keys = await f.Db.Sequences.ListSessionKeysAsync(f.Archive.Archive.Id, SyncService.SessionKey(remote.Id) + "%", f.Ct);
Assert.True(keys.All(k => k.SequenceId == local), "no second local sequence holds a key of the server sequence");
Assert.Equal(1, (await f.Db.Sequences.ListAsync(new SequenceQuery(), f.Ct)).Items.Count);
}
[Test]
public async Task DownloadingAgainIntoTheOwnLocalCopyOfAPairAskedAgainKeepsItsDeletionsAndDetails()
{
await using var f = Capable(await SyncFixture.CreateAsync(context));
var photos = f.NewPhotos(3);
var remote = f.Remote.Add("Comet", photos);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Comet", SyncDownloadMode.Originals, f.Archive.Archive.Id);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var local = Assert.NotNull(Assert.NotNull(await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct)).LocalId);
await WaitIdleAsync(f, local);
// While linked: one photo deleted here (the server keeps it, owner Q6) and the title edited; then unlinked and
// asked about again (Ask again, or a Keep apart undone).
var sequence = (await f.Db.Sequences.GetAsync(local, f.Ct))!;
var gone = (await f.Db.Frames.ListBySequenceAsync(local, cancellationToken: f.Ct))[0];
var delete = await f.Archive.Service.FrameTrash.PlanDeleteAsync(local, [gone.Id], sequence.FramesVersion, f.Ct);
await f.Archive.Service.FrameTrash.DeleteAsync(delete, cancellationToken: f.Ct);
await f.Db.Sequences.EditAsync(local, new SequenceEdit { Title = "Mine" }, null, f.Ct);
await f.Sync.UnlinkAsync(f.Server, remote.Id, local, f.Ct);
var pair = (await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(c => c.RemoteId == remote.Id && c.LocalId == local);
Assert.True(await f.Sync.Copies.SetStateAsync(pair.Id, CopyState.KeptApart, CopyState.Candidate, null, f.Ct));
// Download originals of the server sequence: it refills its own local copy, as a linked copy.
job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Comet", SyncDownloadMode.Originals, f.Archive.Archive.Id);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
// The photo deleted here stays deleted, and the title is not overwritten by the server's.
Assert.Equal(local, Assert.NotNull(await f.Sync.Links.GetAsync(f.Server, remote.Id, f.Ct)).LocalId);
Assert.Count(2, await f.Db.Frames.ListBySequenceAsync(local, cancellationToken: f.Ct));
Assert.Contains(gone.Sha256, (await f.Db.FrameRemovals.ListBySequenceAsync(local, f.Ct)).Select(r => r.Sha256));
Assert.Equal("Mine", (await f.Db.Sequences.GetAsync(local, f.Ct))!.Title);
Assert.Equal("Comet", remote.Title);
Assert.Equal(1, (await f.Db.Sequences.ListAsync(new SequenceQuery(), f.Ct)).Items.Count);
}
[Test]
public async Task AnUploadOfASequenceAskedAboutAgainMakesANewServerSequenceAndLeavesTheOtherAlone()
{
await using var f = Capable(await SyncFixture.CreateAsync(context));
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", [photos[0], photos[1]]);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var first = Assert.NotNull(await f.Sync.Links.FindByLocalAsync(f.Server, local.Id, f.Ct)).RemoteId;
// Unlinked, then asked about again: the pair is a Candidate and nothing links the two.
await f.Sync.UnlinkAsync(f.Server, first, local.Id, f.Ct);
var pair = (await f.Sync.Copies.ListByServerAsync(f.Server, f.Ct)).Single(c => c.RemoteId == first);
Assert.True(await f.Sync.Copies.SetStateAsync(pair.Id, CopyState.KeptApart, CopyState.Candidate, null, f.Ct));
// Renamed on the server (the web, another desktop); a new photo here.
var server = f.Remote.Sequences[first];
server.Title = "Comet NAS";
server.SettingsVersion++;
server.UpdatedAt = DateTimeOffset.UtcNow;
await f.AddLocalAsync(local.Id, photos[2]);
job = await f.Sync.EnqueueUploadAsync(engine, f.Server, (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
// The other is not joined through the key, linked unasked or overwritten: a new server sequence holds every photo.
Assert.Equal("Comet NAS", server.Title);
Assert.True(server.Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals([Sha(photos[0]), Sha(photos[1])]));
Assert.Equal(CopyState.Candidate, (await f.Sync.Copies.GetAsync(pair.Id, f.Ct))!.State);
Assert.Null((await f.Sync.Links.GetAsync(f.Server, first, f.Ct))!.LocalId);
var link = Assert.NotNull(await f.Sync.Links.FindByLocalAsync(f.Server, local.Id, f.Ct));
Assert.NotEqual(first, link.RemoteId);
Assert.True(f.Remote.Sequences[link.RemoteId].Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)));
}
[Test]
public async Task AnUploadThatJoinsAServerSequenceItKnowsAlreadyComparesItsDetailsInsteadOfOverwritingThem()
{
await using var f = Capable(await SyncFixture.CreateAsync(context));
var photos = f.NewPhotos(3);
var local = await f.LocalAsync("Comet", [photos[0], photos[1]]);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var first = Assert.NotNull(await f.Sync.Links.FindByLocalAsync(f.Server, local.Id, f.Ct)).RemoteId;
// The link and its pair were forgotten (a sweep of 404s: the server served another archive for a while); the
// server sequence is listed again, as an observation.
var observed = Assert.NotNull(await f.Sync.Links.GetAsync(f.Server, first, f.Ct)) with { LocalId = null };
await f.Sync.Links.UpsertAsync(observed, f.Ct);
await using (var session = await f.Db.Database.OpenAsync(f.Ct))
await session.ExecuteAsync("DELETE FROM found_copies WHERE remote_id = $1", [first], f.Ct);
var server = f.Remote.Sequences[first];
server.Title = "Comet NAS";
server.SettingsVersion++;
server.UpdatedAt = DateTimeOffset.UtcNow;
await f.AddLocalAsync(local.Id, photos[2]);
job = await f.Sync.EnqueueUploadAsync(engine, f.Server, (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
// Joined through this sequence's key, not made by this upload: the server's title is not overwritten unseen.
Assert.Equal(first, Assert.NotNull(await f.Sync.Links.FindByLocalAsync(f.Server, local.Id, f.Ct)).RemoteId);
Assert.True(server.Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)));
Assert.Equal("Comet NAS", server.Title);
Assert.Equal("Comet", (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Title);
}
[Test]
public async Task ARetriedUploadThatJoinedAServerSequenceItKnewStillComparesItsDetails()
{
await using var f = Capable(await SyncFixture.CreateAsync(context));
var photos = f.NewPhotos(4);
var local = await f.LocalAsync("Comet", [photos[0], photos[1]]);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
var first = Assert.NotNull(await f.Sync.Links.FindByLocalAsync(f.Server, local.Id, f.Ct)).RemoteId;
// The link and its pair were forgotten; the server sequence is listed again and renamed there.
var observed = Assert.NotNull(await f.Sync.Links.GetAsync(f.Server, first, f.Ct)) with { LocalId = null };
await f.Sync.Links.UpsertAsync(observed, f.Ct);
await using (var session = await f.Db.Database.OpenAsync(f.Ct))
await session.ExecuteAsync("DELETE FROM found_copies WHERE remote_id = $1", [first], f.Ct);
var server = f.Remote.Sequences[first];
server.Title = "Comet NAS";
server.SettingsVersion++;
server.UpdatedAt = DateTimeOffset.UtcNow;
await f.AddLocalAsync(local.Id, photos[2]);
await f.AddLocalAsync(local.Id, photos[3]);
// One original unreadable for the first attempt: it fails after joining the known sequence.
var damaged = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Single(fr => fr.Sha256 == Sha(photos[3]));
var path = f.Archive.Paths.FromRelative(damaged.RelativePath);
var bytes = await File.ReadAllBytesAsync(path, f.Ct);
var broken = bytes.ToArray();
broken[^3] ^= 0xFF;
await File.WriteAllBytesAsync(path, broken, f.Ct);
var row = new Job
{
Id = Guid.CreateVersion7(),
Kind = JobKind.UploadCommit,
TargetIds = [local.Id],
State = JobState.Running,
NodeId = "t#0",
Created = DateTimeOffset.UtcNow,
Started = DateTimeOffset.UtcNow,
};
await f.Db.Jobs.EnqueueAsync(row, f.Ct);
var jobs = new JobEngine(f.Db, [], new JobEngineOptions { WorkerId = "t" });
var attempt = new JobContext(jobs, row, f.Ct);
await Assert.ThrowsAsync<InvalidDataException>(() => new SyncTransfers(f.Sync).UploadAsync(attempt, f.Remote, local.Id));
Assert.Equal(first, Assert.NotNull(await f.Sync.Links.FindByLocalAsync(f.Server, local.Id, f.Ct)).RemoteId);
// Retried from that attempt's checkpoint, the original readable again.
await File.WriteAllBytesAsync(path, bytes, f.Ct);
var retry = new JobContext(jobs, row with { Checkpoint = attempt.Checkpoint }, f.Ct);
await new SyncTransfers(f.Sync).UploadAsync(retry, f.Remote, local.Id);
// Still joined, not made by this upload: the server's title is not overwritten unseen.
Assert.True(server.Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)));
Assert.Equal("Comet NAS", server.Title);
Assert.Equal("Comet", (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Title);
}
[Test]
public async Task AnAutomaticUploadAfterTheServerCopyIsGoneFailsAndCreatesNothing()
{
await using var f = Capable(await SyncFixture.CreateAsync(context));
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Comet", photos);
var remote = f.Remote.Add("Comet", [photos[0]]);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
var pair = await PairAsync(f, f.Server, local.Id, remote.Id);
Assert.True(await f.Sync.Copies.SetStateAsync(pair.Id, null, CopyState.Integrated, new CopyAnswer(null, null, null, true, DateTimeOffset.UtcNow), f.Ct));
f.Remote.Sequences.Remove(remote.Id);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueAutomaticUploadAsync(engine, f.Server, local, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Failed);
// Never a new server sequence: the automatic photos move only between the two the pair joins.
Assert.Empty(f.Remote.Sequences);
}
[Test]
public async Task AnAutomaticUploadOfAPairNoLongerIntegratedSendsNothing()
{
await using var f = Capable(await SyncFixture.CreateAsync(context));
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Comet", photos);
var remote = f.Remote.Add("Comet", [photos[0]]);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
await PairAsync(f, f.Server, local.Id, remote.Id, CopyState.Linked);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueAutomaticUploadAsync(engine, f.Server, local, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Count(1, remote.Frames);
}
[Test]
public async Task AnAutomaticDownloadWithoutTheLocalCopyFailsAndCreatesNoSequence()
{
await using var f = Capable(await SyncFixture.CreateAsync(context));
var remote = f.Remote.Add("Comet", f.NewPhotos(2));
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueAutomaticDownloadAsync(engine, f.Server, remote.Id, "Comet", f.Archive.Archive.Id, f.Ct);
await f.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(0, await f.Db.Sequences.CountAsync(new() { ArchiveId = f.Archive.Archive.Id }, f.Ct));
}
private static Task<FoundCopy> PairAsync(SyncFixture f, string server, Guid localId, Guid remoteId, CopyState state = CopyState.Candidate) =>
f.Sync.Copies.UpsertFoundAsync(new FoundCopy
{
Id = Guid.NewGuid(),
Kind = CopyKind.LocalServer,
LocalId = localId,
Server = server,
RemoteId = remoteId,
Evidence = CopyEvidence.Hashes,
Fingerprint = "f",
State = state,
}, f.Ct);
[Test]
public async Task ALinkedUploadFilesIntoItsTargetWhenTheServerCan()
{
await using var f = Capable(await SyncFixture.CreateAsync(context));
var photos = f.NewPhotos(2);
var local = await f.LocalAsync("Comet", photos);
var remote = f.Remote.Add("Comet", [photos[0]]);
// The server holds the second photo in another sequence: filed into the target all the same.
f.Remote.Add("Elsewhere", [photos[1]]);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
Assert.True(remote.Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)));
Assert.Equal((1, 1), (f.Remote.IntoUploads, f.Remote.CopiedInto));
}
[Test]
public async Task ALinkedUploadTheServerRefusesIntoItsTargetStillFilesEveryOriginal()
{
await using var f = Capable(await SyncFixture.CreateAsync(context));
// An account with Read and Upload but not Edit, whose local sequence is linked to another account's: 403 for "into".
f.Remote.ForbidInto = true;
var photos = f.NewPhotos(4);
var local = await f.LocalAsync("Comet", photos);
var remote = f.Remote.Add("Comet", [photos[0]]);
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
var engine = await f.StartEngineAsync();
var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local);
await f.Archive.WaitForAsync(job.Id, JobState.Done);
// The refusal falls back to the upload as before the server could file into a target: every original is there.
Assert.True(remote.Frames.Select(fr => fr.Sha256).ToHashSet().SetEquals(photos.Select(Sha)));
Assert.True(f.Remote.RefusedInto >= 1);
Assert.Equal(0, f.Remote.IntoUploads);
}
}