Owner review 6 asks Chronos to recognise the same time-lapse held under different ids on the server and on this PC, compare the copies, and keep them apart or integrate and sync them. This lays the shared foundation the later parcels build on, without any UI or detection yet. - Migration 0012 adds found_copies (local-server and local-local pairs with evidence, scores, answer, policy and the 404 record) and copy_probes; every existing link with a local sequence becomes a Linked pair, so a server deletion and the per-link policy have a place. - FoundCopyStore keeps the pairs: upserts that never touch the answer, a compare-and-set of the state with an Integrating backstop (a failed or cancelled job reads as the previous state), link promotion and demotion, the once-per-refresh 404 count, the merge's links stage (session keys, links reset, pairs moved, refused when both are linked on one server), the local overlap queries and the probes. SyncLinkStore.RekeyAsync moves pairs and probes with the links in the same transaction. - Contracts for detection, review, plans, the Integrate and MergeSequences jobs (kinds 25/26, High level, new detail steps) and the Upload all rule that keeps suspected, kept-apart, deleted, busy and server-born sequences here. - Wire: POST photos/lookup, uploads into a target (IntoTarget/Copied, trailing and optional), frames/delete on the client, capabilities photo-lookup and upload-into, cached per remote. - The trash plans and deletes accept the job that holds the sequence (its own hold is not Busy) and a frames version checked under the lease; a Background job never interrupts a running one. - Settings for linking, the photo policy and projects; the en and it keys of design section 9. Refs #73 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
316 lines
17 KiB
C#
316 lines
17 KiB
C#
using Chronos.Core.IO;
|
|
using Chronos.Core.Project;
|
|
using Chronos.Data.Copies;
|
|
using Chronos.Data.Homes;
|
|
using Chronos.Data.Projects;
|
|
using Chronos.Data.Sync;
|
|
using Chronos.Data.Tests.Infrastructure;
|
|
using Chronos.Data.Tests.Repositories;
|
|
using Chronos.Data.Workspaces;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Data.Tests.Homes;
|
|
|
|
/// <summary>A server's home moves from its address to its id when it first announces one (owner review 5, #66).</summary>
|
|
[Category("homes")]
|
|
public sealed class HomeRekeyTests(TestContext context)
|
|
{
|
|
private static readonly Guid Id = Guid.Parse("0192a4c1-7b3e-7cc0-8a51-2f9d0e6b1a01");
|
|
private static string IdFolder => Id.ToString("N");
|
|
|
|
private (string Data, WorkspaceHomes Homes, string Cache) Make()
|
|
{
|
|
var data = context.CreateTempDirectory("data");
|
|
return (data, new WorkspaceHomes(Path.Combine(data, "workspace")), Path.Combine(data, "cache"));
|
|
}
|
|
|
|
private static HomeLayoutMigration Migration(string data, WorkspaceHomes homes, string cache) =>
|
|
new(new HomeLayoutOptions { DataDirectory = data, WorkspaceRoot = homes.Root, CacheRoot = cache });
|
|
|
|
private static string AddressHome(string root, string key) => Path.Combine(root, "servers", HomeFolders.ServerFolder(key));
|
|
|
|
private static ProjectBundle Bundle(string home, Guid id) => ProjectBundle.Load(Path.Combine(home, id.ToString("N"), WorkspaceProject.BundleFile));
|
|
|
|
[Test]
|
|
public async Task TheHomeOfTheAddressBecomesTheHomeOfTheId()
|
|
{
|
|
var (data, homes, cache) = Make();
|
|
var home = HomeFixtures.Server(HomeFixtures.Nas);
|
|
var store = homes.StoreFor(home);
|
|
var (a, b) = (HomeFixtures.Project(store.Root, home), HomeFixtures.Project(store.Root, HomeFixtures.Server(HomeFixtures.Nas)));
|
|
HomeFixtures.TrashEntry(store.Root, a, home);
|
|
var before = HomeFixtures.Manifest(homes.Root);
|
|
var (sorted, old) = (Guid.CreateVersion7(), Guid.CreateVersion7());
|
|
var cached = Path.Combine(AddressHome(cache, HomeFixtures.Nas), "proxies", sorted.ToString("N"));
|
|
Directory.CreateDirectory(cached);
|
|
File.WriteAllText(Path.Combine(cached, "remote.json"), "{}");
|
|
HomeFixtures.LegacyProxies(cache, HomeFixtures.Nas, old);
|
|
|
|
var result = await Migration(data, homes, cache).RekeyAsync(Id, [HomeFixtures.Nas], context.Cancellation);
|
|
Assert.Equal(new HomeRekeyResult(2, 0, 0, 2, 0, 0, 0), result);
|
|
var idHome = Path.Combine(homes.Root, "servers", IdFolder);
|
|
Assert.False(Directory.Exists(AddressHome(homes.Root, HomeFixtures.Nas)));
|
|
var idKey = ServerIdentity.IdKey(Id);
|
|
foreach (var id in new[] { a, b })
|
|
{
|
|
var bundle = Bundle(idHome, id);
|
|
Assert.Equal(idKey, bundle.Workspace!.Home!.Server);
|
|
Assert.Equal(ServerProjectHome.ArchiveIdOf(idKey), bundle.Archive.Id);
|
|
}
|
|
var record = Assert.NotNull(ServerHomeRecord.Read(idHome));
|
|
Assert.Equal((idKey, (Guid?)Id, HomeFixtures.Nas), (record.Key, record.Id, record.Address));
|
|
Assert.Count(1, Directory.EnumerateDirectories(Path.Combine(idHome, WorkspaceTrash.FolderName)));
|
|
// Only the working states were rewritten: every other file is as it was.
|
|
var after = HomeFixtures.Manifest(homes.Root);
|
|
Assert.SequenceEqual(before.Keys, after.Keys);
|
|
Assert.SequenceEqual(before.Where(f => !f.Key.EndsWith(WorkspaceProject.BundleFile, StringComparison.Ordinal)),
|
|
after.Where(f => !f.Key.EndsWith(WorkspaceProject.BundleFile, StringComparison.Ordinal)));
|
|
|
|
Assert.FileExists(Path.Combine(cache, "servers", IdFolder, "proxies", sorted.ToString("N"), "remote.json"));
|
|
Assert.FileExists(Path.Combine(cache, "servers", IdFolder, "proxies", old.ToString("N"), "remote.json"));
|
|
Assert.False(Directory.Exists(AddressHome(cache, HomeFixtures.Nas)));
|
|
// The store now finds the id home by either key.
|
|
var identity = new ServerIdentity([new KnownServerKey(Id, [HomeFixtures.Nas])]);
|
|
Assert.Equal(idHome, new WorkspaceHomes(homes.Root, identity).ServerRoot(HomeFixtures.Nas));
|
|
|
|
// Again: nothing left to do.
|
|
Assert.Equal(new HomeRekeyResult(0, 0, 0, 0, 0, 0, 0), await Migration(data, homes, cache).RekeyAsync(Id, [HomeFixtures.Nas], context.Cancellation));
|
|
}
|
|
|
|
[Test]
|
|
public async Task TwoAddressesOfOneServerMergeAndTheOlderCopyGoesToTheTrash()
|
|
{
|
|
var (data, homes, cache) = Make();
|
|
var byName = HomeFixtures.Server(HomeFixtures.Nas);
|
|
var byIp = HomeFixtures.Server(HomeFixtures.NasByIp);
|
|
var first = homes.StoreFor(byName).Root;
|
|
var second = homes.StoreFor(byIp).Root;
|
|
var both = Guid.CreateVersion7();
|
|
HomeFixtures.Project(first, byName, "older", DateTimeOffset.UtcNow.AddDays(-1), both);
|
|
HomeFixtures.Project(second, byIp, "newer", DateTimeOffset.UtcNow, both);
|
|
var onlyFirst = HomeFixtures.Project(first, byName);
|
|
var onlySecond = HomeFixtures.Project(second, byIp);
|
|
HomeFixtures.TrashEntry(first, onlyFirst, byName);
|
|
HomeFixtures.TrashEntry(second, onlyFirst, byIp);
|
|
var files = Directory.EnumerateFiles(homes.Root, "*", SearchOption.AllDirectories).Count(IsProjectFile);
|
|
// Each address had a proxy cache too.
|
|
Directory.CreateDirectory(Path.Combine(AddressHome(cache, HomeFixtures.Nas), "proxies", "a"));
|
|
Directory.CreateDirectory(Path.Combine(AddressHome(cache, HomeFixtures.NasByIp), "proxies", "a"));
|
|
Directory.CreateDirectory(Path.Combine(AddressHome(cache, HomeFixtures.NasByIp), "proxies", "b"));
|
|
|
|
var result = await Migration(data, homes, cache).RekeyAsync(Id, [HomeFixtures.Nas, HomeFixtures.NasByIp], context.Cancellation);
|
|
Assert.Equal((2, 2, 1, 3), (result.HomesMoved, result.ProjectsMoved, result.ProjectsTrashed, result.BundlesRewritten));
|
|
var idHome = Path.Combine(homes.Root, "servers", IdFolder);
|
|
Assert.Equal("newer", Bundle(idHome, both).Workspace!.Name);
|
|
foreach (var id in new[] { onlyFirst, onlySecond }) Assert.FileExists(Path.Combine(idHome, id.ToString("N"), WorkspaceProject.BundleFile));
|
|
var trash = Directory.EnumerateDirectories(Path.Combine(idHome, WorkspaceTrash.FolderName)).ToList();
|
|
Assert.Equal(3, trash.Count);
|
|
Assert.Contains(trash, t => ProjectBundle.Load(Path.Combine(t, WorkspaceProject.BundleFile)).Workspace!.Name == "older");
|
|
// The second address's home held only its manifest and record afterwards: it went. Nothing else was deleted.
|
|
Assert.False(Directory.Exists(second));
|
|
Assert.Equal(files, Directory.EnumerateFiles(homes.Root, "*", SearchOption.AllDirectories).Count(IsProjectFile));
|
|
Assert.Count(1, Directory.EnumerateDirectories(Path.Combine(homes.Root, "servers")));
|
|
Assert.True(Directory.Exists(Path.Combine(cache, "servers", IdFolder, "proxies", "b")));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AProjectInUseStaysAndKeepsItsAliasWhichStillMatches()
|
|
{
|
|
var (data, homes, cache) = Make();
|
|
var home = HomeFixtures.Server(HomeFixtures.Nas);
|
|
var root = homes.StoreFor(home).Root;
|
|
var (held, free) = (HomeFixtures.Project(root, home), HomeFixtures.Project(root, HomeFixtures.Server(HomeFixtures.Nas)));
|
|
HomeRekeyResult result;
|
|
using (HomeFixtures.Hold(Path.Combine(root, held.ToString("N"))))
|
|
{
|
|
// A project in use keeps the whole home where it is until a later re-key; the id home is made beside it.
|
|
_ = new WorkspaceStore(Path.Combine(homes.Root, "servers", IdFolder));
|
|
result = await Migration(data, homes, cache).RekeyAsync(Id, [HomeFixtures.Nas], context.Cancellation);
|
|
}
|
|
Assert.Equal((0, 1, 1), (result.HomesMoved, result.ProjectsMoved, result.BundlesRewritten));
|
|
Assert.FileExists(Path.Combine(root, held.ToString("N"), WorkspaceProject.BundleFile));
|
|
Assert.Equal(HomeFixtures.Nas, Bundle(root, held).Workspace!.Home!.Server);
|
|
var idHome = Path.Combine(homes.Root, "servers", IdFolder);
|
|
Assert.Equal(ServerIdentity.IdKey(Id), Bundle(idHome, free).Workspace!.Home!.Server);
|
|
Assert.True(ServerProjectHome.IsArchiveIdOf(Bundle(root, held).Archive.Id, [ServerIdentity.IdKey(Id), HomeFixtures.Nas]));
|
|
Assert.NotNull(homes.Find(held));
|
|
|
|
result = await Migration(data, homes, cache).RekeyAsync(Id, [HomeFixtures.Nas], context.Cancellation);
|
|
Assert.Equal((1, 1), (result.ProjectsMoved, result.BundlesRewritten));
|
|
Assert.False(Directory.Exists(root));
|
|
Assert.Equal(ServerIdentity.IdKey(Id), Bundle(idHome, held).Workspace!.Home!.Server);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AWorkingStateInUseInTheIdHomeIsLeftNamingTheAlias()
|
|
{
|
|
var (data, homes, cache) = Make();
|
|
var home = HomeFixtures.Server(HomeFixtures.Nas);
|
|
var root = homes.StoreFor(home).Root;
|
|
var id = HomeFixtures.Project(root, home);
|
|
// Renamed whole first (nothing held), then held while its working state would be rewritten.
|
|
var idHome = Path.Combine(homes.Root, "servers", IdFolder);
|
|
Directory.Move(root, idHome);
|
|
HomeRekeyResult result;
|
|
using (HomeFixtures.Hold(Path.Combine(idHome, id.ToString("N"))))
|
|
result = await Migration(data, homes, cache).RekeyAsync(Id, [HomeFixtures.Nas], context.Cancellation);
|
|
Assert.Equal((0, 1), (result.BundlesRewritten, result.BundlesInUse));
|
|
Assert.Equal(HomeFixtures.Nas, Bundle(idHome, id).Workspace!.Home!.Server);
|
|
Assert.Equal(ServerIdentity.IdKey(Id), Assert.NotNull(ServerHomeRecord.Read(idHome)).Key);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnIdKeyIsNeverAnAliasAndAnEmptyIdIsRefused()
|
|
{
|
|
var (data, homes, cache) = Make();
|
|
await Assert.ThrowsAsync<ArgumentException>(() => Migration(data, homes, cache).RekeyAsync(Guid.Empty, [HomeFixtures.Nas], context.Cancellation));
|
|
var other = ServerIdentity.IdKey(Guid.CreateVersion7());
|
|
var otherHome = homes.StoreFor(HomeFixtures.Server(other)).Root;
|
|
Assert.Equal(new HomeRekeyResult(0, 0, 0, 0, 0, 0, 0), await Migration(data, homes, cache).RekeyAsync(Id, [other], context.Cancellation));
|
|
Assert.True(Directory.Exists(otherHome));
|
|
}
|
|
|
|
private static bool IsProjectFile(string path) => Path.GetFileName(path) is not (WorkspaceStore.ManifestFile or HomeFolders.ServerFile);
|
|
}
|
|
|
|
/// <summary>A re-key moves the server's links in the index too.</summary>
|
|
[Category("homes")]
|
|
public abstract class HomeRekeyLinkTests(TestContext context) : RepositoryTest(context)
|
|
{
|
|
private static readonly Guid Id = Guid.Parse("0192a4c1-7b3e-7cc0-8a51-2f9d0e6b1a01");
|
|
|
|
[Test]
|
|
public async Task TheLinksOfEveryAliasMoveToTheId()
|
|
{
|
|
var db = await OpenChronosAsync();
|
|
var store = new SyncLinkStore(db);
|
|
foreach (var key in new[] { HomeFixtures.Nas, HomeFixtures.NasByIp, HomeFixtures.Other })
|
|
{
|
|
await store.UpsertAsync(new SyncLink
|
|
{
|
|
Server = key,
|
|
RemoteId = Guid.CreateVersion7(),
|
|
Title = "Harbour",
|
|
RemoteUpdatedAt = TestData.Now,
|
|
SyncedAt = TestData.Now,
|
|
}, Ct);
|
|
}
|
|
var data = Context.CreateTempDirectory("data");
|
|
var migration = new HomeLayoutMigration(new HomeLayoutOptions { DataDirectory = data, Database = db });
|
|
var result = await migration.RekeyAsync(Id, [HomeFixtures.Nas, HomeFixtures.NasByIp], Ct);
|
|
Assert.Equal((2, 0), (result.LinksMoved, result.LinksDropped));
|
|
Assert.Count(2, await store.ListAsync(ServerIdentity.IdKey(Id), Ct));
|
|
Assert.Count(1, await store.ListAsync(HomeFixtures.Other, Ct));
|
|
Assert.Empty(await store.ListAsync(HomeFixtures.Nas, Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheMatchesAndProbesOfAnAliasMoveWithItsLinks()
|
|
{
|
|
var db = await OpenChronosAsync();
|
|
var (linked, found) = await TwoSequencesAsync(db);
|
|
var store = new SyncLinkStore(db);
|
|
var copies = new FoundCopyStore(db);
|
|
var (r1, r2) = (Guid.CreateVersion7(), Guid.CreateVersion7());
|
|
await store.UpsertAsync(Link(HomeFixtures.Nas, r1, linked, TestData.Now), Ct);
|
|
await copies.EnsureLinkedAsync(HomeFixtures.Nas, r1, linked, Ct);
|
|
await copies.UpsertFoundAsync(Candidate(HomeFixtures.Nas, r2, found), Ct);
|
|
await copies.SetProbeAsync(new CopyProbe(found, HomeFixtures.Nas, 3, 3, TestData.Now), Ct);
|
|
|
|
var migration = new HomeLayoutMigration(new HomeLayoutOptions { DataDirectory = Context.CreateTempDirectory("data"), Database = db });
|
|
var result = await migration.RekeyAsync(Id, [HomeFixtures.Nas, HomeFixtures.NasByIp], Ct);
|
|
|
|
Assert.Equal((1, 0), (result.LinksMoved, result.LinksDropped));
|
|
var key = ServerIdentity.IdKey(Id);
|
|
Assert.Empty(await copies.ListByServerAsync(HomeFixtures.Nas, Ct));
|
|
var moved = await copies.ListByServerAsync(key, Ct);
|
|
Assert.Count(2, moved);
|
|
Assert.Equal(CopyState.Linked, moved.Single(c => c.RemoteId == r1).State);
|
|
Assert.Equal(CopyState.Candidate, moved.Single(c => c.RemoteId == r2).State);
|
|
Assert.Null(await copies.GetProbeAsync(found, HomeFixtures.Nas, Ct));
|
|
Assert.Equal(3, (await copies.GetProbeAsync(found, key, Ct))!.FramesVersion);
|
|
}
|
|
|
|
[Test]
|
|
public async Task OnAClashThePairOfTheSurvivingLinkWinsElseTheHigherState()
|
|
{
|
|
var db = await OpenChronosAsync();
|
|
var (linked, found) = await TwoSequencesAsync(db);
|
|
var key = ServerIdentity.IdKey(Id);
|
|
var store = new SyncLinkStore(db);
|
|
var copies = new FoundCopyStore(db);
|
|
var (r1, r2) = (Guid.CreateVersion7(), Guid.CreateVersion7());
|
|
// The same link under both keys: the one compared last (the alias's) survives, and so does its pair, though the pair
|
|
// under the id is further on.
|
|
await store.UpsertAsync(Link(HomeFixtures.Nas, r1, linked, TestData.Now), Ct);
|
|
await store.UpsertAsync(Link(key, r1, linked, TestData.Now.AddHours(-1)), Ct);
|
|
var aliasPair = await copies.EnsureLinkedAsync(HomeFixtures.Nas, r1, linked, Ct);
|
|
var idPair = await copies.EnsureLinkedAsync(key, r1, linked, Ct);
|
|
Assert.True(await copies.SetStateAsync(idPair.Id, CopyState.Linked, CopyState.Integrated, new CopyAnswer(null, null, null, true, TestData.Now), Ct));
|
|
// A match without a link under both keys: the answered one wins over the newer candidate.
|
|
var keptApart = await copies.UpsertFoundAsync(Candidate(HomeFixtures.Nas, r2, found), Ct);
|
|
Assert.True(await copies.SetStateAsync(keptApart.Id, CopyState.Candidate, CopyState.KeptApart, new CopyAnswer("f", 4, null, null, TestData.Now), Ct));
|
|
var candidate = await copies.UpsertFoundAsync(Candidate(key, r2, found), Ct);
|
|
await copies.SetProbeAsync(new CopyProbe(found, HomeFixtures.Nas, 1, 1, TestData.Now), Ct);
|
|
await copies.SetProbeAsync(new CopyProbe(found, key, 2, 2, TestData.Now), Ct);
|
|
|
|
var (moved, dropped) = await store.RekeyAsync(HomeFixtures.Nas, key, Ct);
|
|
|
|
Assert.Equal((1, 1), (moved, dropped));
|
|
var rows = await copies.ListByServerAsync(key, Ct);
|
|
Assert.Count(2, rows);
|
|
var pair = rows.Single(c => c.RemoteId == r1);
|
|
Assert.Equal(aliasPair.Id, pair.Id);
|
|
Assert.Equal(CopyState.Linked, pair.State);
|
|
var match = rows.Single(c => c.RemoteId == r2);
|
|
Assert.Equal(keptApart.Id, match.Id);
|
|
Assert.NotEqual(candidate.Id, match.Id);
|
|
Assert.Equal(CopyState.KeptApart, match.State);
|
|
Assert.Empty(await copies.ListByServerAsync(HomeFixtures.Nas, Ct));
|
|
Assert.Equal(2, (await copies.GetProbeAsync(found, key, Ct))!.FramesVersion);
|
|
Assert.Null(await copies.GetProbeAsync(found, HomeFixtures.Nas, Ct));
|
|
}
|
|
|
|
private async Task<(Guid Linked, Guid Found)> TwoSequencesAsync(ChronosDatabase db)
|
|
{
|
|
var archive = TestData.Archive();
|
|
await db.Archives.UpsertAsync(archive, Ct);
|
|
var linked = TestData.Sequence(archive.Id);
|
|
var found = TestData.Sequence(archive.Id);
|
|
await db.Sequences.UpsertAsync(linked, Ct);
|
|
await db.Sequences.UpsertAsync(found, Ct);
|
|
return (linked.Id, found.Id);
|
|
}
|
|
|
|
private static SyncLink Link(string server, Guid remoteId, Guid localId, DateTimeOffset synced) => new()
|
|
{
|
|
Server = server,
|
|
RemoteId = remoteId,
|
|
LocalId = localId,
|
|
Title = "Harbour",
|
|
RemoteUpdatedAt = TestData.Now,
|
|
SyncedAt = synced,
|
|
};
|
|
|
|
private static FoundCopy Candidate(string server, Guid remoteId, Guid localId) => new()
|
|
{
|
|
Id = Guid.NewGuid(),
|
|
Kind = CopyKind.LocalServer,
|
|
LocalId = localId,
|
|
Server = server,
|
|
RemoteId = remoteId,
|
|
Evidence = CopyEvidence.Hashes,
|
|
Fingerprint = "f",
|
|
};
|
|
}
|
|
|
|
public sealed class SqliteHomeRekeyLinkTests(TestContext context) : HomeRekeyLinkTests(context)
|
|
{
|
|
protected override Backend Backend => Backend.SqliteMemory;
|
|
}
|
|
|
|
public sealed class PostgresHomeRekeyLinkTests(TestContext context) : HomeRekeyLinkTests(context)
|
|
{
|
|
protected override Backend Backend => Backend.Postgres;
|
|
}
|