using System.Security.Cryptography; using Chronos.Core.Archiving; using Chronos.Core.Model; using Chronos.Core.Project; using Chronos.Data.Archiving; using Chronos.Data.Projects; using Chronos.Data.Tests.Archiving; using Chronos.Data.Tests.Infrastructure; using Chronos.Data.Tests.Workspaces; using Chronos.Data.Workspaces; using Chronos.Testing; using static Chronos.Data.Tests.Archiving.ArchiveFixture; using static Chronos.Data.Tests.Projects.ProjectServiceTests; namespace Chronos.Data.Tests.Projects; /// /// The sidecars stay the truth of a sequence's projects when they cannot be read, are missing, are a newer Chronos's, or /// sit on a read-only archive, and an index that meets them late never takes its own old copy for the sequence's settings. /// [Category("projects")] public abstract class ProjectRecoveryTests(TestContext context) : DatabaseTest(context) { private async Task FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true))); /// The live project folders of a sequence (one per project). private static string[] ProjectFolders(SequencePaths folder) => [.. Directory.GetDirectories(folder.ProjectsDirectory).Where(d => Guid.TryParseExact(Path.GetFileName(d), "N", out _))]; private static int MainRecords(SequencePaths folder) => ProjectFolders(folder).Count(d => ProjectSidecars.ReadRecord(Path.Combine(d, "project.json")).IsMain); private static ProjectSettings Settings(ArchiveFixture fx, Sequence sequence) => ProjectSettings.FromJson(File.ReadAllText(fx.Paths.Sequence(sequence.Folder).ProjectPath)); [Test] public async Task ARecordThatCannotBeReadNeverTakesItsProjectForGoneNorMakesASecondMain() { var fx = await FixtureAsync(); var (sequence, frames) = await UploadedAsync(fx, 3); var projects = fx.Service.Projects; var main = await projects.MainAsync(sequence.Id, fx.Ct); var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); var folder = fx.Paths.Sequence(sequence.Folder); var mainRecord = File.ReadAllText(folder.ProjectRecordPath(main.Id)); var verticalRecord = File.ReadAllText(folder.ProjectRecordPath(vertical.Id)); // Main's record as a newer Chronos writes it, Vertical's damaged. var newer = mainRecord.Replace("\"formatVersion\": 1", "\"formatVersion\": 2", StringComparison.Ordinal); File.WriteAllText(folder.ProjectRecordPath(main.Id), newer); File.WriteAllText(folder.ProjectRecordPath(vertical.Id), "{ not json"); Assert.SequenceEqual([main.Id, vertical.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); Assert.Equal(main.Id, (await projects.MainAsync(sequence.Id, fx.Ct)).Id); Assert.Count(2, ProjectFolders(folder)); Assert.Null((await fx.Db.Projects.GetAsync(vertical.Id))!.DeletedAt); // Nothing is written over a record that cannot be read: the versions sent wait. var head = await projects.HeadAsync(vertical.Id, fx.Ct); await Assert.ThrowsAsync(() => projects.PushAsync(vertical.Id, Draft(head, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)); var home = new ArchiveProjectHome(fx.Db, projects, sequence.Id); await Assert.ThrowsAsync(() => home.PushAsync(vertical.Id, Draft(head, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)); await Assert.ThrowsAsync(() => projects.RenameAsync(vertical.Id, "Square", fx.Ct)); await Assert.ThrowsAsync(() => projects.DeleteAsync(vertical.Id, fx.Ct)); Assert.Equal("{ not json", File.ReadAllText(folder.ProjectRecordPath(vertical.Id))); Assert.Equal(newer, File.ReadAllText(folder.ProjectRecordPath(main.Id))); // Readable again: everything is as it was, and goes on. File.WriteAllText(folder.ProjectRecordPath(main.Id), mainRecord); File.WriteAllText(folder.ProjectRecordPath(vertical.Id), verticalRecord); Assert.Equal(PushOutcome.Stored, (await projects.PushAsync(vertical.Id, Draft(head, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)).Outcome); Assert.SequenceEqual([main.Id, vertical.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); // An index that meets Main's record unreadable makes no Main of its own either, and takes it once it reads. File.WriteAllText(folder.ProjectRecordPath(main.Id), newer); await fx.ReopenWithEmptyIndexAsync(); Assert.Empty((await fx.ScanAsync(ScanMode.Rebuild)).Failed); var rebuilt = fx.Service.Projects; Assert.SequenceEqual([vertical.Id], (await rebuilt.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); await Assert.ThrowsAsync(() => rebuilt.MainAsync(sequence.Id, fx.Ct)); await Assert.ThrowsAsync(() => new ArchiveProjectHome(fx.Db, rebuilt, sequence.Id).ListAsync(fx.Ct)); Assert.Count(2, ProjectFolders(folder)); File.WriteAllText(folder.ProjectRecordPath(main.Id), mainRecord); Assert.Equal(main.Id, (await rebuilt.MainAsync(sequence.Id, fx.Ct)).Id); Assert.Equal(1, MainRecords(folder)); } [Test] public async Task AMissingProjectsFolderIsWrittenAgainFromTheIndex() { var fx = await FixtureAsync(); var (sequence, frames) = await UploadedAsync(fx, 3); var projects = fx.Service.Projects; var main = await projects.MainAsync(sequence.Id, fx.Ct); var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); var pushed = (await projects.PushAsync(vertical.Id, Draft(await projects.HeadAsync(vertical.Id, fx.Ct), new ProjectSettings { ExcludedFrames = [2] }, frames), fx.Ct)).Version!; var folder = fx.Paths.Sequence(sequence.Folder); // Moved away for a moment (a sync tool, a backup put back without it): no project is taken for gone, none made anew. Directory.Move(folder.ProjectsDirectory, folder.ProjectsDirectory + "-aside"); Assert.SequenceEqual([main.Id, vertical.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); Assert.Equal(1, MainRecords(folder)); Assert.Equal(pushed.Id, ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)).Head!.Id); Assert.Equal(pushed.ContentHash, ProjectSidecars.ReadVersion(folder.VersionPath(vertical.Id, 2, pushed.Id)).ContentHash); Assert.Equal(PushOutcome.Stored, (await projects.PushAsync(vertical.Id, Draft(pushed, new ProjectSettings(), frames), fx.Ct)).Outcome); // The sidecars written again are the truth again: a rebuilt index has every project and version back. await fx.ReopenWithEmptyIndexAsync(); await fx.ScanAsync(ScanMode.Rebuild); var listed = await fx.Service.Projects.ListAsync(sequence.Id, fx.Ct); Assert.SequenceEqual([main.Id, vertical.Id], listed.Select(p => p.Id)); Assert.Equal(3, listed[1].HeadNumber); } [Test] public async Task AMainAnOlderChronosTookForGoneIsListedAgain() { var fx = await FixtureAsync(); var (sequence, _) = await UploadedAsync(fx, 2); var projects = fx.Service.Projects; var main = await projects.MainAsync(sequence.Id, fx.Ct); var folder = fx.Paths.Sequence(sequence.Folder); // Its live record says it is not deleted: Main again, no second one. await fx.Db.Projects.UpdateAsync((await fx.Db.Projects.GetAsync(main.Id))! with { DeletedAt = DateTimeOffset.UtcNow }); Assert.SequenceEqual([main.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); Assert.Equal(main.Id, (await projects.MainAsync(sequence.Id, fx.Ct)).Id); // And with its folder gone too: written again from the index. await fx.Db.Projects.UpdateAsync((await fx.Db.Projects.GetAsync(main.Id))! with { DeletedAt = DateTimeOffset.UtcNow }); Directory.Move(folder.ProjectDirectory(main.Id), folder.ProjectDirectory(main.Id) + "-aside"); Assert.SequenceEqual([main.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); Assert.True(ProjectSidecars.ReadRecord(folder.ProjectRecordPath(main.Id)).IsMain); Assert.Equal(1, MainRecords(folder)); } [Test] public async Task ASecondIndexMeetingMainBetweenTwoPushesRevertsNothing() { var fx = await FixtureAsync(); var (sequence, frames) = await UploadedAsync(fx, 3); var projects = fx.Service.Projects; var main = await projects.MainAsync(sequence.Id, fx.Ct); var second = (await projects.PushAsync(main.Id, Draft(await projects.HeadAsync(main.Id, fx.Ct), new ProjectSettings { Exposure = new ExposureSettings { ExposureOffsetEv = 0.7 } }, frames), fx.Ct)).Version!; // Another PC indexes the archive (its copy of the settings: version 2's), then this one pushes version 3. var (other, _) = await fx.OtherWriterAsync(); var third = (await projects.PushAsync(main.Id, Draft(second, new ProjectSettings { Exposure = new ExposureSettings { ExposureOffsetEv = 1.4 } }, frames), fx.Ct)).Version!; // Its first look takes the settings from the archive, not from its old copy: no version is added. await other.Projects.ListAsync(sequence.Id, fx.Ct); Assert.SequenceEqual([3, 2, 1], (await other.Projects.HistoryAsync(main.Id, cancellationToken: fx.Ct)).Select(v => v.Number)); Assert.Equal(1.4, Settings(fx, sequence).Exposure.ExposureOffsetEv); Assert.Equal(third.Id, (await projects.HeadAsync(main.Id, fx.Ct)).Id); Assert.Equal(1.4, ProjectSettings.FromJson((await fx.Db.Sequences.GetAsync(sequence.Id))!.SettingsJson!).Exposure.ExposureOffsetEv); } [Test] public async Task AnIndexWithAnOldCopyOfTheSettingsBringsItUpToMain() { var fx = await FixtureAsync(); var (sequence, frames) = await UploadedAsync(fx, 3); var projects = fx.Service.Projects; var main = await projects.MainAsync(sequence.Id, fx.Ct); var pushed = (await projects.PushAsync(main.Id, Draft(await projects.HeadAsync(main.Id, fx.Ct), new ProjectSettings { ExcludedFrames = [0] }, frames), fx.Ct)).Version!; // An index taken back from a backup made before: it holds the sequence with the settings it had then, and no projects. await fx.ReopenWithEmptyIndexAsync(); await fx.ScanAsync(); await WriteSettingsAsync(fx, sequence.Id, new ProjectSettings { ExcludedFrames = [1] }); var listed = await fx.Service.Projects.ListAsync(sequence.Id, fx.Ct); Assert.Equal((pushed.Id, 2), (listed[0].HeadVersionId, listed[0].HeadNumber)); Assert.SequenceEqual([2, 1], (await fx.Service.Projects.HistoryAsync(main.Id, cancellationToken: fx.Ct)).Select(v => v.Number)); Assert.SequenceEqual([0], ProjectSettings.FromJson((await fx.Db.Sequences.GetAsync(sequence.Id))!.SettingsJson!).ExcludedFrames); Assert.SequenceEqual([0], Settings(fx, sequence).ExcludedFrames); } /// The settings of an index that may hold newer ones than project.json, as the server's does (an earlier release's PATCH wrote the index alone). private async Task ServerFixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true), configure: _ => new ArchiveServiceOptions { Owner = "server", LeaseWait = TimeSpan.FromSeconds(20), IndexSettingsMayLead = true })); private static ProjectSettings Exposure(double ev) => new() { Exposure = new ExposureSettings { ExposureOffsetEv = ev } }; private static double Ev(string json) => ProjectSettings.FromJson(json).Exposure.ExposureOffsetEv; [Test] public async Task OnThisPcMainIsBornFromProjectJsonAloneWhenTheIndexDiffers() { var fx = await FixtureAsync(); var (sequence, _) = await UploadedAsync(fx, 3); var folder = fx.Paths.Sequence(sequence.Folder); Directory.CreateDirectory(folder.MetaDirectory); File.WriteAllText(folder.ProjectPath, Exposure(0.2).ToJson()); await WriteSettingsAsync(fx, sequence.Id, Exposure(0.7)); // The desktop's every writer writes the file: an index that differs is an old copy, brought up to it. var main = await fx.Service.Projects.MainAsync(sequence.Id, fx.Ct); var history = await fx.Service.Projects.HistoryAsync(main.Id, cancellationToken: fx.Ct); Assert.Equal((1, ProjectVersionKind.Imported), (history.Single().Number, history.Single().Kind)); Assert.Equal(0.2, Ev(history.Single().SettingsJson)); Assert.Equal(0.2, Ev((await fx.Db.Sequences.GetAsync(sequence.Id))!.SettingsJson!)); Assert.Equal(0.2, Settings(fx, sequence).Exposure.ExposureOffsetEv); } [Test] public async Task OnAServerMainIsBornWithTheIndexsNewerSettingsInForceAndProjectJsonsKept() { foreach (var readOnly in new[] { false, true }) { var fx = await ServerFixtureAsync(); var (sequence, _) = await UploadedAsync(fx, 3); var folder = fx.Paths.Sequence(sequence.Folder); Directory.CreateDirectory(folder.MetaDirectory); File.WriteAllText(folder.ProjectPath, Exposure(0.2).ToJson()); // A settings PATCH of an earlier release: the index alone. await WriteSettingsAsync(fx, sequence.Id, Exposure(0.7)); var projects = fx.Service.Projects; if (readOnly) { // On a read-only archive Main is made in the index only, both versions in it, nothing written; its sidecars // and the mirror follow once the archive takes writes. projects.Availability = _ => ArchiveAvailability.ReadOnly; Assert.Equal(2, (await projects.ListAsync(sequence.Id, fx.Ct)).Single().HeadNumber); Assert.False(Directory.Exists(folder.ProjectsDirectory)); Assert.Equal(0.2, Settings(fx, sequence).Exposure.ExposureOffsetEv); projects.Availability = ArchiveVolume.Probe; } var main = await projects.MainAsync(sequence.Id, fx.Ct); var history = await projects.HistoryAsync(main.Id, cancellationToken: fx.Ct); Assert.SequenceEqual([(2, ProjectVersionKind.External), (1, ProjectVersionKind.Imported)], history.Select(v => (v.Number, v.Kind))); Assert.Equal((0.7, 0.2), (Ev(history[0].SettingsJson), Ev(history[1].SettingsJson))); Assert.Equal(0.7, Ev((await fx.Db.Sequences.GetAsync(sequence.Id))!.SettingsJson!)); Assert.Equal(0.7, Settings(fx, sequence).Exposure.ExposureOffsetEv); // In step now: read again, nothing is added. await projects.ListAsync(sequence.Id, fx.Ct); Assert.Count(2, await projects.HistoryAsync(main.Id, cancellationToken: fx.Ct)); Assert.Equal(main.Id, ProjectFolders(folder).Select(d => Guid.ParseExact(Path.GetFileName(d), "N")).Single()); } } [Test] public async Task OnAServerAnIndexMeetingMainKeepsItsOwnSettingsAsAVersion() { var fx = await ServerFixtureAsync(); var (sequence, frames) = await UploadedAsync(fx, 3); var projects = fx.Service.Projects; var main = await projects.MainAsync(sequence.Id, fx.Ct); var pushed = (await projects.PushAsync(main.Id, Draft(await projects.HeadAsync(main.Id, fx.Ct), Exposure(0.4), frames), fx.Ct)).Version!; // The index meets Main with no mirror known and its own settings, which no version holds: which is newer cannot be // told, so both are kept, the index's as a version and project.json's (the pushed head) in force on top of it. var row = await WriteSettingsAsync(fx, sequence.Id, Exposure(0.7)); await fx.Db.Projects.SetMirroredAsync(main.Id, null, row.SettingsVersion, fx.Ct); var history = await projects.HistoryAsync(main.Id, cancellationToken: fx.Ct); Assert.SequenceEqual([(4, ProjectVersionKind.Restore), (3, ProjectVersionKind.External), (2, ProjectVersionKind.Commit)], history.Take(3).Select(v => (v.Number, v.Kind))); Assert.Equal((0.4, 0.7, pushed.Id), (Ev(history[0].SettingsJson), Ev(history[1].SettingsJson), history[0].RestoredFrom)); Assert.Equal(0.4, Ev((await fx.Db.Sequences.GetAsync(sequence.Id))!.SettingsJson!)); Assert.Equal(0.4, Settings(fx, sequence).Exposure.ExposureOffsetEv); // An index copy a version holds already (a backup's) adds nothing: the head is mirrored. row = await WriteSettingsAsync(fx, sequence.Id, Exposure(0.7)); await fx.Db.Projects.SetMirroredAsync(main.Id, null, row.SettingsVersion, fx.Ct); Assert.Count(4, await projects.HistoryAsync(main.Id, cancellationToken: fx.Ct)); Assert.Equal(0.4, Ev((await fx.Db.Sequences.GetAsync(sequence.Id))!.SettingsJson!)); } [Test] public async Task AMirrorACrashCutShortBeforeProjectJsonIsCompletedAfterARebuild() { foreach (var olderRecord in new[] { false, true }) { var fx = await FixtureAsync(); var (sequence, frames) = await UploadedAsync(fx, 3); var projects = fx.Service.Projects; var main = await projects.MainAsync(sequence.Id, fx.Ct); var first = await projects.HeadAsync(main.Id, fx.Ct); await projects.PushAsync(main.Id, Draft(first, new ProjectSettings { Stack = new StackSettings { Mode = StackSetting.Median } }, frames), fx.Ct); // As if the push had stopped before its mirror wrote project.json (an older record names no mirror at all). var folder = fx.Paths.Sequence(sequence.Folder); File.WriteAllText(folder.ProjectPath, ProjectSettings.FromJson(ProjectContent.Place(first.SettingsJson, Photos(frames))).ToJson()); var record = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(main.Id)); ProjectSidecars.WriteRecord(folder.ProjectRecordPath(main.Id), record with { Mirrored = olderRecord ? null : new ProjectHead(first.Id, 1) }); await fx.ReopenWithEmptyIndexAsync(); await fx.ScanAsync(ScanMode.Rebuild); Assert.SequenceEqual([2, 1], (await fx.Service.Projects.HistoryAsync(main.Id, cancellationToken: fx.Ct)).Select(v => v.Number)); Assert.Equal(StackSetting.Median, Settings(fx, sequence).Stack.Mode); Assert.Equal(StackSetting.Median, ProjectSettings.FromJson((await fx.Db.Sequences.GetAsync(sequence.Id))!.SettingsJson!).Stack.Mode); } } [Test] public async Task SettingsAnOlderChronosWroteIntoProjectJsonBecomeAnExternalVersionForAnIndexMeetingMain() { var fx = await FixtureAsync(); var (sequence, frames) = await UploadedAsync(fx, 3); var projects = fx.Service.Projects; var main = await projects.MainAsync(sequence.Id, fx.Ct); await projects.PushAsync(main.Id, Draft(await projects.HeadAsync(main.Id, fx.Ct), new ProjectSettings { Exposure = new ExposureSettings { ExposureOffsetEv = 0.7 } }, frames), fx.Ct); // An older Chronos on another PC edits the sequence (project.json); this PC's index is then rebuilt. File.WriteAllText(fx.Paths.Sequence(sequence.Folder).ProjectPath, new ProjectSettings { Exposure = new ExposureSettings { ExposureOffsetEv = -1 } }.ToJson()); await fx.ReopenWithEmptyIndexAsync(); await fx.ScanAsync(ScanMode.Rebuild); var history = await fx.Service.Projects.HistoryAsync(main.Id, cancellationToken: fx.Ct); Assert.SequenceEqual([3, 2, 1], history.Select(v => v.Number)); Assert.Equal(ProjectVersionKind.External, history[0].Kind); Assert.Equal(-1, ProjectSettings.FromJson(history[0].SettingsJson).Exposure.ExposureOffsetEv); } [Test] public async Task AMainHeadOfANewerChronosIsListedAndPulledButNeverMirroredNorBuiltOn() { var fx = await FixtureAsync(); var (sequence, frames) = await UploadedAsync(fx, 3); var projects = fx.Service.Projects; var main = await projects.MainAsync(sequence.Id, fx.Ct); var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); var head = await projects.HeadAsync(main.Id, fx.Ct); var folder = fx.Paths.Sequence(sequence.Folder); // A newer Chronos on another PC sharing the archive pushes version 2 in its settings format, and mirrors it. var settings = """{ "version": 3, "exposure": { "exposureOffsetEv": 0.5 }, "futureThing": true }"""; var now = DateTimeOffset.UtcNow; var future = new ProjectVersion { Id = Guid.CreateVersion7(), ProjectId = main.Id, ParentId = head.Id, Number = 2, Kind = ProjectVersionKind.Commit, ClientName = "newer", SettingsJson = settings, ContentHash = ProjectContent.Hash(settings), SettingsFormat = 3, FramesVersion = head.FramesVersion, CreatedAt = now, StoredAt = now, }; ProjectSidecars.WriteVersion(folder.VersionPath(main.Id, 2, future.Id), future); var record = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(main.Id)); ProjectSidecars.WriteRecord(folder.ProjectRecordPath(main.Id), record with { Head = new ProjectHead(future.Id, 2), Mirrored = new ProjectHead(future.Id, 2) }); File.WriteAllText(folder.ProjectPath, settings); var copy = (await fx.Db.Sequences.GetAsync(sequence.Id))!.SettingsJson; Assert.SequenceEqual([main.Id, vertical.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); Assert.Equal(future.Id, (await projects.HeadAsync(main.Id, fx.Ct)).Id); Assert.Equal(future.Id, (await new ArchiveProjectHome(fx.Db, projects, sequence.Id).VersionsAfterAsync(main.Id, 1, 10, fx.Ct)).Single().Id); Assert.Equal(PushOutcome.Stored, (await projects.PushAsync(vertical.Id, Draft(await projects.HeadAsync(vertical.Id, fx.Ct), new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)).Outcome); Assert.Count(2, await projects.HistoryAsync(main.Id, cancellationToken: fx.Ct)); Assert.Equal(settings, File.ReadAllText(folder.ProjectPath)); Assert.Equal(copy, (await fx.Db.Sequences.GetAsync(sequence.Id))!.SettingsJson); // An older path writes the settings here meanwhile: nothing is merged into, or added over, the newer head. await WriteSettingsAsync(fx, sequence.Id, new ProjectSettings { ExcludedFrames = [2] }); await projects.ListAsync(sequence.Id, fx.Ct); await projects.ListAsync(sequence.Id, fx.Ct); Assert.Count(2, await projects.HistoryAsync(main.Id, cancellationToken: fx.Ct)); Assert.Equal(settings, File.ReadAllText(folder.ProjectPath)); // Nor by an index meeting it for the first time. await fx.ReopenWithEmptyIndexAsync(); await fx.ScanAsync(ScanMode.Rebuild); Assert.Equal(future.Id, (await fx.Service.Projects.HeadAsync(main.Id, fx.Ct)).Id); Assert.Count(2, await fx.Service.Projects.HistoryAsync(main.Id, cancellationToken: fx.Ct)); Assert.Equal(settings, File.ReadAllText(folder.ProjectPath)); } [Test] public async Task APushThisChronosCannotReadStrictlyIsRefused() { var fx = await FixtureAsync(); var (sequence, frames) = await UploadedAsync(fx, 2); var projects = fx.Service.Projects; var main = await projects.MainAsync(sequence.Id, fx.Ct); var head = await projects.HeadAsync(main.Id, fx.Ct); var wrong = """{ "exposure": 5 }"""; var result = await projects.PushAsync(main.Id, Draft(head, new ProjectSettings(), frames) with { SettingsJson = wrong, ContentHash = ProjectContent.Hash(wrong) }, fx.Ct); Assert.Equal(PushOutcome.Invalid, result.Outcome); Assert.Count(1, await projects.HistoryAsync(main.Id, cancellationToken: fx.Ct)); Assert.Equal(PushOutcome.Stored, (await projects.PushAsync(main.Id, Draft(head, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)).Outcome); } [Test] public async Task AHeadThisChronosCannotReadLeavesTheRecordAndTheHistoryUntouched() { foreach (var missing in new[] { false, true }) { var fx = await FixtureAsync(); var (sequence, frames) = await UploadedAsync(fx, 3); var projects = fx.Service.Projects; var main = await projects.MainAsync(sequence.Id, fx.Ct); var first = await projects.HeadAsync(main.Id, fx.Ct); var second = (await projects.PushAsync(main.Id, Draft(first, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)).Version!; var folder = fx.Paths.Sequence(sequence.Folder); // Version 2's file in a newer Chronos's format, or not here yet (a sync tool that brought the record first). var path = folder.VersionPath(main.Id, 2, second.Id); var text = File.ReadAllText(path); if (missing) File.Delete(path); else File.WriteAllText(path, text.Replace("\"formatVersion\": 1", "\"formatVersion\": 2", StringComparison.Ordinal)); var recordBytes = File.ReadAllBytes(folder.ProjectRecordPath(main.Id)); await fx.ReopenWithEmptyIndexAsync(); Assert.Empty((await fx.ScanAsync(ScanMode.Rebuild)).Failed); var rebuilt = fx.Service.Projects; var listed = await rebuilt.ListAsync(sequence.Id, fx.Ct); Assert.Equal((main.Id, first.Id), (listed.Single().Id, listed.Single().HeadVersionId)); Assert.Equal(first.Id, (await rebuilt.HeadAsync(main.Id, fx.Ct)).Id); await Assert.ThrowsAsync(() => rebuilt.PushAsync(main.Id, Draft(first, new ProjectSettings(), frames), fx.Ct)); Assert.SequenceEqual(recordBytes, File.ReadAllBytes(folder.ProjectRecordPath(main.Id))); Assert.False(Directory.Exists(Path.Combine(folder.ProjectVersionsDirectory(main.Id), ".stray"))); Assert.Count(missing ? 1 : 2, ProjectSidecars.ListVersions(folder.ProjectVersionsDirectory(main.Id))); Assert.Empty((await fx.ScanAsync(ScanMode.Rebuild)).Failed); // Once it reads, the history goes on from it. File.WriteAllText(path, text); Assert.Equal(second.Id, (await rebuilt.HeadAsync(main.Id, fx.Ct)).Id); Assert.Equal(PushOutcome.Stored, (await rebuilt.PushAsync(main.Id, Draft(second, new ProjectSettings(), frames), fx.Ct)).Outcome); } } [Test] public async Task VersionFilesOfOneNumberAndRecordsThatAllSayMainNeverStopARebuild() { var fx = await FixtureAsync(); var (sequence, frames) = await UploadedAsync(fx, 3); var other = (await UploadedAsync(fx, 1, seed: 300)).Sequence; var projects = fx.Service.Projects; var main = await projects.MainAsync(sequence.Id, fx.Ct); var otherMain = await projects.MainAsync(other.Id, fx.Ct); var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); var first = await projects.HeadAsync(vertical.Id, fx.Ct); var second = (await projects.PushAsync(vertical.Id, Draft(first, new ProjectSettings { ExcludedFrames = [2] }, frames), fx.Ct)).Version!; var folder = fx.Paths.Sequence(sequence.Folder); // Another file numbered 2, off the record's chain, and two more records that say they are Main and are named so. var twin = Draft(first, new ProjectSettings { ExcludedFrames = [0] }, frames) with { StoredAt = DateTimeOffset.UtcNow }; ProjectSidecars.WriteVersion(folder.VersionPath(vertical.Id, 2, twin.Id), twin); foreach (var _ in new[] { 1, 2 }) { var id = Guid.CreateVersion7(); ProjectSidecars.WriteRecord(folder.ProjectRecordPath(id), new ProjectRecord(id, sequence.Id, ProjectService.MainName, true, null, DateTimeOffset.UtcNow, null, DateTimeOffset.UtcNow)); } await fx.ReopenWithEmptyIndexAsync(); Assert.Empty((await fx.ScanAsync(ScanMode.Rebuild)).Failed); var listed = await fx.Service.Projects.ListAsync(sequence.Id, fx.Ct); Assert.Equal(main.Id, listed.Single(p => p.IsMain).Id); Assert.SequenceEqual(["Main", "Main (2)", "Main (3)", "Vertical"], listed.Select(p => p.Name).Order(StringComparer.Ordinal)); Assert.Equal(second.Id, listed.Single(p => p.Id == vertical.Id).HeadVersionId); Assert.Count(1, Directory.GetFiles(Path.Combine(folder.ProjectVersionsDirectory(vertical.Id), ".stray"))); Assert.NotNull(await fx.Db.Projects.GetAsync(otherMain.Id)); } [Test] public async Task ALargeVersionIsStoredAndReadBackAndOneTooLargeIsNeverStored() { var fx = await FixtureAsync(); var (sequence, frames) = await UploadedAsync(fx, 2); var projects = fx.Service.Projects; var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); var head = await projects.HeadAsync(vertical.Id, fx.Ct); var folder = fx.Paths.Sequence(sequence.Folder); // Some 30,000 photos parked by a long sequence (its nights excluded): past the old 8 MB read limit. var pending = Enumerable.Range(0, 30_000).Select(i => new PendingFrame { Sha256 = Convert.ToHexStringLower(SHA256.HashData(BitConverter.GetBytes(i))), Size = 1_000_000 + i, CapturedAt = Start.AddSeconds(i), Excluded = true, }).ToList(); var settings = ProjectContent.Canonical(new ProjectSettings { PendingFrames = pending }.ToJson()); var large = Draft(head, new ProjectSettings(), frames) with { SettingsJson = settings, ContentHash = ProjectContent.Hash(settings) }; Assert.Equal(PushOutcome.Stored, (await projects.PushAsync(vertical.Id, large, fx.Ct)).Outcome); var path = folder.VersionPath(vertical.Id, 2, large.Id); Assert.Greater(new FileInfo(path).Length, 8L << 20); Assert.Equal(large.ContentHash, ProjectSidecars.ReadVersion(path).ContentHash); await fx.ReopenWithEmptyIndexAsync(); await fx.ScanAsync(ScanMode.Rebuild); var rebuilt = fx.Service.Projects; Assert.Equal(large.Id, (await rebuilt.HeadAsync(vertical.Id, fx.Ct)).Id); // One whose file would be past the limit is refused, and nothing is written. var huge = Draft(large, new ProjectSettings(), frames) with { Note = new string('x', ProjectSidecars.MaxVersionBytes) }; Assert.Equal(PushOutcome.TooLarge, (await rebuilt.PushAsync(vertical.Id, huge, fx.Ct)).Outcome); Assert.False(File.Exists(folder.VersionPath(vertical.Id, 3, huge.Id))); Assert.Throws(() => ProjectSidecars.WriteVersion(Path.Combine(Context.CreateTempDirectory("huge"), "v.json"), huge)); } [Test] public async Task TheHomeHoldsAVersionOnlyWhileItsFileReadsBack() { var fx = await FixtureAsync(); var (sequence, frames) = await UploadedAsync(fx, 2); var projects = fx.Service.Projects; var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); var first = await projects.HeadAsync(vertical.Id, fx.Ct); var second = (await projects.PushAsync(vertical.Id, Draft(first, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)).Version!; var folder = fx.Paths.Sequence(sequence.Folder); IProjectHome home = new ArchiveProjectHome(fx.Db, projects, sequence.Id); Assert.True(await home.HoldsAsync(vertical.Id, first, fx.Ct)); Assert.True(await home.HoldsAsync(vertical.Id, second, fx.Ct)); Assert.False(await home.HoldsAsync(vertical.Id, second with { ContentHash = new string('0', 64) }, fx.Ct)); // The index still has both rows, but the archive no longer holds them as they were. File.WriteAllText(folder.VersionPath(vertical.Id, 1, first.Id), "damaged"); File.Delete(folder.VersionPath(vertical.Id, 2, second.Id)); Assert.NotNull(await home.GetVersionAsync(vertical.Id, first.Id, fx.Ct)); Assert.False(await home.HoldsAsync(vertical.Id, first, fx.Ct)); Assert.False(await home.HoldsAsync(vertical.Id, second, fx.Ct)); } [Test] public async Task ADeleteThatCannotMoveTheFolderChangesNothing() { var fx = await FixtureAsync(); var (sequence, _) = await UploadedAsync(fx, 2); var projects = fx.Service.Projects; var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); var folder = fx.Paths.Sequence(sequence.Folder); // The projects' trash cannot take the folder (a file in its way): the delete fails and says so, and nothing changed. File.WriteAllText(folder.ProjectsTrashDirectory, "in the way"); await Assert.ThrowsAsync(() => projects.DeleteAsync(vertical.Id, fx.Ct)); Assert.Null(ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)).Deleted); Assert.SequenceEqual(["Main", "Vertical"], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Name)); File.Delete(folder.ProjectsTrashDirectory); await projects.DeleteAsync(vertical.Id, fx.Ct); Assert.False(Directory.Exists(folder.ProjectDirectory(vertical.Id))); Assert.NotNull(ProjectSidecars.ReadRecord(Path.Combine(Directory.GetDirectories(folder.ProjectsTrashDirectory).Single(), "project.json")).Deleted); Assert.SequenceEqual(["Main"], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Name)); } [Test] public async Task ARescanTellsTheProjectChangesItFoundOnDisk() { var sets = new System.Collections.Concurrent.ConcurrentQueue(); var fx = Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true), configure: o => o with { ProjectsChanged = (set, _) => { sets.Enqueue(set); return Task.CompletedTask; }, })); var (sequence, _) = await UploadedAsync(fx, 2); var vertical = await fx.Service.Projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); Assert.Contains(sets, s => s.Changes.Any(c => c.ProjectId == vertical.Id && c.Change == Chronos.Protocol.ProjectChanges.Created)); // Renamed by another PC sharing the archive (its record rewritten): the full rescan reconciles it and says so. sets.Clear(); var folder = fx.Paths.Sequence(sequence.Folder); var record = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)); ProjectSidecars.WriteRecord(folder.ProjectRecordPath(vertical.Id), record with { Name = "Square", Updated = record.Updated.AddMinutes(1) }); await fx.ScanAsync(ScanMode.Full); Assert.Equal("Square", (await fx.Db.Projects.GetAsync(vertical.Id))!.Name); Assert.Contains(sets, s => s.SequenceId == sequence.Id && s.Changes.Any(c => c.ProjectId == vertical.Id && c.Change == Chronos.Protocol.ProjectChanges.Renamed)); } [Test] public async Task ASequenceOfAReadOnlyArchiveOpensFromItsSidecarsWithoutWriting() { var fx = await FixtureAsync(); var (fresh, freshFrames) = await UploadedAsync(fx, 2); var (edited, frames) = await UploadedAsync(fx, 2, seed: 300); var vertical = await fx.Service.Projects.CreateAsync(edited.Id, "Vertical", cancellationToken: fx.Ct); var pushed = (await fx.Service.Projects.PushAsync(vertical.Id, Draft(await fx.Service.Projects.HeadAsync(vertical.Id, fx.Ct), new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)).Version!; await fx.ReopenWithEmptyIndexAsync(); await fx.ScanAsync(); var projects = fx.Service.Projects; projects.Availability = _ => ArchiveAvailability.ReadOnly; var locks = Directory.Exists(fx.Paths.LocksDirectory) ? Directory.GetFiles(fx.Paths.LocksDirectory).Length : 0; // A sequence with projects: read from its sidecars as they are. var home = new ArchiveProjectHome(fx.Db, projects, edited.Id); Assert.SequenceEqual(["Main", "Vertical"], (await home.ListAsync(fx.Ct)).Select(p => p.Name)); Assert.Equal(pushed.Id, (await home.HeadAsync(vertical.Id, fx.Ct)).Id); await Assert.ThrowsAsync(() => home.PushAsync(vertical.Id, Draft(pushed, new ProjectSettings(), frames), fx.Ct)); // A sequence with none yet: a Main in the index only, nothing written to the archive. var freshHome = new ArchiveProjectHome(fx.Db, projects, fresh.Id); var main = (await freshHome.ListAsync(fx.Ct)).Single(); Assert.True(main.IsMain); var head = await freshHome.HeadAsync(main.Id, fx.Ct); Assert.False(Directory.Exists(fx.Paths.Sequence(fresh.Folder).ProjectsDirectory)); Assert.Equal(locks, Directory.Exists(fx.Paths.LocksDirectory) ? Directory.GetFiles(fx.Paths.LocksDirectory).Length : 0); await Assert.ThrowsAsync(() => freshHome.PushAsync(main.Id, Draft(head, new ProjectSettings(), freshFrames), fx.Ct)); // Once the archive takes writes, that Main's sidecars are written, under the same id, and it goes on. projects.Availability = ArchiveVolume.Probe; Assert.Equal(main.Id, (await freshHome.ListAsync(fx.Ct)).Single().Id); var folder = fx.Paths.Sequence(fresh.Folder); Assert.Equal((main.Id, true), (ProjectSidecars.ReadRecord(folder.ProjectRecordPath(main.Id)).Id, ProjectSidecars.ReadRecord(folder.ProjectRecordPath(main.Id)).IsMain)); Assert.FileExists(folder.VersionPath(main.Id, 1, head.Id)); Assert.Equal(PushOutcome.Stored, (await freshHome.PushAsync(main.Id, Draft(head, new ProjectSettings { ExcludedFrames = [0] }, freshFrames), fx.Ct)).Outcome); } [Test] public async Task AnIndexOnlyMainMeetingAMainAnotherIndexMadeIsWrittenBackAsAnotherProject() { var fx = await FixtureAsync(); var (fresh, frames) = await UploadedAsync(fx, 2); var projects = fx.Service.Projects; projects.Availability = _ => ArchiveAvailability.ReadOnly; var ours = (await projects.ListAsync(fresh.Id, fx.Ct)).Single(); // Meanwhile another PC's index makes its own Main into the archive; then this archive takes writes again. var (other, _) = await fx.OtherWriterAsync(); var theirs = await other.Projects.MainAsync(fresh.Id, fx.Ct); projects.Availability = ArchiveVolume.Probe; // Theirs is Main, ours another project: in the index and in the sidecars alike, and it stays so. foreach (var _ in new[] { 1, 2 }) { Assert.SequenceEqual([(theirs.Id, "Main", true), (ours.Id, "Main (2)", false)], (await projects.ListAsync(fresh.Id, fx.Ct)).Select(p => (p.Id, p.Name, p.IsMain))); } var folder = fx.Paths.Sequence(fresh.Folder); Assert.Equal(1, MainRecords(folder)); var ourRecord = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(ours.Id)); Assert.Equal(("Main (2)", false), (ourRecord.Name, ourRecord.IsMain)); Assert.Equal(theirs.Id, (await other.Projects.MainAsync(fresh.Id, fx.Ct)).Id); Assert.SequenceEqual([theirs.Id, ours.Id], (await other.Projects.ListAsync(fresh.Id, fx.Ct)).Select(p => p.Id)); Assert.Equal(PushOutcome.Stored, (await projects.PushAsync(theirs.Id, Draft(await projects.HeadAsync(theirs.Id, fx.Ct), new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)).Outcome); Assert.SequenceEqual([1], Settings(fx, fresh).ExcludedFrames); } [Test] public async Task AMainAnotherIndexPassedOnIsTakenFromTheRecords() { var fx = await FixtureAsync(); var (sequence, _) = await UploadedAsync(fx, 2); var projects = fx.Service.Projects; var main = await projects.MainAsync(sequence.Id, fx.Ct); var second = await projects.CreateAsync(sequence.Id, "Main (2)", cancellationToken: fx.Ct); var folder = fx.Paths.Sequence(sequence.Folder); // Another index passed Main on, names and all: the records say the opposite of this index. var mainRecord = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(main.Id)); var secondRecord = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(second.Id)); ProjectSidecars.WriteRecord(folder.ProjectRecordPath(main.Id), mainRecord with { IsMain = false, Name = "Main (2)", Mirrored = null }); ProjectSidecars.WriteRecord(folder.ProjectRecordPath(second.Id), secondRecord with { IsMain = true, Name = "Main" }); Assert.SequenceEqual([(second.Id, "Main", true), (main.Id, "Main (2)", false)], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => (p.Id, p.Name, p.IsMain))); Assert.Equal(second.Id, (await projects.MainAsync(sequence.Id, fx.Ct)).Id); Assert.Count(1, (await fx.Db.Projects.ListBySequenceAsync(sequence.Id, includeDeleted: true, fx.Ct)).Where(p => p.IsMain)); Assert.Equal(1, MainRecords(folder)); } [Test] public async Task AProjectCreationCutBeforeItsRecordHoldsNothingBack() { var fx = await FixtureAsync(); var (sequence, frames) = await UploadedAsync(fx, 2); var folder = fx.Paths.Sequence(sequence.Folder); // As making Main leaves it when its record cannot be written after its version (a lost disk, a crash): a folder with // version 1 and no record, of a project no index row names. var id = Guid.CreateVersion7(); var settings = ProjectContent.ToPortable(new ProjectSettings(), Photos(frames)); var now = DateTimeOffset.UtcNow; var cut = new ProjectVersion { Id = Guid.CreateVersion7(), ProjectId = id, Number = 1, Kind = ProjectVersionKind.Imported, ClientName = "cut", SettingsJson = settings, ContentHash = ProjectContent.Hash(settings), SettingsFormat = ProjectContent.FormatOf(settings), FramesVersion = sequence.FramesVersion, CreatedAt = now, StoredAt = now, }; ProjectSidecars.WriteVersion(folder.VersionPath(id, 1, cut.Id), cut); var main = await fx.Service.Projects.MainAsync(sequence.Id, fx.Ct); Assert.NotEqual(id, main.Id); Assert.SequenceEqual([main.Id], (await new ArchiveProjectHome(fx.Db, fx.Service.Projects, sequence.Id).ListAsync(fx.Ct)).Select(p => p.Id)); Assert.FileExists(folder.VersionPath(id, 1, cut.Id)); Assert.Null(await fx.Db.Projects.GetAsync(id)); // Its record there at last (a sync tool bringing another PC's Main, made before this one): Main, as in every index; the // one made here meanwhile is another project, its version kept. ProjectSidecars.WriteRecord(folder.ProjectRecordPath(id), new ProjectRecord(id, sequence.Id, ProjectService.MainName, true, new ProjectHead(cut.Id, 1), now, null, now)); Assert.SequenceEqual([(id, "Main", true), (main.Id, "Main (2)", false)], (await fx.Service.Projects.ListAsync(sequence.Id, fx.Ct)).Select(p => (p.Id, p.Name, p.IsMain))); Assert.Equal(1, MainRecords(folder)); Assert.Count(1, await fx.Service.Projects.HistoryAsync(main.Id, cancellationToken: fx.Ct)); } [Test] public async Task AMainMadeWhileAnotherMainsRecordWasNotThereYetGivesWayInEveryIndex() { var fx = await FixtureAsync(); var (sequence, frames) = await UploadedAsync(fx, 3); var projects = fx.Service.Projects; var main = await projects.MainAsync(sequence.Id, fx.Ct); var pushed = (await projects.PushAsync(main.Id, Draft(await projects.HeadAsync(main.Id, fx.Ct), new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)) .Version!; var folder = fx.Paths.Sequence(sequence.Folder); // Another PC's index meets the sequence while a sync tool is still bringing Main's record: it makes a Main of its own. var record = folder.ProjectRecordPath(main.Id); File.Move(record, record + ".syncing"); var (other, _) = await fx.OtherWriterAsync(); var theirs = await other.Projects.MainAsync(sequence.Id, fx.Ct); Assert.NotEqual(main.Id, theirs.Id); File.Move(record + ".syncing", record); // The record there, both indexes take the older Main, in either order, and the other is an ordinary project with its version. foreach (var service in new[] { other, fx.Service, other }) { Assert.SequenceEqual([(main.Id, "Main", true), (theirs.Id, "Main (2)", false)], (await service.Projects.ListAsync(sequence.Id, fx.Ct)).Select(p => (p.Id, p.Name, p.IsMain))); } Assert.Equal(1, MainRecords(folder)); Assert.Equal(("Main (2)", false), (ProjectSidecars.ReadRecord(folder.ProjectRecordPath(theirs.Id)).Name, ProjectSidecars.ReadRecord(folder.ProjectRecordPath(theirs.Id)).IsMain)); Assert.Count(1, await other.Projects.HistoryAsync(theirs.Id, cancellationToken: fx.Ct)); Assert.Equal(pushed.Id, (await other.Projects.HeadAsync(main.Id, fx.Ct)).Id); Assert.Equal(PushOutcome.Stored, (await other.Projects.PushAsync(main.Id, Draft(pushed, new ProjectSettings { ExcludedFrames = [2] }, frames), fx.Ct)).Outcome); Assert.SequenceEqual([2], Settings(fx, sequence).ExcludedFrames); Assert.Equal(main.Id, (await projects.MainAsync(sequence.Id, fx.Ct)).Id); } [Test] public async Task AProjectWhoseRecordIsLostIsWrittenAgainFromTheIndex() { var fx = await FixtureAsync(); var (sequence, frames) = await UploadedAsync(fx, 3); var projects = fx.Service.Projects; var main = await projects.MainAsync(sequence.Id, fx.Ct); var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); var first = await projects.HeadAsync(vertical.Id, fx.Ct); var second = (await projects.PushAsync(vertical.Id, Draft(first, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)).Version!; var mainSecond = (await projects.PushAsync(main.Id, Draft(await projects.HeadAsync(main.Id, fx.Ct), new ProjectSettings { ExcludedFrames = [2] }, frames), fx.Ct)) .Version!; var folder = fx.Paths.Sequence(sequence.Folder); // Version 3 of Vertical is in its folder (another PC's push), then both records and version 1's file are lost. var third = Draft(second, new ProjectSettings { ExcludedFrames = [0] }, frames) with { StoredAt = DateTimeOffset.UtcNow }; ProjectSidecars.WriteVersion(folder.VersionPath(vertical.Id, 3, third.Id), third); File.Delete(folder.ProjectRecordPath(vertical.Id)); File.Delete(folder.ProjectRecordPath(main.Id)); File.Delete(folder.VersionPath(vertical.Id, 1, first.Id)); // The first look holds them back and writes nothing (a sync tool may still be bringing the records); once the folders // looked the same twice, nothing is held back: the records are written again from the index, the version on disk past // its head adopted. projects.RecordSettle = TimeSpan.Zero; Assert.SequenceEqual([(main.Id, true, 2), (vertical.Id, false, 2)], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => (p.Id, p.IsMain, p.HeadNumber))); Assert.False(File.Exists(folder.ProjectRecordPath(vertical.Id))); Assert.False(File.Exists(folder.ProjectRecordPath(main.Id))); Assert.SequenceEqual([(main.Id, true, 2), (vertical.Id, false, 3)], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => (p.Id, p.IsMain, p.HeadNumber))); var verticalRecord = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)); Assert.Equal(("Vertical", third.Id), (verticalRecord.Name, verticalRecord.Head!.Id)); var mainRecord = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(main.Id)); Assert.Equal((true, mainSecond.Id, mainSecond.Id), (mainRecord.IsMain, mainRecord.Head!.Id, mainRecord.Mirrored?.Id)); Assert.FileExists(folder.VersionPath(vertical.Id, 1, first.Id)); Assert.Equal(PushOutcome.Stored, (await projects.PushAsync(main.Id, Draft(mainSecond, new ProjectSettings { ExcludedFrames = [0] }, frames), fx.Ct)).Outcome); Assert.SequenceEqual([0], Settings(fx, sequence).ExcludedFrames); Assert.Equal(PushOutcome.Stored, (await projects.PushAsync(vertical.Id, Draft(third, new ProjectSettings(), frames), fx.Ct)).Outcome); await fx.ReopenWithEmptyIndexAsync(); await fx.ScanAsync(ScanMode.Rebuild); Assert.SequenceEqual([4, 3, 2, 1], (await fx.Service.Projects.HistoryAsync(vertical.Id, cancellationToken: fx.Ct)).Select(v => v.Number)); Assert.SequenceEqual([3, 2, 1], (await fx.Service.Projects.HistoryAsync(main.Id, cancellationToken: fx.Ct)).Select(v => v.Number)); } [Test] public async Task ARenameMadeAfterABackupStaysWhenTheBackupIsPutBack() { var fx = await FixtureAsync(); var (sequence, frames) = await UploadedAsync(fx, 3); var projects = fx.Service.Projects; var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); var folder = fx.Paths.Sequence(sequence.Folder); var backup = folder.ProjectsDirectory + "-backup"; CopyDirectory(folder.ProjectsDirectory, backup); await projects.PushAsync(vertical.Id, Draft(await projects.HeadAsync(vertical.Id, fx.Ct), new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct); await projects.RenameAsync(vertical.Id, "Square", fx.Ct); // The backup made before is put back: its record is behind the index and names the old name. Directory.Delete(folder.ProjectsDirectory, recursive: true); CopyDirectory(backup, folder.ProjectsDirectory); Assert.Equal("Square", (await projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id).Name); var record = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)); Assert.Equal(("Square", 2), (record.Name, record.Head!.Number)); await fx.ReopenWithEmptyIndexAsync(); await fx.ScanAsync(ScanMode.Rebuild); Assert.Equal("Square", (await fx.Service.Projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id).Name); } [Test] public async Task ARenameOnlyMadeAfterABackupStaysWhenTheBackupIsPutBack() { var fx = await FixtureAsync(); var (sequence, _) = await UploadedAsync(fx, 3); var projects = fx.Service.Projects; var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); var folder = fx.Paths.Sequence(sequence.Folder); var backup = folder.ProjectsDirectory + "-backup"; CopyDirectory(folder.ProjectsDirectory, backup); await projects.RenameAsync(vertical.Id, "Square", fx.Ct); // The backup made before the rename (the same head) is put back: its record was written before the index's name. Directory.Delete(folder.ProjectsDirectory, recursive: true); CopyDirectory(backup, folder.ProjectsDirectory); Assert.Equal("Square", (await projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id).Name); var record = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)); Assert.Equal(("Square", 1), (record.Name, record.Head!.Number)); await fx.ReopenWithEmptyIndexAsync(); await fx.ScanAsync(ScanMode.Rebuild); Assert.Equal("Square", (await fx.Service.Projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id).Name); // A rename another PC makes is newer than what this index knows, even from a clock behind the one this index wrote // with (ten minutes ahead here): it is taken. var (other, _) = await fx.OtherWriterAsync(); Assert.Equal("Square", (await other.Projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id).Name); var ahead = DateTimeOffset.UtcNow.AddMinutes(10); var written = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)); ProjectSidecars.WriteRecord(folder.ProjectRecordPath(vertical.Id), written with { Updated = ahead }); await fx.Db.Projects.UpdateAsync((await fx.Db.Projects.GetAsync(vertical.Id))! with { UpdatedAt = ahead }, fx.Ct); await other.Projects.RenameAsync(vertical.Id, "Wide", fx.Ct); Assert.True(ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)).Updated > ahead); Assert.Equal("Wide", (await fx.Service.Projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id).Name); Assert.Equal("Wide", ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)).Name); } [Test] public async Task ARecordOnAnotherChainBelowTheHeadKeepsItsName() { var fx = await FixtureAsync(); var (sequence, frames) = await UploadedAsync(fx, 3); var projects = fx.Service.Projects; var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); var first = await projects.HeadAsync(vertical.Id, fx.Ct); var second = (await projects.PushAsync(vertical.Id, Draft(first, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)).Version!; await projects.PushAsync(vertical.Id, Draft(second, new ProjectSettings { ExcludedFrames = [2] }, frames), fx.Ct); var folder = fx.Paths.Sequence(sequence.Folder); // Another PC's record, on a chain of its own from version 1, renamed there: its head number is below the index's, but // it is not a version the index holds, so the record is not behind and its name is taken. var theirs = Draft(first, new ProjectSettings { ExcludedFrames = [0] }, frames) with { StoredAt = DateTimeOffset.UtcNow }; ProjectSidecars.WriteVersion(folder.VersionPath(vertical.Id, 2, theirs.Id), theirs); var record = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)); var apart = record with { Name = "Square", Head = new ProjectHead(theirs.Id, 2), Updated = DateTimeOffset.UtcNow.AddSeconds(1) }; ProjectSidecars.WriteRecord(folder.ProjectRecordPath(vertical.Id), apart); var listed = (await projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id); Assert.Equal(("Square", 3), (listed.Name, listed.HeadNumber)); // Index and record agree on the name, and the record is never written over. Assert.Equal(apart, ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id))); Assert.Equal("Square", (await projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id).Name); } [Test] public async Task ADeletionArrivingHalfSyncedIsTakenAndNeverUndone() { var fx = await FixtureAsync(); var (sequence, _) = await UploadedAsync(fx, 3); var projects = fx.Service.Projects; var main = await projects.MainAsync(sequence.Id, fx.Ct); var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); var square = await projects.CreateAsync(sequence.Id, "Square", cancellationToken: fx.Ct); var folder = fx.Paths.Sequence(sequence.Folder); var (other, _) = await fx.OtherWriterAsync(); Assert.SequenceEqual([main.Id, vertical.Id, square.Id], (await other.Projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); projects.RecordSettle = TimeSpan.Zero; other.Projects.RecordSettle = TimeSpan.Zero; // This PC deletes Vertical; on the other PC a sync tool moving it file by file into the trash has left its live folder // with a version file and no record. However often the other index looks, it takes the deletion and writes nothing. await projects.DeleteAsync(vertical.Id, fx.Ct); var trashed = Directory.GetDirectories(folder.ProjectsTrashDirectory).Single(); Directory.CreateDirectory(folder.ProjectVersionsDirectory(vertical.Id)); foreach (var file in Directory.GetFiles(Path.Combine(trashed, ArchiveLayout.ProjectVersionsFolder))) File.Copy(file, Path.Combine(folder.ProjectVersionsDirectory(vertical.Id), Path.GetFileName(file))); for (var i = 0; i < 2; i++) { Assert.SequenceEqual([main.Id, square.Id], (await other.Projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); Assert.SequenceEqual([main.Id, square.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); } Assert.NotNull((await other.Projects.GetAsync(vertical.Id, fx.Ct))!.DeletedAt); Assert.False(File.Exists(folder.ProjectRecordPath(vertical.Id))); // Square's deletion arrives with its versions in the trash before its record, and its live record gone already: it is // held back (listed, nothing written), never written again as live, and deleted once the trash's record is there. var squareTrash = Path.Combine(folder.ProjectsTrashDirectory, square.Id.ToString("N") + "-" + DateTimeOffset.UtcNow.UtcTicks); CopyDirectory(folder.ProjectDirectory(square.Id), squareTrash); var squareRecord = ProjectSidecars.ReadRecord(Path.Combine(squareTrash, ArchiveLayout.ProjectFile)); File.Delete(Path.Combine(squareTrash, ArchiveLayout.ProjectFile)); File.Delete(folder.ProjectRecordPath(square.Id)); for (var i = 0; i < 2; i++) Assert.SequenceEqual([main.Id, square.Id], (await other.Projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); Assert.False(File.Exists(folder.ProjectRecordPath(square.Id))); ProjectSidecars.WriteRecord(Path.Combine(squareTrash, ArchiveLayout.ProjectFile), squareRecord with { Deleted = DateTimeOffset.UtcNow }); Assert.SequenceEqual([main.Id], (await other.Projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); Assert.SequenceEqual([main.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); Assert.False(File.Exists(folder.ProjectRecordPath(square.Id))); // A rebuilt index has both deleted too. await fx.ReopenWithEmptyIndexAsync(); await fx.ScanAsync(ScanMode.Rebuild); Assert.SequenceEqual([main.Id], (await fx.Service.Projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); } [Test] public async Task APushMadeIntoTheLiveFolderBeforeADeletionArrivedIsKeptBesideTheDeletedRecord() { var fx = await FixtureAsync(); var (sequence, frames) = await UploadedAsync(fx, 3); var projects = fx.Service.Projects; var main = await projects.MainAsync(sequence.Id, fx.Ct); var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); var first = await projects.HeadAsync(vertical.Id, fx.Ct); var folder = fx.Paths.Sequence(sequence.Folder); var (other, _) = await fx.OtherWriterAsync(); var before = folder.ProjectsDirectory + "-before"; CopyDirectory(folder.ProjectDirectory(vertical.Id), before); await projects.DeleteAsync(vertical.Id, fx.Ct); // The other PC pushed version 2 into its live folder before the deletion reached it: a sync tool brings that folder // back beside the trash. The deletion wins, and version 2 goes beside the deleted record, never left where nothing reads it. CopyDirectory(before, folder.ProjectDirectory(vertical.Id)); var theirs = Draft(first, new ProjectSettings { ExcludedFrames = [2] }, frames) with { StoredAt = DateTimeOffset.UtcNow }; var pushed = folder.VersionPath(vertical.Id, 2, theirs.Id); ProjectSidecars.WriteVersion(pushed, theirs); var record = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)); ProjectSidecars.WriteRecord(folder.ProjectRecordPath(vertical.Id), record with { Head = new ProjectHead(theirs.Id, 2), Updated = DateTimeOffset.UtcNow }); Assert.SequenceEqual([main.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); var trashed = Directory.GetDirectories(folder.ProjectsTrashDirectory).Single(); var kept = Path.Combine(trashed, ArchiveLayout.ProjectVersionsFolder, Path.GetFileName(pushed)); Assert.FileExists(kept); Assert.FileMissing(pushed); Assert.Equal(theirs.Id, ProjectSidecars.ReadVersion(kept).Id); Assert.FileExists(Path.Combine(trashed, ArchiveLayout.ProjectVersionsFolder, Path.GetFileName(folder.VersionPath(vertical.Id, 1, first.Id)))); Assert.NotNull(ProjectSidecars.ReadRecord(Path.Combine(trashed, ArchiveLayout.ProjectFile)).Deleted); Assert.SequenceEqual([main.Id], (await other.Projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); Assert.SequenceEqual([main.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); } [Test] public async Task ATrashFolderOfAnotherSequenceOrLongWithoutARecordNoLongerHoldsBackALostRecord() { var fx = await FixtureAsync(); var (sequence, frames) = await UploadedAsync(fx, 3); var projects = fx.Service.Projects; var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); var first = await projects.HeadAsync(vertical.Id, fx.Ct); var folder = fx.Paths.Sequence(sequence.Folder); projects.RecordSettle = TimeSpan.Zero; var path = folder.ProjectRecordPath(vertical.Id); var lost = ProjectSidecars.ReadRecord(path); // A folder of the projects' trash with no record, just made (a deletion on its way): the lost record is not written. var leftover = Path.Combine(folder.ProjectsTrashDirectory, vertical.Id.ToString("N") + "-1"); Directory.CreateDirectory(leftover); File.Delete(path); for (var i = 0; i < 2; i++) await projects.ListAsync(sequence.Id, fx.Ct); Assert.False(File.Exists(path), "held back while a deletion may be on its way"); await Assert.ThrowsAsync(() => projects.PushAsync(vertical.Id, Draft(first, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)); // Nothing written in it for longer than the grace: a leftover, which holds nothing back any more. Directory.SetLastWriteTimeUtc(leftover, DateTime.UtcNow - ProjectIndexer.TrashRecordGrace - TimeSpan.FromHours(1)); for (var i = 0; i < 2; i++) await projects.ListAsync(sequence.Id, fx.Ct); Assert.Equal(("Vertical", first.Id), (ProjectSidecars.ReadRecord(path).Name, ProjectSidecars.ReadRecord(path).Head!.Id)); Assert.Equal(PushOutcome.Stored, (await projects.PushAsync(vertical.Id, Draft(first, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)).Outcome); Directory.Delete(leftover); // A trash folder whose record names another sequence (a sequence folder copied by hand): never this one's deletion. var foreign = Path.Combine(folder.ProjectsTrashDirectory, vertical.Id.ToString("N") + "-2"); Directory.CreateDirectory(foreign); ProjectSidecars.WriteRecord(Path.Combine(foreign, ArchiveLayout.ProjectFile), lost with { SequenceId = Guid.CreateVersion7(), Deleted = DateTimeOffset.UtcNow }); File.Delete(path); for (var i = 0; i < 2; i++) await projects.ListAsync(sequence.Id, fx.Ct); Assert.FileExists(path); Assert.Contains(vertical.Id, (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); // One whose record names this sequence is its deletion. Assert.True(ProjectIndexer.DeletionOnItsWay(foreign, vertical.Id, sequence.Id, DateTime.UtcNow) == false); ProjectSidecars.WriteRecord(Path.Combine(foreign, ArchiveLayout.ProjectFile), lost with { Deleted = DateTimeOffset.UtcNow }); Assert.True(ProjectIndexer.DeletionOnItsWay(foreign, vertical.Id, sequence.Id, DateTime.UtcNow)); } [Test] public async Task AFolderWithNoRecordThatWentAndCameBackSettlesAgainFromWhenItCameBack() { var fx = await FixtureAsync(); var (sequence, _) = await UploadedAsync(fx, 3); var vertical = await fx.Service.Projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); var folder = fx.Paths.Sequence(sequence.Folder); var settle = TimeSpan.FromMilliseconds(TestTiming.Budget(400)); var indexer = new ProjectIndexer(fx.Db) { RecordSettle = settle }; var path = folder.ProjectRecordPath(vertical.Id); File.Delete(path); Assert.Contains(vertical.Id, (await indexer.ReconcileAsync(folder, sequence.Id, fx.Ct)).HeldBack); // The folder goes for a while (a sync tool moving it): what was seen of it is forgotten, so once it is back it must // look the same for a settling time again before its record is written from the index. var away = folder.ProjectsDirectory + "-away"; Directory.Move(folder.ProjectDirectory(vertical.Id), away); Assert.False((await indexer.ReconcileAsync(folder, sequence.Id, fx.Ct)).Seen.Contains(vertical.Id)); await Task.Delay(settle * 2, fx.Ct); Directory.Move(away, folder.ProjectDirectory(vertical.Id)); Assert.Contains(vertical.Id, (await indexer.ReconcileAsync(folder, sequence.Id, fx.Ct)).HeldBack); Assert.False(File.Exists(path), "not written at once on what was seen before it went"); await Task.Delay(settle * 2, fx.Ct); Assert.False((await indexer.ReconcileAsync(folder, sequence.Id, fx.Ct)).HeldBack.Contains(vertical.Id)); Assert.FileExists(path); } [Test] public async Task AHealedRecordIsStampedPastWhatTheIndexKnowsSoTheNextRenameIsTaken() { var fx = await FixtureAsync(); var (sequence, frames) = await UploadedAsync(fx, 3); var projects = fx.Service.Projects; var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); var folder = fx.Paths.Sequence(sequence.Folder); var backup = folder.ProjectsDirectory + "-backup"; CopyDirectory(folder.ProjectsDirectory, backup); await projects.PushAsync(vertical.Id, Draft(await projects.HeadAsync(vertical.Id, fx.Ct), new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct); // A PC whose clock is ten minutes ahead renames it: this index takes the name and that time. var ahead = DateTimeOffset.UtcNow.AddMinutes(10); ProjectSidecars.WriteRecord(folder.ProjectRecordPath(vertical.Id), ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)) with { Name = "Square", Updated = ahead }); Assert.Equal("Square", (await projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id).Name); // The backup is put back (behind): healed, and stamped past what the index knows. Directory.Delete(folder.ProjectsDirectory, recursive: true); CopyDirectory(backup, folder.ProjectsDirectory); await projects.CreateAsync(sequence.Id, "Other", cancellationToken: fx.Ct); var healed = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)); Assert.Equal(("Square", 2), (healed.Name, healed.Head!.Number)); Assert.True(healed.Updated > ahead, "past the time the index took"); // Another PC with a right clock renames it: every index takes it, and nothing writes the old name back. var (other, _) = await fx.OtherWriterAsync(); Assert.Equal("Square", (await other.Projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id).Name); await other.Projects.RenameAsync(vertical.Id, "Wide", fx.Ct); Assert.Equal("Wide", (await projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id).Name); await projects.CreateAsync(sequence.Id, "Other2", cancellationToken: fx.Ct); Assert.Equal("Wide", ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)).Name); Assert.Equal("Wide", (await other.Projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id).Name); } [Test] public async Task ALostRecordIsWrittenAgainOnlyOnceItsFolderHasSettledSoAnIncomingOneWins() { var fx = await FixtureAsync(); var (sequence, frames) = await UploadedAsync(fx, 3); var projects = fx.Service.Projects; var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); var first = await projects.HeadAsync(vertical.Id, fx.Ct); var folder = fx.Paths.Sequence(sequence.Folder); var path = folder.ProjectRecordPath(vertical.Id); // A sync tool is bringing the record another PC wrote (a rename, a push): meanwhile the folder holds none. However often // the index looks within the settling time, it writes none of its own and takes nothing for the project. var incoming = ProjectSidecars.ReadRecord(path); File.Delete(path); for (var i = 0; i < 2; i++) { var held = (await projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id); Assert.Equal(("Vertical", 1), (held.Name, held.HeadNumber)); } Assert.False(File.Exists(path)); await Assert.ThrowsAsync(() => projects.PushAsync(vertical.Id, Draft(first, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)); // The record arrives, renamed and moved on there: it wins. var theirs = Draft(first, new ProjectSettings { ExcludedFrames = [2] }, frames) with { StoredAt = DateTimeOffset.UtcNow }; ProjectSidecars.WriteVersion(folder.VersionPath(vertical.Id, 2, theirs.Id), theirs); ProjectSidecars.WriteRecord(path, incoming with { Name = "Square", Head = new ProjectHead(theirs.Id, 2), Updated = DateTimeOffset.UtcNow }); var listed = (await projects.ListAsync(sequence.Id, fx.Ct)).Single(p => p.Id == vertical.Id); Assert.Equal(("Square", theirs.Id), (listed.Name, listed.HeadVersionId)); // A record really lost is written again from the index once the folder looked the same on two reconciles a settling time apart. File.Delete(path); projects.RecordSettle = TimeSpan.Zero; await projects.ListAsync(sequence.Id, fx.Ct); Assert.False(File.Exists(path)); await projects.ListAsync(sequence.Id, fx.Ct); var written = ProjectSidecars.ReadRecord(path); Assert.Equal(("Square", theirs.Id), (written.Name, written.Head!.Id)); Assert.Equal(PushOutcome.Stored, (await projects.PushAsync(vertical.Id, Draft(theirs, new ProjectSettings(), frames), fx.Ct)).Outcome); } [Test] public async Task AProjectsFolderPutBackFromAnOlderBackupIsMadeWholeFromTheIndex() { var fx = await FixtureAsync(); var (sequence, frames) = await UploadedAsync(fx, 3); var projects = fx.Service.Projects; var main = await projects.MainAsync(sequence.Id, fx.Ct); var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); var folder = fx.Paths.Sequence(sequence.Folder); var backup = folder.ProjectsDirectory + "-backup"; CopyDirectory(folder.ProjectsDirectory, backup); var v2 = (await projects.PushAsync(vertical.Id, Draft(await projects.HeadAsync(vertical.Id, fx.Ct), new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)) .Version!; var m2 = (await projects.PushAsync(main.Id, Draft(await projects.HeadAsync(main.Id, fx.Ct), new ProjectSettings { ExcludedFrames = [2] }, frames), fx.Ct)).Version!; // The backup made before those pushes is put back: the records name version 1, the index holds version 2. Directory.Delete(folder.ProjectsDirectory, recursive: true); CopyDirectory(backup, folder.ProjectsDirectory); Assert.Equal(PushOutcome.Stored, (await projects.PushAsync(vertical.Id, Draft(v2, new ProjectSettings { ExcludedFrames = [0] }, frames), fx.Ct)).Outcome); Assert.Equal(PushOutcome.Stored, (await projects.PushAsync(main.Id, Draft(m2, new ProjectSettings { ExcludedFrames = [0] }, frames), fx.Ct)).Outcome); Assert.Equal(3, ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)).Head!.Number); Assert.Equal(v2.ContentHash, ProjectSidecars.ReadVersion(folder.VersionPath(vertical.Id, 2, v2.Id)).ContentHash); Assert.Equal(m2.ContentHash, ProjectSidecars.ReadVersion(folder.VersionPath(main.Id, 2, m2.Id)).ContentHash); // The sidecars hold every version again: a rebuilt index has them all, and nothing was added. await fx.ReopenWithEmptyIndexAsync(); await fx.ScanAsync(ScanMode.Rebuild); Assert.SequenceEqual([3, 2, 1], (await fx.Service.Projects.HistoryAsync(vertical.Id, cancellationToken: fx.Ct)).Select(v => v.Number)); Assert.SequenceEqual([ProjectVersionKind.Commit, ProjectVersionKind.Commit, ProjectVersionKind.Imported], (await fx.Service.Projects.HistoryAsync(main.Id, cancellationToken: fx.Ct)).Select(v => v.Kind)); Assert.SequenceEqual([0], Settings(fx, sequence).ExcludedFrames); // A read alone makes the records whole too. Directory.Delete(folder.ProjectsDirectory, recursive: true); CopyDirectory(backup, folder.ProjectsDirectory); await fx.Service.Projects.ListAsync(sequence.Id, fx.Ct); Assert.Equal(3, ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)).Head!.Number); Assert.Count(3, ProjectSidecars.ListVersions(folder.ProjectVersionsDirectory(main.Id))); } [Test] public async Task AMirrorRecordThatLagsProjectJsonNeverRevertsANewerPush() { var fx = await FixtureAsync(); var (sequence, frames) = await UploadedAsync(fx, 3); var projects = fx.Service.Projects; var main = await projects.MainAsync(sequence.Id, fx.Ct); var first = await projects.HeadAsync(main.Id, fx.Ct); var second = (await projects.PushAsync(main.Id, Draft(first, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)).Version!; var folder = fx.Paths.Sequence(sequence.Folder); var afterSecond = File.ReadAllText(folder.ProjectPath); await projects.PushAsync(main.Id, Draft(second, new ProjectSettings { ExcludedFrames = [2] }, frames), fx.Ct); // Version 2's mirror wrote project.json but not the record, and version 3's mirror was cut before project.json. File.WriteAllText(folder.ProjectPath, afterSecond); var record = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(main.Id)); ProjectSidecars.WriteRecord(folder.ProjectRecordPath(main.Id), record with { Mirrored = new ProjectHead(first.Id, 1) }); await fx.ReopenWithEmptyIndexAsync(); await fx.ScanAsync(ScanMode.Rebuild); // The mirror completes: no external version brings version 2 back over version 3. Assert.SequenceEqual([3, 2, 1], (await fx.Service.Projects.HistoryAsync(main.Id, cancellationToken: fx.Ct)).Select(v => v.Number)); Assert.SequenceEqual([2], Settings(fx, sequence).ExcludedFrames); } [Test] public async Task AnIndexMeetingMainMergesSettingsWrittenSinceWithAPushNoMirrorWrote() { var fx = await FixtureAsync(); var (sequence, frames) = await UploadedAsync(fx, 3); var projects = fx.Service.Projects; var main = await projects.MainAsync(sequence.Id, fx.Ct); var first = await projects.HeadAsync(main.Id, fx.Ct); var pushed = (await projects.PushAsync(main.Id, Draft(first, new ProjectSettings { Stack = new StackSettings { Mode = StackSetting.Median } }, frames), fx.Ct)) .Version!; // Version 2's mirror cut short (the record names version 1), and an older Chronos then writes an offset into project.json. var folder = fx.Paths.Sequence(sequence.Folder); File.WriteAllText(folder.ProjectPath, new ProjectSettings { Exposure = new ExposureSettings { ExposureOffsetEv = -1 } }.ToJson()); var record = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(main.Id)); ProjectSidecars.WriteRecord(folder.ProjectRecordPath(main.Id), record with { Mirrored = new ProjectHead(first.Id, 1) }); await fx.ReopenWithEmptyIndexAsync(); await fx.ScanAsync(ScanMode.Rebuild); // Both are in force, as an index that knew its mirror would have merged them. var history = await fx.Service.Projects.HistoryAsync(main.Id, cancellationToken: fx.Ct); Assert.SequenceEqual([ProjectVersionKind.Merge, ProjectVersionKind.Commit, ProjectVersionKind.Imported], history.Select(v => v.Kind)); Assert.Equal(pushed.Id, history[0].ParentId); Assert.Equal((StackSetting.Median, -1.0), (Settings(fx, sequence).Stack.Mode, Settings(fx, sequence).Exposure.ExposureOffsetEv)); } [Test] public async Task ANewerChronosHeadOverAMirrorCutShortIsLeftAsItIs() { var fx = await FixtureAsync(); var (sequence, frames) = await UploadedAsync(fx, 3); var projects = fx.Service.Projects; var main = await projects.MainAsync(sequence.Id, fx.Ct); var first = await projects.HeadAsync(main.Id, fx.Ct); var pushed = (await projects.PushAsync(main.Id, Draft(first, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)).Version!; // Version 2's mirror cut short, nobody writing the settings since; then a newer Chronos pushes version 3. var reverted = await WriteSettingsAsync(fx, sequence.Id, new ProjectSettings()); await fx.Db.Projects.SetMirroredAsync(main.Id, first.Id, reverted.SettingsVersion, fx.Ct); var newer = NewerChronos.Push(fx, sequence, pushed); Assert.SequenceEqual([main.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id)); Assert.Equal(newer.Id, (await projects.HeadAsync(main.Id, fx.Ct)).Id); Assert.Count(3, await projects.HistoryAsync(main.Id, cancellationToken: fx.Ct)); Assert.Empty(ProjectSettings.FromJson((await fx.Db.Sequences.GetAsync(sequence.Id))!.SettingsJson!).ExcludedFrames); } [Test] public async Task AProjectDeletedAtItsHomeHasNoHeadThere() { var fx = await FixtureAsync(); var (sequence, _) = await UploadedAsync(fx, 2); var projects = fx.Service.Projects; var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); var first = await projects.HeadAsync(vertical.Id, fx.Ct); IProjectHome home = new ArchiveProjectHome(fx.Db, projects, sequence.Id); // Deleted (by another PC, say): a workspace asking for its head learns it is gone, never that it is in step. await projects.DeleteAsync(vertical.Id, fx.Ct); Assert.Equal(ProjectError.NotFound, (await Assert.ThrowsAsync(() => home.HeadAsync(vertical.Id, fx.Ct))).Reason); Assert.False(await home.HoldsAsync(vertical.Id, first, fx.Ct)); Assert.Equal(first.Id, (await home.HistoryAsync(vertical.Id, null, 10, fx.Ct)).Single().Id); } [Test] public async Task AVersionParkingFortyThousandPhotosGoesFromAWorkspaceToTheArchiveAndBack() { var fx = await FixtureAsync(); var bed = Own(await WorkspaceFixture.CreateAsync(Context, fx, 3)); var vertical = await fx.Service.Projects.CreateAsync(bed.Sequence.Id, "Vertical", cancellationToken: fx.Ct); var (_, sync) = bed.Workspace(); var project = await bed.CheckoutAsync(sync, vertical.Id); // A long sequence's nights excluded: 40,000 photos parked by a version, past the old 8 MB limit of both files. var pending = Enumerable.Range(0, 40_000).Select(i => new PendingFrame { Sha256 = Convert.ToHexStringLower(SHA256.HashData(BitConverter.GetBytes(i))), Size = 1_000_000 + i, CapturedAt = Start.AddSeconds(i), Excluded = true, }).ToList(); var session = await sync.OpenAsync(vertical.Id, fx.Ct); await sync.SaveAsync(vertical.Id, session.Working with { PendingFrames = pending }, fx.Ct); var version = Assert.NotNull(await sync.CommitAsync(vertical.Id, "nights", cancellationToken: fx.Ct)); Assert.Greater(new FileInfo(project.VersionPath(version)).Length, 8L << 20); Assert.Equal(1, (await sync.StatusAsync(vertical.Id, bed.Home, fx.Ct)).Ahead); var pushed = await sync.PushAsync(vertical.Id, bed.Home, fx.Ct); Assert.Equal(version.Id, pushed.Pushed.Single().Id); Assert.Equal(WorkspaceState.InSync, pushed.Status.State); await fx.ReopenWithEmptyIndexAsync(); await fx.ScanAsync(ScanMode.Rebuild); var head = await fx.Service.Projects.HeadAsync(vertical.Id, fx.Ct); Assert.Equal((version.Id, "nights"), (head.Id, head.Note)); Assert.Count(40_000, ProjectSettings.FromJson(head.SettingsJson).PendingFrames ?? []); } private static void CopyDirectory(string from, string to) { Directory.CreateDirectory(to); foreach (var file in Directory.GetFiles(from)) File.Copy(file, Path.Combine(to, Path.GetFileName(file))); foreach (var directory in Directory.GetDirectories(from)) CopyDirectory(directory, Path.Combine(to, Path.GetFileName(directory))); } } public sealed class SqliteProjectRecoveryTests(TestContext context) : ProjectRecoveryTests(context) { protected override Backend Backend => Backend.SqliteFile; } [RequiresEnvironment(PgTestServer.Variable)] public sealed class PostgresProjectRecoveryTests(TestContext context) : ProjectRecoveryTests(context) { protected override Backend Backend => Backend.Postgres; }