CI / Build and test (push) Successful in 8m14s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 7s
Release / Build, test and publish (push) Successful in 13m19s
CI / PostgreSQL 16 integration (push) Successful in 4m49s
CI / PostgreSQL 17 integration (push) Successful in 5m28s
CI / Docker image (push) Successful in 53s
Release / Server image (push) Failing after 10s
The owner renamed the product and moved the repository to the self-hosted Gitea (https://gitea.encke-hake.ts.net/BalboSoft/Chronos) for more control, and asked for Actions and a first automatic release with the server package and the Windows exe (docs/requests/2026-10-23-chronos-gitea.md). - Chronos everywhere: a byte-exact rename of contents and paths, then a per-language review of the forms the old name's ending had dictated. - Gitea instead of GitHub: the update check, licence, issue and support links read ChronosReleasesUrl; the update check takes the highest stable version, offers a pre-release build its final release and reports a repository it cannot see as a failed check. EULA, security policy and Gitea issue forms live at the repository root. - Gitea Actions for the Linux act_runner (.gitea/workflows): CI, release, nightly image and performance gates in the .NET SDK container with git checkout and the system SQLite. The release cross-compiles the Windows programs, checks their icon, version and DPI manifest, packs installers and zips, publishes the Gitea release and pushes the server image (REGISTRY_TOKEN). Windows-only tests run in an opt-in job on a Windows runner (WINDOWS_RUNNER). - New Chronos.Tools commands: zip, gitea-release, postgres-ci (replaces the docker-exec script, which Gitea's service containers do not allow; the throw-away server runs without durability) and pe-check. - Timed tests take their budget from TestTiming (CHRONOS_TEST_TIME_FACTOR, 3 on the shared runner; the weekly performance gates keep theirs). BREAKING CHANGE: programs, settings, data folders, file types, environment variables and the Docker image use the new name; nothing is migrated from Titano, which never had a release. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
226 lines
12 KiB
C#
226 lines
12 KiB
C#
using Chronos.Core.Archiving;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Data.Archiving;
|
|
using Chronos.Data.Jobs;
|
|
using Chronos.Testing;
|
|
using static Chronos.Data.Tests.Archiving.ArchiveFixture;
|
|
|
|
namespace Chronos.Data.Tests.Archiving;
|
|
|
|
[Category("archive")]
|
|
public sealed class ArchiveIntegrityTests(TestContext context)
|
|
{
|
|
private static Task<VerifyReport> VerifyAsync(ArchiveFixture fx, IReadOnlyList<Guid>? ids = null) =>
|
|
fx.Service.Verifier.VerifyAsync(fx.Archive, ids, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
|
|
|
|
private static async Task<Guid> IngestSeriesAsync(ArchiveFixture fx, string key, int count, int seed)
|
|
{
|
|
Guid sequence = default;
|
|
for (var i = 0; i < count; i++)
|
|
{
|
|
var staged = fx.Stage(Start.AddSeconds(10 * i), seed + i);
|
|
sequence = (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, key, staged.Path, $"IMG_{i:0000}.JPG",
|
|
staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId;
|
|
}
|
|
return sequence;
|
|
}
|
|
|
|
[Test]
|
|
public async Task VerifyFlagsAFlippedByteAndAMissingFileInSidecarAndIndex()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var id = await IngestSeriesAsync(fx, "a", 5, 1);
|
|
var clean = await VerifyAsync(fx);
|
|
Assert.Equal(5L, clean.Ok);
|
|
Assert.Empty(clean.Problems);
|
|
Assert.True((await fx.Db.Sequences.GetAsync(id))!.Status.HasFlag(SequenceStatus.Verified));
|
|
|
|
var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder);
|
|
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
|
|
var damaged = folder.File(manifest.Frames[1].File);
|
|
var bytes = File.ReadAllBytes(damaged);
|
|
bytes[bytes.Length / 3] ^= 0x10;
|
|
File.WriteAllBytes(damaged, bytes);
|
|
File.Delete(folder.File(manifest.Frames[3].File));
|
|
|
|
var report = await VerifyAsync(fx);
|
|
Assert.SequenceEqual([(manifest.Frames[1].Id, FrameHealth.Corrupt), (manifest.Frames[3].Id, FrameHealth.Missing)],
|
|
report.Problems.Select(p => (p.FrameId, p.Health)));
|
|
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
|
|
Assert.Equal(FrameFlags.Corrupt, frames[1].Flags);
|
|
Assert.Equal(FrameFlags.Missing, frames[3].Flags);
|
|
var after = ManifestStore.ReadSequence(folder.ManifestPath);
|
|
Assert.Equal(FrameFlags.Corrupt, after.Frames[1].Flags);
|
|
Assert.False(after.Status.HasFlag(SequenceStatus.Verified));
|
|
var row = (await fx.Db.Sequences.GetAsync(id))!;
|
|
Assert.True(row.Status.HasFlag(SequenceStatus.HasErrors));
|
|
Assert.NotNull(row.VerifiedAt);
|
|
var health = (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id)).Health;
|
|
Assert.Equal((1L, 1L), (health.MissingFrames, health.CorruptFrames));
|
|
}
|
|
|
|
[Test]
|
|
public async Task VerifyGivesAdoptedPhotosTheirFirstHash()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "Adopted"), Start, 3);
|
|
await fx.ScanAsync();
|
|
var report = await VerifyAsync(fx);
|
|
Assert.Equal(3L, report.Hashed);
|
|
var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence("Adopted").ManifestPath);
|
|
Assert.True(manifest.Frames.All(f => f.Sha256 is { Length: 64 }));
|
|
var frames = await fx.Db.Frames.ListBySequenceAsync(manifest.Id);
|
|
Assert.SequenceEqual(manifest.Frames.Select(f => f.Sha256), frames.Select(f => f.Sha256));
|
|
Assert.Equal(3L, (await VerifyAsync(fx)).Ok);
|
|
}
|
|
|
|
[Test]
|
|
public async Task DuplicatesAcrossSequencesAreFoundAndFlaggedAfterADryRun()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var a = await IngestSeriesAsync(fx, "a", 3, 1);
|
|
var b = await IngestSeriesAsync(fx, "b", 2, 1); // the same two photos again, in another sequence
|
|
var groups = await fx.Service.FindDuplicatesAsync(fx.Archive.Id, fx.Ct);
|
|
Assert.Equal(2, groups.Count);
|
|
Assert.True(groups.All(g => g.Frames.Count == 2 && g.Frames[0].SequenceId == a && g.Frames[1].SequenceId == b));
|
|
|
|
var dry = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: true, 0, _ => ValueTask.CompletedTask, fx.Ct);
|
|
Assert.Equal(2, dry.DuplicateFlagChanges);
|
|
Assert.Equal(0L, (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id)).Health.DuplicateFrames);
|
|
|
|
var run = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
|
|
Assert.Equal(2, run.DuplicateFlagChanges);
|
|
Assert.Equal(2L, (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id)).Health.DuplicateFrames);
|
|
var folder = (await fx.Db.Sequences.GetAsync(b))!.Folder;
|
|
Assert.True(ManifestStore.ReadSequence(fx.Paths.Sequence(folder).ManifestPath).Frames.All(f => f.Flags.HasFlag(FrameFlags.Duplicate)));
|
|
Assert.Equal(0, (await fx.Service.Maintenance.RunAsync(fx.Archive, false, 0, _ => ValueTask.CompletedTask, fx.Ct)).DuplicateFlagChanges);
|
|
}
|
|
|
|
[Test]
|
|
public async Task OrphansListUnindexedAndMissingFilesAndTheFixIndexesThem()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var id = await IngestSeriesAsync(fx, "a", 3, 1);
|
|
var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder);
|
|
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
|
|
File.Delete(folder.File(manifest.Frames[0].File));
|
|
File.WriteAllBytes(Path.Combine(folder.Originals, "dropped.JPG"), Photos.Jpeg(Start.AddMinutes(5), 50));
|
|
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "New folder"), Start, 1, firstSeed: 60);
|
|
|
|
var report = await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct);
|
|
var orphans = report.Sequences.Single();
|
|
Assert.SequenceEqual(["originals/dropped.JPG"], orphans.UnindexedFiles);
|
|
Assert.Equal(manifest.Frames[0].Id, orphans.MissingFiles.Single().Id);
|
|
Assert.SequenceEqual(["New folder"], report.UnindexedFolders);
|
|
|
|
await fx.Service.ReconcileAsync(id, null, deepAnalysis: false, fx.Ct);
|
|
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
|
|
Assert.Equal(4, frames.Count);
|
|
Assert.Equal(FrameFlags.Missing, frames[0].Flags);
|
|
Assert.Equal("dropped.JPG", frames[^1].FileName);
|
|
Assert.Empty((await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct)).Sequences.Where(s => s.UnindexedFiles.Count > 0));
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheOrphansFixOnAServerDrawsTheFramesItIndexes()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var id = await IngestSeriesAsync(fx, "a", 3, 1);
|
|
// The archive's server, which draws what its clients load; its engine runs no analysis.
|
|
var server = new ArchiveService(fx.Db, new ArchiveServiceOptions { Owner = "server", LeaseWait = TimeSpan.FromSeconds(20), ImagesAfterScan = true });
|
|
await using var engine = new JobEngine(fx.Db, server.CreateJobHandlers(),
|
|
new JobEngineOptions { Workers = 1, WorkerId = "server", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
|
|
await engine.StartAsync(fx.Ct);
|
|
foreach (var pass in new[] { ArchiveJobs.Thumbnails([id]), ArchiveJobs.Proxies([id]) })
|
|
await fx.WaitForAsync((await engine.EnqueueAsync(pass, cancellationToken: fx.Ct)).Id, JobState.Done);
|
|
var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id, fx.Ct))!.Folder);
|
|
File.WriteAllBytes(Path.Combine(folder.Originals, "dropped.JPG"), Photos.Jpeg(Start.AddMinutes(5), 50));
|
|
|
|
// The fix indexes the photo copied in by hand, and queues the thumbnails and proxies of the frames as they are now;
|
|
// no analysis, even when asked for one.
|
|
var before = (await fx.Db.Jobs.ListAsync(null, null, 100, null, fx.Ct)).Items.Select(j => j.Id).ToHashSet();
|
|
await server.ReconcileAsync(id, engine, deepAnalysis: true, fx.Ct);
|
|
var queued = (await fx.Db.Jobs.ListAsync(null, null, 100, null, fx.Ct)).Items.Where(j => !before.Contains(j.Id)).ToList();
|
|
Assert.SequenceEqual([JobKind.Proxies, JobKind.Thumbs], queued.Select(j => j.Kind).Order());
|
|
foreach (var job in queued)
|
|
{
|
|
Assert.SequenceEqual([id], job.TargetIds);
|
|
await fx.WaitForAsync(job.Id, JobState.Done);
|
|
}
|
|
var row = (await fx.Db.Sequences.GetAsync(id, fx.Ct))!;
|
|
Assert.Equal(4, row.FrameCount);
|
|
Assert.True(row.Status.HasFlag(SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt), row.Status.ToString());
|
|
Assert.True(File.Exists(folder.Proxy(3)), "proxy of the new frame");
|
|
}
|
|
|
|
[Test]
|
|
public async Task MaintenanceRemovesOnlyStaleLeftovers()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var id = await IngestSeriesAsync(fx, "a", 2, 1);
|
|
var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder);
|
|
var oldTemp = Path.Combine(folder.MetaDirectory, ".sequence.json.1234abcd.tmp");
|
|
var freshTemp = Path.Combine(folder.MetaDirectory, ".sequence.json.5678abcd.tmp");
|
|
var oldStaged = Path.Combine(fx.Paths.StagingDirectory, "old.part");
|
|
File.WriteAllText(oldTemp, "x");
|
|
File.WriteAllText(freshTemp, "x");
|
|
File.WriteAllText(oldStaged, "x");
|
|
File.SetLastWriteTimeUtc(oldTemp, DateTime.UtcNow.AddHours(-2));
|
|
File.SetLastWriteTimeUtc(oldStaged, DateTime.UtcNow.AddDays(-2));
|
|
var past = DateTimeOffset.UtcNow.AddHours(-1);
|
|
File.WriteAllText(Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease"),
|
|
$$"""{"token":"t","owner":"crashed","host":"h","processId":1,"acquired":"{{past:O}}","expires":"{{past.AddMinutes(2):O}}"}""");
|
|
|
|
var dry = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: true, 0, _ => ValueTask.CompletedTask, fx.Ct);
|
|
Assert.Equal(2, dry.TemporaryFiles.Count);
|
|
Assert.SequenceEqual(["seq-dead"], dry.StaleLeases);
|
|
Assert.FileExists(oldTemp);
|
|
|
|
await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
|
|
Assert.FileMissing(oldTemp);
|
|
Assert.FileMissing(oldStaged);
|
|
Assert.FileExists(freshTemp);
|
|
Assert.FileMissing(Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease"));
|
|
Assert.True(folder.Frames().All(File.Exists));
|
|
}
|
|
|
|
[Test]
|
|
public async Task PackCompactionWaitsForTheThumbnailWriter()
|
|
{
|
|
await using var fx = await CreateAsync(context);
|
|
var id = await IngestSeriesAsync(fx, "a", 2, 1);
|
|
var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder);
|
|
var hash = new byte[ThumbPack.HashLength];
|
|
using (var pack = ThumbPackWriter.Open(folder.ThumbsPackPath))
|
|
{
|
|
// The same entry written again and again: everything but the last copy is garbage.
|
|
for (var i = 0; i < 4; i++) pack.Append(0, 48, 32, hash, Photos.Jpeg(Start, 70 + i));
|
|
pack.Commit();
|
|
}
|
|
var length = new FileInfo(folder.ThumbsPackPath).Length;
|
|
|
|
// A thumbnail pass holds the pack's own lease (not the sequence's) while it appends.
|
|
Task<MaintenanceReport> maintenance;
|
|
using (await LeaseFile.AcquireAsync(fx.Paths.LocksDirectory, $"thumbs-{id:N}", "thumbnails", TimeSpan.FromSeconds(5), cancellationToken: fx.Ct))
|
|
{
|
|
maintenance = fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
|
|
await Task.Delay(500, fx.Ct);
|
|
Assert.False(maintenance.IsCompleted);
|
|
Assert.Equal(length, new FileInfo(folder.ThumbsPackPath).Length);
|
|
}
|
|
|
|
var report = await maintenance;
|
|
Assert.SequenceEqual([folder.RelativeFolder], report.CompactedPacks);
|
|
using var compacted = ThumbPackReader.Open(folder.ThumbsPackPath);
|
|
Assert.Equal(0L, compacted.GarbageBytes);
|
|
Assert.NotNull(compacted.Read(0));
|
|
}
|
|
}
|
|
|
|
internal static class SequencePathsExtensions
|
|
{
|
|
/// <summary>Absolute paths of every frame listed in the sidecar.</summary>
|
|
public static IEnumerable<string> Frames(this SequencePaths sequence) =>
|
|
ManifestStore.ReadSequence(sequence.ManifestPath).Frames.Select(f => sequence.File(f.File));
|
|
}
|