using Chronos.Core.Localization; using Chronos.Core.Model; using Chronos.Data.Archiving; using Chronos.Data.Archiving.Jobs; using Chronos.Data.Housekeeping; using Chronos.Data.Jobs; using Chronos.Data.Sqlite; using Chronos.Data.Tests.Archiving; using Chronos.Testing; namespace Chronos.Data.Tests.Scheduling; /// The jobs schedules add (owner review 2, R10): index backups, clean-up steps, and the read-only duplicates and orphans reports. [Category("schedules")] public sealed class HousekeepingJobTests(TestContext context) { /// A host step that counts its runs, or fails. private sealed class TestStep(string name, bool fails = false) : ICleanupStep { public int Runs; public string Name => name; public Task RunAsync(CleanupContext context, CleanupParameters parameters) { Interlocked.Increment(ref Runs); return fails ? throw new IOException("disk gone") : Task.FromResult($"{name} ran for {parameters.OlderThanDays} days"); } } private static async Task IngestAsync(ArchiveFixture fx, string key, int count, int seed) { Guid sequence = default; for (var i = 0; i < count; i++) { var staged = fx.Stage(ArchiveFixture.Start.AddSeconds(10 * i), seed + i); sequence = (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, key, staged.Path, $"IMG_{i:0000}.JPG", staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId; } return sequence; } private static async Task RunAsync(ArchiveFixture fx, JobEngine engine, ArchiveJobRequest request, JobState state = JobState.Done) { var job = await engine.EnqueueAsync(request, cancellationToken: fx.Ct); return await fx.WaitForAsync(job.Id, state); } [Test] public async Task BackupsGoNextToTheIndexAndKeepTheNewest() { await using var fx = await ArchiveFixture.CreateAsync(context); var engine = await fx.StartEngineAsync(); var path = ((SqliteDatabase)fx.Db.Database).Path!; Job done = null!; for (var i = 0; i < 3; i++) done = await RunAsync(fx, engine, ArchiveJobs.Backup(keep: 2)); var backups = DatabaseBackups.List(DatabaseBackups.DefaultDirectory(path), path); Assert.Equal(2, backups.Count); Assert.Equal(Path.Combine(Path.GetDirectoryName(path)!, "backups"), DatabaseBackups.DefaultDirectory(path)); Assert.Contains(backups[0].Path, done.LogTail); Assert.Equal((1L, 1L), (done.ProgressDone, done.ProgressTotal)); Assert.Null(DatabaseBackups.Unavailable(fx.Db)); // One folder, one retention: a backup that keeps fewer never deletes the copies the index backup schedule keeps. await fx.Db.Schedules.EnsureSystemAsync([new Data.Scheduling.SystemSchedule(DatabaseBackups.ScheduleKey, "Index backup", Data.Scheduling.ScheduleActions.Backup, Data.Scheduling.ScheduleRecurrence.Parse("daily:03:00"), new Protocol.ScheduleParametersDto(Keep: 4))], DateTimeOffset.UtcNow, fx.Ct); for (var i = 0; i < 3; i++) await RunAsync(fx, engine, ArchiveJobs.Backup(keep: 1)); Assert.Count(4, DatabaseBackups.List(DatabaseBackups.DefaultDirectory(path), path)); } [Test] public async Task ABackupOfAnIndexWithoutAFileFails() { await using var db = await ChronosDatabase.OpenAsync(DatabaseOptions.SqliteInMemory(), context.Cancellation); Assert.Equal(Data.Scheduling.ScheduleUnavailable.NoIndexFile, DatabaseBackups.Unavailable(db)); await using var engine = new JobEngine(db, new ArchiveService(db).CreateJobHandlers(), new JobEngineOptions { WorkerId = "memory", PollInterval = TimeSpan.FromMilliseconds(50) }); await engine.StartAsync(context.Cancellation); var job = await engine.EnqueueAsync(ArchiveJobs.Backup(), cancellationToken: context.Cancellation); for (var i = 0; i < 500 && (await db.Jobs.GetAsync(job.Id))!.State is JobState.Queued or JobState.Running; i++) await Task.Delay(20); var failed = (await db.Jobs.GetAsync(job.Id))!; Assert.Equal(JobState.Failed, failed.State); Assert.Contains("without a file", failed.Error); } [Test] public async Task TheJobHistoryStepRemovesOnlyOldFinishedJobs() { await using var fx = await ArchiveFixture.CreateAsync(context); var now = DateTimeOffset.UtcNow; Job Old(JobState state, int days, JobKind kind = JobKind.Scan) => new() { Id = Guid.CreateVersion7(), Kind = kind, State = state, Created = now.AddDays(-days), Finished = state is JobState.Done or JobState.Failed or JobState.Cancelled ? now.AddDays(-days) : null, }; // A render waits: the archive engine has no render handler, so it stays queued however old. var old = Old(JobState.Done, 100); var oldFailed = Old(JobState.Failed, 91); var recent = Old(JobState.Done, 10); var waiting = Old(JobState.Queued, 200, JobKind.Render); foreach (var job in new[] { old, oldFailed, recent, waiting }) await fx.Db.Jobs.EnqueueAsync(job, fx.Ct); var engine = await fx.StartEngineAsync(); var kept = await RunAsync(fx, engine, ArchiveJobs.Cleanup(fx.Archive, [CleanupSteps.JobHistory], 0)); Assert.Contains(Loc.L("Archive.Log.HistoryKept"), kept.LogTail); Assert.NotNull(await fx.Db.Jobs.GetAsync(old.Id, fx.Ct)); var purged = await RunAsync(fx, engine, ArchiveJobs.Cleanup(fx.Archive, [CleanupSteps.JobHistory], 90)); Assert.Contains(Loc.L("Archive.Log.HistoryPurged", 2), purged.LogTail); Assert.Null(await fx.Db.Jobs.GetAsync(old.Id, fx.Ct)); Assert.Null(await fx.Db.Jobs.GetAsync(oldFailed.Id, fx.Ct)); Assert.NotNull(await fx.Db.Jobs.GetAsync(recent.Id, fx.Ct)); Assert.NotNull(await fx.Db.Jobs.GetAsync(waiting.Id, fx.Ct)); } [Test] public async Task TheAuditStepKeepsEverythingAtZeroDays() { await using var fx = await ArchiveFixture.CreateAsync(context); var now = DateTimeOffset.UtcNow; foreach (var days in new[] { 400, 1 }) await fx.Db.Audit.AppendAsync(new AuditEntry { Id = Guid.CreateVersion7(), At = now.AddDays(-days), Action = "user.login" }, fx.Ct); var engine = await fx.StartEngineAsync(); var kept = await RunAsync(fx, engine, ArchiveJobs.Cleanup(null, [CleanupSteps.Audit], 0)); Assert.Contains(Loc.L("Archive.Log.AuditKept"), kept.LogTail); Assert.Count(2, (await fx.Db.Audit.ListAsync(cancellationToken: fx.Ct)).Items); var purged = await RunAsync(fx, engine, ArchiveJobs.Cleanup(null, [CleanupSteps.Audit], 365)); Assert.Contains(Loc.L("Archive.Log.AuditPurged", 1), purged.LogTail); Assert.Count(1, (await fx.Db.Audit.ListAsync(cancellationToken: fx.Ct)).Items); } [Test] public async Task TheTrashReportListsOldEntriesAndNeverDeletes() { await using var fx = await ArchiveFixture.CreateAsync(context); var id = await IngestAsync(fx, "a", 2, 1); var entry = (await fx.Service.DeleteAsync(await fx.Service.PlanDeleteAsync([id], cancellationToken: fx.Ct), "test", cancellationToken: fx.Ct)).Single(); var folder = Path.Combine(SequenceTrash.TrashDirectory(fx.Paths), entry.Name); var files = Directory.GetFiles(folder, "*", SearchOption.AllDirectories).Length; var engine = await fx.StartEngineAsync(); var all = await RunAsync(fx, engine, ArchiveJobs.Cleanup(fx.Archive, [CleanupSteps.TrashReport, "shred"], 0)); Assert.Contains(Loc.L("Archive.Log.TrashReport", 1, CleanupSteps.Megabytes(entry.Bytes)), all.LogTail); Assert.Contains(entry.Title, all.LogTail); Assert.Contains(Loc.L("Archive.Log.StepUnknown", "shred"), all.LogTail); var none = await RunAsync(fx, engine, ArchiveJobs.Cleanup(fx.Archive, [CleanupSteps.TrashReport], 30)); Assert.Contains(Loc.L("Archive.Log.TrashReport", 0, CleanupSteps.Megabytes(0)), none.LogTail); // Reported, never deleted: emptying the trash stays the user's action. Assert.Equal(files, Directory.GetFiles(folder, "*", SearchOption.AllDirectories).Length); Assert.Equal(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name); var offline = await RunAsync(fx, engine, ArchiveJobs.Cleanup(null, [CleanupSteps.TrashReport], 0)); Assert.Contains(Loc.L("Archive.Log.TrashOffline"), offline.LogTail); } [Test] public async Task AFailingStepFailsTheJobAfterTheOthersRan() { var count = new TestStep("count"); var explode = new TestStep("explode", fails: true); await using var db = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "chronos.db")), context.Cancellation); var service = new ArchiveService(db, new ArchiveServiceOptions { CleanupSteps = [explode, count, new TestStep(CleanupSteps.Audit)] }); Assert.SequenceEqual([CleanupSteps.JobHistory, CleanupSteps.Audit, CleanupSteps.TrashReport, "explode", "count"], service.CleanupSteps.Select(s => s.Name)); await using var engine = new JobEngine(db, service.CreateJobHandlers(), new JobEngineOptions { WorkerId = "steps", PollInterval = TimeSpan.FromMilliseconds(50) }); await engine.StartAsync(context.Cancellation); var job = await engine.EnqueueAsync(ArchiveJobs.Cleanup(null, ["explode", "count"], 7), cancellationToken: context.Cancellation); for (var i = 0; i < 500 && (await db.Jobs.GetAsync(job.Id))!.State is JobState.Queued or JobState.Running; i++) await Task.Delay(20); var failed = (await db.Jobs.GetAsync(job.Id))!; Assert.Equal(JobState.Failed, failed.State); Assert.Contains("explode", failed.Error); Assert.Contains(Loc.L("Archive.Log.StepFailed", "explode", "disk gone"), failed.LogTail); Assert.Contains("count ran for 7 days", failed.LogTail); Assert.Equal((1, 1), (explode.Runs, count.Runs)); } [Test] public async Task DuplicatesAndOrphansReportWithoutChangingAnything() { await using var fx = await ArchiveFixture.CreateAsync(context); await IngestAsync(fx, "a", 3, 1); await IngestAsync(fx, "b", 2, 1); Photos.WriteSeries(Path.Combine(fx.Paths.Root, "New folder"), ArchiveFixture.Start, 1, firstSeed: 60); var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id); var engine = await fx.StartEngineAsync(); var duplicates = await RunAsync(fx, engine, ArchiveJobs.Duplicates(fx.Archive)); Assert.Contains(Loc.L("Archive.Log.DuplicatesDone", 2, 2), duplicates.LogTail); Assert.Contains(Loc.L("Archive.Log.ReportOnly"), duplicates.LogTail); var orphans = await RunAsync(fx, engine, ArchiveJobs.Orphans(fx.Archive)); Assert.Contains(Loc.L("Archive.Log.FolderUnindexed", "New folder"), orphans.LogTail); Assert.Contains(Loc.L("Archive.Log.OrphanFolders", 0, 1), orphans.LogTail); Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id)); Assert.Equal(0L, (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id, fx.Ct)).Health.DuplicateFrames); } }