using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Data.Archiving;
using Chronos.Data.Archiving.Jobs;
using Chronos.Data.Housekeeping;
using Chronos.Data.Jobs;
using Chronos.Data.Sqlite;
using Chronos.Data.Tests.Archiving;
using Chronos.Testing;
namespace Chronos.Data.Tests.Scheduling;
/// The jobs schedules add (owner review 2, R10): index backups, clean-up steps, and the read-only duplicates and orphans reports.
[Category("schedules")]
public sealed class HousekeepingJobTests(TestContext context)
{
/// A host step that counts its runs, or fails.
private sealed class TestStep(string name, bool fails = false) : ICleanupStep
{
public int Runs;
public string Name => name;
public Task RunAsync(CleanupContext context, CleanupParameters parameters)
{
Interlocked.Increment(ref Runs);
return fails ? throw new IOException("disk gone") : Task.FromResult($"{name} ran for {parameters.OlderThanDays} days");
}
}
private static async Task IngestAsync(ArchiveFixture fx, string key, int count, int seed)
{
Guid sequence = default;
for (var i = 0; i < count; i++)
{
var staged = fx.Stage(ArchiveFixture.Start.AddSeconds(10 * i), seed + i);
sequence = (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, key, staged.Path, $"IMG_{i:0000}.JPG",
staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId;
}
return sequence;
}
private static async Task RunAsync(ArchiveFixture fx, JobEngine engine, ArchiveJobRequest request, JobState state = JobState.Done)
{
var job = await engine.EnqueueAsync(request, cancellationToken: fx.Ct);
return await fx.WaitForAsync(job.Id, state);
}
[Test]
public async Task BackupsGoNextToTheIndexAndKeepTheNewest()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
var engine = await fx.StartEngineAsync();
var path = ((SqliteDatabase)fx.Db.Database).Path!;
Job done = null!;
for (var i = 0; i < 3; i++) done = await RunAsync(fx, engine, ArchiveJobs.Backup(keep: 2));
var backups = DatabaseBackups.List(DatabaseBackups.DefaultDirectory(path), path);
Assert.Equal(2, backups.Count);
Assert.Equal(Path.Combine(Path.GetDirectoryName(path)!, "backups"), DatabaseBackups.DefaultDirectory(path));
Assert.Contains(backups[0].Path, done.LogTail);
Assert.Equal((1L, 1L), (done.ProgressDone, done.ProgressTotal));
Assert.Null(DatabaseBackups.Unavailable(fx.Db));
// One folder, one retention: a backup that keeps fewer never deletes the copies the index backup schedule keeps.
await fx.Db.Schedules.EnsureSystemAsync([new Data.Scheduling.SystemSchedule(DatabaseBackups.ScheduleKey, "Index backup", Data.Scheduling.ScheduleActions.Backup,
Data.Scheduling.ScheduleRecurrence.Parse("daily:03:00"), new Protocol.ScheduleParametersDto(Keep: 4))], DateTimeOffset.UtcNow, fx.Ct);
for (var i = 0; i < 3; i++) await RunAsync(fx, engine, ArchiveJobs.Backup(keep: 1));
Assert.Count(4, DatabaseBackups.List(DatabaseBackups.DefaultDirectory(path), path));
}
[Test]
public async Task ABackupOfAnIndexWithoutAFileFails()
{
await using var db = await ChronosDatabase.OpenAsync(DatabaseOptions.SqliteInMemory(), context.Cancellation);
Assert.Equal(Data.Scheduling.ScheduleUnavailable.NoIndexFile, DatabaseBackups.Unavailable(db));
await using var engine = new JobEngine(db, new ArchiveService(db).CreateJobHandlers(),
new JobEngineOptions { WorkerId = "memory", PollInterval = TimeSpan.FromMilliseconds(50) });
await engine.StartAsync(context.Cancellation);
var job = await engine.EnqueueAsync(ArchiveJobs.Backup(), cancellationToken: context.Cancellation);
for (var i = 0; i < 500 && (await db.Jobs.GetAsync(job.Id))!.State is JobState.Queued or JobState.Running; i++) await Task.Delay(20);
var failed = (await db.Jobs.GetAsync(job.Id))!;
Assert.Equal(JobState.Failed, failed.State);
Assert.Contains("without a file", failed.Error);
}
[Test]
public async Task TheJobHistoryStepRemovesOnlyOldFinishedJobs()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
var now = DateTimeOffset.UtcNow;
Job Old(JobState state, int days, JobKind kind = JobKind.Scan) => new()
{
Id = Guid.CreateVersion7(),
Kind = kind,
State = state,
Created = now.AddDays(-days),
Finished = state is JobState.Done or JobState.Failed or JobState.Cancelled ? now.AddDays(-days) : null,
};
// A render waits: the archive engine has no render handler, so it stays queued however old.
var old = Old(JobState.Done, 100);
var oldFailed = Old(JobState.Failed, 91);
var recent = Old(JobState.Done, 10);
var waiting = Old(JobState.Queued, 200, JobKind.Render);
foreach (var job in new[] { old, oldFailed, recent, waiting }) await fx.Db.Jobs.EnqueueAsync(job, fx.Ct);
var engine = await fx.StartEngineAsync();
var kept = await RunAsync(fx, engine, ArchiveJobs.Cleanup(fx.Archive, [CleanupSteps.JobHistory], 0));
Assert.Contains(Loc.L("Archive.Log.HistoryKept"), kept.LogTail);
Assert.NotNull(await fx.Db.Jobs.GetAsync(old.Id, fx.Ct));
var purged = await RunAsync(fx, engine, ArchiveJobs.Cleanup(fx.Archive, [CleanupSteps.JobHistory], 90));
Assert.Contains(Loc.L("Archive.Log.HistoryPurged", 2), purged.LogTail);
Assert.Null(await fx.Db.Jobs.GetAsync(old.Id, fx.Ct));
Assert.Null(await fx.Db.Jobs.GetAsync(oldFailed.Id, fx.Ct));
Assert.NotNull(await fx.Db.Jobs.GetAsync(recent.Id, fx.Ct));
Assert.NotNull(await fx.Db.Jobs.GetAsync(waiting.Id, fx.Ct));
}
[Test]
public async Task TheAuditStepKeepsEverythingAtZeroDays()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
var now = DateTimeOffset.UtcNow;
foreach (var days in new[] { 400, 1 })
await fx.Db.Audit.AppendAsync(new AuditEntry { Id = Guid.CreateVersion7(), At = now.AddDays(-days), Action = "user.login" }, fx.Ct);
var engine = await fx.StartEngineAsync();
var kept = await RunAsync(fx, engine, ArchiveJobs.Cleanup(null, [CleanupSteps.Audit], 0));
Assert.Contains(Loc.L("Archive.Log.AuditKept"), kept.LogTail);
Assert.Count(2, (await fx.Db.Audit.ListAsync(cancellationToken: fx.Ct)).Items);
var purged = await RunAsync(fx, engine, ArchiveJobs.Cleanup(null, [CleanupSteps.Audit], 365));
Assert.Contains(Loc.L("Archive.Log.AuditPurged", 1), purged.LogTail);
Assert.Count(1, (await fx.Db.Audit.ListAsync(cancellationToken: fx.Ct)).Items);
}
[Test]
public async Task TheTrashReportListsOldEntriesAndNeverDeletes()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
var id = await IngestAsync(fx, "a", 2, 1);
var entry = (await fx.Service.DeleteAsync(await fx.Service.PlanDeleteAsync([id], cancellationToken: fx.Ct), "test", cancellationToken: fx.Ct)).Single();
var folder = Path.Combine(SequenceTrash.TrashDirectory(fx.Paths), entry.Name);
var files = Directory.GetFiles(folder, "*", SearchOption.AllDirectories).Length;
var engine = await fx.StartEngineAsync();
var all = await RunAsync(fx, engine, ArchiveJobs.Cleanup(fx.Archive, [CleanupSteps.TrashReport, "shred"], 0));
Assert.Contains(Loc.L("Archive.Log.TrashReport", 1, CleanupSteps.Megabytes(entry.Bytes)), all.LogTail);
Assert.Contains(entry.Title, all.LogTail);
Assert.Contains(Loc.L("Archive.Log.StepUnknown", "shred"), all.LogTail);
var none = await RunAsync(fx, engine, ArchiveJobs.Cleanup(fx.Archive, [CleanupSteps.TrashReport], 30));
Assert.Contains(Loc.L("Archive.Log.TrashReport", 0, CleanupSteps.Megabytes(0)), none.LogTail);
// Reported, never deleted: emptying the trash stays the user's action.
Assert.Equal(files, Directory.GetFiles(folder, "*", SearchOption.AllDirectories).Length);
Assert.Equal(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name);
var offline = await RunAsync(fx, engine, ArchiveJobs.Cleanup(null, [CleanupSteps.TrashReport], 0));
Assert.Contains(Loc.L("Archive.Log.TrashOffline"), offline.LogTail);
}
[Test]
public async Task AFailingStepFailsTheJobAfterTheOthersRan()
{
var count = new TestStep("count");
var explode = new TestStep("explode", fails: true);
await using var db = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "chronos.db")), context.Cancellation);
var service = new ArchiveService(db, new ArchiveServiceOptions { CleanupSteps = [explode, count, new TestStep(CleanupSteps.Audit)] });
Assert.SequenceEqual([CleanupSteps.JobHistory, CleanupSteps.Audit, CleanupSteps.TrashReport, "explode", "count"], service.CleanupSteps.Select(s => s.Name));
await using var engine = new JobEngine(db, service.CreateJobHandlers(), new JobEngineOptions { WorkerId = "steps", PollInterval = TimeSpan.FromMilliseconds(50) });
await engine.StartAsync(context.Cancellation);
var job = await engine.EnqueueAsync(ArchiveJobs.Cleanup(null, ["explode", "count"], 7), cancellationToken: context.Cancellation);
for (var i = 0; i < 500 && (await db.Jobs.GetAsync(job.Id))!.State is JobState.Queued or JobState.Running; i++) await Task.Delay(20);
var failed = (await db.Jobs.GetAsync(job.Id))!;
Assert.Equal(JobState.Failed, failed.State);
Assert.Contains("explode", failed.Error);
Assert.Contains(Loc.L("Archive.Log.StepFailed", "explode", "disk gone"), failed.LogTail);
Assert.Contains("count ran for 7 days", failed.LogTail);
Assert.Equal((1, 1), (explode.Runs, count.Runs));
}
[Test]
public async Task DuplicatesAndOrphansReportWithoutChangingAnything()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await IngestAsync(fx, "a", 3, 1);
await IngestAsync(fx, "b", 2, 1);
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "New folder"), ArchiveFixture.Start, 1, firstSeed: 60);
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
var engine = await fx.StartEngineAsync();
var duplicates = await RunAsync(fx, engine, ArchiveJobs.Duplicates(fx.Archive));
Assert.Contains(Loc.L("Archive.Log.DuplicatesDone", 2, 2), duplicates.LogTail);
Assert.Contains(Loc.L("Archive.Log.ReportOnly"), duplicates.LogTail);
var orphans = await RunAsync(fx, engine, ArchiveJobs.Orphans(fx.Archive));
Assert.Contains(Loc.L("Archive.Log.FolderUnindexed", "New folder"), orphans.LogTail);
Assert.Contains(Loc.L("Archive.Log.OrphanFolders", 0, 1), orphans.LogTail);
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
Assert.Equal(0L, (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id, fx.Ct)).Health.DuplicateFrames);
}
}