Owner review 2, phase 2 review findings (scheduler core). - Log.Write caught only IOException. Microsoft.Extensions.Logging wraps provider failures in an AggregateException, so a log line that could not be written (in tests: the process-wide sink of a stopped server whose log folder was gone) escaped JobEngine.RunAsync's catch before the failure was stored: the job stayed running and the worker faulted. This was the cause of the intermittent JobApiTests/ArchiveModuleTests failures. Log.Write now drops any non-fatal sink failure, TeeLogSink keeps a line from the other sinks, FileLogSink drops a line it cannot write and reopens the file at the next one, and LoggerSink drops what the providers throw. The scheduler's loop relies on the same guarantee. - The daily backup loop (RunDailyAsync, IsDue, Interval, CheckInterval) had no caller since the index-backup schedule replaced it; it is removed and the DatabaseBackups summary describes the schedule. - The first copy after the upgrade no longer waits a day (or for 03:00): SystemSchedule.LastDone seeds a new row due at once when its work was last done a period or more before, and both hosts pass the newest copy in the backups folder for index-backup. - Back up now (Settings and Archive), the copy kept before a restore and a backup job that keeps fewer prune to the largest Copies kept of the backup schedules (DatabaseBackups.KeptAsync), not a fixed three. The four tooltips that promised three daily copies are reworded (en, it) and their stale translations deleted. - New tests for the scheduler loop itself: a tick, Wake, a pass that threw, and dispose during a pass. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> (cherry picked from commit c5ad0c4c3f636129afe5dd1ce039185516da5621)
201 lines
11 KiB
C#
201 lines
11 KiB
C#
using Chronos.Core.Localization;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Data.Archiving;
|
|
using Chronos.Data.Archiving.Jobs;
|
|
using Chronos.Data.Housekeeping;
|
|
using Chronos.Data.Jobs;
|
|
using Chronos.Data.Sqlite;
|
|
using Chronos.Data.Tests.Archiving;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Data.Tests.Scheduling;
|
|
|
|
/// <summary>The jobs schedules add (owner review 2, R10): index backups, clean-up steps, and the read-only duplicates and orphans reports.</summary>
|
|
[Category("schedules")]
|
|
public sealed class HousekeepingJobTests(TestContext context)
|
|
{
|
|
/// <summary>A host step that counts its runs, or fails.</summary>
|
|
private sealed class TestStep(string name, bool fails = false) : ICleanupStep
|
|
{
|
|
public int Runs;
|
|
|
|
public string Name => name;
|
|
|
|
public Task<string> RunAsync(CleanupContext context, CleanupParameters parameters)
|
|
{
|
|
Interlocked.Increment(ref Runs);
|
|
return fails ? throw new IOException("disk gone") : Task.FromResult($"{name} ran for {parameters.OlderThanDays} days");
|
|
}
|
|
}
|
|
|
|
private static async Task<Guid> IngestAsync(ArchiveFixture fx, string key, int count, int seed)
|
|
{
|
|
Guid sequence = default;
|
|
for (var i = 0; i < count; i++)
|
|
{
|
|
var staged = fx.Stage(ArchiveFixture.Start.AddSeconds(10 * i), seed + i);
|
|
sequence = (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, key, staged.Path, $"IMG_{i:0000}.JPG",
|
|
staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId;
|
|
}
|
|
return sequence;
|
|
}
|
|
|
|
private static async Task<Job> RunAsync(ArchiveFixture fx, JobEngine engine, ArchiveJobRequest request, JobState state = JobState.Done)
|
|
{
|
|
var job = await engine.EnqueueAsync(request, cancellationToken: fx.Ct);
|
|
return await fx.WaitForAsync(job.Id, state);
|
|
}
|
|
|
|
[Test]
|
|
public async Task BackupsGoNextToTheIndexAndKeepTheNewest()
|
|
{
|
|
await using var fx = await ArchiveFixture.CreateAsync(context);
|
|
var engine = await fx.StartEngineAsync();
|
|
var path = ((SqliteDatabase)fx.Db.Database).Path!;
|
|
Job done = null!;
|
|
for (var i = 0; i < 3; i++) done = await RunAsync(fx, engine, ArchiveJobs.Backup(keep: 2));
|
|
var backups = DatabaseBackups.List(DatabaseBackups.DefaultDirectory(path), path);
|
|
Assert.Equal(2, backups.Count);
|
|
Assert.Equal(Path.Combine(Path.GetDirectoryName(path)!, "backups"), DatabaseBackups.DefaultDirectory(path));
|
|
Assert.Contains(backups[0].Path, done.LogTail);
|
|
Assert.Equal((1L, 1L), (done.ProgressDone, done.ProgressTotal));
|
|
Assert.Null(DatabaseBackups.Unavailable(fx.Db));
|
|
|
|
// One folder, one retention: a backup that keeps fewer never deletes the copies the index backup schedule keeps.
|
|
await fx.Db.Schedules.EnsureSystemAsync([new Data.Scheduling.SystemSchedule(DatabaseBackups.ScheduleKey, "Index backup", Data.Scheduling.ScheduleActions.Backup,
|
|
Data.Scheduling.ScheduleRecurrence.Parse("daily:03:00"), new Protocol.ScheduleParametersDto(Keep: 4))], DateTimeOffset.UtcNow, fx.Ct);
|
|
for (var i = 0; i < 3; i++) await RunAsync(fx, engine, ArchiveJobs.Backup(keep: 1));
|
|
Assert.Count(4, DatabaseBackups.List(DatabaseBackups.DefaultDirectory(path), path));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ABackupOfAnIndexWithoutAFileFails()
|
|
{
|
|
await using var db = await ChronosDatabase.OpenAsync(DatabaseOptions.SqliteInMemory(), context.Cancellation);
|
|
Assert.Equal(Data.Scheduling.ScheduleUnavailable.NoIndexFile, DatabaseBackups.Unavailable(db));
|
|
await using var engine = new JobEngine(db, new ArchiveService(db).CreateJobHandlers(),
|
|
new JobEngineOptions { WorkerId = "memory", PollInterval = TimeSpan.FromMilliseconds(50) });
|
|
await engine.StartAsync(context.Cancellation);
|
|
var job = await engine.EnqueueAsync(ArchiveJobs.Backup(), cancellationToken: context.Cancellation);
|
|
for (var i = 0; i < 500 && (await db.Jobs.GetAsync(job.Id))!.State is JobState.Queued or JobState.Running; i++) await Task.Delay(20);
|
|
var failed = (await db.Jobs.GetAsync(job.Id))!;
|
|
Assert.Equal(JobState.Failed, failed.State);
|
|
Assert.Contains("without a file", failed.Error);
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheJobHistoryStepRemovesOnlyOldFinishedJobs()
|
|
{
|
|
await using var fx = await ArchiveFixture.CreateAsync(context);
|
|
var now = DateTimeOffset.UtcNow;
|
|
Job Old(JobState state, int days, JobKind kind = JobKind.Scan) => new()
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
Kind = kind,
|
|
State = state,
|
|
Created = now.AddDays(-days),
|
|
Finished = state is JobState.Done or JobState.Failed or JobState.Cancelled ? now.AddDays(-days) : null,
|
|
};
|
|
// A render waits: the archive engine has no render handler, so it stays queued however old.
|
|
var old = Old(JobState.Done, 100);
|
|
var oldFailed = Old(JobState.Failed, 91);
|
|
var recent = Old(JobState.Done, 10);
|
|
var waiting = Old(JobState.Queued, 200, JobKind.Render);
|
|
foreach (var job in new[] { old, oldFailed, recent, waiting }) await fx.Db.Jobs.EnqueueAsync(job, fx.Ct);
|
|
var engine = await fx.StartEngineAsync();
|
|
|
|
var kept = await RunAsync(fx, engine, ArchiveJobs.Cleanup(fx.Archive, [CleanupSteps.JobHistory], 0));
|
|
Assert.Contains(Loc.L("Archive.Log.HistoryKept"), kept.LogTail);
|
|
Assert.NotNull(await fx.Db.Jobs.GetAsync(old.Id, fx.Ct));
|
|
|
|
var purged = await RunAsync(fx, engine, ArchiveJobs.Cleanup(fx.Archive, [CleanupSteps.JobHistory], 90));
|
|
Assert.Contains(Loc.L("Archive.Log.HistoryPurged", 2), purged.LogTail);
|
|
Assert.Null(await fx.Db.Jobs.GetAsync(old.Id, fx.Ct));
|
|
Assert.Null(await fx.Db.Jobs.GetAsync(oldFailed.Id, fx.Ct));
|
|
Assert.NotNull(await fx.Db.Jobs.GetAsync(recent.Id, fx.Ct));
|
|
Assert.NotNull(await fx.Db.Jobs.GetAsync(waiting.Id, fx.Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheAuditStepKeepsEverythingAtZeroDays()
|
|
{
|
|
await using var fx = await ArchiveFixture.CreateAsync(context);
|
|
var now = DateTimeOffset.UtcNow;
|
|
foreach (var days in new[] { 400, 1 })
|
|
await fx.Db.Audit.AppendAsync(new AuditEntry { Id = Guid.CreateVersion7(), At = now.AddDays(-days), Action = "user.login" }, fx.Ct);
|
|
var engine = await fx.StartEngineAsync();
|
|
|
|
var kept = await RunAsync(fx, engine, ArchiveJobs.Cleanup(null, [CleanupSteps.Audit], 0));
|
|
Assert.Contains(Loc.L("Archive.Log.AuditKept"), kept.LogTail);
|
|
Assert.Count(2, (await fx.Db.Audit.ListAsync(cancellationToken: fx.Ct)).Items);
|
|
var purged = await RunAsync(fx, engine, ArchiveJobs.Cleanup(null, [CleanupSteps.Audit], 365));
|
|
Assert.Contains(Loc.L("Archive.Log.AuditPurged", 1), purged.LogTail);
|
|
Assert.Count(1, (await fx.Db.Audit.ListAsync(cancellationToken: fx.Ct)).Items);
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheTrashReportListsOldEntriesAndNeverDeletes()
|
|
{
|
|
await using var fx = await ArchiveFixture.CreateAsync(context);
|
|
var id = await IngestAsync(fx, "a", 2, 1);
|
|
var entry = (await fx.Service.DeleteAsync(await fx.Service.PlanDeleteAsync([id], cancellationToken: fx.Ct), "test", cancellationToken: fx.Ct)).Single();
|
|
var folder = Path.Combine(SequenceTrash.TrashDirectory(fx.Paths), entry.Name);
|
|
var files = Directory.GetFiles(folder, "*", SearchOption.AllDirectories).Length;
|
|
var engine = await fx.StartEngineAsync();
|
|
|
|
var all = await RunAsync(fx, engine, ArchiveJobs.Cleanup(fx.Archive, [CleanupSteps.TrashReport, "shred"], 0));
|
|
Assert.Contains(Loc.L("Archive.Log.TrashReport", 1, CleanupSteps.Megabytes(entry.Bytes)), all.LogTail);
|
|
Assert.Contains(entry.Title, all.LogTail);
|
|
Assert.Contains(Loc.L("Archive.Log.StepUnknown", "shred"), all.LogTail);
|
|
var none = await RunAsync(fx, engine, ArchiveJobs.Cleanup(fx.Archive, [CleanupSteps.TrashReport], 30));
|
|
Assert.Contains(Loc.L("Archive.Log.TrashReport", 0, CleanupSteps.Megabytes(0)), none.LogTail);
|
|
|
|
// Reported, never deleted: emptying the trash stays the user's action.
|
|
Assert.Equal(files, Directory.GetFiles(folder, "*", SearchOption.AllDirectories).Length);
|
|
Assert.Equal(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name);
|
|
var offline = await RunAsync(fx, engine, ArchiveJobs.Cleanup(null, [CleanupSteps.TrashReport], 0));
|
|
Assert.Contains(Loc.L("Archive.Log.TrashOffline"), offline.LogTail);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AFailingStepFailsTheJobAfterTheOthersRan()
|
|
{
|
|
var count = new TestStep("count");
|
|
var explode = new TestStep("explode", fails: true);
|
|
await using var db = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "chronos.db")), context.Cancellation);
|
|
var service = new ArchiveService(db, new ArchiveServiceOptions { CleanupSteps = [explode, count, new TestStep(CleanupSteps.Audit)] });
|
|
Assert.SequenceEqual([CleanupSteps.JobHistory, CleanupSteps.Audit, CleanupSteps.TrashReport, "explode", "count"], service.CleanupSteps.Select(s => s.Name));
|
|
await using var engine = new JobEngine(db, service.CreateJobHandlers(), new JobEngineOptions { WorkerId = "steps", PollInterval = TimeSpan.FromMilliseconds(50) });
|
|
await engine.StartAsync(context.Cancellation);
|
|
var job = await engine.EnqueueAsync(ArchiveJobs.Cleanup(null, ["explode", "count"], 7), cancellationToken: context.Cancellation);
|
|
for (var i = 0; i < 500 && (await db.Jobs.GetAsync(job.Id))!.State is JobState.Queued or JobState.Running; i++) await Task.Delay(20);
|
|
var failed = (await db.Jobs.GetAsync(job.Id))!;
|
|
Assert.Equal(JobState.Failed, failed.State);
|
|
Assert.Contains("explode", failed.Error);
|
|
Assert.Contains(Loc.L("Archive.Log.StepFailed", "explode", "disk gone"), failed.LogTail);
|
|
Assert.Contains("count ran for 7 days", failed.LogTail);
|
|
Assert.Equal((1, 1), (explode.Runs, count.Runs));
|
|
}
|
|
|
|
[Test]
|
|
public async Task DuplicatesAndOrphansReportWithoutChangingAnything()
|
|
{
|
|
await using var fx = await ArchiveFixture.CreateAsync(context);
|
|
await IngestAsync(fx, "a", 3, 1);
|
|
await IngestAsync(fx, "b", 2, 1);
|
|
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "New folder"), ArchiveFixture.Start, 1, firstSeed: 60);
|
|
var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id);
|
|
var engine = await fx.StartEngineAsync();
|
|
|
|
var duplicates = await RunAsync(fx, engine, ArchiveJobs.Duplicates(fx.Archive));
|
|
Assert.Contains(Loc.L("Archive.Log.DuplicatesDone", 2, 2), duplicates.LogTail);
|
|
Assert.Contains(Loc.L("Archive.Log.ReportOnly"), duplicates.LogTail);
|
|
var orphans = await RunAsync(fx, engine, ArchiveJobs.Orphans(fx.Archive));
|
|
Assert.Contains(Loc.L("Archive.Log.FolderUnindexed", "New folder"), orphans.LogTail);
|
|
Assert.Contains(Loc.L("Archive.Log.OrphanFolders", 0, 1), orphans.LogTail);
|
|
|
|
Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id));
|
|
Assert.Equal(0L, (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id, fx.Ct)).Health.DuplicateFrames);
|
|
}
|
|
}
|