Files
Chronos/tests/Chronos.Data.Tests/Scheduling/SchedulerTests.cs
T
Alby96 e384564784 fix: logging never fails a job, one backup retention, first copy after the upgrade
Owner review 2, phase 2 review findings (scheduler core).

- Log.Write caught only IOException. Microsoft.Extensions.Logging wraps
  provider failures in an AggregateException, so a log line that could not
  be written (in tests: the process-wide sink of a stopped server whose log
  folder was gone) escaped JobEngine.RunAsync's catch before the failure was
  stored: the job stayed running and the worker faulted. This was the cause
  of the intermittent JobApiTests/ArchiveModuleTests failures. Log.Write now
  drops any non-fatal sink failure, TeeLogSink keeps a line from the other
  sinks, FileLogSink drops a line it cannot write and reopens the file at the
  next one, and LoggerSink drops what the providers throw. The scheduler's
  loop relies on the same guarantee.
- The daily backup loop (RunDailyAsync, IsDue, Interval, CheckInterval) had
  no caller since the index-backup schedule replaced it; it is removed and
  the DatabaseBackups summary describes the schedule.
- The first copy after the upgrade no longer waits a day (or for 03:00):
  SystemSchedule.LastDone seeds a new row due at once when its work was last
  done a period or more before, and both hosts pass the newest copy in the
  backups folder for index-backup.
- Back up now (Settings and Archive), the copy kept before a restore and a
  backup job that keeps fewer prune to the largest Copies kept of the backup
  schedules (DatabaseBackups.KeptAsync), not a fixed three. The four
  tooltips that promised three daily copies are reworded (en, it) and their
  stale translations deleted.
- New tests for the scheduler loop itself: a tick, Wake, a pass that threw,
  and dispose during a pass.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit c5ad0c4c3f636129afe5dd1ce039185516da5621)
2026-10-27 09:56:47 +01:00

723 lines
42 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using System.Threading.Channels;
using Chronos.Core.Archiving;
using Chronos.Core.Model;
using Chronos.Data.Archiving;
using Chronos.Data.Archiving.Jobs;
using Chronos.Data.Jobs;
using Chronos.Data.Processing;
using Chronos.Data.Scheduling;
using Chronos.Data.Sqlite;
using Chronos.Data.Tests.Archiving;
using Chronos.Protocol;
using Chronos.Testing;
namespace Chronos.Data.Tests.Scheduling;
/// <summary>
/// A clock the test sets by hand; its timers are real ones, so the scheduler's loop waits in real time (most tests drive
/// passes themselves; the loop's own tests give it a short tick or wake it).
/// </summary>
internal sealed class ScheduleClock(DateTimeOffset start) : TimeProvider
{
private long _ticks = start.UtcTicks;
public DateTimeOffset Now => GetUtcNow();
public override DateTimeOffset GetUtcNow() => new(Interlocked.Read(ref _ticks), TimeSpan.Zero);
public void Advance(TimeSpan by) => Interlocked.Add(ref _ticks, by.Ticks);
public void Set(DateTimeOffset at) => Interlocked.Exchange(ref _ticks, at.UtcTicks);
}
/// <summary>A host whose archive, system schedules and refusals the test sets.</summary>
internal sealed class TestScheduleHost(Archive? archive) : IScheduleHost
{
public Archive? Archive { get; set; } = archive;
public TimeZoneInfo TimeZone { get; set; } = TimeZoneInfo.Utc;
public List<SystemSchedule> Systems { get; } = [];
public Dictionary<string, string> Refused { get; } = [];
public IReadOnlyList<SystemSchedule> SystemSchedules => Systems;
public IReadOnlyList<string> CleanupSteps { get; set; } = ["job-history", "audit", "trash-report"];
public bool DeepAnalysisAfterScan { get; set; }
/// <summary>Awaited before the archive is handed out (a slow drive, a pass the test holds).</summary>
public Func<CancellationToken, Task>? BeforeArchive { get; set; }
public async ValueTask<Archive?> GetArchiveAsync(CancellationToken cancellationToken)
{
if (BeforeArchive is { } wait) await wait(cancellationToken);
return Archive;
}
public string? Unavailable(string action) => Refused.GetValueOrDefault(action);
}
/// <summary>The scheduler (owner review 2, R10): due runs fire once, pending work is not queued twice, catch-up, clock changes, edits.</summary>
[Category("schedules")]
public sealed class SchedulerTests(TestContext context)
{
private static readonly DateTimeOffset T0 = new(2026, 10, 2, 9, 0, 0, TimeSpan.Zero);
private static readonly RecurrenceDto Every10 = new("every", Every: 10, Unit: "minutes");
private CancellationToken Ct => context.Cancellation;
/// <summary>An engine with the archive and analysis handlers that is never started, so queued jobs stay queued.</summary>
private static JobEngine Engine(ArchiveFixture fx) =>
new(fx.Db, [.. fx.Service.CreateJobHandlers(), .. new ProcessingService(fx.Db).CreateJobHandlers()], new JobEngineOptions { WorkerId = "test" });
private async Task<Scheduler> StartAsync(ChronosDatabase db, JobEngine engine, IScheduleHost host, ScheduleClock clock)
{
var scheduler = new Scheduler(db, engine, new SchedulerOptions { Time = clock, Tick = TimeSpan.FromHours(1) });
await scheduler.StartAsync(host, Ct);
return scheduler;
}
private async Task<ScheduleDto> CreateAsync(Scheduler scheduler, string action, RecurrenceDto? recurrence = null, ScheduleParametersDto? parameters = null,
bool enabled = true, bool catchUp = true)
{
var result = await scheduler.CreateAsync(new CreateScheduleRequest("Test " + action, action, recurrence ?? Every10, parameters, catchUp, Enabled: enabled), null, Ct);
Assert.Null(result.Problem);
return result.Schedule!;
}
[Test]
public async Task ADueScheduleFiresOnceAtItsPriorityWithItsId()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(T0);
await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock);
var schedule = await CreateAsync(scheduler, ScheduleActions.Scan);
Assert.Equal(T0.AddMinutes(10), schedule.NextRun);
Assert.Equal(0, await scheduler.RunDueAsync(Ct));
clock.Advance(TimeSpan.FromMinutes(10));
Assert.Equal(1, await scheduler.RunDueAsync(Ct));
Assert.Equal(0, await scheduler.RunDueAsync(Ct));
var job = (await fx.Db.Jobs.ListByScheduleAsync(schedule.Id, cancellationToken: Ct)).Single();
Assert.Equal((JobKind.Scan, JobPriority.Background, JobState.Queued), (job.Kind, job.Priority, job.State));
Assert.SequenceEqual([fx.Archive.Id], job.TargetIds);
var after = (await scheduler.GetAsync(schedule.Id, Ct))!;
Assert.Equal((ScheduleOutcome.Queued, job.Id, T0.AddMinutes(20), clock.Now), (after.LastOutcome!, after.LastJobId!.Value, after.NextRun!.Value, after.LastRun!.Value));
}
[Test]
public async Task ARunWhosePreviousJobIsPendingIsSkipped()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(T0);
await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock);
var schedule = await CreateAsync(scheduler, ScheduleActions.Duplicates);
clock.Advance(TimeSpan.FromMinutes(10));
await scheduler.RunDueAsync(Ct);
var first = (await scheduler.GetAsync(schedule.Id, Ct))!.LastJobId;
clock.Advance(TimeSpan.FromMinutes(10));
Assert.Equal(1, await scheduler.RunDueAsync(Ct));
var skipped = (await scheduler.GetAsync(schedule.Id, Ct))!;
Assert.Equal((ScheduleOutcome.SkippedPending, first), (skipped.LastOutcome!, skipped.LastJobId));
Assert.Count(1, await fx.Db.Jobs.ListByScheduleAsync(schedule.Id, cancellationToken: Ct));
// Once it ends, the next run queues again; a scan another user queued counts as the same work.
await fx.Db.Jobs.SetStateAsync(first!.Value, JobState.Done, cancellationToken: Ct);
clock.Advance(TimeSpan.FromMinutes(10));
await scheduler.RunDueAsync(Ct);
Assert.Equal(ScheduleOutcome.Queued, (await scheduler.GetAsync(schedule.Id, Ct))!.LastOutcome);
await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: Ct);
var scan = await CreateAsync(scheduler, ScheduleActions.Scan);
Assert.Equal(ScheduleOutcome.SkippedPending, (await scheduler.RunNowAsync(scan.Id, null, Ct))!.Outcome);
}
[Test]
public async Task MissedRunsFireOnceWithCatchUpAndAreSkippedWithout()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(T0);
var host = new TestScheduleHost(fx.Archive);
ScheduleDto catchUp, skip;
await using (var before = await StartAsync(fx.Db, engine, host, clock))
{
catchUp = await CreateAsync(before, ScheduleActions.Scan, new RecurrenceDto("every", Every: 1, Unit: "hours"), catchUp: true);
skip = await CreateAsync(before, ScheduleActions.Orphans, new RecurrenceDto("every", Every: 1, Unit: "hours"), catchUp: false);
}
// The host was off for 5 h 30: five runs of each were missed.
clock.Advance(TimeSpan.FromMinutes(330));
await using var after = await StartAsync(fx.Db, engine, host, clock);
Assert.Count(1, await fx.Db.Jobs.ListByScheduleAsync(catchUp.Id, cancellationToken: Ct));
Assert.Empty(await fx.Db.Jobs.ListByScheduleAsync(skip.Id, cancellationToken: Ct));
var caught = (await after.GetAsync(catchUp.Id, Ct))!;
var skipped = (await after.GetAsync(skip.Id, Ct))!;
Assert.Equal(ScheduleOutcome.Queued, caught.LastOutcome);
Assert.Null(skipped.LastOutcome);
// Both move to the next occurrence on their grid, in the future.
Assert.Equal(T0.AddHours(6), caught.NextRun);
Assert.Equal(T0.AddHours(6), skipped.NextRun);
Assert.Equal(0, await after.RunDueAsync(Ct));
}
/// <summary>The server fixes an age limit from its environment there: a run caught up at start must already use it.</summary>
[Test]
public async Task AnEditBeforeTheFirstPassAppliesToTheRunCaughtUpAtStart()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(T0);
var host = new TestScheduleHost(fx.Archive);
host.Systems.Add(new SystemSchedule("audit-cleanup", "Audit log clean-up", ScheduleActions.Cleanup, ScheduleRecurrence.Parse("daily:04:30"),
new ScheduleParametersDto(OlderThanDays: 90, Steps: ["audit"])));
await using (await StartAsync(fx.Db, engine, host, clock))
{
}
// The host was off over the run of 04:30.
clock.Advance(TimeSpan.FromDays(1));
await using var scheduler = new Scheduler(fx.Db, engine, new SchedulerOptions { Time = clock, Tick = TimeSpan.FromHours(1) });
var id = Guid.Empty;
await scheduler.StartAsync(host, async ct =>
{
var row = (await fx.Db.Schedules.GetBySystemKeyAsync("audit-cleanup", ct))!;
id = row.Id;
Assert.Empty(await fx.Db.Jobs.ListByScheduleAsync(id, cancellationToken: ct));
var edit = await scheduler.UpdateAsync(id, new UpdateScheduleRequest(Parameters: new ScheduleParametersDto(OlderThanDays: 365, Steps: ["audit"])), ct);
Assert.Null(edit.Problem);
}, Ct);
var jobs = await fx.Db.Jobs.ListByScheduleAsync(id, cancellationToken: Ct);
Assert.Count(1, jobs);
using var parameters = System.Text.Json.JsonDocument.Parse(jobs[0].ParametersJson!);
Assert.Equal(365, parameters.RootElement.GetProperty("olderThanDays").GetInt32());
}
[Test]
public async Task DisabledSchedulesNeverRunAndGetANewNextRunWhenEnabled()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(T0);
await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock);
var disabled = await CreateAsync(scheduler, ScheduleActions.Scan, enabled: false);
Assert.Null(disabled.NextRun);
var enabled = await CreateAsync(scheduler, ScheduleActions.Orphans);
Assert.Null((await scheduler.UpdateAsync(enabled.Id, new UpdateScheduleRequest(Enabled: false), Ct)).Schedule!.NextRun);
clock.Advance(TimeSpan.FromDays(1));
Assert.Equal(0, await scheduler.RunDueAsync(Ct));
Assert.Empty((await fx.Db.Jobs.ListAsync(cancellationToken: Ct)).Items);
var back = (await scheduler.UpdateAsync(enabled.Id, new UpdateScheduleRequest(Enabled: true), Ct)).Schedule!;
Assert.Equal(clock.Now.AddMinutes(10), back.NextRun);
Assert.Equal(0, await scheduler.RunDueAsync(Ct));
// An edit that keeps the recurrence keeps the next run; a new recurrence moves it.
Assert.Equal(back.NextRun, (await scheduler.UpdateAsync(enabled.Id, new UpdateScheduleRequest(Name: "Renamed"), Ct)).Schedule!.NextRun);
var hourly = (await scheduler.UpdateAsync(enabled.Id, new UpdateScheduleRequest(Recurrence: new RecurrenceDto("daily", Time: "03:00")), Ct)).Schedule!;
Assert.Equal(new DateTimeOffset(2026, 10, 4, 3, 0, 0, TimeSpan.Zero), hourly.NextRun);
}
[Test]
public async Task RunNowQueuesForTheUserWithoutMovingTheNextRun()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(T0);
await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock);
var user = Guid.CreateVersion7();
var disabled = await CreateAsync(scheduler, ScheduleActions.Duplicates, enabled: false);
var run = (await scheduler.RunNowAsync(disabled.Id, user, Ct))!;
Assert.Equal(ScheduleOutcome.Queued, run.Outcome);
var job = (await fx.Db.Jobs.GetAsync(run.JobId!.Value, Ct))!;
Assert.Equal((JobKind.Duplicates, user, disabled.Id), (job.Kind, job.CreatedBy!.Value, job.ScheduleId!.Value));
Assert.Null(run.Schedule.NextRun);
Assert.Equal(run.JobId, run.Schedule.LastJobId);
var enabled = await CreateAsync(scheduler, ScheduleActions.Orphans);
Assert.Equal(ScheduleOutcome.Queued, (await scheduler.RunNowAsync(enabled.Id, null, Ct))!.Outcome);
Assert.Equal(enabled.NextRun, (await scheduler.GetAsync(enabled.Id, Ct))!.NextRun);
Assert.Equal(ScheduleOutcome.SkippedPending, (await scheduler.RunNowAsync(enabled.Id, null, Ct))!.Outcome);
Assert.Null(await scheduler.RunNowAsync(Guid.CreateVersion7(), null, Ct));
}
[Test]
public async Task TwoSchedulersOnOneDatabaseFireEachRunOnce()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
var path = ((SqliteDatabase)fx.Db.Database).Path!;
await using var otherDb = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(path), Ct);
await using var engine = Engine(fx);
await using var otherEngine = new JobEngine(otherDb, fx.Service.CreateJobHandlers(), new JobEngineOptions { WorkerId = "other" });
var clock = new ScheduleClock(T0);
await using var a = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock);
await using var b = await StartAsync(otherDb, otherEngine, new TestScheduleHost(fx.Archive), clock);
var schedule = await CreateAsync(a, ScheduleActions.Backup);
for (var round = 1; round <= 4; round++)
{
clock.Advance(TimeSpan.FromMinutes(10));
var fired = await Task.WhenAll(Task.Run(() => a.RunDueAsync(Ct)), Task.Run(() => b.RunDueAsync(Ct)));
Assert.Equal(1, fired.Sum(), $"round {round}");
// The backup jobs never run here: each round queues once, then the next is skipped while it waits.
Assert.Count(1, await fx.Db.Jobs.ListByScheduleAsync(schedule.Id, cancellationToken: Ct));
}
}
[Test]
public async Task ANextRunFarAheadAfterTheClockWentBackIsComputedAgain()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(T0);
await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock);
var schedule = await CreateAsync(scheduler, ScheduleActions.Scan);
clock.Set(T0.AddHours(-5).AddSeconds(20));
Assert.Equal(0, await scheduler.RunDueAsync(Ct));
Assert.Equal(T0.AddHours(-5).AddMinutes(10), (await scheduler.GetAsync(schedule.Id, Ct))!.NextRun);
// Within two periods it is left alone.
clock.Advance(TimeSpan.FromMinutes(-5));
await scheduler.RunDueAsync(Ct);
Assert.Equal(T0.AddHours(-5).AddMinutes(10), (await scheduler.GetAsync(schedule.Id, Ct))!.NextRun);
}
[Test]
public async Task SystemSchedulesAreSeededOnceEditableAndResetNeverDeleted()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(T0);
var host = new TestScheduleHost(fx.Archive);
host.Systems.Add(new SystemSchedule("archive-scan", "Archive scan", ScheduleActions.Scan, ScheduleRecurrence.Parse("every:10m")));
host.Systems.Add(new SystemSchedule("index-backup", "Index backup", ScheduleActions.Backup, ScheduleRecurrence.Parse("daily:03:00"), Enabled: false));
Guid id;
await using (var first = await StartAsync(fx.Db, engine, host, clock))
{
var list = await first.ListAsync(Ct);
Assert.SequenceEqual(["archive-scan", "index-backup"], list.Items.Select(s => s.SystemKey!));
var scan = list.Items[0];
id = scan.Id;
Assert.Equal(T0.AddMinutes(10), scan.NextRun);
Assert.Null(list.Items[1].NextRun);
Assert.Equal("UTC", list.TimeZone);
Assert.SequenceEqual(host.CleanupSteps, list.Steps!);
Assert.Equal("name", (await first.UpdateAsync(id, new UpdateScheduleRequest(Name: "Mine"), Ct)).Problem);
Assert.Equal("action", (await first.UpdateAsync(id, new UpdateScheduleRequest(Action: ScheduleActions.Verify), Ct)).Problem);
Assert.Null((await first.UpdateAsync(id, new UpdateScheduleRequest(Name: "Archive scan", Recurrence: new RecurrenceDto("every", Every: 30, Unit: "minutes")), Ct)).Problem);
Assert.Equal(ScheduleDeleteOutcome.System, await first.DeleteAsync(id, Ct));
}
host.Systems[0] = host.Systems[0] with { Recurrence = ScheduleRecurrence.Parse("every:5m") };
await using var second = await StartAsync(fx.Db, engine, host, clock);
Assert.Equal(new RecurrenceDto("every", Every: 30, Unit: "minutes"), (await second.GetAsync(id, Ct))!.Recurrence);
var reset = (await second.ResetAsync(id, Ct)).Schedule!;
Assert.Equal((new RecurrenceDto("every", Every: 5, Unit: "minutes"), T0.AddMinutes(5)), (reset.Recurrence, reset.NextRun!.Value));
var mine = await CreateAsync(second, ScheduleActions.Orphans);
Assert.Equal("systemKey", (await second.ResetAsync(mine.Id, Ct)).Problem);
Assert.Equal(ScheduleDeleteOutcome.Deleted, await second.DeleteAsync(mine.Id, Ct));
Assert.Equal(ScheduleDeleteOutcome.NotFound, await second.DeleteAsync(mine.Id, Ct));
Assert.True((await second.UpdateAsync(mine.Id, new UpdateScheduleRequest(Enabled: false), Ct)).NotFound);
}
[Test]
public async Task InvalidSchedulesNameTheirField()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), new ScheduleClock(T0));
async Task<string?> Problem(CreateScheduleRequest request) => (await scheduler.CreateAsync(request, null, Ct)).Problem;
Assert.Equal("name", await Problem(new CreateScheduleRequest(" ", ScheduleActions.Scan, Every10)));
Assert.Equal("name", await Problem(new CreateScheduleRequest(new string('n', Scheduler.MaxNameLength + 1), ScheduleActions.Scan, Every10)));
Assert.Equal("action", await Problem(new CreateScheduleRequest("Rebuild", "rebuild", Every10)));
Assert.Equal("recurrence", await Problem(new CreateScheduleRequest("Fast", ScheduleActions.Scan, new RecurrenceDto("every", Every: 2, Unit: "minutes"))));
Assert.Equal("parameters.percent", await Problem(new CreateScheduleRequest("Verify", ScheduleActions.Verify, Every10, new ScheduleParametersDto(Percent: 0))));
Assert.Equal("parameters.olderThanDays", await Problem(new CreateScheduleRequest("Verify", ScheduleActions.Verify, Every10, new ScheduleParametersDto(OlderThanDays: -1))));
Assert.Equal("parameters.keep", await Problem(new CreateScheduleRequest("Backup", ScheduleActions.Backup, Every10, new ScheduleParametersDto(Keep: 0))));
Assert.Equal("parameters.steps", await Problem(new CreateScheduleRequest("Clean", ScheduleActions.Cleanup, Every10, new ScheduleParametersDto(Steps: ["shred"]))));
Assert.Equal("parameters.steps", await Problem(new CreateScheduleRequest("Clean", ScheduleActions.Cleanup, Every10, new ScheduleParametersDto(Steps: []))));
Assert.Equal("priority", await Problem(new CreateScheduleRequest("Scan", ScheduleActions.Scan, Every10, Priority: "urgent")));
Assert.Equal("target", await Problem(new CreateScheduleRequest("Scan", ScheduleActions.Scan, Every10, Target: "elsewhere")));
// Parameters are stored normalised: the action's own fields, its defaults for those left out.
var verify = (await scheduler.CreateAsync(new CreateScheduleRequest("Verify", ScheduleActions.Verify, Every10,
new ScheduleParametersDto(Percent: 25, DryRun: true, Steps: ["audit"]), Priority: "user"), null, Ct)).Schedule!;
Assert.Equal((new ScheduleParametersDto(OlderThanDays: 30, Percent: 25), "user"), (verify.Parameters, verify.Priority));
// Another action starts from its own defaults.
var changed = (await scheduler.UpdateAsync(verify.Id, new UpdateScheduleRequest(Action: ScheduleActions.Backup), Ct)).Schedule!;
Assert.Equal(new ScheduleParametersDto(Keep: 3), changed.Parameters);
}
[Test]
public async Task UnavailableActionsAndAMissingArchiveQueueNothing()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var host = new TestScheduleHost(fx.Archive);
host.Refused[ScheduleActions.Thumbnails] = "not-here";
await using var scheduler = await StartAsync(fx.Db, engine, host, new ScheduleClock(T0));
var actions = (await scheduler.ListAsync(Ct)).Actions.ToDictionary(a => a.Name);
Assert.SequenceEqual(ScheduleActions.All, actions.Keys);
Assert.Equal(new ScheduleActionDto(ScheduleActions.Thumbnails, false, "not-here"), actions[ScheduleActions.Thumbnails]);
Assert.True(actions[ScheduleActions.Backup].Available, "an SQLite index file is backed up");
Assert.True(actions[ScheduleActions.Proxies].Available);
var thumbnails = await CreateAsync(scheduler, ScheduleActions.Thumbnails);
Assert.Equal(ScheduleOutcome.Unavailable, (await scheduler.RunNowAsync(thumbnails.Id, null, Ct))!.Outcome);
var proxies = await CreateAsync(scheduler, ScheduleActions.Proxies);
host.Archive = await fx.Service.UpdateSettingsAsync(fx.Archive.Id, s => s with { Proxies = s.Proxies with { LongEdge = ProxyPolicy.Disabled } }, Ct);
Assert.Equal(new ScheduleActionDto(ScheduleActions.Proxies, false, ScheduleUnavailable.ProxiesDisabled),
(await scheduler.ListAsync(Ct)).Actions.Single(a => a.Name == ScheduleActions.Proxies));
Assert.Equal(ScheduleOutcome.Unavailable, (await scheduler.RunNowAsync(proxies.Id, null, Ct))!.Outcome);
// No engine handler for the kind: a desktop without the archive engine, a server without the processing module.
await using var bare = new JobEngine(fx.Db, [], new JobEngineOptions { WorkerId = "bare" });
await using var other = new Scheduler(fx.Db, bare, new SchedulerOptions { Time = new ScheduleClock(T0), Tick = TimeSpan.FromHours(1) });
await other.StartAsync(new TestScheduleHost(fx.Archive), Ct);
Assert.True((await other.ListAsync(Ct)).Actions.All(a => a is { Available: false, Reason: ScheduleUnavailable.NoHandler }));
host.Archive = null;
var scan = await CreateAsync(scheduler, ScheduleActions.Scan);
Assert.Equal(ScheduleOutcome.SkippedOffline, (await scheduler.RunNowAsync(scan.Id, null, Ct))!.Outcome);
// The backup and the clean-up need no archive.
var backup = await CreateAsync(scheduler, ScheduleActions.Backup);
Assert.Equal(ScheduleOutcome.Queued, (await scheduler.RunNowAsync(backup.Id, null, Ct))!.Outcome);
var cleanup = await CreateAsync(scheduler, ScheduleActions.Cleanup);
var cleanupRun = (await scheduler.RunNowAsync(cleanup.Id, null, Ct))!;
Assert.Equal(ScheduleOutcome.Queued, cleanupRun.Outcome);
Assert.Empty((await fx.Db.Jobs.GetAsync(cleanupRun.JobId!.Value, Ct))!.TargetIds);
}
[Test]
public async Task BackupsAreUnavailableWithoutAnIndexFile()
{
await using var db = await ChronosDatabase.OpenAsync(DatabaseOptions.SqliteInMemory(), Ct);
await using var engine = new JobEngine(db, new ArchiveService(db).CreateJobHandlers(), new JobEngineOptions { WorkerId = "memory" });
await using var scheduler = new Scheduler(db, engine, new SchedulerOptions { Time = new ScheduleClock(T0) });
await scheduler.StartAsync(new TestScheduleHost(null), Ct);
Assert.Equal(new ScheduleActionDto(ScheduleActions.Backup, false, ScheduleUnavailable.NoIndexFile),
(await scheduler.ListAsync(Ct)).Actions.Single(a => a.Name == ScheduleActions.Backup));
Assert.Equal(ScheduleOutcome.Unavailable,
(await scheduler.RunNowAsync((await CreateAsync(scheduler, ScheduleActions.Backup)).Id, null, Ct))!.Outcome);
Assert.Equal(ScheduleOutcome.SkippedOffline,
(await scheduler.RunNowAsync((await CreateAsync(scheduler, ScheduleActions.Duplicates)).Id, null, Ct))!.Outcome);
}
[Test]
public async Task VerifyTakesTheLeastRecentlyVerifiedShareOfTheArchive()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(DateTimeOffset.UtcNow);
await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock);
var ids = new List<Guid>();
for (var i = 0; i < 4; i++) ids.Add(await IngestAsync(fx, "s" + i, 10 * i + 1));
await fx.Db.Sequences.RecordVerifiedAsync(ids[0], SequenceStatus.Verified, SequenceStatus.None, clock.Now.AddDays(-60), Ct);
await fx.Db.Sequences.RecordVerifiedAsync(ids[1], SequenceStatus.Verified, SequenceStatus.None, clock.Now.AddDays(-2), Ct);
var half = await CreateAsync(scheduler, ScheduleActions.Verify, parameters: new ScheduleParametersDto(OlderThanDays: 30, Percent: 50));
var job = (await fx.Db.Jobs.GetAsync((await scheduler.RunNowAsync(half.Id, null, Ct))!.JobId!.Value, Ct))!;
var never = ids.Skip(2).Order().ToList();
Assert.SequenceEqual(never, ArchiveJobJson.Read(job, ArchiveJobJson.Default.VerifyParameters, new VerifyParameters(null)).SequenceIds!.Order());
await fx.Db.Jobs.SetStateAsync(job.Id, JobState.Done, cancellationToken: Ct);
var all = await CreateAsync(scheduler, ScheduleActions.Verify, parameters: new ScheduleParametersDto(OlderThanDays: 0, Percent: 100));
var whole = (await fx.Db.Jobs.GetAsync((await scheduler.RunNowAsync(all.Id, null, Ct))!.JobId!.Value, Ct))!;
Assert.Null(ArchiveJobJson.Read(whole, ArchiveJobJson.Default.VerifyParameters, new VerifyParameters([])).SequenceIds);
await fx.Db.Jobs.SetStateAsync(whole.Id, JobState.Done, cancellationToken: Ct);
foreach (var id in ids) await fx.Db.Sequences.RecordVerifiedAsync(id, SequenceStatus.Verified, SequenceStatus.None, clock.Now.AddDays(-1), Ct);
Assert.Equal(ScheduleOutcome.SkippedNothing, (await scheduler.RunNowAsync(half.Id, null, Ct))!.Outcome);
}
[Test]
public async Task ImagePassesCoverTheSequencesWithoutThemOnce()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), new ScheduleClock(T0));
var a = await IngestAsync(fx, "a", 1);
var b = await IngestAsync(fx, "b", 11);
await fx.Db.Sequences.UpdateStatusAsync(b, SequenceStatus.ThumbsBuilt, SequenceStatus.None, Ct);
var thumbnails = await CreateAsync(scheduler, ScheduleActions.Thumbnails);
var run = (await scheduler.RunNowAsync(thumbnails.Id, null, Ct))!;
var job = (await fx.Db.Jobs.GetAsync(run.JobId!.Value, Ct))!;
Assert.Equal((JobKind.Thumbs, ScheduleOutcome.Queued), (job.Kind, run.Outcome));
Assert.SequenceEqual([a], job.TargetIds);
// Another schedule finds that work pending.
var again = await CreateAsync(scheduler, ScheduleActions.Thumbnails);
Assert.Equal(ScheduleOutcome.SkippedPending, (await scheduler.RunNowAsync(again.Id, null, Ct))!.Outcome);
var forced = await CreateAsync(scheduler, ScheduleActions.Thumbnails, parameters: new ScheduleParametersDto(Force: true));
var forcedRun = (await scheduler.RunNowAsync(forced.Id, null, Ct))!;
Assert.Equal(ScheduleOutcome.Queued, forcedRun.Outcome);
var forcedJob = (await fx.Db.Jobs.GetAsync(forcedRun.JobId!.Value, Ct))!;
Assert.SequenceEqual([b], forcedJob.TargetIds);
var analysis = await CreateAsync(scheduler, ScheduleActions.Analysis);
var light = (await fx.Db.Jobs.GetAsync((await scheduler.RunNowAsync(analysis.Id, null, Ct))!.JobId!.Value, Ct))!;
Assert.Equal(JobKind.LightAnalyse, light.Kind);
Assert.SequenceEqual(new[] { a, b }.Order(), light.TargetIds.Order());
var deep = await CreateAsync(scheduler, ScheduleActions.Analysis, parameters: new ScheduleParametersDto(Deep: true));
var deepJob = (await fx.Db.Jobs.GetAsync((await scheduler.RunNowAsync(deep.Id, null, Ct))!.JobId!.Value, Ct))!;
Assert.Equal((JobKind.DeepAnalyse, JobPriority.Background), (deepJob.Kind, deepJob.Priority));
// Without the analysis engine (a host that registers only the archive's handlers) analysis is not available.
await using var archiveOnly = new JobEngine(fx.Db, fx.Service.CreateJobHandlers(), new JobEngineOptions { WorkerId = "archive" });
await using var other = new Scheduler(fx.Db, archiveOnly, new SchedulerOptions { Time = new ScheduleClock(T0), Tick = TimeSpan.FromHours(1) });
await other.StartAsync(new TestScheduleHost(fx.Archive), Ct);
Assert.Equal(ScheduleOutcome.Unavailable, (await other.RunNowAsync((await CreateAsync(other, ScheduleActions.Analysis)).Id, null, Ct))!.Outcome);
}
/// <summary>
/// Owner review 2, R10: both hosts scan right after they start (the desktop also when its archive is back), as their
/// polling did, through the archive scan system schedule: once, only while it is enabled, never twice for one start.
/// </summary>
[Test]
public async Task ASystemScheduleRunsNowOnceWhenEnabledAndNotRunSince()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(T0);
var host = new TestScheduleHost(fx.Archive);
host.Systems.Add(new SystemSchedule("archive-scan", "Archive scan", ScheduleActions.Scan, ScheduleRecurrence.Interval(10, RecurrenceUnit.Minutes), CatchUp: false));
await using var scheduler = await StartAsync(fx.Db, engine, host, clock);
Assert.Null(await scheduler.RunSystemNowAsync("no-such-key", null, Ct));
// Right after the start: the scan is queued, without waiting for its first interval.
var started = clock.Now;
var run = Assert.NotNull(await scheduler.RunSystemNowAsync("archive-scan", started, Ct));
Assert.Equal(ScheduleOutcome.Queued, run.Outcome);
var job = (await fx.Db.Jobs.GetAsync(run.JobId!.Value, Ct))!;
Assert.Equal((JobKind.Scan, JobPriority.Background, (Guid?)run.Schedule.Id), (job.Kind, job.Priority, job.ScheduleId));
Assert.Equal(T0.AddMinutes(10), run.Schedule.NextRun);
// It ran since that start: not again. Asked again later (the archive is back), its pending scan is not queued twice.
Assert.Null(await scheduler.RunSystemNowAsync("archive-scan", started, Ct));
Assert.Equal(ScheduleOutcome.SkippedPending, (await scheduler.RunSystemNowAsync("archive-scan", null, Ct))!.Outcome);
Assert.Count(1, await fx.Db.Jobs.ListByScheduleAsync(run.Schedule.Id, cancellationToken: Ct));
// Turned off, it does not run at start.
Assert.Null((await scheduler.UpdateAsync(run.Schedule.Id, new UpdateScheduleRequest(Enabled: false), Ct)).Problem);
await fx.Db.Jobs.SetStateAsync(job.Id, JobState.Done, cancellationToken: Ct);
Assert.Null(await scheduler.RunSystemNowAsync("archive-scan", null, Ct));
}
/// <summary>
/// Owner review 2, R10: a scan whose schedule leaves deep unset follows the host at every run (the desktop's Settings ›
/// Analysis › After a scan as it is then), one that sets it keeps its own choice.
/// </summary>
[Test]
public async Task AScanWithoutItsOwnDeepFollowsTheHostAtEveryRun()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var host = new TestScheduleHost(fx.Archive);
await using var scheduler = await StartAsync(fx.Db, engine, host, new ScheduleClock(T0));
var follows = await CreateAsync(scheduler, ScheduleActions.Scan);
Assert.Null(follows.Parameters.Deep);
var own = await CreateAsync(scheduler, ScheduleActions.Scan, parameters: new ScheduleParametersDto(Deep: false));
Assert.Equal(false, own.Parameters.Deep);
async Task<bool> DeepAsync(ScheduleDto schedule)
{
var resolution = await ScheduleActions.ResolveAsync(new ScheduleActionContext(fx.Db, host, _ => true, T0), schedule.Action, schedule.Parameters, Ct);
var request = resolution.Requests.Single();
return System.Text.Json.JsonSerializer.Deserialize(request.ParametersJson!, ArchiveJobJson.Default.ScanParameters)!.DeepAnalysis;
}
Assert.False(await DeepAsync(follows));
host.DeepAnalysisAfterScan = true;
Assert.True(await DeepAsync(follows));
Assert.False(await DeepAsync(own));
// The queued job carries it too.
var job = (await fx.Db.Jobs.GetAsync((await scheduler.RunNowAsync(follows.Id, null, Ct))!.JobId!.Value, Ct))!;
Assert.True(ArchiveJobJson.Read(job, ArchiveJobJson.Default.ScanParameters, new ScanParameters(false)).DeepAnalysis);
}
/// <summary>
/// Owner review 2, R10: a system schedule that replaced a timer is seeded due when the timer's work was last done a period
/// or more before (the newest index copy a week old: the first start after the upgrade copies at once, as the daily loop
/// did); done more recently, or never, it waits for the recurrence's first occurrence. Rows that exist keep their runs.
/// </summary>
[Test]
public async Task ASystemScheduleIsSeededDueWhenItsWorkIsAPeriodOld()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(T0);
var host = new TestScheduleHost(fx.Archive);
var daily = ScheduleRecurrence.Interval(1, RecurrenceUnit.Days);
host.Systems.Add(new SystemSchedule("index-backup", "Index backup", ScheduleActions.Backup, daily, LastDone: T0.AddDays(-7)));
host.Systems.Add(new SystemSchedule("nightly-backup", "Nightly", ScheduleActions.Backup, ScheduleRecurrence.DailyAt(new TimeOnly(3, 0)), LastDone: T0.AddHours(-23)));
host.Systems.Add(new SystemSchedule("new-backup", "New", ScheduleActions.Backup, daily));
host.Systems.Add(new SystemSchedule("off-backup", "Off", ScheduleActions.Backup, daily, Enabled: false, LastDone: T0.AddDays(-7)));
await using (var first = await StartAsync(fx.Db, engine, host, clock))
{
var due = (await fx.Db.Schedules.GetBySystemKeyAsync("index-backup", Ct))!;
Assert.Equal((ScheduleOutcome.Queued, T0, T0.AddDays(1)), (due.LastOutcome!, due.LastRun!.Value, due.NextRun!.Value));
var nightly = (await fx.Db.Schedules.GetBySystemKeyAsync("nightly-backup", Ct))!;
Assert.Equal(((string?)null, new DateTimeOffset(2026, 10, 3, 3, 0, 0, TimeSpan.Zero)), (nightly.LastOutcome, nightly.NextRun!.Value));
var fresh = (await fx.Db.Schedules.GetBySystemKeyAsync("new-backup", Ct))!;
Assert.Equal(((string?)null, T0.AddDays(1)), (fresh.LastOutcome, fresh.NextRun!.Value));
var off = (await fx.Db.Schedules.GetBySystemKeyAsync("off-backup", Ct))!;
Assert.Equal(((string?)null, (DateTimeOffset?)null), (off.LastOutcome, off.NextRun));
}
// The next start finds the rows: nothing is due again.
clock.Advance(TimeSpan.FromHours(1));
await using var second = await StartAsync(fx.Db, engine, host, clock);
Assert.Count(1, (await fx.Db.Jobs.ListAsync(cancellationToken: Ct)).Items);
}
/// <summary>The loop itself (no pass by hand): its tick fires a due schedule.</summary>
[Test]
public async Task TheLoopFiresADueScheduleAtItsTick()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(T0);
await using var scheduler = new Scheduler(fx.Db, engine, new SchedulerOptions { Time = clock, Tick = TimeSpan.FromMilliseconds(100) });
await scheduler.StartAsync(new TestScheduleHost(fx.Archive), Ct);
var changes = Changes(scheduler);
var schedule = await CreateAsync(scheduler, ScheduleActions.Scan);
clock.Advance(TimeSpan.FromMinutes(10));
var ran = await RanAsync(changes, schedule.Id, null);
Assert.Equal((ScheduleOutcome.Queued, T0.AddMinutes(20)), (ran.LastOutcome!, ran.NextRun!.Value));
Assert.Count(1, await fx.Db.Jobs.ListByScheduleAsync(schedule.Id, cancellationToken: Ct));
}
/// <summary><see cref="Scheduler.Wake"/> makes the loop look now, long before its tick.</summary>
[Test]
public async Task WakeFiresADueScheduleBeforeTheTick()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(T0);
await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock);
var changes = Changes(scheduler);
var schedule = await CreateAsync(scheduler, ScheduleActions.Scan);
clock.Advance(TimeSpan.FromMinutes(10));
await Task.Delay(200, Ct);
Assert.Null((await scheduler.GetAsync(schedule.Id, Ct))!.LastOutcome, "the tick is an hour away");
scheduler.Wake();
Assert.Equal(ScheduleOutcome.Queued, (await RanAsync(changes, schedule.Id, null)).LastOutcome);
}
/// <summary>Nothing a pass throws (here a subscriber of <see cref="Scheduler.ScheduleChanged"/>) ends the loop: the next run still fires.</summary>
[Test]
public async Task TheLoopSurvivesAPassThatThrew()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(T0);
await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock);
var changes = Changes(scheduler);
var thrown = 0;
scheduler.ScheduleChanged += dto =>
{
if (dto.LastOutcome is not null && Interlocked.Exchange(ref thrown, 1) == 0) throw new InvalidOperationException("A subscriber failed");
};
var schedule = await CreateAsync(scheduler, ScheduleActions.Duplicates);
clock.Advance(TimeSpan.FromMinutes(10));
scheduler.Wake();
var first = await RanAsync(changes, schedule.Id, null);
Assert.Equal(1, Volatile.Read(ref thrown));
await fx.Db.Jobs.SetStateAsync(first.LastJobId!.Value, JobState.Done, cancellationToken: Ct);
clock.Advance(TimeSpan.FromMinutes(10));
scheduler.Wake();
var second = await RanAsync(changes, schedule.Id, first.LastJobId);
Assert.Equal((ScheduleOutcome.Queued, T0.AddMinutes(20)), (second.LastOutcome!, second.LastRun!.Value));
Assert.Count(2, await fx.Db.Jobs.ListByScheduleAsync(schedule.Id, cancellationToken: Ct));
}
/// <summary>
/// Disposing stops the loop during a pass: it waits for a host call that does not watch its token, so the database is
/// never used after it returns, and cancels one that does.
/// </summary>
[Test]
public async Task DisposeDuringAPassWaitsForItAndStopsIt()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(T0);
var host = new TestScheduleHost(fx.Archive);
var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var scheduler = await StartAsync(fx.Db, engine, host, clock);
var disposed = false;
try
{
await CreateAsync(scheduler, ScheduleActions.Scan);
var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
host.BeforeArchive = async _ =>
{
entered.TrySetResult();
await gate.Task;
};
clock.Advance(TimeSpan.FromMinutes(10));
scheduler.Wake();
await entered.Task.WaitAsync(Budget, Ct);
var disposing = scheduler.DisposeAsync().AsTask();
disposed = true;
await Task.Delay(200, Ct);
Assert.False(disposing.IsCompleted, "disposed while a pass was still running");
gate.SetResult();
await disposing.WaitAsync(Budget, Ct);
}
finally
{
gate.TrySetResult();
if (!disposed) await scheduler.DisposeAsync();
}
// A host call that watches the token ends with it.
var again = await StartAsync(fx.Db, engine, host, clock);
var waiting = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
host.BeforeArchive = async ct =>
{
waiting.TrySetResult();
await Task.Delay(Timeout.Infinite, ct);
};
clock.Advance(TimeSpan.FromMinutes(10));
again.Wake();
await waiting.Task.WaitAsync(Budget, Ct);
await again.DisposeAsync().AsTask().WaitAsync(Budget, Ct);
}
private static TimeSpan Budget => TimeSpan.FromSeconds(TestTiming.Budget(10));
/// <summary>Every <see cref="Scheduler.ScheduleChanged"/> from now on.</summary>
private static Channel<ScheduleDto> Changes(Scheduler scheduler)
{
var changes = Channel.CreateUnbounded<ScheduleDto>();
scheduler.ScheduleChanged += dto => changes.Writer.TryWrite(dto);
return changes;
}
/// <summary>The first change of <paramref name="id"/> that records a run whose job is not <paramref name="previousJob"/>.</summary>
private async Task<ScheduleDto> RanAsync(Channel<ScheduleDto> changes, Guid id, Guid? previousJob)
{
while (true)
{
var dto = await changes.Reader.ReadAsync(Ct).AsTask().WaitAsync(Budget, Ct);
if (dto.Id == id && dto.LastOutcome is not null && dto.LastJobId != previousJob) return dto;
}
}
private static async Task<Guid> IngestAsync(ArchiveFixture fx, string key, int seed)
{
var staged = fx.Stage(ArchiveFixture.Start.AddMinutes(seed), seed);
return (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, key, staged.Path, "IMG_0001.JPG",
staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId;
}
}