using System.Threading.Channels;
using Chronos.Core.Archiving;
using Chronos.Core.Model;
using Chronos.Data.Archiving;
using Chronos.Data.Archiving.Jobs;
using Chronos.Data.Jobs;
using Chronos.Data.Processing;
using Chronos.Data.Scheduling;
using Chronos.Data.Sqlite;
using Chronos.Data.Tests.Archiving;
using Chronos.Protocol;
using Chronos.Testing;
namespace Chronos.Data.Tests.Scheduling;
///
/// A clock the test sets by hand; its timers are real ones, so the scheduler's loop waits in real time (most tests drive
/// passes themselves; the loop's own tests give it a short tick or wake it).
///
internal sealed class ScheduleClock(DateTimeOffset start) : TimeProvider
{
private long _ticks = start.UtcTicks;
public DateTimeOffset Now => GetUtcNow();
public override DateTimeOffset GetUtcNow() => new(Interlocked.Read(ref _ticks), TimeSpan.Zero);
public void Advance(TimeSpan by) => Interlocked.Add(ref _ticks, by.Ticks);
public void Set(DateTimeOffset at) => Interlocked.Exchange(ref _ticks, at.UtcTicks);
}
/// A host whose archive, system schedules and refusals the test sets.
internal sealed class TestScheduleHost(Archive? archive) : IScheduleHost
{
public Archive? Archive { get; set; } = archive;
public TimeZoneInfo TimeZone { get; set; } = TimeZoneInfo.Utc;
public List Systems { get; } = [];
public Dictionary Refused { get; } = [];
public IReadOnlyList SystemSchedules => Systems;
public IReadOnlyList CleanupSteps { get; set; } = ["job-history", "audit", "trash-report"];
public bool DeepAnalysisAfterScan { get; set; }
/// Awaited before the archive is handed out (a slow drive, a pass the test holds).
public Func? BeforeArchive { get; set; }
public async ValueTask GetArchiveAsync(CancellationToken cancellationToken)
{
if (BeforeArchive is { } wait) await wait(cancellationToken);
return Archive;
}
public string? Unavailable(string action) => Refused.GetValueOrDefault(action);
}
/// The scheduler (owner review 2, R10): due runs fire once, pending work is not queued twice, catch-up, clock changes, edits.
[Category("schedules")]
public sealed class SchedulerTests(TestContext context)
{
private static readonly DateTimeOffset T0 = new(2026, 10, 2, 9, 0, 0, TimeSpan.Zero);
private static readonly RecurrenceDto Every10 = new("every", Every: 10, Unit: "minutes");
private CancellationToken Ct => context.Cancellation;
/// An engine with the archive and analysis handlers that is never started, so queued jobs stay queued.
private static JobEngine Engine(ArchiveFixture fx) =>
new(fx.Db, [.. fx.Service.CreateJobHandlers(), .. new ProcessingService(fx.Db).CreateJobHandlers()], new JobEngineOptions { WorkerId = "test" });
private async Task StartAsync(ChronosDatabase db, JobEngine engine, IScheduleHost host, ScheduleClock clock)
{
var scheduler = new Scheduler(db, engine, new SchedulerOptions { Time = clock, Tick = TimeSpan.FromHours(1) });
await scheduler.StartAsync(host, Ct);
return scheduler;
}
private async Task CreateAsync(Scheduler scheduler, string action, RecurrenceDto? recurrence = null, ScheduleParametersDto? parameters = null,
bool enabled = true, bool catchUp = true)
{
var result = await scheduler.CreateAsync(new CreateScheduleRequest("Test " + action, action, recurrence ?? Every10, parameters, catchUp, Enabled: enabled), null, Ct);
Assert.Null(result.Problem);
return result.Schedule!;
}
[Test]
public async Task ADueScheduleFiresOnceAtItsPriorityWithItsId()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(T0);
await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock);
var schedule = await CreateAsync(scheduler, ScheduleActions.Scan);
Assert.Equal(T0.AddMinutes(10), schedule.NextRun);
Assert.Equal(0, await scheduler.RunDueAsync(Ct));
clock.Advance(TimeSpan.FromMinutes(10));
Assert.Equal(1, await scheduler.RunDueAsync(Ct));
Assert.Equal(0, await scheduler.RunDueAsync(Ct));
var job = (await fx.Db.Jobs.ListByScheduleAsync(schedule.Id, cancellationToken: Ct)).Single();
Assert.Equal((JobKind.Scan, JobPriority.Background, JobState.Queued), (job.Kind, job.Priority, job.State));
Assert.SequenceEqual([fx.Archive.Id], job.TargetIds);
var after = (await scheduler.GetAsync(schedule.Id, Ct))!;
Assert.Equal((ScheduleOutcome.Queued, job.Id, T0.AddMinutes(20), clock.Now), (after.LastOutcome!, after.LastJobId!.Value, after.NextRun!.Value, after.LastRun!.Value));
}
[Test]
public async Task ARunWhosePreviousJobIsPendingIsSkipped()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(T0);
await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock);
var schedule = await CreateAsync(scheduler, ScheduleActions.Duplicates);
clock.Advance(TimeSpan.FromMinutes(10));
await scheduler.RunDueAsync(Ct);
var first = (await scheduler.GetAsync(schedule.Id, Ct))!.LastJobId;
clock.Advance(TimeSpan.FromMinutes(10));
Assert.Equal(1, await scheduler.RunDueAsync(Ct));
var skipped = (await scheduler.GetAsync(schedule.Id, Ct))!;
Assert.Equal((ScheduleOutcome.SkippedPending, first), (skipped.LastOutcome!, skipped.LastJobId));
Assert.Count(1, await fx.Db.Jobs.ListByScheduleAsync(schedule.Id, cancellationToken: Ct));
// Once it ends, the next run queues again; a scan another user queued counts as the same work.
await fx.Db.Jobs.SetStateAsync(first!.Value, JobState.Done, cancellationToken: Ct);
clock.Advance(TimeSpan.FromMinutes(10));
await scheduler.RunDueAsync(Ct);
Assert.Equal(ScheduleOutcome.Queued, (await scheduler.GetAsync(schedule.Id, Ct))!.LastOutcome);
await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: Ct);
var scan = await CreateAsync(scheduler, ScheduleActions.Scan);
Assert.Equal(ScheduleOutcome.SkippedPending, (await scheduler.RunNowAsync(scan.Id, null, Ct))!.Outcome);
}
[Test]
public async Task MissedRunsFireOnceWithCatchUpAndAreSkippedWithout()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(T0);
var host = new TestScheduleHost(fx.Archive);
ScheduleDto catchUp, skip;
await using (var before = await StartAsync(fx.Db, engine, host, clock))
{
catchUp = await CreateAsync(before, ScheduleActions.Scan, new RecurrenceDto("every", Every: 1, Unit: "hours"), catchUp: true);
skip = await CreateAsync(before, ScheduleActions.Orphans, new RecurrenceDto("every", Every: 1, Unit: "hours"), catchUp: false);
}
// The host was off for 5 h 30: five runs of each were missed.
clock.Advance(TimeSpan.FromMinutes(330));
await using var after = await StartAsync(fx.Db, engine, host, clock);
Assert.Count(1, await fx.Db.Jobs.ListByScheduleAsync(catchUp.Id, cancellationToken: Ct));
Assert.Empty(await fx.Db.Jobs.ListByScheduleAsync(skip.Id, cancellationToken: Ct));
var caught = (await after.GetAsync(catchUp.Id, Ct))!;
var skipped = (await after.GetAsync(skip.Id, Ct))!;
Assert.Equal(ScheduleOutcome.Queued, caught.LastOutcome);
Assert.Null(skipped.LastOutcome);
// Both move to the next occurrence on their grid, in the future.
Assert.Equal(T0.AddHours(6), caught.NextRun);
Assert.Equal(T0.AddHours(6), skipped.NextRun);
Assert.Equal(0, await after.RunDueAsync(Ct));
}
/// The server fixes an age limit from its environment there: a run caught up at start must already use it.
[Test]
public async Task AnEditBeforeTheFirstPassAppliesToTheRunCaughtUpAtStart()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(T0);
var host = new TestScheduleHost(fx.Archive);
host.Systems.Add(new SystemSchedule("audit-cleanup", "Audit log clean-up", ScheduleActions.Cleanup, ScheduleRecurrence.Parse("daily:04:30"),
new ScheduleParametersDto(OlderThanDays: 90, Steps: ["audit"])));
await using (await StartAsync(fx.Db, engine, host, clock))
{
}
// The host was off over the run of 04:30.
clock.Advance(TimeSpan.FromDays(1));
await using var scheduler = new Scheduler(fx.Db, engine, new SchedulerOptions { Time = clock, Tick = TimeSpan.FromHours(1) });
var id = Guid.Empty;
await scheduler.StartAsync(host, async ct =>
{
var row = (await fx.Db.Schedules.GetBySystemKeyAsync("audit-cleanup", ct))!;
id = row.Id;
Assert.Empty(await fx.Db.Jobs.ListByScheduleAsync(id, cancellationToken: ct));
var edit = await scheduler.UpdateAsync(id, new UpdateScheduleRequest(Parameters: new ScheduleParametersDto(OlderThanDays: 365, Steps: ["audit"])), ct);
Assert.Null(edit.Problem);
}, Ct);
var jobs = await fx.Db.Jobs.ListByScheduleAsync(id, cancellationToken: Ct);
Assert.Count(1, jobs);
using var parameters = System.Text.Json.JsonDocument.Parse(jobs[0].ParametersJson!);
Assert.Equal(365, parameters.RootElement.GetProperty("olderThanDays").GetInt32());
}
[Test]
public async Task DisabledSchedulesNeverRunAndGetANewNextRunWhenEnabled()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(T0);
await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock);
var disabled = await CreateAsync(scheduler, ScheduleActions.Scan, enabled: false);
Assert.Null(disabled.NextRun);
var enabled = await CreateAsync(scheduler, ScheduleActions.Orphans);
Assert.Null((await scheduler.UpdateAsync(enabled.Id, new UpdateScheduleRequest(Enabled: false), Ct)).Schedule!.NextRun);
clock.Advance(TimeSpan.FromDays(1));
Assert.Equal(0, await scheduler.RunDueAsync(Ct));
Assert.Empty((await fx.Db.Jobs.ListAsync(cancellationToken: Ct)).Items);
var back = (await scheduler.UpdateAsync(enabled.Id, new UpdateScheduleRequest(Enabled: true), Ct)).Schedule!;
Assert.Equal(clock.Now.AddMinutes(10), back.NextRun);
Assert.Equal(0, await scheduler.RunDueAsync(Ct));
// An edit that keeps the recurrence keeps the next run; a new recurrence moves it.
Assert.Equal(back.NextRun, (await scheduler.UpdateAsync(enabled.Id, new UpdateScheduleRequest(Name: "Renamed"), Ct)).Schedule!.NextRun);
var hourly = (await scheduler.UpdateAsync(enabled.Id, new UpdateScheduleRequest(Recurrence: new RecurrenceDto("daily", Time: "03:00")), Ct)).Schedule!;
Assert.Equal(new DateTimeOffset(2026, 10, 4, 3, 0, 0, TimeSpan.Zero), hourly.NextRun);
}
[Test]
public async Task RunNowQueuesForTheUserWithoutMovingTheNextRun()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(T0);
await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock);
var user = Guid.CreateVersion7();
var disabled = await CreateAsync(scheduler, ScheduleActions.Duplicates, enabled: false);
var run = (await scheduler.RunNowAsync(disabled.Id, user, Ct))!;
Assert.Equal(ScheduleOutcome.Queued, run.Outcome);
var job = (await fx.Db.Jobs.GetAsync(run.JobId!.Value, Ct))!;
Assert.Equal((JobKind.Duplicates, user, disabled.Id), (job.Kind, job.CreatedBy!.Value, job.ScheduleId!.Value));
Assert.Null(run.Schedule.NextRun);
Assert.Equal(run.JobId, run.Schedule.LastJobId);
var enabled = await CreateAsync(scheduler, ScheduleActions.Orphans);
Assert.Equal(ScheduleOutcome.Queued, (await scheduler.RunNowAsync(enabled.Id, null, Ct))!.Outcome);
Assert.Equal(enabled.NextRun, (await scheduler.GetAsync(enabled.Id, Ct))!.NextRun);
Assert.Equal(ScheduleOutcome.SkippedPending, (await scheduler.RunNowAsync(enabled.Id, null, Ct))!.Outcome);
Assert.Null(await scheduler.RunNowAsync(Guid.CreateVersion7(), null, Ct));
}
[Test]
public async Task TwoSchedulersOnOneDatabaseFireEachRunOnce()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
var path = ((SqliteDatabase)fx.Db.Database).Path!;
await using var otherDb = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(path), Ct);
await using var engine = Engine(fx);
await using var otherEngine = new JobEngine(otherDb, fx.Service.CreateJobHandlers(), new JobEngineOptions { WorkerId = "other" });
var clock = new ScheduleClock(T0);
await using var a = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock);
await using var b = await StartAsync(otherDb, otherEngine, new TestScheduleHost(fx.Archive), clock);
var schedule = await CreateAsync(a, ScheduleActions.Backup);
for (var round = 1; round <= 4; round++)
{
clock.Advance(TimeSpan.FromMinutes(10));
var fired = await Task.WhenAll(Task.Run(() => a.RunDueAsync(Ct)), Task.Run(() => b.RunDueAsync(Ct)));
Assert.Equal(1, fired.Sum(), $"round {round}");
// The backup jobs never run here: each round queues once, then the next is skipped while it waits.
Assert.Count(1, await fx.Db.Jobs.ListByScheduleAsync(schedule.Id, cancellationToken: Ct));
}
}
[Test]
public async Task ANextRunFarAheadAfterTheClockWentBackIsComputedAgain()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(T0);
await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock);
var schedule = await CreateAsync(scheduler, ScheduleActions.Scan);
clock.Set(T0.AddHours(-5).AddSeconds(20));
Assert.Equal(0, await scheduler.RunDueAsync(Ct));
Assert.Equal(T0.AddHours(-5).AddMinutes(10), (await scheduler.GetAsync(schedule.Id, Ct))!.NextRun);
// Within two periods it is left alone.
clock.Advance(TimeSpan.FromMinutes(-5));
await scheduler.RunDueAsync(Ct);
Assert.Equal(T0.AddHours(-5).AddMinutes(10), (await scheduler.GetAsync(schedule.Id, Ct))!.NextRun);
}
[Test]
public async Task SystemSchedulesAreSeededOnceEditableAndResetNeverDeleted()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(T0);
var host = new TestScheduleHost(fx.Archive);
host.Systems.Add(new SystemSchedule("archive-scan", "Archive scan", ScheduleActions.Scan, ScheduleRecurrence.Parse("every:10m")));
host.Systems.Add(new SystemSchedule("index-backup", "Index backup", ScheduleActions.Backup, ScheduleRecurrence.Parse("daily:03:00"), Enabled: false));
Guid id;
await using (var first = await StartAsync(fx.Db, engine, host, clock))
{
var list = await first.ListAsync(Ct);
Assert.SequenceEqual(["archive-scan", "index-backup"], list.Items.Select(s => s.SystemKey!));
var scan = list.Items[0];
id = scan.Id;
Assert.Equal(T0.AddMinutes(10), scan.NextRun);
Assert.Null(list.Items[1].NextRun);
Assert.Equal("UTC", list.TimeZone);
Assert.SequenceEqual(host.CleanupSteps, list.Steps!);
Assert.Equal("name", (await first.UpdateAsync(id, new UpdateScheduleRequest(Name: "Mine"), Ct)).Problem);
Assert.Equal("action", (await first.UpdateAsync(id, new UpdateScheduleRequest(Action: ScheduleActions.Verify), Ct)).Problem);
Assert.Null((await first.UpdateAsync(id, new UpdateScheduleRequest(Name: "Archive scan", Recurrence: new RecurrenceDto("every", Every: 30, Unit: "minutes")), Ct)).Problem);
Assert.Equal(ScheduleDeleteOutcome.System, await first.DeleteAsync(id, Ct));
}
host.Systems[0] = host.Systems[0] with { Recurrence = ScheduleRecurrence.Parse("every:5m") };
await using var second = await StartAsync(fx.Db, engine, host, clock);
Assert.Equal(new RecurrenceDto("every", Every: 30, Unit: "minutes"), (await second.GetAsync(id, Ct))!.Recurrence);
var reset = (await second.ResetAsync(id, Ct)).Schedule!;
Assert.Equal((new RecurrenceDto("every", Every: 5, Unit: "minutes"), T0.AddMinutes(5)), (reset.Recurrence, reset.NextRun!.Value));
var mine = await CreateAsync(second, ScheduleActions.Orphans);
Assert.Equal("systemKey", (await second.ResetAsync(mine.Id, Ct)).Problem);
Assert.Equal(ScheduleDeleteOutcome.Deleted, await second.DeleteAsync(mine.Id, Ct));
Assert.Equal(ScheduleDeleteOutcome.NotFound, await second.DeleteAsync(mine.Id, Ct));
Assert.True((await second.UpdateAsync(mine.Id, new UpdateScheduleRequest(Enabled: false), Ct)).NotFound);
}
[Test]
public async Task InvalidSchedulesNameTheirField()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), new ScheduleClock(T0));
async Task Problem(CreateScheduleRequest request) => (await scheduler.CreateAsync(request, null, Ct)).Problem;
Assert.Equal("name", await Problem(new CreateScheduleRequest(" ", ScheduleActions.Scan, Every10)));
Assert.Equal("name", await Problem(new CreateScheduleRequest(new string('n', Scheduler.MaxNameLength + 1), ScheduleActions.Scan, Every10)));
Assert.Equal("action", await Problem(new CreateScheduleRequest("Rebuild", "rebuild", Every10)));
Assert.Equal("recurrence", await Problem(new CreateScheduleRequest("Fast", ScheduleActions.Scan, new RecurrenceDto("every", Every: 2, Unit: "minutes"))));
Assert.Equal("parameters.percent", await Problem(new CreateScheduleRequest("Verify", ScheduleActions.Verify, Every10, new ScheduleParametersDto(Percent: 0))));
Assert.Equal("parameters.olderThanDays", await Problem(new CreateScheduleRequest("Verify", ScheduleActions.Verify, Every10, new ScheduleParametersDto(OlderThanDays: -1))));
Assert.Equal("parameters.keep", await Problem(new CreateScheduleRequest("Backup", ScheduleActions.Backup, Every10, new ScheduleParametersDto(Keep: 0))));
Assert.Equal("parameters.steps", await Problem(new CreateScheduleRequest("Clean", ScheduleActions.Cleanup, Every10, new ScheduleParametersDto(Steps: ["shred"]))));
Assert.Equal("parameters.steps", await Problem(new CreateScheduleRequest("Clean", ScheduleActions.Cleanup, Every10, new ScheduleParametersDto(Steps: []))));
Assert.Equal("priority", await Problem(new CreateScheduleRequest("Scan", ScheduleActions.Scan, Every10, Priority: "urgent")));
Assert.Equal("target", await Problem(new CreateScheduleRequest("Scan", ScheduleActions.Scan, Every10, Target: "elsewhere")));
// Parameters are stored normalised: the action's own fields, its defaults for those left out.
var verify = (await scheduler.CreateAsync(new CreateScheduleRequest("Verify", ScheduleActions.Verify, Every10,
new ScheduleParametersDto(Percent: 25, DryRun: true, Steps: ["audit"]), Priority: "user"), null, Ct)).Schedule!;
Assert.Equal((new ScheduleParametersDto(OlderThanDays: 30, Percent: 25), "user"), (verify.Parameters, verify.Priority));
// Another action starts from its own defaults.
var changed = (await scheduler.UpdateAsync(verify.Id, new UpdateScheduleRequest(Action: ScheduleActions.Backup), Ct)).Schedule!;
Assert.Equal(new ScheduleParametersDto(Keep: 3), changed.Parameters);
}
[Test]
public async Task UnavailableActionsAndAMissingArchiveQueueNothing()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var host = new TestScheduleHost(fx.Archive);
host.Refused[ScheduleActions.Thumbnails] = "not-here";
await using var scheduler = await StartAsync(fx.Db, engine, host, new ScheduleClock(T0));
var actions = (await scheduler.ListAsync(Ct)).Actions.ToDictionary(a => a.Name);
Assert.SequenceEqual(ScheduleActions.All, actions.Keys);
Assert.Equal(new ScheduleActionDto(ScheduleActions.Thumbnails, false, "not-here"), actions[ScheduleActions.Thumbnails]);
Assert.True(actions[ScheduleActions.Backup].Available, "an SQLite index file is backed up");
Assert.True(actions[ScheduleActions.Proxies].Available);
var thumbnails = await CreateAsync(scheduler, ScheduleActions.Thumbnails);
Assert.Equal(ScheduleOutcome.Unavailable, (await scheduler.RunNowAsync(thumbnails.Id, null, Ct))!.Outcome);
var proxies = await CreateAsync(scheduler, ScheduleActions.Proxies);
host.Archive = await fx.Service.UpdateSettingsAsync(fx.Archive.Id, s => s with { Proxies = s.Proxies with { LongEdge = ProxyPolicy.Disabled } }, Ct);
Assert.Equal(new ScheduleActionDto(ScheduleActions.Proxies, false, ScheduleUnavailable.ProxiesDisabled),
(await scheduler.ListAsync(Ct)).Actions.Single(a => a.Name == ScheduleActions.Proxies));
Assert.Equal(ScheduleOutcome.Unavailable, (await scheduler.RunNowAsync(proxies.Id, null, Ct))!.Outcome);
// No engine handler for the kind: a desktop without the archive engine, a server without the processing module.
await using var bare = new JobEngine(fx.Db, [], new JobEngineOptions { WorkerId = "bare" });
await using var other = new Scheduler(fx.Db, bare, new SchedulerOptions { Time = new ScheduleClock(T0), Tick = TimeSpan.FromHours(1) });
await other.StartAsync(new TestScheduleHost(fx.Archive), Ct);
Assert.True((await other.ListAsync(Ct)).Actions.All(a => a is { Available: false, Reason: ScheduleUnavailable.NoHandler }));
host.Archive = null;
var scan = await CreateAsync(scheduler, ScheduleActions.Scan);
Assert.Equal(ScheduleOutcome.SkippedOffline, (await scheduler.RunNowAsync(scan.Id, null, Ct))!.Outcome);
// The backup and the clean-up need no archive.
var backup = await CreateAsync(scheduler, ScheduleActions.Backup);
Assert.Equal(ScheduleOutcome.Queued, (await scheduler.RunNowAsync(backup.Id, null, Ct))!.Outcome);
var cleanup = await CreateAsync(scheduler, ScheduleActions.Cleanup);
var cleanupRun = (await scheduler.RunNowAsync(cleanup.Id, null, Ct))!;
Assert.Equal(ScheduleOutcome.Queued, cleanupRun.Outcome);
Assert.Empty((await fx.Db.Jobs.GetAsync(cleanupRun.JobId!.Value, Ct))!.TargetIds);
}
[Test]
public async Task BackupsAreUnavailableWithoutAnIndexFile()
{
await using var db = await ChronosDatabase.OpenAsync(DatabaseOptions.SqliteInMemory(), Ct);
await using var engine = new JobEngine(db, new ArchiveService(db).CreateJobHandlers(), new JobEngineOptions { WorkerId = "memory" });
await using var scheduler = new Scheduler(db, engine, new SchedulerOptions { Time = new ScheduleClock(T0) });
await scheduler.StartAsync(new TestScheduleHost(null), Ct);
Assert.Equal(new ScheduleActionDto(ScheduleActions.Backup, false, ScheduleUnavailable.NoIndexFile),
(await scheduler.ListAsync(Ct)).Actions.Single(a => a.Name == ScheduleActions.Backup));
Assert.Equal(ScheduleOutcome.Unavailable,
(await scheduler.RunNowAsync((await CreateAsync(scheduler, ScheduleActions.Backup)).Id, null, Ct))!.Outcome);
Assert.Equal(ScheduleOutcome.SkippedOffline,
(await scheduler.RunNowAsync((await CreateAsync(scheduler, ScheduleActions.Duplicates)).Id, null, Ct))!.Outcome);
}
[Test]
public async Task VerifyTakesTheLeastRecentlyVerifiedShareOfTheArchive()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(DateTimeOffset.UtcNow);
await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock);
var ids = new List();
for (var i = 0; i < 4; i++) ids.Add(await IngestAsync(fx, "s" + i, 10 * i + 1));
await fx.Db.Sequences.RecordVerifiedAsync(ids[0], SequenceStatus.Verified, SequenceStatus.None, clock.Now.AddDays(-60), Ct);
await fx.Db.Sequences.RecordVerifiedAsync(ids[1], SequenceStatus.Verified, SequenceStatus.None, clock.Now.AddDays(-2), Ct);
var half = await CreateAsync(scheduler, ScheduleActions.Verify, parameters: new ScheduleParametersDto(OlderThanDays: 30, Percent: 50));
var job = (await fx.Db.Jobs.GetAsync((await scheduler.RunNowAsync(half.Id, null, Ct))!.JobId!.Value, Ct))!;
var never = ids.Skip(2).Order().ToList();
Assert.SequenceEqual(never, ArchiveJobJson.Read(job, ArchiveJobJson.Default.VerifyParameters, new VerifyParameters(null)).SequenceIds!.Order());
await fx.Db.Jobs.SetStateAsync(job.Id, JobState.Done, cancellationToken: Ct);
var all = await CreateAsync(scheduler, ScheduleActions.Verify, parameters: new ScheduleParametersDto(OlderThanDays: 0, Percent: 100));
var whole = (await fx.Db.Jobs.GetAsync((await scheduler.RunNowAsync(all.Id, null, Ct))!.JobId!.Value, Ct))!;
Assert.Null(ArchiveJobJson.Read(whole, ArchiveJobJson.Default.VerifyParameters, new VerifyParameters([])).SequenceIds);
await fx.Db.Jobs.SetStateAsync(whole.Id, JobState.Done, cancellationToken: Ct);
foreach (var id in ids) await fx.Db.Sequences.RecordVerifiedAsync(id, SequenceStatus.Verified, SequenceStatus.None, clock.Now.AddDays(-1), Ct);
Assert.Equal(ScheduleOutcome.SkippedNothing, (await scheduler.RunNowAsync(half.Id, null, Ct))!.Outcome);
}
[Test]
public async Task ImagePassesCoverTheSequencesWithoutThemOnce()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), new ScheduleClock(T0));
var a = await IngestAsync(fx, "a", 1);
var b = await IngestAsync(fx, "b", 11);
await fx.Db.Sequences.UpdateStatusAsync(b, SequenceStatus.ThumbsBuilt, SequenceStatus.None, Ct);
var thumbnails = await CreateAsync(scheduler, ScheduleActions.Thumbnails);
var run = (await scheduler.RunNowAsync(thumbnails.Id, null, Ct))!;
var job = (await fx.Db.Jobs.GetAsync(run.JobId!.Value, Ct))!;
Assert.Equal((JobKind.Thumbs, ScheduleOutcome.Queued), (job.Kind, run.Outcome));
Assert.SequenceEqual([a], job.TargetIds);
// Another schedule finds that work pending.
var again = await CreateAsync(scheduler, ScheduleActions.Thumbnails);
Assert.Equal(ScheduleOutcome.SkippedPending, (await scheduler.RunNowAsync(again.Id, null, Ct))!.Outcome);
var forced = await CreateAsync(scheduler, ScheduleActions.Thumbnails, parameters: new ScheduleParametersDto(Force: true));
var forcedRun = (await scheduler.RunNowAsync(forced.Id, null, Ct))!;
Assert.Equal(ScheduleOutcome.Queued, forcedRun.Outcome);
var forcedJob = (await fx.Db.Jobs.GetAsync(forcedRun.JobId!.Value, Ct))!;
Assert.SequenceEqual([b], forcedJob.TargetIds);
var analysis = await CreateAsync(scheduler, ScheduleActions.Analysis);
var light = (await fx.Db.Jobs.GetAsync((await scheduler.RunNowAsync(analysis.Id, null, Ct))!.JobId!.Value, Ct))!;
Assert.Equal(JobKind.LightAnalyse, light.Kind);
Assert.SequenceEqual(new[] { a, b }.Order(), light.TargetIds.Order());
var deep = await CreateAsync(scheduler, ScheduleActions.Analysis, parameters: new ScheduleParametersDto(Deep: true));
var deepJob = (await fx.Db.Jobs.GetAsync((await scheduler.RunNowAsync(deep.Id, null, Ct))!.JobId!.Value, Ct))!;
Assert.Equal((JobKind.DeepAnalyse, JobPriority.Background), (deepJob.Kind, deepJob.Priority));
// Without the analysis engine (a host that registers only the archive's handlers) analysis is not available.
await using var archiveOnly = new JobEngine(fx.Db, fx.Service.CreateJobHandlers(), new JobEngineOptions { WorkerId = "archive" });
await using var other = new Scheduler(fx.Db, archiveOnly, new SchedulerOptions { Time = new ScheduleClock(T0), Tick = TimeSpan.FromHours(1) });
await other.StartAsync(new TestScheduleHost(fx.Archive), Ct);
Assert.Equal(ScheduleOutcome.Unavailable, (await other.RunNowAsync((await CreateAsync(other, ScheduleActions.Analysis)).Id, null, Ct))!.Outcome);
}
///
/// Owner review 2, R10: both hosts scan right after they start (the desktop also when its archive is back), as their
/// polling did, through the archive scan system schedule: once, only while it is enabled, never twice for one start.
///
[Test]
public async Task ASystemScheduleRunsNowOnceWhenEnabledAndNotRunSince()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(T0);
var host = new TestScheduleHost(fx.Archive);
host.Systems.Add(new SystemSchedule("archive-scan", "Archive scan", ScheduleActions.Scan, ScheduleRecurrence.Interval(10, RecurrenceUnit.Minutes), CatchUp: false));
await using var scheduler = await StartAsync(fx.Db, engine, host, clock);
Assert.Null(await scheduler.RunSystemNowAsync("no-such-key", null, Ct));
// Right after the start: the scan is queued, without waiting for its first interval.
var started = clock.Now;
var run = Assert.NotNull(await scheduler.RunSystemNowAsync("archive-scan", started, Ct));
Assert.Equal(ScheduleOutcome.Queued, run.Outcome);
var job = (await fx.Db.Jobs.GetAsync(run.JobId!.Value, Ct))!;
Assert.Equal((JobKind.Scan, JobPriority.Background, (Guid?)run.Schedule.Id), (job.Kind, job.Priority, job.ScheduleId));
Assert.Equal(T0.AddMinutes(10), run.Schedule.NextRun);
// It ran since that start: not again. Asked again later (the archive is back), its pending scan is not queued twice.
Assert.Null(await scheduler.RunSystemNowAsync("archive-scan", started, Ct));
Assert.Equal(ScheduleOutcome.SkippedPending, (await scheduler.RunSystemNowAsync("archive-scan", null, Ct))!.Outcome);
Assert.Count(1, await fx.Db.Jobs.ListByScheduleAsync(run.Schedule.Id, cancellationToken: Ct));
// Turned off, it does not run at start.
Assert.Null((await scheduler.UpdateAsync(run.Schedule.Id, new UpdateScheduleRequest(Enabled: false), Ct)).Problem);
await fx.Db.Jobs.SetStateAsync(job.Id, JobState.Done, cancellationToken: Ct);
Assert.Null(await scheduler.RunSystemNowAsync("archive-scan", null, Ct));
}
///
/// Owner review 2, R10: a scan whose schedule leaves deep unset follows the host at every run (the desktop's Settings ›
/// Analysis › After a scan as it is then), one that sets it keeps its own choice.
///
[Test]
public async Task AScanWithoutItsOwnDeepFollowsTheHostAtEveryRun()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var host = new TestScheduleHost(fx.Archive);
await using var scheduler = await StartAsync(fx.Db, engine, host, new ScheduleClock(T0));
var follows = await CreateAsync(scheduler, ScheduleActions.Scan);
Assert.Null(follows.Parameters.Deep);
var own = await CreateAsync(scheduler, ScheduleActions.Scan, parameters: new ScheduleParametersDto(Deep: false));
Assert.Equal(false, own.Parameters.Deep);
async Task DeepAsync(ScheduleDto schedule)
{
var resolution = await ScheduleActions.ResolveAsync(new ScheduleActionContext(fx.Db, host, _ => true, T0), schedule.Action, schedule.Parameters, Ct);
var request = resolution.Requests.Single();
return System.Text.Json.JsonSerializer.Deserialize(request.ParametersJson!, ArchiveJobJson.Default.ScanParameters)!.DeepAnalysis;
}
Assert.False(await DeepAsync(follows));
host.DeepAnalysisAfterScan = true;
Assert.True(await DeepAsync(follows));
Assert.False(await DeepAsync(own));
// The queued job carries it too.
var job = (await fx.Db.Jobs.GetAsync((await scheduler.RunNowAsync(follows.Id, null, Ct))!.JobId!.Value, Ct))!;
Assert.True(ArchiveJobJson.Read(job, ArchiveJobJson.Default.ScanParameters, new ScanParameters(false)).DeepAnalysis);
}
///
/// Owner review 2, R10: a system schedule that replaced a timer is seeded due when the timer's work was last done a period
/// or more before (the newest index copy a week old: the first start after the upgrade copies at once, as the daily loop
/// did); done more recently, or never, it waits for the recurrence's first occurrence. Rows that exist keep their runs.
///
[Test]
public async Task ASystemScheduleIsSeededDueWhenItsWorkIsAPeriodOld()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(T0);
var host = new TestScheduleHost(fx.Archive);
var daily = ScheduleRecurrence.Interval(1, RecurrenceUnit.Days);
host.Systems.Add(new SystemSchedule("index-backup", "Index backup", ScheduleActions.Backup, daily, LastDone: T0.AddDays(-7)));
host.Systems.Add(new SystemSchedule("nightly-backup", "Nightly", ScheduleActions.Backup, ScheduleRecurrence.DailyAt(new TimeOnly(3, 0)), LastDone: T0.AddHours(-23)));
host.Systems.Add(new SystemSchedule("new-backup", "New", ScheduleActions.Backup, daily));
host.Systems.Add(new SystemSchedule("off-backup", "Off", ScheduleActions.Backup, daily, Enabled: false, LastDone: T0.AddDays(-7)));
await using (var first = await StartAsync(fx.Db, engine, host, clock))
{
var due = (await fx.Db.Schedules.GetBySystemKeyAsync("index-backup", Ct))!;
Assert.Equal((ScheduleOutcome.Queued, T0, T0.AddDays(1)), (due.LastOutcome!, due.LastRun!.Value, due.NextRun!.Value));
var nightly = (await fx.Db.Schedules.GetBySystemKeyAsync("nightly-backup", Ct))!;
Assert.Equal(((string?)null, new DateTimeOffset(2026, 10, 3, 3, 0, 0, TimeSpan.Zero)), (nightly.LastOutcome, nightly.NextRun!.Value));
var fresh = (await fx.Db.Schedules.GetBySystemKeyAsync("new-backup", Ct))!;
Assert.Equal(((string?)null, T0.AddDays(1)), (fresh.LastOutcome, fresh.NextRun!.Value));
var off = (await fx.Db.Schedules.GetBySystemKeyAsync("off-backup", Ct))!;
Assert.Equal(((string?)null, (DateTimeOffset?)null), (off.LastOutcome, off.NextRun));
}
// The next start finds the rows: nothing is due again.
clock.Advance(TimeSpan.FromHours(1));
await using var second = await StartAsync(fx.Db, engine, host, clock);
Assert.Count(1, (await fx.Db.Jobs.ListAsync(cancellationToken: Ct)).Items);
}
/// The loop itself (no pass by hand): its tick fires a due schedule.
[Test]
public async Task TheLoopFiresADueScheduleAtItsTick()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(T0);
await using var scheduler = new Scheduler(fx.Db, engine, new SchedulerOptions { Time = clock, Tick = TimeSpan.FromMilliseconds(100) });
await scheduler.StartAsync(new TestScheduleHost(fx.Archive), Ct);
var changes = Changes(scheduler);
var schedule = await CreateAsync(scheduler, ScheduleActions.Scan);
clock.Advance(TimeSpan.FromMinutes(10));
var ran = await RanAsync(changes, schedule.Id, null);
Assert.Equal((ScheduleOutcome.Queued, T0.AddMinutes(20)), (ran.LastOutcome!, ran.NextRun!.Value));
Assert.Count(1, await fx.Db.Jobs.ListByScheduleAsync(schedule.Id, cancellationToken: Ct));
}
/// makes the loop look now, long before its tick.
[Test]
public async Task WakeFiresADueScheduleBeforeTheTick()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(T0);
await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock);
var changes = Changes(scheduler);
var schedule = await CreateAsync(scheduler, ScheduleActions.Scan);
clock.Advance(TimeSpan.FromMinutes(10));
await Task.Delay(200, Ct);
Assert.Null((await scheduler.GetAsync(schedule.Id, Ct))!.LastOutcome, "the tick is an hour away");
scheduler.Wake();
Assert.Equal(ScheduleOutcome.Queued, (await RanAsync(changes, schedule.Id, null)).LastOutcome);
}
/// Nothing a pass throws (here a subscriber of ) ends the loop: the next run still fires.
[Test]
public async Task TheLoopSurvivesAPassThatThrew()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(T0);
await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock);
var changes = Changes(scheduler);
var thrown = 0;
scheduler.ScheduleChanged += dto =>
{
if (dto.LastOutcome is not null && Interlocked.Exchange(ref thrown, 1) == 0) throw new InvalidOperationException("A subscriber failed");
};
var schedule = await CreateAsync(scheduler, ScheduleActions.Duplicates);
clock.Advance(TimeSpan.FromMinutes(10));
scheduler.Wake();
var first = await RanAsync(changes, schedule.Id, null);
Assert.Equal(1, Volatile.Read(ref thrown));
await fx.Db.Jobs.SetStateAsync(first.LastJobId!.Value, JobState.Done, cancellationToken: Ct);
clock.Advance(TimeSpan.FromMinutes(10));
scheduler.Wake();
var second = await RanAsync(changes, schedule.Id, first.LastJobId);
Assert.Equal((ScheduleOutcome.Queued, T0.AddMinutes(20)), (second.LastOutcome!, second.LastRun!.Value));
Assert.Count(2, await fx.Db.Jobs.ListByScheduleAsync(schedule.Id, cancellationToken: Ct));
}
///
/// Disposing stops the loop during a pass: it waits for a host call that does not watch its token, so the database is
/// never used after it returns, and cancels one that does.
///
[Test]
public async Task DisposeDuringAPassWaitsForItAndStopsIt()
{
await using var fx = await ArchiveFixture.CreateAsync(context);
await using var engine = Engine(fx);
var clock = new ScheduleClock(T0);
var host = new TestScheduleHost(fx.Archive);
var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var scheduler = await StartAsync(fx.Db, engine, host, clock);
var disposed = false;
try
{
await CreateAsync(scheduler, ScheduleActions.Scan);
var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
host.BeforeArchive = async _ =>
{
entered.TrySetResult();
await gate.Task;
};
clock.Advance(TimeSpan.FromMinutes(10));
scheduler.Wake();
await entered.Task.WaitAsync(Budget, Ct);
var disposing = scheduler.DisposeAsync().AsTask();
disposed = true;
await Task.Delay(200, Ct);
Assert.False(disposing.IsCompleted, "disposed while a pass was still running");
gate.SetResult();
await disposing.WaitAsync(Budget, Ct);
}
finally
{
gate.TrySetResult();
if (!disposed) await scheduler.DisposeAsync();
}
// A host call that watches the token ends with it.
var again = await StartAsync(fx.Db, engine, host, clock);
var waiting = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
host.BeforeArchive = async ct =>
{
waiting.TrySetResult();
await Task.Delay(Timeout.Infinite, ct);
};
clock.Advance(TimeSpan.FromMinutes(10));
again.Wake();
await waiting.Task.WaitAsync(Budget, Ct);
await again.DisposeAsync().AsTask().WaitAsync(Budget, Ct);
}
private static TimeSpan Budget => TimeSpan.FromSeconds(TestTiming.Budget(10));
/// Every from now on.
private static Channel Changes(Scheduler scheduler)
{
var changes = Channel.CreateUnbounded();
scheduler.ScheduleChanged += dto => changes.Writer.TryWrite(dto);
return changes;
}
/// The first change of that records a run whose job is not .
private async Task RanAsync(Channel changes, Guid id, Guid? previousJob)
{
while (true)
{
var dto = await changes.Reader.ReadAsync(Ct).AsTask().WaitAsync(Budget, Ct);
if (dto.Id == id && dto.LastOutcome is not null && dto.LastJobId != previousJob) return dto;
}
}
private static async Task IngestAsync(ArchiveFixture fx, string key, int seed)
{
var staged = fx.Stage(ArchiveFixture.Start.AddMinutes(seed), seed);
return (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, key, staged.Path, "IMG_0001.JPG",
staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId;
}
}