using System.Threading.Channels; using Chronos.Core.Archiving; using Chronos.Core.Model; using Chronos.Data.Archiving; using Chronos.Data.Archiving.Jobs; using Chronos.Data.Jobs; using Chronos.Data.Processing; using Chronos.Data.Scheduling; using Chronos.Data.Sqlite; using Chronos.Data.Tests.Archiving; using Chronos.Protocol; using Chronos.Testing; namespace Chronos.Data.Tests.Scheduling; /// /// A clock the test sets by hand; its timers are real ones, so the scheduler's loop waits in real time (most tests drive /// passes themselves; the loop's own tests give it a short tick or wake it). /// internal sealed class ScheduleClock(DateTimeOffset start) : TimeProvider { private long _ticks = start.UtcTicks; public DateTimeOffset Now => GetUtcNow(); public override DateTimeOffset GetUtcNow() => new(Interlocked.Read(ref _ticks), TimeSpan.Zero); public void Advance(TimeSpan by) => Interlocked.Add(ref _ticks, by.Ticks); public void Set(DateTimeOffset at) => Interlocked.Exchange(ref _ticks, at.UtcTicks); } /// A host whose archive, system schedules and refusals the test sets. internal sealed class TestScheduleHost(Archive? archive) : IScheduleHost { public Archive? Archive { get; set; } = archive; public TimeZoneInfo TimeZone { get; set; } = TimeZoneInfo.Utc; public List Systems { get; } = []; public Dictionary Refused { get; } = []; public IReadOnlyList SystemSchedules => Systems; public IReadOnlyList CleanupSteps { get; set; } = ["job-history", "audit", "trash-report"]; public bool DeepAnalysisAfterScan { get; set; } /// Awaited before the archive is handed out (a slow drive, a pass the test holds). public Func? BeforeArchive { get; set; } public async ValueTask GetArchiveAsync(CancellationToken cancellationToken) { if (BeforeArchive is { } wait) await wait(cancellationToken); return Archive; } public string? Unavailable(string action) => Refused.GetValueOrDefault(action); } /// The scheduler (owner review 2, R10): due runs fire once, pending work is not queued twice, catch-up, clock changes, edits. [Category("schedules")] public sealed class SchedulerTests(TestContext context) { private static readonly DateTimeOffset T0 = new(2026, 10, 2, 9, 0, 0, TimeSpan.Zero); private static readonly RecurrenceDto Every10 = new("every", Every: 10, Unit: "minutes"); private CancellationToken Ct => context.Cancellation; /// An engine with the archive and analysis handlers that is never started, so queued jobs stay queued. private static JobEngine Engine(ArchiveFixture fx) => new(fx.Db, [.. fx.Service.CreateJobHandlers(), .. new ProcessingService(fx.Db).CreateJobHandlers()], new JobEngineOptions { WorkerId = "test" }); private async Task StartAsync(ChronosDatabase db, JobEngine engine, IScheduleHost host, ScheduleClock clock) { var scheduler = new Scheduler(db, engine, new SchedulerOptions { Time = clock, Tick = TimeSpan.FromHours(1) }); await scheduler.StartAsync(host, Ct); return scheduler; } private async Task CreateAsync(Scheduler scheduler, string action, RecurrenceDto? recurrence = null, ScheduleParametersDto? parameters = null, bool enabled = true, bool catchUp = true) { var result = await scheduler.CreateAsync(new CreateScheduleRequest("Test " + action, action, recurrence ?? Every10, parameters, catchUp, Enabled: enabled), null, Ct); Assert.Null(result.Problem); return result.Schedule!; } [Test] public async Task ADueScheduleFiresOnceAtItsPriorityWithItsId() { await using var fx = await ArchiveFixture.CreateAsync(context); await using var engine = Engine(fx); var clock = new ScheduleClock(T0); await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock); var schedule = await CreateAsync(scheduler, ScheduleActions.Scan); Assert.Equal(T0.AddMinutes(10), schedule.NextRun); Assert.Equal(0, await scheduler.RunDueAsync(Ct)); clock.Advance(TimeSpan.FromMinutes(10)); Assert.Equal(1, await scheduler.RunDueAsync(Ct)); Assert.Equal(0, await scheduler.RunDueAsync(Ct)); var job = (await fx.Db.Jobs.ListByScheduleAsync(schedule.Id, cancellationToken: Ct)).Single(); Assert.Equal((JobKind.Scan, JobPriority.Background, JobState.Queued), (job.Kind, job.Priority, job.State)); Assert.SequenceEqual([fx.Archive.Id], job.TargetIds); var after = (await scheduler.GetAsync(schedule.Id, Ct))!; Assert.Equal((ScheduleOutcome.Queued, job.Id, T0.AddMinutes(20), clock.Now), (after.LastOutcome!, after.LastJobId!.Value, after.NextRun!.Value, after.LastRun!.Value)); } [Test] public async Task ARunWhosePreviousJobIsPendingIsSkipped() { await using var fx = await ArchiveFixture.CreateAsync(context); await using var engine = Engine(fx); var clock = new ScheduleClock(T0); await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock); var schedule = await CreateAsync(scheduler, ScheduleActions.Duplicates); clock.Advance(TimeSpan.FromMinutes(10)); await scheduler.RunDueAsync(Ct); var first = (await scheduler.GetAsync(schedule.Id, Ct))!.LastJobId; clock.Advance(TimeSpan.FromMinutes(10)); Assert.Equal(1, await scheduler.RunDueAsync(Ct)); var skipped = (await scheduler.GetAsync(schedule.Id, Ct))!; Assert.Equal((ScheduleOutcome.SkippedPending, first), (skipped.LastOutcome!, skipped.LastJobId)); Assert.Count(1, await fx.Db.Jobs.ListByScheduleAsync(schedule.Id, cancellationToken: Ct)); // Once it ends, the next run queues again; a scan another user queued counts as the same work. await fx.Db.Jobs.SetStateAsync(first!.Value, JobState.Done, cancellationToken: Ct); clock.Advance(TimeSpan.FromMinutes(10)); await scheduler.RunDueAsync(Ct); Assert.Equal(ScheduleOutcome.Queued, (await scheduler.GetAsync(schedule.Id, Ct))!.LastOutcome); await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: Ct); var scan = await CreateAsync(scheduler, ScheduleActions.Scan); Assert.Equal(ScheduleOutcome.SkippedPending, (await scheduler.RunNowAsync(scan.Id, null, Ct))!.Outcome); } [Test] public async Task MissedRunsFireOnceWithCatchUpAndAreSkippedWithout() { await using var fx = await ArchiveFixture.CreateAsync(context); await using var engine = Engine(fx); var clock = new ScheduleClock(T0); var host = new TestScheduleHost(fx.Archive); ScheduleDto catchUp, skip; await using (var before = await StartAsync(fx.Db, engine, host, clock)) { catchUp = await CreateAsync(before, ScheduleActions.Scan, new RecurrenceDto("every", Every: 1, Unit: "hours"), catchUp: true); skip = await CreateAsync(before, ScheduleActions.Orphans, new RecurrenceDto("every", Every: 1, Unit: "hours"), catchUp: false); } // The host was off for 5 h 30: five runs of each were missed. clock.Advance(TimeSpan.FromMinutes(330)); await using var after = await StartAsync(fx.Db, engine, host, clock); Assert.Count(1, await fx.Db.Jobs.ListByScheduleAsync(catchUp.Id, cancellationToken: Ct)); Assert.Empty(await fx.Db.Jobs.ListByScheduleAsync(skip.Id, cancellationToken: Ct)); var caught = (await after.GetAsync(catchUp.Id, Ct))!; var skipped = (await after.GetAsync(skip.Id, Ct))!; Assert.Equal(ScheduleOutcome.Queued, caught.LastOutcome); Assert.Null(skipped.LastOutcome); // Both move to the next occurrence on their grid, in the future. Assert.Equal(T0.AddHours(6), caught.NextRun); Assert.Equal(T0.AddHours(6), skipped.NextRun); Assert.Equal(0, await after.RunDueAsync(Ct)); } /// The server fixes an age limit from its environment there: a run caught up at start must already use it. [Test] public async Task AnEditBeforeTheFirstPassAppliesToTheRunCaughtUpAtStart() { await using var fx = await ArchiveFixture.CreateAsync(context); await using var engine = Engine(fx); var clock = new ScheduleClock(T0); var host = new TestScheduleHost(fx.Archive); host.Systems.Add(new SystemSchedule("audit-cleanup", "Audit log clean-up", ScheduleActions.Cleanup, ScheduleRecurrence.Parse("daily:04:30"), new ScheduleParametersDto(OlderThanDays: 90, Steps: ["audit"]))); await using (await StartAsync(fx.Db, engine, host, clock)) { } // The host was off over the run of 04:30. clock.Advance(TimeSpan.FromDays(1)); await using var scheduler = new Scheduler(fx.Db, engine, new SchedulerOptions { Time = clock, Tick = TimeSpan.FromHours(1) }); var id = Guid.Empty; await scheduler.StartAsync(host, async ct => { var row = (await fx.Db.Schedules.GetBySystemKeyAsync("audit-cleanup", ct))!; id = row.Id; Assert.Empty(await fx.Db.Jobs.ListByScheduleAsync(id, cancellationToken: ct)); var edit = await scheduler.UpdateAsync(id, new UpdateScheduleRequest(Parameters: new ScheduleParametersDto(OlderThanDays: 365, Steps: ["audit"])), ct); Assert.Null(edit.Problem); }, Ct); var jobs = await fx.Db.Jobs.ListByScheduleAsync(id, cancellationToken: Ct); Assert.Count(1, jobs); using var parameters = System.Text.Json.JsonDocument.Parse(jobs[0].ParametersJson!); Assert.Equal(365, parameters.RootElement.GetProperty("olderThanDays").GetInt32()); } [Test] public async Task DisabledSchedulesNeverRunAndGetANewNextRunWhenEnabled() { await using var fx = await ArchiveFixture.CreateAsync(context); await using var engine = Engine(fx); var clock = new ScheduleClock(T0); await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock); var disabled = await CreateAsync(scheduler, ScheduleActions.Scan, enabled: false); Assert.Null(disabled.NextRun); var enabled = await CreateAsync(scheduler, ScheduleActions.Orphans); Assert.Null((await scheduler.UpdateAsync(enabled.Id, new UpdateScheduleRequest(Enabled: false), Ct)).Schedule!.NextRun); clock.Advance(TimeSpan.FromDays(1)); Assert.Equal(0, await scheduler.RunDueAsync(Ct)); Assert.Empty((await fx.Db.Jobs.ListAsync(cancellationToken: Ct)).Items); var back = (await scheduler.UpdateAsync(enabled.Id, new UpdateScheduleRequest(Enabled: true), Ct)).Schedule!; Assert.Equal(clock.Now.AddMinutes(10), back.NextRun); Assert.Equal(0, await scheduler.RunDueAsync(Ct)); // An edit that keeps the recurrence keeps the next run; a new recurrence moves it. Assert.Equal(back.NextRun, (await scheduler.UpdateAsync(enabled.Id, new UpdateScheduleRequest(Name: "Renamed"), Ct)).Schedule!.NextRun); var hourly = (await scheduler.UpdateAsync(enabled.Id, new UpdateScheduleRequest(Recurrence: new RecurrenceDto("daily", Time: "03:00")), Ct)).Schedule!; Assert.Equal(new DateTimeOffset(2026, 10, 4, 3, 0, 0, TimeSpan.Zero), hourly.NextRun); } [Test] public async Task RunNowQueuesForTheUserWithoutMovingTheNextRun() { await using var fx = await ArchiveFixture.CreateAsync(context); await using var engine = Engine(fx); var clock = new ScheduleClock(T0); await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock); var user = Guid.CreateVersion7(); var disabled = await CreateAsync(scheduler, ScheduleActions.Duplicates, enabled: false); var run = (await scheduler.RunNowAsync(disabled.Id, user, Ct))!; Assert.Equal(ScheduleOutcome.Queued, run.Outcome); var job = (await fx.Db.Jobs.GetAsync(run.JobId!.Value, Ct))!; Assert.Equal((JobKind.Duplicates, user, disabled.Id), (job.Kind, job.CreatedBy!.Value, job.ScheduleId!.Value)); Assert.Null(run.Schedule.NextRun); Assert.Equal(run.JobId, run.Schedule.LastJobId); var enabled = await CreateAsync(scheduler, ScheduleActions.Orphans); Assert.Equal(ScheduleOutcome.Queued, (await scheduler.RunNowAsync(enabled.Id, null, Ct))!.Outcome); Assert.Equal(enabled.NextRun, (await scheduler.GetAsync(enabled.Id, Ct))!.NextRun); Assert.Equal(ScheduleOutcome.SkippedPending, (await scheduler.RunNowAsync(enabled.Id, null, Ct))!.Outcome); Assert.Null(await scheduler.RunNowAsync(Guid.CreateVersion7(), null, Ct)); } [Test] public async Task TwoSchedulersOnOneDatabaseFireEachRunOnce() { await using var fx = await ArchiveFixture.CreateAsync(context); var path = ((SqliteDatabase)fx.Db.Database).Path!; await using var otherDb = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(path), Ct); await using var engine = Engine(fx); await using var otherEngine = new JobEngine(otherDb, fx.Service.CreateJobHandlers(), new JobEngineOptions { WorkerId = "other" }); var clock = new ScheduleClock(T0); await using var a = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock); await using var b = await StartAsync(otherDb, otherEngine, new TestScheduleHost(fx.Archive), clock); var schedule = await CreateAsync(a, ScheduleActions.Backup); for (var round = 1; round <= 4; round++) { clock.Advance(TimeSpan.FromMinutes(10)); var fired = await Task.WhenAll(Task.Run(() => a.RunDueAsync(Ct)), Task.Run(() => b.RunDueAsync(Ct))); Assert.Equal(1, fired.Sum(), $"round {round}"); // The backup jobs never run here: each round queues once, then the next is skipped while it waits. Assert.Count(1, await fx.Db.Jobs.ListByScheduleAsync(schedule.Id, cancellationToken: Ct)); } } [Test] public async Task ANextRunFarAheadAfterTheClockWentBackIsComputedAgain() { await using var fx = await ArchiveFixture.CreateAsync(context); await using var engine = Engine(fx); var clock = new ScheduleClock(T0); await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock); var schedule = await CreateAsync(scheduler, ScheduleActions.Scan); clock.Set(T0.AddHours(-5).AddSeconds(20)); Assert.Equal(0, await scheduler.RunDueAsync(Ct)); Assert.Equal(T0.AddHours(-5).AddMinutes(10), (await scheduler.GetAsync(schedule.Id, Ct))!.NextRun); // Within two periods it is left alone. clock.Advance(TimeSpan.FromMinutes(-5)); await scheduler.RunDueAsync(Ct); Assert.Equal(T0.AddHours(-5).AddMinutes(10), (await scheduler.GetAsync(schedule.Id, Ct))!.NextRun); } [Test] public async Task SystemSchedulesAreSeededOnceEditableAndResetNeverDeleted() { await using var fx = await ArchiveFixture.CreateAsync(context); await using var engine = Engine(fx); var clock = new ScheduleClock(T0); var host = new TestScheduleHost(fx.Archive); host.Systems.Add(new SystemSchedule("archive-scan", "Archive scan", ScheduleActions.Scan, ScheduleRecurrence.Parse("every:10m"))); host.Systems.Add(new SystemSchedule("index-backup", "Index backup", ScheduleActions.Backup, ScheduleRecurrence.Parse("daily:03:00"), Enabled: false)); Guid id; await using (var first = await StartAsync(fx.Db, engine, host, clock)) { var list = await first.ListAsync(Ct); Assert.SequenceEqual(["archive-scan", "index-backup"], list.Items.Select(s => s.SystemKey!)); var scan = list.Items[0]; id = scan.Id; Assert.Equal(T0.AddMinutes(10), scan.NextRun); Assert.Null(list.Items[1].NextRun); Assert.Equal("UTC", list.TimeZone); Assert.SequenceEqual(host.CleanupSteps, list.Steps!); Assert.Equal("name", (await first.UpdateAsync(id, new UpdateScheduleRequest(Name: "Mine"), Ct)).Problem); Assert.Equal("action", (await first.UpdateAsync(id, new UpdateScheduleRequest(Action: ScheduleActions.Verify), Ct)).Problem); Assert.Null((await first.UpdateAsync(id, new UpdateScheduleRequest(Name: "Archive scan", Recurrence: new RecurrenceDto("every", Every: 30, Unit: "minutes")), Ct)).Problem); Assert.Equal(ScheduleDeleteOutcome.System, await first.DeleteAsync(id, Ct)); } host.Systems[0] = host.Systems[0] with { Recurrence = ScheduleRecurrence.Parse("every:5m") }; await using var second = await StartAsync(fx.Db, engine, host, clock); Assert.Equal(new RecurrenceDto("every", Every: 30, Unit: "minutes"), (await second.GetAsync(id, Ct))!.Recurrence); var reset = (await second.ResetAsync(id, Ct)).Schedule!; Assert.Equal((new RecurrenceDto("every", Every: 5, Unit: "minutes"), T0.AddMinutes(5)), (reset.Recurrence, reset.NextRun!.Value)); var mine = await CreateAsync(second, ScheduleActions.Orphans); Assert.Equal("systemKey", (await second.ResetAsync(mine.Id, Ct)).Problem); Assert.Equal(ScheduleDeleteOutcome.Deleted, await second.DeleteAsync(mine.Id, Ct)); Assert.Equal(ScheduleDeleteOutcome.NotFound, await second.DeleteAsync(mine.Id, Ct)); Assert.True((await second.UpdateAsync(mine.Id, new UpdateScheduleRequest(Enabled: false), Ct)).NotFound); } [Test] public async Task InvalidSchedulesNameTheirField() { await using var fx = await ArchiveFixture.CreateAsync(context); await using var engine = Engine(fx); await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), new ScheduleClock(T0)); async Task Problem(CreateScheduleRequest request) => (await scheduler.CreateAsync(request, null, Ct)).Problem; Assert.Equal("name", await Problem(new CreateScheduleRequest(" ", ScheduleActions.Scan, Every10))); Assert.Equal("name", await Problem(new CreateScheduleRequest(new string('n', Scheduler.MaxNameLength + 1), ScheduleActions.Scan, Every10))); Assert.Equal("action", await Problem(new CreateScheduleRequest("Rebuild", "rebuild", Every10))); Assert.Equal("recurrence", await Problem(new CreateScheduleRequest("Fast", ScheduleActions.Scan, new RecurrenceDto("every", Every: 2, Unit: "minutes")))); Assert.Equal("parameters.percent", await Problem(new CreateScheduleRequest("Verify", ScheduleActions.Verify, Every10, new ScheduleParametersDto(Percent: 0)))); Assert.Equal("parameters.olderThanDays", await Problem(new CreateScheduleRequest("Verify", ScheduleActions.Verify, Every10, new ScheduleParametersDto(OlderThanDays: -1)))); Assert.Equal("parameters.keep", await Problem(new CreateScheduleRequest("Backup", ScheduleActions.Backup, Every10, new ScheduleParametersDto(Keep: 0)))); Assert.Equal("parameters.steps", await Problem(new CreateScheduleRequest("Clean", ScheduleActions.Cleanup, Every10, new ScheduleParametersDto(Steps: ["shred"])))); Assert.Equal("parameters.steps", await Problem(new CreateScheduleRequest("Clean", ScheduleActions.Cleanup, Every10, new ScheduleParametersDto(Steps: [])))); Assert.Equal("priority", await Problem(new CreateScheduleRequest("Scan", ScheduleActions.Scan, Every10, Priority: "urgent"))); Assert.Equal("target", await Problem(new CreateScheduleRequest("Scan", ScheduleActions.Scan, Every10, Target: "elsewhere"))); // Parameters are stored normalised: the action's own fields, its defaults for those left out. var verify = (await scheduler.CreateAsync(new CreateScheduleRequest("Verify", ScheduleActions.Verify, Every10, new ScheduleParametersDto(Percent: 25, DryRun: true, Steps: ["audit"]), Priority: "user"), null, Ct)).Schedule!; Assert.Equal((new ScheduleParametersDto(OlderThanDays: 30, Percent: 25), "user"), (verify.Parameters, verify.Priority)); // Another action starts from its own defaults. var changed = (await scheduler.UpdateAsync(verify.Id, new UpdateScheduleRequest(Action: ScheduleActions.Backup), Ct)).Schedule!; Assert.Equal(new ScheduleParametersDto(Keep: 3), changed.Parameters); } [Test] public async Task UnavailableActionsAndAMissingArchiveQueueNothing() { await using var fx = await ArchiveFixture.CreateAsync(context); await using var engine = Engine(fx); var host = new TestScheduleHost(fx.Archive); host.Refused[ScheduleActions.Thumbnails] = "not-here"; await using var scheduler = await StartAsync(fx.Db, engine, host, new ScheduleClock(T0)); var actions = (await scheduler.ListAsync(Ct)).Actions.ToDictionary(a => a.Name); Assert.SequenceEqual(ScheduleActions.All, actions.Keys); Assert.Equal(new ScheduleActionDto(ScheduleActions.Thumbnails, false, "not-here"), actions[ScheduleActions.Thumbnails]); Assert.True(actions[ScheduleActions.Backup].Available, "an SQLite index file is backed up"); Assert.True(actions[ScheduleActions.Proxies].Available); var thumbnails = await CreateAsync(scheduler, ScheduleActions.Thumbnails); Assert.Equal(ScheduleOutcome.Unavailable, (await scheduler.RunNowAsync(thumbnails.Id, null, Ct))!.Outcome); var proxies = await CreateAsync(scheduler, ScheduleActions.Proxies); host.Archive = await fx.Service.UpdateSettingsAsync(fx.Archive.Id, s => s with { Proxies = s.Proxies with { LongEdge = ProxyPolicy.Disabled } }, Ct); Assert.Equal(new ScheduleActionDto(ScheduleActions.Proxies, false, ScheduleUnavailable.ProxiesDisabled), (await scheduler.ListAsync(Ct)).Actions.Single(a => a.Name == ScheduleActions.Proxies)); Assert.Equal(ScheduleOutcome.Unavailable, (await scheduler.RunNowAsync(proxies.Id, null, Ct))!.Outcome); // No engine handler for the kind: a desktop without the archive engine, a server without the processing module. await using var bare = new JobEngine(fx.Db, [], new JobEngineOptions { WorkerId = "bare" }); await using var other = new Scheduler(fx.Db, bare, new SchedulerOptions { Time = new ScheduleClock(T0), Tick = TimeSpan.FromHours(1) }); await other.StartAsync(new TestScheduleHost(fx.Archive), Ct); Assert.True((await other.ListAsync(Ct)).Actions.All(a => a is { Available: false, Reason: ScheduleUnavailable.NoHandler })); host.Archive = null; var scan = await CreateAsync(scheduler, ScheduleActions.Scan); Assert.Equal(ScheduleOutcome.SkippedOffline, (await scheduler.RunNowAsync(scan.Id, null, Ct))!.Outcome); // The backup and the clean-up need no archive. var backup = await CreateAsync(scheduler, ScheduleActions.Backup); Assert.Equal(ScheduleOutcome.Queued, (await scheduler.RunNowAsync(backup.Id, null, Ct))!.Outcome); var cleanup = await CreateAsync(scheduler, ScheduleActions.Cleanup); var cleanupRun = (await scheduler.RunNowAsync(cleanup.Id, null, Ct))!; Assert.Equal(ScheduleOutcome.Queued, cleanupRun.Outcome); Assert.Empty((await fx.Db.Jobs.GetAsync(cleanupRun.JobId!.Value, Ct))!.TargetIds); } [Test] public async Task BackupsAreUnavailableWithoutAnIndexFile() { await using var db = await ChronosDatabase.OpenAsync(DatabaseOptions.SqliteInMemory(), Ct); await using var engine = new JobEngine(db, new ArchiveService(db).CreateJobHandlers(), new JobEngineOptions { WorkerId = "memory" }); await using var scheduler = new Scheduler(db, engine, new SchedulerOptions { Time = new ScheduleClock(T0) }); await scheduler.StartAsync(new TestScheduleHost(null), Ct); Assert.Equal(new ScheduleActionDto(ScheduleActions.Backup, false, ScheduleUnavailable.NoIndexFile), (await scheduler.ListAsync(Ct)).Actions.Single(a => a.Name == ScheduleActions.Backup)); Assert.Equal(ScheduleOutcome.Unavailable, (await scheduler.RunNowAsync((await CreateAsync(scheduler, ScheduleActions.Backup)).Id, null, Ct))!.Outcome); Assert.Equal(ScheduleOutcome.SkippedOffline, (await scheduler.RunNowAsync((await CreateAsync(scheduler, ScheduleActions.Duplicates)).Id, null, Ct))!.Outcome); } [Test] public async Task VerifyTakesTheLeastRecentlyVerifiedShareOfTheArchive() { await using var fx = await ArchiveFixture.CreateAsync(context); await using var engine = Engine(fx); var clock = new ScheduleClock(DateTimeOffset.UtcNow); await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock); var ids = new List(); for (var i = 0; i < 4; i++) ids.Add(await IngestAsync(fx, "s" + i, 10 * i + 1)); await fx.Db.Sequences.RecordVerifiedAsync(ids[0], SequenceStatus.Verified, SequenceStatus.None, clock.Now.AddDays(-60), Ct); await fx.Db.Sequences.RecordVerifiedAsync(ids[1], SequenceStatus.Verified, SequenceStatus.None, clock.Now.AddDays(-2), Ct); var half = await CreateAsync(scheduler, ScheduleActions.Verify, parameters: new ScheduleParametersDto(OlderThanDays: 30, Percent: 50)); var job = (await fx.Db.Jobs.GetAsync((await scheduler.RunNowAsync(half.Id, null, Ct))!.JobId!.Value, Ct))!; var never = ids.Skip(2).Order().ToList(); Assert.SequenceEqual(never, ArchiveJobJson.Read(job, ArchiveJobJson.Default.VerifyParameters, new VerifyParameters(null)).SequenceIds!.Order()); await fx.Db.Jobs.SetStateAsync(job.Id, JobState.Done, cancellationToken: Ct); var all = await CreateAsync(scheduler, ScheduleActions.Verify, parameters: new ScheduleParametersDto(OlderThanDays: 0, Percent: 100)); var whole = (await fx.Db.Jobs.GetAsync((await scheduler.RunNowAsync(all.Id, null, Ct))!.JobId!.Value, Ct))!; Assert.Null(ArchiveJobJson.Read(whole, ArchiveJobJson.Default.VerifyParameters, new VerifyParameters([])).SequenceIds); await fx.Db.Jobs.SetStateAsync(whole.Id, JobState.Done, cancellationToken: Ct); foreach (var id in ids) await fx.Db.Sequences.RecordVerifiedAsync(id, SequenceStatus.Verified, SequenceStatus.None, clock.Now.AddDays(-1), Ct); Assert.Equal(ScheduleOutcome.SkippedNothing, (await scheduler.RunNowAsync(half.Id, null, Ct))!.Outcome); } [Test] public async Task ImagePassesCoverTheSequencesWithoutThemOnce() { await using var fx = await ArchiveFixture.CreateAsync(context); await using var engine = Engine(fx); await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), new ScheduleClock(T0)); var a = await IngestAsync(fx, "a", 1); var b = await IngestAsync(fx, "b", 11); await fx.Db.Sequences.UpdateStatusAsync(b, SequenceStatus.ThumbsBuilt, SequenceStatus.None, Ct); var thumbnails = await CreateAsync(scheduler, ScheduleActions.Thumbnails); var run = (await scheduler.RunNowAsync(thumbnails.Id, null, Ct))!; var job = (await fx.Db.Jobs.GetAsync(run.JobId!.Value, Ct))!; Assert.Equal((JobKind.Thumbs, ScheduleOutcome.Queued), (job.Kind, run.Outcome)); Assert.SequenceEqual([a], job.TargetIds); // Another schedule finds that work pending. var again = await CreateAsync(scheduler, ScheduleActions.Thumbnails); Assert.Equal(ScheduleOutcome.SkippedPending, (await scheduler.RunNowAsync(again.Id, null, Ct))!.Outcome); var forced = await CreateAsync(scheduler, ScheduleActions.Thumbnails, parameters: new ScheduleParametersDto(Force: true)); var forcedRun = (await scheduler.RunNowAsync(forced.Id, null, Ct))!; Assert.Equal(ScheduleOutcome.Queued, forcedRun.Outcome); var forcedJob = (await fx.Db.Jobs.GetAsync(forcedRun.JobId!.Value, Ct))!; Assert.SequenceEqual([b], forcedJob.TargetIds); var analysis = await CreateAsync(scheduler, ScheduleActions.Analysis); var light = (await fx.Db.Jobs.GetAsync((await scheduler.RunNowAsync(analysis.Id, null, Ct))!.JobId!.Value, Ct))!; Assert.Equal(JobKind.LightAnalyse, light.Kind); Assert.SequenceEqual(new[] { a, b }.Order(), light.TargetIds.Order()); var deep = await CreateAsync(scheduler, ScheduleActions.Analysis, parameters: new ScheduleParametersDto(Deep: true)); var deepJob = (await fx.Db.Jobs.GetAsync((await scheduler.RunNowAsync(deep.Id, null, Ct))!.JobId!.Value, Ct))!; Assert.Equal((JobKind.DeepAnalyse, JobPriority.Background), (deepJob.Kind, deepJob.Priority)); // Without the analysis engine (a host that registers only the archive's handlers) analysis is not available. await using var archiveOnly = new JobEngine(fx.Db, fx.Service.CreateJobHandlers(), new JobEngineOptions { WorkerId = "archive" }); await using var other = new Scheduler(fx.Db, archiveOnly, new SchedulerOptions { Time = new ScheduleClock(T0), Tick = TimeSpan.FromHours(1) }); await other.StartAsync(new TestScheduleHost(fx.Archive), Ct); Assert.Equal(ScheduleOutcome.Unavailable, (await other.RunNowAsync((await CreateAsync(other, ScheduleActions.Analysis)).Id, null, Ct))!.Outcome); } /// /// Owner review 2, R10: both hosts scan right after they start (the desktop also when its archive is back), as their /// polling did, through the archive scan system schedule: once, only while it is enabled, never twice for one start. /// [Test] public async Task ASystemScheduleRunsNowOnceWhenEnabledAndNotRunSince() { await using var fx = await ArchiveFixture.CreateAsync(context); await using var engine = Engine(fx); var clock = new ScheduleClock(T0); var host = new TestScheduleHost(fx.Archive); host.Systems.Add(new SystemSchedule("archive-scan", "Archive scan", ScheduleActions.Scan, ScheduleRecurrence.Interval(10, RecurrenceUnit.Minutes), CatchUp: false)); await using var scheduler = await StartAsync(fx.Db, engine, host, clock); Assert.Null(await scheduler.RunSystemNowAsync("no-such-key", null, Ct)); // Right after the start: the scan is queued, without waiting for its first interval. var started = clock.Now; var run = Assert.NotNull(await scheduler.RunSystemNowAsync("archive-scan", started, Ct)); Assert.Equal(ScheduleOutcome.Queued, run.Outcome); var job = (await fx.Db.Jobs.GetAsync(run.JobId!.Value, Ct))!; Assert.Equal((JobKind.Scan, JobPriority.Background, (Guid?)run.Schedule.Id), (job.Kind, job.Priority, job.ScheduleId)); Assert.Equal(T0.AddMinutes(10), run.Schedule.NextRun); // It ran since that start: not again. Asked again later (the archive is back), its pending scan is not queued twice. Assert.Null(await scheduler.RunSystemNowAsync("archive-scan", started, Ct)); Assert.Equal(ScheduleOutcome.SkippedPending, (await scheduler.RunSystemNowAsync("archive-scan", null, Ct))!.Outcome); Assert.Count(1, await fx.Db.Jobs.ListByScheduleAsync(run.Schedule.Id, cancellationToken: Ct)); // Turned off, it does not run at start. Assert.Null((await scheduler.UpdateAsync(run.Schedule.Id, new UpdateScheduleRequest(Enabled: false), Ct)).Problem); await fx.Db.Jobs.SetStateAsync(job.Id, JobState.Done, cancellationToken: Ct); Assert.Null(await scheduler.RunSystemNowAsync("archive-scan", null, Ct)); } /// /// Owner review 2, R10: a scan whose schedule leaves deep unset follows the host at every run (the desktop's Settings › /// Analysis › After a scan as it is then), one that sets it keeps its own choice. /// [Test] public async Task AScanWithoutItsOwnDeepFollowsTheHostAtEveryRun() { await using var fx = await ArchiveFixture.CreateAsync(context); await using var engine = Engine(fx); var host = new TestScheduleHost(fx.Archive); await using var scheduler = await StartAsync(fx.Db, engine, host, new ScheduleClock(T0)); var follows = await CreateAsync(scheduler, ScheduleActions.Scan); Assert.Null(follows.Parameters.Deep); var own = await CreateAsync(scheduler, ScheduleActions.Scan, parameters: new ScheduleParametersDto(Deep: false)); Assert.Equal(false, own.Parameters.Deep); async Task DeepAsync(ScheduleDto schedule) { var resolution = await ScheduleActions.ResolveAsync(new ScheduleActionContext(fx.Db, host, _ => true, T0), schedule.Action, schedule.Parameters, Ct); var request = resolution.Requests.Single(); return System.Text.Json.JsonSerializer.Deserialize(request.ParametersJson!, ArchiveJobJson.Default.ScanParameters)!.DeepAnalysis; } Assert.False(await DeepAsync(follows)); host.DeepAnalysisAfterScan = true; Assert.True(await DeepAsync(follows)); Assert.False(await DeepAsync(own)); // The queued job carries it too. var job = (await fx.Db.Jobs.GetAsync((await scheduler.RunNowAsync(follows.Id, null, Ct))!.JobId!.Value, Ct))!; Assert.True(ArchiveJobJson.Read(job, ArchiveJobJson.Default.ScanParameters, new ScanParameters(false)).DeepAnalysis); } /// /// Owner review 2, R10: a system schedule that replaced a timer is seeded due when the timer's work was last done a period /// or more before (the newest index copy a week old: the first start after the upgrade copies at once, as the daily loop /// did); done more recently, or never, it waits for the recurrence's first occurrence. Rows that exist keep their runs. /// [Test] public async Task ASystemScheduleIsSeededDueWhenItsWorkIsAPeriodOld() { await using var fx = await ArchiveFixture.CreateAsync(context); await using var engine = Engine(fx); var clock = new ScheduleClock(T0); var host = new TestScheduleHost(fx.Archive); var daily = ScheduleRecurrence.Interval(1, RecurrenceUnit.Days); host.Systems.Add(new SystemSchedule("index-backup", "Index backup", ScheduleActions.Backup, daily, LastDone: T0.AddDays(-7))); host.Systems.Add(new SystemSchedule("nightly-backup", "Nightly", ScheduleActions.Backup, ScheduleRecurrence.DailyAt(new TimeOnly(3, 0)), LastDone: T0.AddHours(-23))); host.Systems.Add(new SystemSchedule("new-backup", "New", ScheduleActions.Backup, daily)); host.Systems.Add(new SystemSchedule("off-backup", "Off", ScheduleActions.Backup, daily, Enabled: false, LastDone: T0.AddDays(-7))); await using (var first = await StartAsync(fx.Db, engine, host, clock)) { var due = (await fx.Db.Schedules.GetBySystemKeyAsync("index-backup", Ct))!; Assert.Equal((ScheduleOutcome.Queued, T0, T0.AddDays(1)), (due.LastOutcome!, due.LastRun!.Value, due.NextRun!.Value)); var nightly = (await fx.Db.Schedules.GetBySystemKeyAsync("nightly-backup", Ct))!; Assert.Equal(((string?)null, new DateTimeOffset(2026, 10, 3, 3, 0, 0, TimeSpan.Zero)), (nightly.LastOutcome, nightly.NextRun!.Value)); var fresh = (await fx.Db.Schedules.GetBySystemKeyAsync("new-backup", Ct))!; Assert.Equal(((string?)null, T0.AddDays(1)), (fresh.LastOutcome, fresh.NextRun!.Value)); var off = (await fx.Db.Schedules.GetBySystemKeyAsync("off-backup", Ct))!; Assert.Equal(((string?)null, (DateTimeOffset?)null), (off.LastOutcome, off.NextRun)); } // The next start finds the rows: nothing is due again. clock.Advance(TimeSpan.FromHours(1)); await using var second = await StartAsync(fx.Db, engine, host, clock); Assert.Count(1, (await fx.Db.Jobs.ListAsync(cancellationToken: Ct)).Items); } /// The loop itself (no pass by hand): its tick fires a due schedule. [Test] public async Task TheLoopFiresADueScheduleAtItsTick() { await using var fx = await ArchiveFixture.CreateAsync(context); await using var engine = Engine(fx); var clock = new ScheduleClock(T0); await using var scheduler = new Scheduler(fx.Db, engine, new SchedulerOptions { Time = clock, Tick = TimeSpan.FromMilliseconds(100) }); await scheduler.StartAsync(new TestScheduleHost(fx.Archive), Ct); var changes = Changes(scheduler); var schedule = await CreateAsync(scheduler, ScheduleActions.Scan); clock.Advance(TimeSpan.FromMinutes(10)); var ran = await RanAsync(changes, schedule.Id, null); Assert.Equal((ScheduleOutcome.Queued, T0.AddMinutes(20)), (ran.LastOutcome!, ran.NextRun!.Value)); Assert.Count(1, await fx.Db.Jobs.ListByScheduleAsync(schedule.Id, cancellationToken: Ct)); } /// makes the loop look now, long before its tick. [Test] public async Task WakeFiresADueScheduleBeforeTheTick() { await using var fx = await ArchiveFixture.CreateAsync(context); await using var engine = Engine(fx); var clock = new ScheduleClock(T0); await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock); var changes = Changes(scheduler); var schedule = await CreateAsync(scheduler, ScheduleActions.Scan); clock.Advance(TimeSpan.FromMinutes(10)); await Task.Delay(200, Ct); Assert.Null((await scheduler.GetAsync(schedule.Id, Ct))!.LastOutcome, "the tick is an hour away"); scheduler.Wake(); Assert.Equal(ScheduleOutcome.Queued, (await RanAsync(changes, schedule.Id, null)).LastOutcome); } /// Nothing a pass throws (here a subscriber of ) ends the loop: the next run still fires. [Test] public async Task TheLoopSurvivesAPassThatThrew() { await using var fx = await ArchiveFixture.CreateAsync(context); await using var engine = Engine(fx); var clock = new ScheduleClock(T0); await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock); var changes = Changes(scheduler); var thrown = 0; scheduler.ScheduleChanged += dto => { if (dto.LastOutcome is not null && Interlocked.Exchange(ref thrown, 1) == 0) throw new InvalidOperationException("A subscriber failed"); }; var schedule = await CreateAsync(scheduler, ScheduleActions.Duplicates); clock.Advance(TimeSpan.FromMinutes(10)); scheduler.Wake(); var first = await RanAsync(changes, schedule.Id, null); Assert.Equal(1, Volatile.Read(ref thrown)); await fx.Db.Jobs.SetStateAsync(first.LastJobId!.Value, JobState.Done, cancellationToken: Ct); clock.Advance(TimeSpan.FromMinutes(10)); scheduler.Wake(); var second = await RanAsync(changes, schedule.Id, first.LastJobId); Assert.Equal((ScheduleOutcome.Queued, T0.AddMinutes(20)), (second.LastOutcome!, second.LastRun!.Value)); Assert.Count(2, await fx.Db.Jobs.ListByScheduleAsync(schedule.Id, cancellationToken: Ct)); } /// /// Disposing stops the loop during a pass: it waits for a host call that does not watch its token, so the database is /// never used after it returns, and cancels one that does. /// [Test] public async Task DisposeDuringAPassWaitsForItAndStopsIt() { await using var fx = await ArchiveFixture.CreateAsync(context); await using var engine = Engine(fx); var clock = new ScheduleClock(T0); var host = new TestScheduleHost(fx.Archive); var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var scheduler = await StartAsync(fx.Db, engine, host, clock); var disposed = false; try { await CreateAsync(scheduler, ScheduleActions.Scan); var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); host.BeforeArchive = async _ => { entered.TrySetResult(); await gate.Task; }; clock.Advance(TimeSpan.FromMinutes(10)); scheduler.Wake(); await entered.Task.WaitAsync(Budget, Ct); var disposing = scheduler.DisposeAsync().AsTask(); disposed = true; await Task.Delay(200, Ct); Assert.False(disposing.IsCompleted, "disposed while a pass was still running"); gate.SetResult(); await disposing.WaitAsync(Budget, Ct); } finally { gate.TrySetResult(); if (!disposed) await scheduler.DisposeAsync(); } // A host call that watches the token ends with it. var again = await StartAsync(fx.Db, engine, host, clock); var waiting = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); host.BeforeArchive = async ct => { waiting.TrySetResult(); await Task.Delay(Timeout.Infinite, ct); }; clock.Advance(TimeSpan.FromMinutes(10)); again.Wake(); await waiting.Task.WaitAsync(Budget, Ct); await again.DisposeAsync().AsTask().WaitAsync(Budget, Ct); } private static TimeSpan Budget => TimeSpan.FromSeconds(TestTiming.Budget(10)); /// Every from now on. private static Channel Changes(Scheduler scheduler) { var changes = Channel.CreateUnbounded(); scheduler.ScheduleChanged += dto => changes.Writer.TryWrite(dto); return changes; } /// The first change of that records a run whose job is not . private async Task RanAsync(Channel changes, Guid id, Guid? previousJob) { while (true) { var dto = await changes.Reader.ReadAsync(Ct).AsTask().WaitAsync(Budget, Ct); if (dto.Id == id && dto.LastOutcome is not null && dto.LastJobId != previousJob) return dto; } } private static async Task IngestAsync(ArchiveFixture fx, string key, int seed) { var staged = fx.Stage(ArchiveFixture.Start.AddMinutes(seed), seed); return (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, key, staged.Path, "IMG_0001.JPG", staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId; } }