using System.Net; using System.Net.Http.Json; using System.Text.Json; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging.Abstractions; using Chronos.Core.Archiving; using Chronos.Core.Model; using Chronos.Data; using Chronos.Data.Jobs; using Chronos.Data.Scheduling; using Chronos.Protocol; using Chronos.Server.Configuration; using Chronos.Server.Events; using Chronos.Server.Jobs; using Chronos.Server.Storage; using Chronos.Testing; namespace Chronos.Server.Tests.Jobs; /// /// /api/v1/schedules and the server's system schedules (owner review 2, R10): every timer the server ran is a visible /// schedule, seeded once, edited, reset and run like the others; the old timers are gone. /// [Category("server")] public sealed class ScheduleEndpointTests(TestContext context) { private static readonly string[] SystemKeys = [ServerScheduleHost.ArchiveScan, ServerScheduleHost.UploadCleanup, ServerScheduleHost.IndexBackup, ServerScheduleHost.Housekeeping, ServerScheduleHost.AuditCleanup]; private CancellationToken Ct => context.Cancellation; [Test] public async Task TheSystemSchedulesTheActionsAndTheTimeZoneAreListed() { await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); var list = await ListAsync(await server.AdminAsync()); // The timers the server ran before, with their timings: nothing runs that this list does not show. Assert.SequenceEqual(SystemKeys, list.Items.Select(s => s.SystemKey)); var scan = SystemRow(list, ServerScheduleHost.ArchiveScan); Assert.Equal(("scan", new RecurrenceDto("every", 10, "minutes"), false, "background", true), (scan.Action, scan.Recurrence, scan.CatchUp, scan.Priority, scan.Enabled)); Assert.NotNull(scan.NextRun); var uploads = SystemRow(list, ServerScheduleHost.UploadCleanup); // Every 6 hours, not more often: four clean-up jobs a day keep the job history readable. Assert.Equal(("cleanup", new RecurrenceDto("every", 6, "hours"), true, (int?)90), (uploads.Action, uploads.Recurrence, uploads.CatchUp, uploads.Parameters.OlderThanDays)); Assert.SequenceEqual([UploadCleanupStep.StepName], uploads.Parameters.Steps!); var backup = SystemRow(list, ServerScheduleHost.IndexBackup); Assert.Equal(("backup", new RecurrenceDto("daily", Time: "03:00"), true, (int?)3), (backup.Action, backup.Recurrence, backup.CatchUp, backup.Parameters.Keep)); var history = SystemRow(list, ServerScheduleHost.Housekeeping); Assert.Equal((new RecurrenceDto("daily", Time: "04:00"), (int?)90), (history.Recurrence, history.Parameters.OlderThanDays)); Assert.SequenceEqual(["job-history"], history.Parameters.Steps!); var audit = SystemRow(list, ServerScheduleHost.AuditCleanup); // The audit log is kept forever by default: no deletion after an upgrade. Assert.Equal((new RecurrenceDto("daily", Time: "04:30"), (int?)0, false), (audit.Recurrence, audit.Parameters.OlderThanDays, audit.Enabled)); Assert.SequenceEqual(["audit"], audit.Parameters.Steps!); // Every action, with whether this server runs it: without the processing module there is no analysis. Assert.SequenceEqual(ScheduleActions.All, list.Actions.Select(a => a.Name)); Assert.Equal((false, "no-handler"), (list.Actions.Single(a => a.Name == "analysis").Available, list.Actions.Single(a => a.Name == "analysis").Reason)); Assert.True(list.Actions.Where(a => a.Name != "analysis").All(a => a.Available), string.Join(", ", list.Actions.Select(a => $"{a.Name}:{a.Reason}"))); Assert.SequenceEqual(["job-history", "audit", "trash-report", UploadCleanupStep.StepName], list.Steps!); Assert.Equal(server.Services.GetRequiredService().TimeZone.Id, list.TimeZone); } [Test] public async Task SchedulesAreCreatedChangedAndDeleted() { await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); var admin = await server.AdminAsync(); var created = await CreateAsync(admin, new CreateScheduleRequest("Weekly check", "verify", new RecurrenceDto("weekly", Time: "02:30", Days: ["thu", "mon"]), new ScheduleParametersDto(OlderThanDays: 60, Percent: 25))); Assert.Equal(("Weekly check", "verify", "current", "background", true, (string?)null), (created.Name, created.Action, created.Target, created.Priority, created.Enabled, created.SystemKey)); Assert.SequenceEqual(["mon", "thu"], created.Recurrence.Days!); Assert.Equal(((int?)60, (int?)25), (created.Parameters.OlderThanDays, created.Parameters.Percent)); Assert.NotNull(created.NextRun); // After the system schedules. Assert.Equal(created.Id, (await ListAsync(admin)).Items[^1].Id); // PUT replaces the fields it gives; disabled, a schedule has no next run. using (var put = await admin.PutAsJsonAsync($"/api/v1/schedules/{created.Id}", new UpdateScheduleRequest(Recurrence: new RecurrenceDto("every", 2, "hours"), Priority: "normal", Enabled: false), ProtocolJson.Default.UpdateScheduleRequest)) { var updated = await put.ReadAsync(ProtocolJson.Default.ScheduleDto); Assert.Equal((new RecurrenceDto("every", 2, "hours"), "normal", false, (DateTimeOffset?)null, (int?)25), (updated.Recurrence, updated.Priority, updated.Enabled, updated.NextRun, updated.Parameters.Percent)); } // PATCH does the same: enabled again, it gets a next run. using (var patch = await admin.PatchAsJsonAsync($"/api/v1/schedules/{created.Id}", new UpdateScheduleRequest(Enabled: true, Name: "Check"), ProtocolJson.Default.UpdateScheduleRequest)) { var enabled = await patch.ReadAsync(ProtocolJson.Default.ScheduleDto); Assert.Equal(("Check", true), (enabled.Name, enabled.Enabled)); Assert.NotNull(enabled.NextRun); } using (var one = await admin.GetAsync($"/api/v1/schedules/{created.Id}")) Assert.Equal("Check", (await one.ReadAsync(ProtocolJson.Default.ScheduleDto)).Name); using (var delete = await admin.DeleteAsync($"/api/v1/schedules/{created.Id}")) await delete.ExpectAsync(HttpStatusCode.NoContent); using (var gone = await admin.GetAsync($"/api/v1/schedules/{created.Id}")) await gone.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found"); using (var again = await admin.DeleteAsync($"/api/v1/schedules/{created.Id}")) await again.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found"); Assert.SequenceEqual(SystemKeys, (await ListAsync(admin)).Items.Select(s => s.SystemKey)); // Every change is in the audit log. using var audit = await admin.GetAsync("/api/v1/audit?action=schedule_created,schedule_updated,schedule_deleted"); var entries = (await audit.ReadAsync(ProtocolJson.Default.PageAuditEntryDto)).Items; Assert.SequenceEqual(["schedule_deleted", "schedule_updated", "schedule_updated", "schedule_created"], entries.Select(e => e.Action)); Assert.True(entries.All(e => e.TargetId == created.Id.ToString() && e.Username == "admin"), "target and user"); Assert.Equal("Weekly check · verify · weekly:mon,thu@02:30", entries[^1].Details); Assert.Equal(("Weekly check · verify · every:2h · off", "Check · verify · every:2h"), (entries[2].Details, entries[1].Details)); } [Test] public async Task OnlyArchiveAdministratorsSeeAndChangeSchedules() { await using var server = await TestServer.StartAsync(context); var (editor, _, _) = await server.UserAsync("alberto", "editor"); using (var list = await editor.GetAsync("/api/v1/schedules")) await list.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden"); using (var create = await editor.PostAsJsonAsync("/api/v1/schedules", new CreateScheduleRequest("Mine", "scan", new RecurrenceDto("daily", Time: "01:00")), ProtocolJson.Default.CreateScheduleRequest)) await create.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden"); var scan = Assert.NotNull(await server.Database.Schedules.GetBySystemKeyAsync(ServerScheduleHost.ArchiveScan, Ct)); using (var run = await editor.PostAsync($"/api/v1/schedules/{scan.Id}/run", null)) await run.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden"); using (var anonymous = await server.Client.GetAsync("/api/v1/schedules")) Assert.Equal(HttpStatusCode.Unauthorized, anonymous.StatusCode); } [Test] public async Task InvalidFieldsAreNamed() { await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); var admin = await server.AdminAsync(); var daily = new RecurrenceDto("daily", Time: "03:00"); var cases = new (CreateScheduleRequest Request, string Field)[] { (new(" ", "scan", daily), "name"), (new(new string('x', Scheduler.MaxNameLength + 1), "scan", daily), "name"), (new("Render", "render", daily), "action"), (new("Often", "scan", new RecurrenceDto("every", 1, "minutes")), "recurrence"), (new("No days", "scan", new RecurrenceDto("weekly", Time: "03:00", Days: [])), "recurrence"), (new("Late", "scan", new RecurrenceDto("daily", Time: "25:00")), "recurrence"), (new("Day 31", "scan", new RecurrenceDto("monthly", Time: "03:00", DayOfMonth: 31)), "recurrence"), (new("None", "verify", daily, new ScheduleParametersDto(Percent: 0)), "parameters.percent"), (new("Old", "verify", daily, new ScheduleParametersDto(OlderThanDays: -1)), "parameters.olderThanDays"), (new("Copies", "backup", daily, new ScheduleParametersDto(Keep: 0)), "parameters.keep"), (new("Unknown step", "cleanup", daily, new ScheduleParametersDto(Steps: ["photos"])), "parameters.steps"), (new("No step", "cleanup", daily, new ScheduleParametersDto(Steps: [])), "parameters.steps"), (new("Urgent", "scan", daily, Priority: "urgent"), "priority"), (new("Elsewhere", "scan", daily, Target: "another-archive"), "target"), }; foreach (var (request, field) in cases) { using var response = await admin.PostAsJsonAsync("/api/v1/schedules", request, ProtocolJson.Default.CreateScheduleRequest); var error = await response.ExpectErrorAsync(HttpStatusCode.BadRequest, "invalid_schedule"); Assert.Equal(field, error.Details, request.Name); Assert.Contains(field, error.Message); } Assert.Count(SystemKeys.Length, (await ListAsync(admin)).Items); } [Test] public async Task SystemSchedulesAreEditedAndResetButNeverDeleted() { await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); var admin = await server.AdminAsync(); var housekeeping = SystemRow(await ListAsync(admin), ServerScheduleHost.Housekeeping); using (var delete = await admin.DeleteAsync($"/api/v1/schedules/{housekeeping.Id}")) await delete.ExpectErrorAsync(HttpStatusCode.Conflict, "schedule_system"); foreach (var (request, field) in new[] { (new UpdateScheduleRequest(Name: "Mine"), "name"), (new UpdateScheduleRequest(Action: "verify"), "action") }) { using var refused = await admin.PutAsJsonAsync($"/api/v1/schedules/{housekeeping.Id}", request, ProtocolJson.Default.UpdateScheduleRequest); Assert.Equal(field, (await refused.ExpectErrorAsync(HttpStatusCode.BadRequest, "invalid_schedule")).Details); } // Its timing, its parameters and its switch change like any schedule's; the same name and action may be sent back. using (var edit = await admin.PutAsJsonAsync($"/api/v1/schedules/{housekeeping.Id}", new UpdateScheduleRequest(housekeeping.Name, "cleanup", new RecurrenceDto("weekly", Time: "05:15", Days: ["sun"]), new ScheduleParametersDto(Steps: ["job-history", "trash-report"], OlderThanDays: 30), CatchUp: false, Enabled: false), ProtocolJson.Default.UpdateScheduleRequest)) { var edited = await edit.ReadAsync(ProtocolJson.Default.ScheduleDto); Assert.Equal(("weekly", (int?)30, false, false), (edited.Recurrence.Kind, edited.Parameters.OlderThanDays, edited.CatchUp, edited.Enabled)); } // Reset puts the server's defaults back, with a new next run. using (var reset = await admin.PostAsync($"/api/v1/schedules/{housekeeping.Id}/reset", null)) { var back = await reset.ReadAsync(ProtocolJson.Default.ScheduleDto); Assert.Equal((new RecurrenceDto("daily", Time: "04:00"), (int?)90, true, true), (back.Recurrence, back.Parameters.OlderThanDays, back.CatchUp, back.Enabled)); Assert.SequenceEqual(["job-history"], back.Parameters.Steps!); Assert.NotNull(back.NextRun); } var custom = await CreateAsync(admin, new CreateScheduleRequest("Mine", "duplicates", new RecurrenceDto("monthly", Time: "01:00", DayOfMonth: 1))); using (var notSystem = await admin.PostAsync($"/api/v1/schedules/{custom.Id}/reset", null)) await notSystem.ExpectErrorAsync(HttpStatusCode.Conflict, "schedule_no_defaults"); using (var unknown = await admin.PostAsync($"/api/v1/schedules/{Guid.CreateVersion7()}/reset", null)) await unknown.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found"); Assert.Equal("queued", (await RunAsync(admin, housekeeping.Id)).Outcome); // A system schedule is audited after its key, so the admin panel names it in its language, as its Scheduled tab does. using var audit = await admin.GetAsync("/api/v1/audit?action=schedule_updated,schedule_reset,schedule_run"); var entries = (await audit.ReadAsync(ProtocolJson.Default.PageAuditEntryDto)).Items; Assert.SequenceEqual(["schedule_run", "schedule_reset", "schedule_updated"], entries.Select(e => e.Action)); Assert.True(entries.All(e => e.TargetId == housekeeping.Id.ToString()), "the target is the schedule"); Assert.SequenceEqual(["system:housekeeping · Housekeeping · queued", "system:housekeeping · Housekeeping · cleanup · daily:04:00", "system:housekeeping · Housekeeping · cleanup · weekly:sun@05:15 · off"], entries.Select(e => e.Details)); } /// /// Every change and run of a schedule goes out on the event socket, also a run that queues no job, so the admin panel's /// Scheduled tab follows what the scheduler does. /// [Test] public async Task ScheduleChangesAndRunsAreEvents() { await using var server = await TestServer.StartAsync(context); var admin = await server.AdminAsync(); using var events = server.Services.GetRequiredService().Subscribe(); // Off, so the scheduler's own passes leave it alone; without a duplicates handler its run queues nothing. var created = await CreateAsync(admin, new CreateScheduleRequest("Duplicates", "duplicates", new RecurrenceDto("daily", Time: "01:00"), Enabled: false)); var run = await RunAsync(admin, created.Id); Assert.Equal(("unavailable", (Guid?)null), (run.Outcome, run.JobId)); using (var delete = await admin.DeleteAsync($"/api/v1/schedules/{created.Id}")) await delete.ExpectAsync(HttpStatusCode.NoContent); var changes = new List(); while (events.Reader.TryRead(out var e)) { if (e is ScheduleChangedEvent changed && changed.ScheduleId == created.Id) changes.Add(changed.Deleted); } Assert.SequenceEqual([false, false, true], changes); Assert.Equal($$"""{"type":"scheduleChanged","scheduleId":"{{created.Id}}","deleted":true}""", JsonSerializer.Serialize(new ScheduleChangedEvent(created.Id, true), ProtocolJson.Default.ServerEvent)); } /// /// Settings a desktop put into archive.json reach the index every 10 minutes whatever the archive scan does, as with the /// former polling: the scan's runs read them too, but they are skipped while a long scan is pending, and the schedule may /// be rare or off. /// [Test] public async Task ArchiveJsonIsReadEveryTenMinutesWhateverTheScanScheduleDoes() { var clock = new SteppedTimeProvider(); await using var server = await TestServer.StartAsync(context, services: s => { ArchiveModule.Register(s); s.AddSingleton(clock); }); var host = server.Services.GetRequiredService(); var archive = server.Services.GetRequiredService(); // The scan at start is over, then the scan schedule is turned off. await ScansAsync(server, 1); var scanId = Assert.NotNull(await server.Database.Schedules.GetBySystemKeyAsync(ServerScheduleHost.ArchiveScan, Ct)).Id; Assert.Null((await host.Scheduler.UpdateAsync(scanId, new UpdateScheduleRequest(Enabled: false), Ct)).Problem); // A desktop sharing the archive changes its folder template; nothing has read it yet. const string template = "{camera}/{date}"; var path = new ArchivePaths(archive.Root).ManifestPath; var manifest = ManifestStore.ReadArchive(path); Assert.NotEqual(template, manifest.LayoutTemplates.Folder); ManifestStore.WriteArchive(path, manifest with { LayoutTemplates = manifest.LayoutTemplates with { Folder = template } }); Assert.Equal(manifest.LayoutTemplates.Folder, Assert.NotNull(await server.Database.Archives.GetAsync(archive.Id, Ct)).FolderTemplate); // Ten minutes on, uploads (the index row) and the admin panel (the served archive) follow it, with no scan. clock.Advance(TimeSpan.FromMinutes(ServerScheduleHost.ScanMinutes)); var until = DateTime.UtcNow + TimeSpan.FromSeconds(TestTiming.Budget(30)); while (Assert.NotNull(await server.Database.Archives.GetAsync(archive.Id, Ct)).FolderTemplate != template) { Assert.True(DateTime.UtcNow < until, "archive.json was not read again"); await Task.Delay(20); } Assert.Equal(template, archive.Current.FolderTemplate); Assert.Count(1, (await server.Database.Jobs.ListAsync(null, JobKind.Scan, 50, null, Ct)).Items); } [Test] public async Task TheJobHistoryOfTheSettingsIsWhatTheHousekeepingScheduleIsResetTo() { await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); var admin = await server.AdminAsync(); using (var read = await admin.GetAsync("/api/v1/server/settings")) { var settings = await read.ReadAsync(ProtocolJson.Default.ServerSettingsDto); // The test server listens on ports the OS chose, which the settings refuse: real ones are sent with the edit. using var saved = await admin.PutAsJsonAsync("/api/v1/server/settings", settings with { JobHistoryDays = 30, HttpPort = 18080, HttpsPort = 18443 }, ProtocolJson.Default.ServerSettingsDto); await saved.ExpectAsync(HttpStatusCode.OK); } // The schedule keeps its own value; a reset takes the new one, without a restart. Assert.Equal(30, server.Options.JobHistoryDays); var housekeeping = SystemRow(await ListAsync(admin), ServerScheduleHost.Housekeeping); Assert.Equal((int?)90, housekeeping.Parameters.OlderThanDays); using var reset = await admin.PostAsync($"/api/v1/schedules/{housekeeping.Id}/reset", null); Assert.Equal((int?)30, (await reset.ReadAsync(ProtocolJson.Default.ScheduleDto)).Parameters.OlderThanDays); } /// /// Owner review 2, R10 and R11: the audit retention has one source of truth, the audit clean-up schedule's age limit. /// Settings › Logs shows it wherever it was set (0 while the schedule is off or without its audit step) and sets it /// (turning the schedule on with its audit step, or off at 0); Reset goes back to the value last saved there, and at 0 /// the audit step deletes nothing. /// [Test] public async Task TheAuditRetentionOfTheSettingsIsTheAgeLimitOfTheAuditCleanUp() { await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); var admin = await server.AdminAsync(); var audit = SystemRow(await ListAsync(admin), ServerScheduleHost.AuditCleanup); Assert.Equal(((int?)0, false), (audit.Parameters.OlderThanDays, audit.Enabled)); // At 0 a run (also of the disabled schedule, on request) keeps the whole audit log. var entries = (await server.Database.Audit.ListByActionsAsync(null, [], 1000, null, Ct)).Items.Select(e => e.Id).ToList(); Assert.True(entries.Count > 0, "the sign-in left audit entries"); var run = await RunAsync(admin, audit.Id); var job = await JobApiTests.WaitForAsync(admin, run.JobId!.Value, "done"); Assert.Contains("Audit log kept in full", job.LogTail); var kept = (await server.Database.Audit.ListByActionsAsync(null, [], 1000, null, Ct)).Items.Select(e => e.Id).ToHashSet(); Assert.True(entries.All(kept.Contains), "no audit entry was deleted"); // Saved in Settings › Logs, a retention becomes the schedule's age limit and turns it on, without a restart. await SaveAuditRetentionAsync(admin, 30); Assert.Equal(30, server.Options.AuditRetentionDays); audit = SystemRow(await ListAsync(admin), ServerScheduleHost.AuditCleanup); Assert.Equal(((int?)30, true), (audit.Parameters.OlderThanDays, audit.Enabled)); // Without an environment variable the schedule stays editable, and Settings shows its limit. await EditAsync(admin, audit.Id, new UpdateScheduleRequest(Parameters: new ScheduleParametersDto(Steps: ["audit"], OlderThanDays: 60))); Assert.Equal(60, (await SettingsAsync(admin)).Logs.AuditRetentionDays); // Saved back as shown, it changes neither the schedule nor the value Reset goes back to. await SaveAuditRetentionAsync(admin, 60); Assert.Equal(30, server.Options.AuditRetentionDays); Assert.Equal((int?)60, SystemRow(await ListAsync(admin), ServerScheduleHost.AuditCleanup).Parameters.OlderThanDays); using (var reset = await admin.PostAsync($"/api/v1/schedules/{audit.Id}/reset", null)) Assert.Equal((int?)30, (await reset.ReadAsync(ProtocolJson.Default.ScheduleDto)).Parameters.OlderThanDays); Assert.Equal(30, (await SettingsAsync(admin)).Logs.AuditRetentionDays); // Turned off on the schedule, nothing is deleted: Settings shows 0, and saving 30 there turns it on again. await EditAsync(admin, audit.Id, new UpdateScheduleRequest(Enabled: false)); Assert.Equal(0, (await SettingsAsync(admin)).Logs.AuditRetentionDays); await SaveAuditRetentionAsync(admin, 30); audit = SystemRow(await ListAsync(admin), ServerScheduleHost.AuditCleanup); Assert.Equal(((int?)30, true), (audit.Parameters.OlderThanDays, audit.Enabled)); // Without its audit step it deletes nothing either: Settings shows 0, and saving 30 there ticks the step again. await EditAsync(admin, audit.Id, new UpdateScheduleRequest(Parameters: new ScheduleParametersDto(Steps: ["job-history"], OlderThanDays: 30))); Assert.Equal(0, (await SettingsAsync(admin)).Logs.AuditRetentionDays); await SaveAuditRetentionAsync(admin, 30); audit = SystemRow(await ListAsync(admin), ServerScheduleHost.AuditCleanup); Assert.Equal(((int?)30, true), (audit.Parameters.OlderThanDays, audit.Enabled)); Assert.SequenceEqual(["job-history", "audit"], audit.Parameters.Steps!); // Back to 0: kept forever, the schedule off. await SaveAuditRetentionAsync(admin, 0); audit = SystemRow(await ListAsync(admin), ServerScheduleHost.AuditCleanup); Assert.Equal(((int?)0, false), (audit.Parameters.OlderThanDays, audit.Enabled)); // A limit set on the schedule while the setting is 0 is shown, and saving 0 turns the schedule off. await EditAsync(admin, audit.Id, new UpdateScheduleRequest(Parameters: new ScheduleParametersDto(Steps: ["audit"], OlderThanDays: 45), Enabled: true)); Assert.Equal(45, (await SettingsAsync(admin)).Logs.AuditRetentionDays); await SaveAuditRetentionAsync(admin, 0); audit = SystemRow(await ListAsync(admin), ServerScheduleHost.AuditCleanup); Assert.Equal(((int?)0, false), (audit.Parameters.OlderThanDays, audit.Enabled)); } private static async Task SettingsAsync(HttpClient admin) { using var read = await admin.GetAsync("/api/v1/server/settings"); return await read.ReadAsync(ProtocolJson.Default.ServerSettingsDto); } /// Saves Settings › Logs › Keep audit (days) as the admin panel does; the answer shows the value saved. private static async Task SaveAuditRetentionAsync(HttpClient admin, int days) { var settings = await SettingsAsync(admin); // The test server listens on ports the OS chose, which the settings refuse: real ones are sent with the edit. using var saved = await admin.PutAsJsonAsync("/api/v1/server/settings", settings with { HttpPort = 18080, HttpsPort = 18443, Logs = settings.Logs with { AuditRetentionDays = days } }, ProtocolJson.Default.ServerSettingsDto); await saved.ExpectAsync(HttpStatusCode.OK); Assert.Equal(days, (await saved.ReadAsync(ProtocolJson.Default.ServerSettingsDto)).Logs.AuditRetentionDays); } private static async Task EditAsync(HttpClient admin, Guid id, UpdateScheduleRequest request) { using var edit = await admin.PutAsJsonAsync($"/api/v1/schedules/{id}", request, ProtocolJson.Default.UpdateScheduleRequest); await edit.ExpectAsync(HttpStatusCode.OK); } [Test] public async Task RunNowQueuesTheJobsOnceAndListsThemAsRuns() { var handler = new BlockingJobHandler(JobKind.Duplicates); await using var server = await TestServer.StartAsync(context, services: s => s.AddSingleton(handler)); var admin = await server.AdminAsync(); // Also a disabled schedule runs now. var schedule = await CreateAsync(admin, new CreateScheduleRequest("Duplicates", "duplicates", new RecurrenceDto("daily", Time: "01:00"), Enabled: false)); var first = await RunAsync(admin, schedule.Id); Assert.Equal("queued", first.Outcome); var job = await JobApiTests.WaitForAsync(admin, first.JobId!.Value, "running"); Assert.Equal(("duplicates", (Guid?)schedule.Id, "background"), (job.Kind, job.ScheduleId, job.Priority)); Assert.Equal((first.JobId, "queued", (DateTimeOffset?)null), (first.Schedule.LastJobId, first.Schedule.LastOutcome, first.Schedule.NextRun)); // While its last job has not ended, it is not run again. using (var busy = await admin.PostAsync($"/api/v1/schedules/{schedule.Id}/run", null)) Assert.Equal(first.JobId.ToString(), (await busy.ExpectErrorAsync(HttpStatusCode.Conflict, "schedule_pending")).Details); handler.Release(); await JobApiTests.WaitForAsync(admin, first.JobId!.Value, "done"); var second = await RunAsync(admin, schedule.Id); Assert.Equal("queued", second.Outcome); await JobApiTests.WaitForAsync(admin, second.JobId!.Value, "running"); handler.Release(); await JobApiTests.WaitForAsync(admin, second.JobId!.Value, "done"); // Its runs, newest first, a page at a time. using (var page1 = await admin.GetAsync($"/api/v1/schedules/{schedule.Id}/runs?limit=1")) { var runs = await page1.ReadAsync(ProtocolJson.Default.PageJobDto); Assert.Equal(second.JobId, runs.Items.Single().Id); using var page2 = await admin.GetAsync($"/api/v1/schedules/{schedule.Id}/runs?limit=1&cursor={Uri.EscapeDataString(Assert.NotNull(runs.NextCursor))}"); var rest = await page2.ReadAsync(ProtocolJson.Default.PageJobDto); Assert.Equal((first.JobId, (string?)null), (rest.Items.Single().Id, rest.NextCursor)); } using (var bad = await admin.GetAsync($"/api/v1/schedules/{schedule.Id}/runs?cursor=nonsense")) await bad.ExpectErrorAsync(HttpStatusCode.BadRequest, "invalid_cursor"); using (var unknown = await admin.GetAsync($"/api/v1/schedules/{Guid.CreateVersion7()}/runs")) await unknown.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found"); using (var other = await admin.PostAsync($"/api/v1/schedules/{Guid.CreateVersion7()}/run", null)) await other.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found"); // The queue shows which jobs a schedule queued. using (var queue = await admin.GetAsync("/api/v1/jobs?kind=duplicates")) Assert.True((await queue.ReadAsync(ProtocolJson.Default.PageJobDto)).Items.All(j => j.ScheduleId == schedule.Id), "scheduleId on the job rows"); using var audit = await admin.GetAsync("/api/v1/audit?action=schedule_run"); Assert.SequenceEqual(["Duplicates · queued", "Duplicates · queued"], (await audit.ReadAsync(ProtocolJson.Default.PageAuditEntryDto)).Items.Select(e => e.Details)); } [Test] public async Task SystemSchedulesAreSeededOnceAndKeepTheirEditsAcrossRestarts() { var root = context.CreateTempDirectory("restart"); void Paths(ServerOptions o) { o.DataPath = Path.Combine(root, "data"); o.ArchivePath = Path.Combine(root, "archive"); Directory.CreateDirectory(o.ArchivePath); } Guid scanId; DateTimeOffset? lastRun; await using (var first = await TestServer.StartAsync(context, Paths)) { var admin = await first.AdminAsync(); scanId = SystemRow(await ListAsync(admin), ServerScheduleHost.ArchiveScan).Id; // Its run at start (no scan handler here: "unavailable") is recorded before the edit. var until = DateTime.UtcNow + TimeSpan.FromSeconds(30); while ((lastRun = (await first.Database.Schedules.GetAsync(scanId, Ct))!.LastRun) is null) { Assert.True(DateTime.UtcNow < until, "no run at start"); await Task.Delay(20); } using var put = await admin.PutAsJsonAsync($"/api/v1/schedules/{scanId}", new UpdateScheduleRequest(Recurrence: new RecurrenceDto("every", 20, "minutes"), Enabled: false), ProtocolJson.Default.UpdateScheduleRequest); await put.ExpectAsync(HttpStatusCode.OK); } await using var second = await TestServer.StartAsync(context, Paths); var rows = await second.Database.Schedules.ListAsync(Ct); Assert.SequenceEqual(SystemKeys, rows.Select(s => s.SystemKey)); var scan = rows.Single(s => s.SystemKey == ServerScheduleHost.ArchiveScan); Assert.Equal((scanId, "every:20m", false, (DateTimeOffset?)null), (scan.Id, scan.Recurrence, scan.Enabled, scan.NextRun)); // Disabled, the scan does not run at start either. Assert.Equal(lastRun, scan.LastRun); } [Test] public async Task TheScanScheduleQueuesAScanWhenDueAndNoTimerScansBesideIt() { var clock = new ManualTimeProvider(); await using var server = await TestServer.StartAsync(context, services: s => { ArchiveModule.Register(s); s.AddSingleton(clock); }); var hosted = server.Services.GetServices().Select(h => h.GetType().Name).ToList(); Assert.Contains(nameof(SchedulerHost), hosted); Assert.True(hosted.IndexOf(nameof(JobEngineHost)) < hosted.IndexOf(nameof(SchedulerHost)), "the scheduler starts after the engine"); Assert.True(!hosted.Contains("ArchivePolling") && !hosted.Contains("DatabaseBackupService"), string.Join(", ", hosted)); var scheduleId = Assert.NotNull(await server.Database.Schedules.GetBySystemKeyAsync(ServerScheduleHost.ArchiveScan, Ct)).Id; // At start the scan schedule runs once, as the polling did; no other scan is queued. var scans = await ScansAsync(server, 1); Assert.Equal(scheduleId, scans.Single().ScheduleId); // Ten minutes later it is due: the next pass queues the second scan. clock.Advance(TimeSpan.FromMinutes(11)); await server.Services.GetRequiredService().Scheduler.RunDueAsync(Ct); scans = await ScansAsync(server, 2); Assert.True(scans.All(j => j.ScheduleId == scheduleId && j.Priority == JobPriority.Background), "scans of the schedule, in the background"); var row = Assert.NotNull(await server.Database.Schedules.GetAsync(scheduleId, Ct)); Assert.Equal(("queued", (Guid?)scans[^1].Id), (row.LastOutcome, row.LastJobId)); Assert.True(row.NextRun > clock.GetUtcNow(), "the next run moved on"); } [Test] public async Task TheUploadCleanupScheduleSweepsAbandonedUploadsLikeTheFormerTimer() { var clock = new ManualTimeProvider(); await using var server = await TestServer.StartAsync(context, o => o.UploadRetentionHours = 48, s => { ArchiveModule.Register(s); s.AddSingleton(clock); }); var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader"); var id = (await Uploads.StartAsync(uploader, Uploads.Photo(1000, 10))).UploadId!.Value; var store = server.Services.GetRequiredService(); Assert.True(store.Exists(id)); var scheduleId = Assert.NotNull(await server.Database.Schedules.GetBySystemKeyAsync(ServerScheduleHost.UploadCleanup, Ct)).Id; // Two days later its run (every 6 hours, with catch-up) expires the upload and deletes its partial file. clock.Advance(TimeSpan.FromHours(49)); await server.Services.GetRequiredService().Scheduler.RunDueAsync(Ct); var until = DateTime.UtcNow + TimeSpan.FromSeconds(30); Job? sweep = null; while (sweep is not { State: JobState.Done }) { Assert.True(DateTime.UtcNow < until, $"the clean-up did not finish: {sweep?.State} {sweep?.Error}"); await Task.Delay(50); sweep = (await server.Database.Jobs.ListByScheduleAsync(scheduleId, cancellationToken: Ct)).FirstOrDefault(); } Assert.Equal(JobKind.Cleanup, sweep.Kind); Assert.Contains("Uploads expired: 1", sweep.LogTail); Assert.False(store.Exists(id)); using var state = await uploader.GetAsync($"/api/v1/uploads/{id}"); Assert.Equal("cancelled", (await state.ReadAsync(ProtocolJson.Default.UploadState)).State); } [Test] public async Task AnAuditRetentionFromTheEnvironmentIsAppliedAtStartAndCannotBeChanged() { var root = context.CreateTempDirectory("locked"); void Paths(ServerOptions o) { o.DataPath = Path.Combine(root, "data"); o.ArchivePath = Path.Combine(root, "archive"); Directory.CreateDirectory(o.ArchivePath); } await using (var first = await TestServer.StartAsync(context, Paths, ArchiveModule.Register)) { var seeded = Assert.NotNull(await first.Database.Schedules.GetBySystemKeyAsync(ServerScheduleHost.AuditCleanup, Ct)); Assert.Equal(((int?)0, false), (ScheduleMapping.Parameters(seeded).OlderThanDays, seeded.Enabled)); // Its audit step unticked before the variable is set. var unticked = await first.Services.GetRequiredService().Scheduler.UpdateAsync(seeded.Id, new UpdateScheduleRequest(Parameters: new ScheduleParametersDto(Steps: ["job-history"], OlderThanDays: 0)), Ct); Assert.Null(unticked.Problem); } // CHRONOS_AUDIT_RETENTION_DAYS=30 at the next start: the limit, the switch and the audit step it deletes with. await using var second = await TestServer.StartAsync(context, o => { Paths(o); o.AuditRetentionDays = 30; o.Locked = [nameof(ServerOptions.AuditRetentionDays)]; }, ArchiveModule.Register); var admin = await second.AdminAsync(); var audit = SystemRow(await ListAsync(admin), ServerScheduleHost.AuditCleanup); Assert.Equal(((int?)30, true), (audit.Parameters.OlderThanDays, audit.Enabled)); Assert.SequenceEqual(["job-history", "audit"], audit.Parameters.Steps!); using (var changed = await admin.PutAsJsonAsync($"/api/v1/schedules/{audit.Id}", new UpdateScheduleRequest(Parameters: new ScheduleParametersDto(Steps: ["audit"], OlderThanDays: 60)), ProtocolJson.Default.UpdateScheduleRequest)) Assert.Equal("AuditRetentionDays", (await changed.ExpectErrorAsync(HttpStatusCode.Conflict, "setting_locked")).Details); // Turned off, or without its audit step, it would keep what the variable says to delete. using (var off = await admin.PutAsJsonAsync($"/api/v1/schedules/{audit.Id}", new UpdateScheduleRequest(Enabled: false), ProtocolJson.Default.UpdateScheduleRequest)) Assert.Equal("AuditRetentionDays", (await off.ExpectErrorAsync(HttpStatusCode.Conflict, "setting_locked")).Details); using (var unticked = await admin.PutAsJsonAsync($"/api/v1/schedules/{audit.Id}", new UpdateScheduleRequest(Parameters: new ScheduleParametersDto(Steps: ["job-history"], OlderThanDays: 30)), ProtocolJson.Default.UpdateScheduleRequest)) Assert.Equal("AuditRetentionDays", (await unticked.ExpectErrorAsync(HttpStatusCode.Conflict, "setting_locked")).Details); Assert.SequenceEqual(["job-history", "audit"], SystemRow(await ListAsync(admin), ServerScheduleHost.AuditCleanup).Parameters.Steps!); // Everything else stays editable. using var timing = await admin.PutAsJsonAsync($"/api/v1/schedules/{audit.Id}", new UpdateScheduleRequest(Recurrence: new RecurrenceDto("daily", Time: "05:00"), Parameters: new ScheduleParametersDto(Steps: ["audit", "job-history"], OlderThanDays: 30)), ProtocolJson.Default.UpdateScheduleRequest); await timing.ExpectAsync(HttpStatusCode.OK); // Settings shows the fixed value, and the other settings still save with it. Assert.Equal(30, (await SettingsAsync(admin)).Logs.AuditRetentionDays); await SaveAuditRetentionAsync(admin, 30); } /// /// A missed audit clean-up caught up at start runs with the retention CHRONOS_AUDIT_RETENTION_DAYS fixes, not with the /// age limit the schedule held before: the variable is applied before the first pass. /// [Test] public async Task ALockedAuditRetentionAppliesToTheRunCaughtUpAtStart() { foreach (var days in new[] { 365, 0 }) { var root = context.CreateTempDirectory($"catch-up-{days}"); void Paths(ServerOptions o) { o.DataPath = Path.Combine(root, "data"); o.ArchivePath = Path.Combine(root, "archive"); Directory.CreateDirectory(o.ArchivePath); } DatabaseOptions database; await using (var first = await TestServer.StartAsync(context, Paths, ArchiveModule.Register)) database = first.Options.ToDatabaseOptions(); // While the server was off: a 90-day audit clean-up whose run passed an hour ago, and an entry 200 days old. var old = Guid.CreateVersion7(); await using (var db = await ChronosDatabase.OpenAsync(database, Ct)) { var row = Assert.NotNull(await db.Schedules.GetBySystemKeyAsync(ServerScheduleHost.AuditCleanup, Ct)); var parameters = ScheduleMapping.Parameters(row) with { OlderThanDays = 90 }; Assert.True(await db.Schedules.UpdateAsync(row with { ParametersJson = ScheduleMapping.ParametersJson(row.Action, parameters), Enabled = true, NextRun = DateTimeOffset.UtcNow.AddHours(-1), }, setNextRun: true, Ct)); await db.Audit.AppendAsync(new AuditEntry { Id = old, At = DateTimeOffset.UtcNow.AddDays(-200), Action = "user.login" }, Ct); } // CHRONOS_AUDIT_RETENTION_DAYS at the next start. await using var second = await TestServer.StartAsync(context, o => { Paths(o); o.AuditRetentionDays = days; o.Locked = [nameof(ServerOptions.AuditRetentionDays)]; }, ArchiveModule.Register); var schedule = Assert.NotNull(await second.Database.Schedules.GetBySystemKeyAsync(ServerScheduleHost.AuditCleanup, Ct)); var queued = await second.Database.Jobs.ListByScheduleAsync(schedule.Id, cancellationToken: Ct); if (days == 0) { // Kept forever: the schedule is off before its missed run could fire. Assert.False(schedule.Enabled); Assert.Empty(queued); } else { Assert.Count(1, queued); var job = queued[0]; using (var json = JsonDocument.Parse(job.ParametersJson!)) Assert.Equal(days, json.RootElement.GetProperty("olderThanDays").GetInt32()); await JobApiTests.WaitForAsync(await second.AdminAsync(), job.Id, "done"); } var kept = (await second.Database.Audit.ListByActionsAsync(null, [], 1000, null, Ct)).Items.Select(e => e.Id); Assert.Contains(old, kept); } } [Test] public void AnUnknownTimeZoneFallsBackToUtc() { var local = ServerScheduleHost.ResolveTimeZone(null, NullLogger.Instance); Assert.Equal(TimeZoneInfo.Local.BaseUtcOffset, local.BaseUtcOffset); Assert.Equal(TimeZoneInfo.Local.SupportsDaylightSavingTime, local.SupportsDaylightSavingTime); // The runtime ignores TZ on Windows; elsewhere a zone it cannot find would leave the schedules in an unknown zone. if (!OperatingSystem.IsWindows()) { Assert.Equal(TimeZoneInfo.Utc, ServerScheduleHost.ResolveTimeZone("Not/AZone", NullLogger.Instance)); Assert.Equal(TimeZoneInfo.Local.Id, ServerScheduleHost.ResolveTimeZone(":" + TimeZoneInfo.Local.Id, NullLogger.Instance).Id); } } [Test] public void TheAuditRetentionVariableIsReadAndChecked() { var data = context.CreateTempDirectory("env"); var options = ServerOptions.Load(new Dictionary { ["CHRONOS_DATA_PATH"] = data, ["CHRONOS_DB"] = "postgres", [ServerOptions.AuditRetentionVariable] = "0" }); Assert.Equal(0, options.AuditRetentionDays); Assert.Contains(nameof(ServerOptions.AuditRetentionDays), options.Locked); Assert.Equal(0, new ServerOptions().AuditRetentionDays); Assert.Equal(3650, ServerOptions.Load(new Dictionary { ["CHRONOS_DATA_PATH"] = data, ["CHRONOS_DB"] = "postgres", [ServerOptions.AuditRetentionVariable] = "3650" }) .AuditRetentionDays); foreach (var invalid in new[] { "-1", "3651", "a year" }) { Assert.Throws(() => ServerOptions.Load(new Dictionary { ["CHRONOS_DATA_PATH"] = data, ["CHRONOS_DB"] = "postgres", [ServerOptions.AuditRetentionVariable] = invalid })); } } /// Scan jobs, oldest first, once exist and none is waiting or running. private async Task> ScansAsync(TestServer server, int count) { var until = DateTime.UtcNow + TimeSpan.FromSeconds(30); while (true) { var scans = (await server.Database.Jobs.ListAsync(null, JobKind.Scan, 50, null, Ct)).Items.OrderBy(j => j.Created).ToList(); if (scans.Count >= count && scans.All(j => j.State is JobState.Done or JobState.Failed or JobState.Cancelled)) { Assert.Count(count, scans); return scans; } Assert.True(DateTime.UtcNow < until, $"{scans.Count} scan(s): {string.Join(", ", scans.Select(j => j.State))}"); await Task.Delay(50); } } private static ScheduleDto SystemRow(SchedulesDto list, string key) => list.Items.Single(s => s.SystemKey == key); private static async Task ListAsync(HttpClient admin) { using var response = await admin.GetAsync("/api/v1/schedules"); return await response.ReadAsync(ProtocolJson.Default.SchedulesDto); } private static async Task CreateAsync(HttpClient admin, CreateScheduleRequest request) { using var response = await admin.PostAsJsonAsync("/api/v1/schedules", request, ProtocolJson.Default.CreateScheduleRequest); await response.ExpectAsync(HttpStatusCode.Created); return await response.ReadAsync(ProtocolJson.Default.ScheduleDto); } private static async Task RunAsync(HttpClient admin, Guid id) { using var response = await admin.PostAsync($"/api/v1/schedules/{id}/run", null); return await response.ReadAsync(ProtocolJson.Default.ScheduleRunDto); } }