Files
Chronos/tests/Chronos.Server.Tests/Jobs/ScheduleEndpointTests.cs
T
Alby96 3f0e6357b1 fix(server): archive.json on its own timer, a live Scheduled tab in one time zone, schedule audit in words
- SchedulerHost reads archive.json every 10 minutes on its own timer. The archive-scan
  runs that used to do it are skipped while a long scan is pending, and the schedule may
  be off or daily, so uploads kept a desktop's old folder template or proxy size.
- Every schedule change and run, a skipped run included, is published as a
  scheduleChanged event. The Scheduled tab reloads on it, so skipped runs, next-run moves
  and other administrators' edits show without waiting for a job.
- The Scheduled tab still draws when the schedule a job links to, or an expanded one, no
  longer exists: its runs are dropped instead of failing the whole load.
- Every time on the Scheduled tab (next run, last result, runs) is in the server's time
  zone, as its "Times in" line says.
- Audit details name a system schedule after its key (system:<key> · name · ...). The
  dashboard and the Logs page show it by its localized name and the recurrence code in
  words, as the Scheduled tab does. Older entries keep their recorded name.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit c20f573d0bdf1cdcf3a86e9702ac76a4bf9508d9)
2026-10-27 09:57:23 +01:00

699 lines
43 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using System.Net;
using System.Net.Http.Json;
using System.Text.Json;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging.Abstractions;
using Chronos.Core.Archiving;
using Chronos.Core.Model;
using Chronos.Data;
using Chronos.Data.Jobs;
using Chronos.Data.Scheduling;
using Chronos.Protocol;
using Chronos.Server.Configuration;
using Chronos.Server.Events;
using Chronos.Server.Jobs;
using Chronos.Server.Storage;
using Chronos.Testing;
namespace Chronos.Server.Tests.Jobs;
/// <summary>
/// <c>/api/v1/schedules</c> and the server's system schedules (owner review 2, R10): every timer the server ran is a visible
/// schedule, seeded once, edited, reset and run like the others; the old timers are gone.
/// </summary>
[Category("server")]
public sealed class ScheduleEndpointTests(TestContext context)
{
private static readonly string[] SystemKeys =
[ServerScheduleHost.ArchiveScan, ServerScheduleHost.UploadCleanup, ServerScheduleHost.IndexBackup, ServerScheduleHost.Housekeeping, ServerScheduleHost.AuditCleanup];
private CancellationToken Ct => context.Cancellation;
[Test]
public async Task TheSystemSchedulesTheActionsAndTheTimeZoneAreListed()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var list = await ListAsync(await server.AdminAsync());
// The timers the server ran before, with their timings: nothing runs that this list does not show.
Assert.SequenceEqual(SystemKeys, list.Items.Select(s => s.SystemKey));
var scan = SystemRow(list, ServerScheduleHost.ArchiveScan);
Assert.Equal(("scan", new RecurrenceDto("every", 10, "minutes"), false, "background", true), (scan.Action, scan.Recurrence, scan.CatchUp, scan.Priority, scan.Enabled));
Assert.NotNull(scan.NextRun);
var uploads = SystemRow(list, ServerScheduleHost.UploadCleanup);
// Every 6 hours, not more often: four clean-up jobs a day keep the job history readable.
Assert.Equal(("cleanup", new RecurrenceDto("every", 6, "hours"), true, (int?)90), (uploads.Action, uploads.Recurrence, uploads.CatchUp, uploads.Parameters.OlderThanDays));
Assert.SequenceEqual([UploadCleanupStep.StepName], uploads.Parameters.Steps!);
var backup = SystemRow(list, ServerScheduleHost.IndexBackup);
Assert.Equal(("backup", new RecurrenceDto("daily", Time: "03:00"), true, (int?)3), (backup.Action, backup.Recurrence, backup.CatchUp, backup.Parameters.Keep));
var history = SystemRow(list, ServerScheduleHost.Housekeeping);
Assert.Equal((new RecurrenceDto("daily", Time: "04:00"), (int?)90), (history.Recurrence, history.Parameters.OlderThanDays));
Assert.SequenceEqual(["job-history"], history.Parameters.Steps!);
var audit = SystemRow(list, ServerScheduleHost.AuditCleanup);
// The audit log is kept forever by default: no deletion after an upgrade.
Assert.Equal((new RecurrenceDto("daily", Time: "04:30"), (int?)0, false), (audit.Recurrence, audit.Parameters.OlderThanDays, audit.Enabled));
Assert.SequenceEqual(["audit"], audit.Parameters.Steps!);
// Every action, with whether this server runs it: without the processing module there is no analysis.
Assert.SequenceEqual(ScheduleActions.All, list.Actions.Select(a => a.Name));
Assert.Equal((false, "no-handler"), (list.Actions.Single(a => a.Name == "analysis").Available, list.Actions.Single(a => a.Name == "analysis").Reason));
Assert.True(list.Actions.Where(a => a.Name != "analysis").All(a => a.Available), string.Join(", ", list.Actions.Select(a => $"{a.Name}:{a.Reason}")));
Assert.SequenceEqual(["job-history", "audit", "trash-report", UploadCleanupStep.StepName], list.Steps!);
Assert.Equal(server.Services.GetRequiredService<ServerScheduleHost>().TimeZone.Id, list.TimeZone);
}
[Test]
public async Task SchedulesAreCreatedChangedAndDeleted()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var admin = await server.AdminAsync();
var created = await CreateAsync(admin, new CreateScheduleRequest("Weekly check", "verify", new RecurrenceDto("weekly", Time: "02:30", Days: ["thu", "mon"]),
new ScheduleParametersDto(OlderThanDays: 60, Percent: 25)));
Assert.Equal(("Weekly check", "verify", "current", "background", true, (string?)null), (created.Name, created.Action, created.Target, created.Priority, created.Enabled, created.SystemKey));
Assert.SequenceEqual(["mon", "thu"], created.Recurrence.Days!);
Assert.Equal(((int?)60, (int?)25), (created.Parameters.OlderThanDays, created.Parameters.Percent));
Assert.NotNull(created.NextRun);
// After the system schedules.
Assert.Equal(created.Id, (await ListAsync(admin)).Items[^1].Id);
// PUT replaces the fields it gives; disabled, a schedule has no next run.
using (var put = await admin.PutAsJsonAsync($"/api/v1/schedules/{created.Id}",
new UpdateScheduleRequest(Recurrence: new RecurrenceDto("every", 2, "hours"), Priority: "normal", Enabled: false), ProtocolJson.Default.UpdateScheduleRequest))
{
var updated = await put.ReadAsync(ProtocolJson.Default.ScheduleDto);
Assert.Equal((new RecurrenceDto("every", 2, "hours"), "normal", false, (DateTimeOffset?)null, (int?)25),
(updated.Recurrence, updated.Priority, updated.Enabled, updated.NextRun, updated.Parameters.Percent));
}
// PATCH does the same: enabled again, it gets a next run.
using (var patch = await admin.PatchAsJsonAsync($"/api/v1/schedules/{created.Id}", new UpdateScheduleRequest(Enabled: true, Name: "Check"),
ProtocolJson.Default.UpdateScheduleRequest))
{
var enabled = await patch.ReadAsync(ProtocolJson.Default.ScheduleDto);
Assert.Equal(("Check", true), (enabled.Name, enabled.Enabled));
Assert.NotNull(enabled.NextRun);
}
using (var one = await admin.GetAsync($"/api/v1/schedules/{created.Id}")) Assert.Equal("Check", (await one.ReadAsync(ProtocolJson.Default.ScheduleDto)).Name);
using (var delete = await admin.DeleteAsync($"/api/v1/schedules/{created.Id}")) await delete.ExpectAsync(HttpStatusCode.NoContent);
using (var gone = await admin.GetAsync($"/api/v1/schedules/{created.Id}")) await gone.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found");
using (var again = await admin.DeleteAsync($"/api/v1/schedules/{created.Id}")) await again.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found");
Assert.SequenceEqual(SystemKeys, (await ListAsync(admin)).Items.Select(s => s.SystemKey));
// Every change is in the audit log.
using var audit = await admin.GetAsync("/api/v1/audit?action=schedule_created,schedule_updated,schedule_deleted");
var entries = (await audit.ReadAsync(ProtocolJson.Default.PageAuditEntryDto)).Items;
Assert.SequenceEqual(["schedule_deleted", "schedule_updated", "schedule_updated", "schedule_created"], entries.Select(e => e.Action));
Assert.True(entries.All(e => e.TargetId == created.Id.ToString() && e.Username == "admin"), "target and user");
Assert.Equal("Weekly check · verify · weekly:mon,thu@02:30", entries[^1].Details);
Assert.Equal(("Weekly check · verify · every:2h · off", "Check · verify · every:2h"), (entries[2].Details, entries[1].Details));
}
[Test]
public async Task OnlyArchiveAdministratorsSeeAndChangeSchedules()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
using (var list = await editor.GetAsync("/api/v1/schedules")) await list.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
using (var create = await editor.PostAsJsonAsync("/api/v1/schedules", new CreateScheduleRequest("Mine", "scan", new RecurrenceDto("daily", Time: "01:00")),
ProtocolJson.Default.CreateScheduleRequest))
await create.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
var scan = Assert.NotNull(await server.Database.Schedules.GetBySystemKeyAsync(ServerScheduleHost.ArchiveScan, Ct));
using (var run = await editor.PostAsync($"/api/v1/schedules/{scan.Id}/run", null)) await run.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
using (var anonymous = await server.Client.GetAsync("/api/v1/schedules")) Assert.Equal(HttpStatusCode.Unauthorized, anonymous.StatusCode);
}
[Test]
public async Task InvalidFieldsAreNamed()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var admin = await server.AdminAsync();
var daily = new RecurrenceDto("daily", Time: "03:00");
var cases = new (CreateScheduleRequest Request, string Field)[]
{
(new(" ", "scan", daily), "name"),
(new(new string('x', Scheduler.MaxNameLength + 1), "scan", daily), "name"),
(new("Render", "render", daily), "action"),
(new("Often", "scan", new RecurrenceDto("every", 1, "minutes")), "recurrence"),
(new("No days", "scan", new RecurrenceDto("weekly", Time: "03:00", Days: [])), "recurrence"),
(new("Late", "scan", new RecurrenceDto("daily", Time: "25:00")), "recurrence"),
(new("Day 31", "scan", new RecurrenceDto("monthly", Time: "03:00", DayOfMonth: 31)), "recurrence"),
(new("None", "verify", daily, new ScheduleParametersDto(Percent: 0)), "parameters.percent"),
(new("Old", "verify", daily, new ScheduleParametersDto(OlderThanDays: -1)), "parameters.olderThanDays"),
(new("Copies", "backup", daily, new ScheduleParametersDto(Keep: 0)), "parameters.keep"),
(new("Unknown step", "cleanup", daily, new ScheduleParametersDto(Steps: ["photos"])), "parameters.steps"),
(new("No step", "cleanup", daily, new ScheduleParametersDto(Steps: [])), "parameters.steps"),
(new("Urgent", "scan", daily, Priority: "urgent"), "priority"),
(new("Elsewhere", "scan", daily, Target: "another-archive"), "target"),
};
foreach (var (request, field) in cases)
{
using var response = await admin.PostAsJsonAsync("/api/v1/schedules", request, ProtocolJson.Default.CreateScheduleRequest);
var error = await response.ExpectErrorAsync(HttpStatusCode.BadRequest, "invalid_schedule");
Assert.Equal(field, error.Details, request.Name);
Assert.Contains(field, error.Message);
}
Assert.Count(SystemKeys.Length, (await ListAsync(admin)).Items);
}
[Test]
public async Task SystemSchedulesAreEditedAndResetButNeverDeleted()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var admin = await server.AdminAsync();
var housekeeping = SystemRow(await ListAsync(admin), ServerScheduleHost.Housekeeping);
using (var delete = await admin.DeleteAsync($"/api/v1/schedules/{housekeeping.Id}")) await delete.ExpectErrorAsync(HttpStatusCode.Conflict, "schedule_system");
foreach (var (request, field) in new[] { (new UpdateScheduleRequest(Name: "Mine"), "name"), (new UpdateScheduleRequest(Action: "verify"), "action") })
{
using var refused = await admin.PutAsJsonAsync($"/api/v1/schedules/{housekeeping.Id}", request, ProtocolJson.Default.UpdateScheduleRequest);
Assert.Equal(field, (await refused.ExpectErrorAsync(HttpStatusCode.BadRequest, "invalid_schedule")).Details);
}
// Its timing, its parameters and its switch change like any schedule's; the same name and action may be sent back.
using (var edit = await admin.PutAsJsonAsync($"/api/v1/schedules/{housekeeping.Id}", new UpdateScheduleRequest(housekeeping.Name, "cleanup",
new RecurrenceDto("weekly", Time: "05:15", Days: ["sun"]), new ScheduleParametersDto(Steps: ["job-history", "trash-report"], OlderThanDays: 30),
CatchUp: false, Enabled: false), ProtocolJson.Default.UpdateScheduleRequest))
{
var edited = await edit.ReadAsync(ProtocolJson.Default.ScheduleDto);
Assert.Equal(("weekly", (int?)30, false, false), (edited.Recurrence.Kind, edited.Parameters.OlderThanDays, edited.CatchUp, edited.Enabled));
}
// Reset puts the server's defaults back, with a new next run.
using (var reset = await admin.PostAsync($"/api/v1/schedules/{housekeeping.Id}/reset", null))
{
var back = await reset.ReadAsync(ProtocolJson.Default.ScheduleDto);
Assert.Equal((new RecurrenceDto("daily", Time: "04:00"), (int?)90, true, true), (back.Recurrence, back.Parameters.OlderThanDays, back.CatchUp, back.Enabled));
Assert.SequenceEqual(["job-history"], back.Parameters.Steps!);
Assert.NotNull(back.NextRun);
}
var custom = await CreateAsync(admin, new CreateScheduleRequest("Mine", "duplicates", new RecurrenceDto("monthly", Time: "01:00", DayOfMonth: 1)));
using (var notSystem = await admin.PostAsync($"/api/v1/schedules/{custom.Id}/reset", null))
await notSystem.ExpectErrorAsync(HttpStatusCode.Conflict, "schedule_no_defaults");
using (var unknown = await admin.PostAsync($"/api/v1/schedules/{Guid.CreateVersion7()}/reset", null)) await unknown.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found");
Assert.Equal("queued", (await RunAsync(admin, housekeeping.Id)).Outcome);
// A system schedule is audited after its key, so the admin panel names it in its language, as its Scheduled tab does.
using var audit = await admin.GetAsync("/api/v1/audit?action=schedule_updated,schedule_reset,schedule_run");
var entries = (await audit.ReadAsync(ProtocolJson.Default.PageAuditEntryDto)).Items;
Assert.SequenceEqual(["schedule_run", "schedule_reset", "schedule_updated"], entries.Select(e => e.Action));
Assert.True(entries.All(e => e.TargetId == housekeeping.Id.ToString()), "the target is the schedule");
Assert.SequenceEqual(["system:housekeeping · Housekeeping · queued", "system:housekeeping · Housekeeping · cleanup · daily:04:00",
"system:housekeeping · Housekeeping · cleanup · weekly:sun@05:15 · off"], entries.Select(e => e.Details));
}
/// <summary>
/// Every change and run of a schedule goes out on the event socket, also a run that queues no job, so the admin panel's
/// Scheduled tab follows what the scheduler does.
/// </summary>
[Test]
public async Task ScheduleChangesAndRunsAreEvents()
{
await using var server = await TestServer.StartAsync(context);
var admin = await server.AdminAsync();
using var events = server.Services.GetRequiredService<EventHub>().Subscribe();
// Off, so the scheduler's own passes leave it alone; without a duplicates handler its run queues nothing.
var created = await CreateAsync(admin, new CreateScheduleRequest("Duplicates", "duplicates", new RecurrenceDto("daily", Time: "01:00"), Enabled: false));
var run = await RunAsync(admin, created.Id);
Assert.Equal(("unavailable", (Guid?)null), (run.Outcome, run.JobId));
using (var delete = await admin.DeleteAsync($"/api/v1/schedules/{created.Id}")) await delete.ExpectAsync(HttpStatusCode.NoContent);
var changes = new List<bool>();
while (events.Reader.TryRead(out var e))
{
if (e is ScheduleChangedEvent changed && changed.ScheduleId == created.Id) changes.Add(changed.Deleted);
}
Assert.SequenceEqual([false, false, true], changes);
Assert.Equal($$"""{"type":"scheduleChanged","scheduleId":"{{created.Id}}","deleted":true}""",
JsonSerializer.Serialize<ServerEvent>(new ScheduleChangedEvent(created.Id, true), ProtocolJson.Default.ServerEvent));
}
/// <summary>
/// Settings a desktop put into archive.json reach the index every 10 minutes whatever the archive scan does, as with the
/// former polling: the scan's runs read them too, but they are skipped while a long scan is pending, and the schedule may
/// be rare or off.
/// </summary>
[Test]
public async Task ArchiveJsonIsReadEveryTenMinutesWhateverTheScanScheduleDoes()
{
var clock = new SteppedTimeProvider();
await using var server = await TestServer.StartAsync(context, services: s =>
{
ArchiveModule.Register(s);
s.AddSingleton<TimeProvider>(clock);
});
var host = server.Services.GetRequiredService<SchedulerHost>();
var archive = server.Services.GetRequiredService<ServerArchive>();
// The scan at start is over, then the scan schedule is turned off.
await ScansAsync(server, 1);
var scanId = Assert.NotNull(await server.Database.Schedules.GetBySystemKeyAsync(ServerScheduleHost.ArchiveScan, Ct)).Id;
Assert.Null((await host.Scheduler.UpdateAsync(scanId, new UpdateScheduleRequest(Enabled: false), Ct)).Problem);
// A desktop sharing the archive changes its folder template; nothing has read it yet.
const string template = "{camera}/{date}";
var path = new ArchivePaths(archive.Root).ManifestPath;
var manifest = ManifestStore.ReadArchive(path);
Assert.NotEqual(template, manifest.LayoutTemplates.Folder);
ManifestStore.WriteArchive(path, manifest with { LayoutTemplates = manifest.LayoutTemplates with { Folder = template } });
Assert.Equal(manifest.LayoutTemplates.Folder, Assert.NotNull(await server.Database.Archives.GetAsync(archive.Id, Ct)).FolderTemplate);
// Ten minutes on, uploads (the index row) and the admin panel (the served archive) follow it, with no scan.
clock.Advance(TimeSpan.FromMinutes(ServerScheduleHost.ScanMinutes));
var until = DateTime.UtcNow + TimeSpan.FromSeconds(TestTiming.Budget(30));
while (Assert.NotNull(await server.Database.Archives.GetAsync(archive.Id, Ct)).FolderTemplate != template)
{
Assert.True(DateTime.UtcNow < until, "archive.json was not read again");
await Task.Delay(20);
}
Assert.Equal(template, archive.Current.FolderTemplate);
Assert.Count(1, (await server.Database.Jobs.ListAsync(null, JobKind.Scan, 50, null, Ct)).Items);
}
[Test]
public async Task TheJobHistoryOfTheSettingsIsWhatTheHousekeepingScheduleIsResetTo()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var admin = await server.AdminAsync();
using (var read = await admin.GetAsync("/api/v1/server/settings"))
{
var settings = await read.ReadAsync(ProtocolJson.Default.ServerSettingsDto);
// The test server listens on ports the OS chose, which the settings refuse: real ones are sent with the edit.
using var saved = await admin.PutAsJsonAsync("/api/v1/server/settings", settings with { JobHistoryDays = 30, HttpPort = 18080, HttpsPort = 18443 },
ProtocolJson.Default.ServerSettingsDto);
await saved.ExpectAsync(HttpStatusCode.OK);
}
// The schedule keeps its own value; a reset takes the new one, without a restart.
Assert.Equal(30, server.Options.JobHistoryDays);
var housekeeping = SystemRow(await ListAsync(admin), ServerScheduleHost.Housekeeping);
Assert.Equal((int?)90, housekeeping.Parameters.OlderThanDays);
using var reset = await admin.PostAsync($"/api/v1/schedules/{housekeeping.Id}/reset", null);
Assert.Equal((int?)30, (await reset.ReadAsync(ProtocolJson.Default.ScheduleDto)).Parameters.OlderThanDays);
}
/// <summary>
/// Owner review 2, R10 and R11: the audit retention has one source of truth, the audit clean-up schedule's age limit.
/// Settings › Logs shows it wherever it was set (0 while the schedule is off) and sets it (turning the schedule on, or
/// off at 0); Reset goes back to the value last saved there, and at 0 the audit step deletes nothing.
/// </summary>
[Test]
public async Task TheAuditRetentionOfTheSettingsIsTheAgeLimitOfTheAuditCleanUp()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var admin = await server.AdminAsync();
var audit = SystemRow(await ListAsync(admin), ServerScheduleHost.AuditCleanup);
Assert.Equal(((int?)0, false), (audit.Parameters.OlderThanDays, audit.Enabled));
// At 0 a run (also of the disabled schedule, on request) keeps the whole audit log.
var entries = (await server.Database.Audit.ListByActionsAsync(null, [], 1000, null, Ct)).Items.Select(e => e.Id).ToList();
Assert.True(entries.Count > 0, "the sign-in left audit entries");
var run = await RunAsync(admin, audit.Id);
var job = await JobApiTests.WaitForAsync(admin, run.JobId!.Value, "done");
Assert.Contains("Audit log kept in full", job.LogTail);
var kept = (await server.Database.Audit.ListByActionsAsync(null, [], 1000, null, Ct)).Items.Select(e => e.Id).ToHashSet();
Assert.True(entries.All(kept.Contains), "no audit entry was deleted");
// Saved in Settings › Logs, a retention becomes the schedule's age limit and turns it on, without a restart.
await SaveAuditRetentionAsync(admin, 30);
Assert.Equal(30, server.Options.AuditRetentionDays);
audit = SystemRow(await ListAsync(admin), ServerScheduleHost.AuditCleanup);
Assert.Equal(((int?)30, true), (audit.Parameters.OlderThanDays, audit.Enabled));
// Without an environment variable the schedule stays editable, and Settings shows its limit.
await EditAsync(admin, audit.Id, new UpdateScheduleRequest(Parameters: new ScheduleParametersDto(Steps: ["audit"], OlderThanDays: 60)));
Assert.Equal(60, (await SettingsAsync(admin)).Logs.AuditRetentionDays);
// Saved back as shown, it changes neither the schedule nor the value Reset goes back to.
await SaveAuditRetentionAsync(admin, 60);
Assert.Equal(30, server.Options.AuditRetentionDays);
Assert.Equal((int?)60, SystemRow(await ListAsync(admin), ServerScheduleHost.AuditCleanup).Parameters.OlderThanDays);
using (var reset = await admin.PostAsync($"/api/v1/schedules/{audit.Id}/reset", null))
Assert.Equal((int?)30, (await reset.ReadAsync(ProtocolJson.Default.ScheduleDto)).Parameters.OlderThanDays);
Assert.Equal(30, (await SettingsAsync(admin)).Logs.AuditRetentionDays);
// Turned off on the schedule, nothing is deleted: Settings shows 0, and saving 30 there turns it on again.
await EditAsync(admin, audit.Id, new UpdateScheduleRequest(Enabled: false));
Assert.Equal(0, (await SettingsAsync(admin)).Logs.AuditRetentionDays);
await SaveAuditRetentionAsync(admin, 30);
audit = SystemRow(await ListAsync(admin), ServerScheduleHost.AuditCleanup);
Assert.Equal(((int?)30, true), (audit.Parameters.OlderThanDays, audit.Enabled));
// Back to 0: kept forever, the schedule off.
await SaveAuditRetentionAsync(admin, 0);
audit = SystemRow(await ListAsync(admin), ServerScheduleHost.AuditCleanup);
Assert.Equal(((int?)0, false), (audit.Parameters.OlderThanDays, audit.Enabled));
// A limit set on the schedule while the setting is 0 is shown, and saving 0 turns the schedule off.
await EditAsync(admin, audit.Id, new UpdateScheduleRequest(Parameters: new ScheduleParametersDto(Steps: ["audit"], OlderThanDays: 45), Enabled: true));
Assert.Equal(45, (await SettingsAsync(admin)).Logs.AuditRetentionDays);
await SaveAuditRetentionAsync(admin, 0);
audit = SystemRow(await ListAsync(admin), ServerScheduleHost.AuditCleanup);
Assert.Equal(((int?)0, false), (audit.Parameters.OlderThanDays, audit.Enabled));
}
private static async Task<ServerSettingsDto> SettingsAsync(HttpClient admin)
{
using var read = await admin.GetAsync("/api/v1/server/settings");
return await read.ReadAsync(ProtocolJson.Default.ServerSettingsDto);
}
/// <summary>Saves Settings › Logs › Keep audit (days) as the admin panel does; the answer shows the value saved.</summary>
private static async Task SaveAuditRetentionAsync(HttpClient admin, int days)
{
var settings = await SettingsAsync(admin);
// The test server listens on ports the OS chose, which the settings refuse: real ones are sent with the edit.
using var saved = await admin.PutAsJsonAsync("/api/v1/server/settings",
settings with { HttpPort = 18080, HttpsPort = 18443, Logs = settings.Logs with { AuditRetentionDays = days } }, ProtocolJson.Default.ServerSettingsDto);
await saved.ExpectAsync(HttpStatusCode.OK);
Assert.Equal(days, (await saved.ReadAsync(ProtocolJson.Default.ServerSettingsDto)).Logs.AuditRetentionDays);
}
private static async Task EditAsync(HttpClient admin, Guid id, UpdateScheduleRequest request)
{
using var edit = await admin.PutAsJsonAsync($"/api/v1/schedules/{id}", request, ProtocolJson.Default.UpdateScheduleRequest);
await edit.ExpectAsync(HttpStatusCode.OK);
}
[Test]
public async Task RunNowQueuesTheJobsOnceAndListsThemAsRuns()
{
var handler = new BlockingJobHandler(JobKind.Duplicates);
await using var server = await TestServer.StartAsync(context, services: s => s.AddSingleton<IJobHandler>(handler));
var admin = await server.AdminAsync();
// Also a disabled schedule runs now.
var schedule = await CreateAsync(admin, new CreateScheduleRequest("Duplicates", "duplicates", new RecurrenceDto("daily", Time: "01:00"), Enabled: false));
var first = await RunAsync(admin, schedule.Id);
Assert.Equal("queued", first.Outcome);
var job = await JobApiTests.WaitForAsync(admin, first.JobId!.Value, "running");
Assert.Equal(("duplicates", (Guid?)schedule.Id, "background"), (job.Kind, job.ScheduleId, job.Priority));
Assert.Equal((first.JobId, "queued", (DateTimeOffset?)null), (first.Schedule.LastJobId, first.Schedule.LastOutcome, first.Schedule.NextRun));
// While its last job has not ended, it is not run again.
using (var busy = await admin.PostAsync($"/api/v1/schedules/{schedule.Id}/run", null))
Assert.Equal(first.JobId.ToString(), (await busy.ExpectErrorAsync(HttpStatusCode.Conflict, "schedule_pending")).Details);
handler.Release();
await JobApiTests.WaitForAsync(admin, first.JobId!.Value, "done");
var second = await RunAsync(admin, schedule.Id);
Assert.Equal("queued", second.Outcome);
await JobApiTests.WaitForAsync(admin, second.JobId!.Value, "running");
handler.Release();
await JobApiTests.WaitForAsync(admin, second.JobId!.Value, "done");
// Its runs, newest first, a page at a time.
using (var page1 = await admin.GetAsync($"/api/v1/schedules/{schedule.Id}/runs?limit=1"))
{
var runs = await page1.ReadAsync(ProtocolJson.Default.PageJobDto);
Assert.Equal(second.JobId, runs.Items.Single().Id);
using var page2 = await admin.GetAsync($"/api/v1/schedules/{schedule.Id}/runs?limit=1&cursor={Uri.EscapeDataString(Assert.NotNull(runs.NextCursor))}");
var rest = await page2.ReadAsync(ProtocolJson.Default.PageJobDto);
Assert.Equal((first.JobId, (string?)null), (rest.Items.Single().Id, rest.NextCursor));
}
using (var bad = await admin.GetAsync($"/api/v1/schedules/{schedule.Id}/runs?cursor=nonsense")) await bad.ExpectErrorAsync(HttpStatusCode.BadRequest, "invalid_cursor");
using (var unknown = await admin.GetAsync($"/api/v1/schedules/{Guid.CreateVersion7()}/runs")) await unknown.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found");
using (var other = await admin.PostAsync($"/api/v1/schedules/{Guid.CreateVersion7()}/run", null)) await other.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found");
// The queue shows which jobs a schedule queued.
using (var queue = await admin.GetAsync("/api/v1/jobs?kind=duplicates"))
Assert.True((await queue.ReadAsync(ProtocolJson.Default.PageJobDto)).Items.All(j => j.ScheduleId == schedule.Id), "scheduleId on the job rows");
using var audit = await admin.GetAsync("/api/v1/audit?action=schedule_run");
Assert.SequenceEqual(["Duplicates · queued", "Duplicates · queued"], (await audit.ReadAsync(ProtocolJson.Default.PageAuditEntryDto)).Items.Select(e => e.Details));
}
[Test]
public async Task SystemSchedulesAreSeededOnceAndKeepTheirEditsAcrossRestarts()
{
var root = context.CreateTempDirectory("restart");
void Paths(ServerOptions o)
{
o.DataPath = Path.Combine(root, "data");
o.ArchivePath = Path.Combine(root, "archive");
Directory.CreateDirectory(o.ArchivePath);
}
Guid scanId;
DateTimeOffset? lastRun;
await using (var first = await TestServer.StartAsync(context, Paths))
{
var admin = await first.AdminAsync();
scanId = SystemRow(await ListAsync(admin), ServerScheduleHost.ArchiveScan).Id;
// Its run at start (no scan handler here: "unavailable") is recorded before the edit.
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
while ((lastRun = (await first.Database.Schedules.GetAsync(scanId, Ct))!.LastRun) is null)
{
Assert.True(DateTime.UtcNow < until, "no run at start");
await Task.Delay(20);
}
using var put = await admin.PutAsJsonAsync($"/api/v1/schedules/{scanId}", new UpdateScheduleRequest(Recurrence: new RecurrenceDto("every", 20, "minutes"), Enabled: false),
ProtocolJson.Default.UpdateScheduleRequest);
await put.ExpectAsync(HttpStatusCode.OK);
}
await using var second = await TestServer.StartAsync(context, Paths);
var rows = await second.Database.Schedules.ListAsync(Ct);
Assert.SequenceEqual(SystemKeys, rows.Select(s => s.SystemKey));
var scan = rows.Single(s => s.SystemKey == ServerScheduleHost.ArchiveScan);
Assert.Equal((scanId, "every:20m", false, (DateTimeOffset?)null), (scan.Id, scan.Recurrence, scan.Enabled, scan.NextRun));
// Disabled, the scan does not run at start either.
Assert.Equal(lastRun, scan.LastRun);
}
[Test]
public async Task TheScanScheduleQueuesAScanWhenDueAndNoTimerScansBesideIt()
{
var clock = new ManualTimeProvider();
await using var server = await TestServer.StartAsync(context, services: s =>
{
ArchiveModule.Register(s);
s.AddSingleton<TimeProvider>(clock);
});
var hosted = server.Services.GetServices<IHostedService>().Select(h => h.GetType().Name).ToList();
Assert.Contains(nameof(SchedulerHost), hosted);
Assert.True(hosted.IndexOf(nameof(JobEngineHost)) < hosted.IndexOf(nameof(SchedulerHost)), "the scheduler starts after the engine");
Assert.True(!hosted.Contains("ArchivePolling") && !hosted.Contains("DatabaseBackupService"), string.Join(", ", hosted));
var scheduleId = Assert.NotNull(await server.Database.Schedules.GetBySystemKeyAsync(ServerScheduleHost.ArchiveScan, Ct)).Id;
// At start the scan schedule runs once, as the polling did; no other scan is queued.
var scans = await ScansAsync(server, 1);
Assert.Equal(scheduleId, scans.Single().ScheduleId);
// Ten minutes later it is due: the next pass queues the second scan.
clock.Advance(TimeSpan.FromMinutes(11));
await server.Services.GetRequiredService<SchedulerHost>().Scheduler.RunDueAsync(Ct);
scans = await ScansAsync(server, 2);
Assert.True(scans.All(j => j.ScheduleId == scheduleId && j.Priority == JobPriority.Background), "scans of the schedule, in the background");
var row = Assert.NotNull(await server.Database.Schedules.GetAsync(scheduleId, Ct));
Assert.Equal(("queued", (Guid?)scans[^1].Id), (row.LastOutcome, row.LastJobId));
Assert.True(row.NextRun > clock.GetUtcNow(), "the next run moved on");
}
[Test]
public async Task TheUploadCleanupScheduleSweepsAbandonedUploadsLikeTheFormerTimer()
{
var clock = new ManualTimeProvider();
await using var server = await TestServer.StartAsync(context, o => o.UploadRetentionHours = 48, s =>
{
ArchiveModule.Register(s);
s.AddSingleton<TimeProvider>(clock);
});
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
var id = (await Uploads.StartAsync(uploader, Uploads.Photo(1000, 10))).UploadId!.Value;
var store = server.Services.GetRequiredService<UploadStore>();
Assert.True(store.Exists(id));
var scheduleId = Assert.NotNull(await server.Database.Schedules.GetBySystemKeyAsync(ServerScheduleHost.UploadCleanup, Ct)).Id;
// Two days later its run (every 6 hours, with catch-up) expires the upload and deletes its partial file.
clock.Advance(TimeSpan.FromHours(49));
await server.Services.GetRequiredService<SchedulerHost>().Scheduler.RunDueAsync(Ct);
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
Job? sweep = null;
while (sweep is not { State: JobState.Done })
{
Assert.True(DateTime.UtcNow < until, $"the clean-up did not finish: {sweep?.State} {sweep?.Error}");
await Task.Delay(50);
sweep = (await server.Database.Jobs.ListByScheduleAsync(scheduleId, cancellationToken: Ct)).FirstOrDefault();
}
Assert.Equal(JobKind.Cleanup, sweep.Kind);
Assert.Contains("Uploads expired: 1", sweep.LogTail);
Assert.False(store.Exists(id));
using var state = await uploader.GetAsync($"/api/v1/uploads/{id}");
Assert.Equal("cancelled", (await state.ReadAsync(ProtocolJson.Default.UploadState)).State);
}
[Test]
public async Task AnAuditRetentionFromTheEnvironmentIsAppliedAtStartAndCannotBeChanged()
{
var root = context.CreateTempDirectory("locked");
void Paths(ServerOptions o)
{
o.DataPath = Path.Combine(root, "data");
o.ArchivePath = Path.Combine(root, "archive");
Directory.CreateDirectory(o.ArchivePath);
}
await using (var first = await TestServer.StartAsync(context, Paths, ArchiveModule.Register))
{
var seeded = Assert.NotNull(await first.Database.Schedules.GetBySystemKeyAsync(ServerScheduleHost.AuditCleanup, Ct));
Assert.Equal(((int?)0, false), (ScheduleMapping.Parameters(seeded).OlderThanDays, seeded.Enabled));
}
// CHRONOS_AUDIT_RETENTION_DAYS=30 at the next start.
await using var second = await TestServer.StartAsync(context, o =>
{
Paths(o);
o.AuditRetentionDays = 30;
o.Locked = [nameof(ServerOptions.AuditRetentionDays)];
}, ArchiveModule.Register);
var admin = await second.AdminAsync();
var audit = SystemRow(await ListAsync(admin), ServerScheduleHost.AuditCleanup);
Assert.Equal(((int?)30, true), (audit.Parameters.OlderThanDays, audit.Enabled));
using (var changed = await admin.PutAsJsonAsync($"/api/v1/schedules/{audit.Id}", new UpdateScheduleRequest(Parameters: new ScheduleParametersDto(Steps: ["audit"], OlderThanDays: 60)),
ProtocolJson.Default.UpdateScheduleRequest))
Assert.Equal("AuditRetentionDays", (await changed.ExpectErrorAsync(HttpStatusCode.Conflict, "setting_locked")).Details);
// Turned off, it would keep what the variable says to delete.
using (var off = await admin.PutAsJsonAsync($"/api/v1/schedules/{audit.Id}", new UpdateScheduleRequest(Enabled: false), ProtocolJson.Default.UpdateScheduleRequest))
Assert.Equal("AuditRetentionDays", (await off.ExpectErrorAsync(HttpStatusCode.Conflict, "setting_locked")).Details);
// Everything else stays editable.
using var timing = await admin.PutAsJsonAsync($"/api/v1/schedules/{audit.Id}", new UpdateScheduleRequest(Recurrence: new RecurrenceDto("daily", Time: "05:00"),
Parameters: new ScheduleParametersDto(Steps: ["audit", "job-history"], OlderThanDays: 30)), ProtocolJson.Default.UpdateScheduleRequest);
await timing.ExpectAsync(HttpStatusCode.OK);
// Settings shows the fixed value, and the other settings still save with it.
Assert.Equal(30, (await SettingsAsync(admin)).Logs.AuditRetentionDays);
await SaveAuditRetentionAsync(admin, 30);
}
/// <summary>
/// A missed audit clean-up caught up at start runs with the retention CHRONOS_AUDIT_RETENTION_DAYS fixes, not with the
/// age limit the schedule held before: the variable is applied before the first pass.
/// </summary>
[Test]
public async Task ALockedAuditRetentionAppliesToTheRunCaughtUpAtStart()
{
foreach (var days in new[] { 365, 0 })
{
var root = context.CreateTempDirectory($"catch-up-{days}");
void Paths(ServerOptions o)
{
o.DataPath = Path.Combine(root, "data");
o.ArchivePath = Path.Combine(root, "archive");
Directory.CreateDirectory(o.ArchivePath);
}
DatabaseOptions database;
await using (var first = await TestServer.StartAsync(context, Paths, ArchiveModule.Register))
database = first.Options.ToDatabaseOptions();
// While the server was off: a 90-day audit clean-up whose run passed an hour ago, and an entry 200 days old.
var old = Guid.CreateVersion7();
await using (var db = await ChronosDatabase.OpenAsync(database, Ct))
{
var row = Assert.NotNull(await db.Schedules.GetBySystemKeyAsync(ServerScheduleHost.AuditCleanup, Ct));
var parameters = ScheduleMapping.Parameters(row) with { OlderThanDays = 90 };
Assert.True(await db.Schedules.UpdateAsync(row with
{
ParametersJson = ScheduleMapping.ParametersJson(row.Action, parameters),
Enabled = true,
NextRun = DateTimeOffset.UtcNow.AddHours(-1),
}, setNextRun: true, Ct));
await db.Audit.AppendAsync(new AuditEntry { Id = old, At = DateTimeOffset.UtcNow.AddDays(-200), Action = "user.login" }, Ct);
}
// CHRONOS_AUDIT_RETENTION_DAYS at the next start.
await using var second = await TestServer.StartAsync(context, o =>
{
Paths(o);
o.AuditRetentionDays = days;
o.Locked = [nameof(ServerOptions.AuditRetentionDays)];
}, ArchiveModule.Register);
var schedule = Assert.NotNull(await second.Database.Schedules.GetBySystemKeyAsync(ServerScheduleHost.AuditCleanup, Ct));
var queued = await second.Database.Jobs.ListByScheduleAsync(schedule.Id, cancellationToken: Ct);
if (days == 0)
{
// Kept forever: the schedule is off before its missed run could fire.
Assert.False(schedule.Enabled);
Assert.Empty(queued);
}
else
{
Assert.Count(1, queued);
var job = queued[0];
using (var json = JsonDocument.Parse(job.ParametersJson!))
Assert.Equal(days, json.RootElement.GetProperty("olderThanDays").GetInt32());
await JobApiTests.WaitForAsync(await second.AdminAsync(), job.Id, "done");
}
var kept = (await second.Database.Audit.ListByActionsAsync(null, [], 1000, null, Ct)).Items.Select(e => e.Id);
Assert.Contains(old, kept);
}
}
[Test]
public void AnUnknownTimeZoneFallsBackToUtc()
{
var local = ServerScheduleHost.ResolveTimeZone(null, NullLogger.Instance);
Assert.Equal(TimeZoneInfo.Local.BaseUtcOffset, local.BaseUtcOffset);
Assert.Equal(TimeZoneInfo.Local.SupportsDaylightSavingTime, local.SupportsDaylightSavingTime);
// The runtime ignores TZ on Windows; elsewhere a zone it cannot find would leave the schedules in an unknown zone.
if (!OperatingSystem.IsWindows())
{
Assert.Equal(TimeZoneInfo.Utc, ServerScheduleHost.ResolveTimeZone("Not/AZone", NullLogger.Instance));
Assert.Equal(TimeZoneInfo.Local.Id, ServerScheduleHost.ResolveTimeZone(":" + TimeZoneInfo.Local.Id, NullLogger.Instance).Id);
}
}
[Test]
public void TheAuditRetentionVariableIsReadAndChecked()
{
var data = context.CreateTempDirectory("env");
var options = ServerOptions.Load(new Dictionary<string, string?> { ["CHRONOS_DATA_PATH"] = data, ["CHRONOS_DB"] = "postgres", [ServerOptions.AuditRetentionVariable] = "0" });
Assert.Equal(0, options.AuditRetentionDays);
Assert.Contains(nameof(ServerOptions.AuditRetentionDays), options.Locked);
Assert.Equal(0, new ServerOptions().AuditRetentionDays);
Assert.Equal(3650, ServerOptions.Load(new Dictionary<string, string?> { ["CHRONOS_DATA_PATH"] = data, ["CHRONOS_DB"] = "postgres", [ServerOptions.AuditRetentionVariable] = "3650" })
.AuditRetentionDays);
foreach (var invalid in new[] { "-1", "3651", "a year" })
{
Assert.Throws<InvalidOperationException>(() =>
ServerOptions.Load(new Dictionary<string, string?> { ["CHRONOS_DATA_PATH"] = data, ["CHRONOS_DB"] = "postgres", [ServerOptions.AuditRetentionVariable] = invalid }));
}
}
/// <summary>Scan jobs, oldest first, once <paramref name="count"/> exist and none is waiting or running.</summary>
private async Task<List<Job>> ScansAsync(TestServer server, int count)
{
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
while (true)
{
var scans = (await server.Database.Jobs.ListAsync(null, JobKind.Scan, 50, null, Ct)).Items.OrderBy(j => j.Created).ToList();
if (scans.Count >= count && scans.All(j => j.State is JobState.Done or JobState.Failed or JobState.Cancelled))
{
Assert.Count(count, scans);
return scans;
}
Assert.True(DateTime.UtcNow < until, $"{scans.Count} scan(s): {string.Join(", ", scans.Select(j => j.State))}");
await Task.Delay(50);
}
}
private static ScheduleDto SystemRow(SchedulesDto list, string key) => list.Items.Single(s => s.SystemKey == key);
private static async Task<SchedulesDto> ListAsync(HttpClient admin)
{
using var response = await admin.GetAsync("/api/v1/schedules");
return await response.ReadAsync(ProtocolJson.Default.SchedulesDto);
}
private static async Task<ScheduleDto> CreateAsync(HttpClient admin, CreateScheduleRequest request)
{
using var response = await admin.PostAsJsonAsync("/api/v1/schedules", request, ProtocolJson.Default.CreateScheduleRequest);
await response.ExpectAsync(HttpStatusCode.Created);
return await response.ReadAsync(ProtocolJson.Default.ScheduleDto);
}
private static async Task<ScheduleRunDto> RunAsync(HttpClient admin, Guid id)
{
using var response = await admin.PostAsync($"/api/v1/schedules/{id}/run", null);
return await response.ReadAsync(ProtocolJson.Default.ScheduleRunDto);
}
}