using System.Net;
using System.Net.Http.Json;
using System.Text.Json;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging.Abstractions;
using Chronos.Core.Archiving;
using Chronos.Core.Model;
using Chronos.Data;
using Chronos.Data.Jobs;
using Chronos.Data.Scheduling;
using Chronos.Protocol;
using Chronos.Server.Configuration;
using Chronos.Server.Events;
using Chronos.Server.Jobs;
using Chronos.Server.Storage;
using Chronos.Testing;
namespace Chronos.Server.Tests.Jobs;
///
/// /api/v1/schedules and the server's system schedules (owner review 2, R10): every timer the server ran is a visible
/// schedule, seeded once, edited, reset and run like the others; the old timers are gone.
///
[Category("server")]
public sealed class ScheduleEndpointTests(TestContext context)
{
private static readonly string[] SystemKeys =
[ServerScheduleHost.ArchiveScan, ServerScheduleHost.UploadCleanup, ServerScheduleHost.IndexBackup, ServerScheduleHost.Housekeeping, ServerScheduleHost.AuditCleanup];
private CancellationToken Ct => context.Cancellation;
[Test]
public async Task TheSystemSchedulesTheActionsAndTheTimeZoneAreListed()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var list = await ListAsync(await server.AdminAsync());
// The timers the server ran before, with their timings: nothing runs that this list does not show.
Assert.SequenceEqual(SystemKeys, list.Items.Select(s => s.SystemKey));
var scan = SystemRow(list, ServerScheduleHost.ArchiveScan);
Assert.Equal(("scan", new RecurrenceDto("every", 10, "minutes"), false, "background", true), (scan.Action, scan.Recurrence, scan.CatchUp, scan.Priority, scan.Enabled));
Assert.NotNull(scan.NextRun);
var uploads = SystemRow(list, ServerScheduleHost.UploadCleanup);
// Every 6 hours, not more often: four clean-up jobs a day keep the job history readable.
Assert.Equal(("cleanup", new RecurrenceDto("every", 6, "hours"), true, (int?)90), (uploads.Action, uploads.Recurrence, uploads.CatchUp, uploads.Parameters.OlderThanDays));
Assert.SequenceEqual([UploadCleanupStep.StepName], uploads.Parameters.Steps!);
var backup = SystemRow(list, ServerScheduleHost.IndexBackup);
Assert.Equal(("backup", new RecurrenceDto("daily", Time: "03:00"), true, (int?)3), (backup.Action, backup.Recurrence, backup.CatchUp, backup.Parameters.Keep));
var history = SystemRow(list, ServerScheduleHost.Housekeeping);
Assert.Equal((new RecurrenceDto("daily", Time: "04:00"), (int?)90), (history.Recurrence, history.Parameters.OlderThanDays));
Assert.SequenceEqual(["job-history"], history.Parameters.Steps!);
var audit = SystemRow(list, ServerScheduleHost.AuditCleanup);
// The audit log is kept forever by default: no deletion after an upgrade.
Assert.Equal((new RecurrenceDto("daily", Time: "04:30"), (int?)0, false), (audit.Recurrence, audit.Parameters.OlderThanDays, audit.Enabled));
Assert.SequenceEqual(["audit"], audit.Parameters.Steps!);
// Every action, with whether this server runs it: without the processing module there is no analysis.
Assert.SequenceEqual(ScheduleActions.All, list.Actions.Select(a => a.Name));
Assert.Equal((false, "no-handler"), (list.Actions.Single(a => a.Name == "analysis").Available, list.Actions.Single(a => a.Name == "analysis").Reason));
Assert.True(list.Actions.Where(a => a.Name != "analysis").All(a => a.Available), string.Join(", ", list.Actions.Select(a => $"{a.Name}:{a.Reason}")));
Assert.SequenceEqual(["job-history", "audit", "trash-report", UploadCleanupStep.StepName], list.Steps!);
Assert.Equal(server.Services.GetRequiredService().TimeZone.Id, list.TimeZone);
}
[Test]
public async Task SchedulesAreCreatedChangedAndDeleted()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var admin = await server.AdminAsync();
var created = await CreateAsync(admin, new CreateScheduleRequest("Weekly check", "verify", new RecurrenceDto("weekly", Time: "02:30", Days: ["thu", "mon"]),
new ScheduleParametersDto(OlderThanDays: 60, Percent: 25)));
Assert.Equal(("Weekly check", "verify", "current", "background", true, (string?)null), (created.Name, created.Action, created.Target, created.Priority, created.Enabled, created.SystemKey));
Assert.SequenceEqual(["mon", "thu"], created.Recurrence.Days!);
Assert.Equal(((int?)60, (int?)25), (created.Parameters.OlderThanDays, created.Parameters.Percent));
Assert.NotNull(created.NextRun);
// After the system schedules.
Assert.Equal(created.Id, (await ListAsync(admin)).Items[^1].Id);
// PUT replaces the fields it gives; disabled, a schedule has no next run.
using (var put = await admin.PutAsJsonAsync($"/api/v1/schedules/{created.Id}",
new UpdateScheduleRequest(Recurrence: new RecurrenceDto("every", 2, "hours"), Priority: "normal", Enabled: false), ProtocolJson.Default.UpdateScheduleRequest))
{
var updated = await put.ReadAsync(ProtocolJson.Default.ScheduleDto);
Assert.Equal((new RecurrenceDto("every", 2, "hours"), "normal", false, (DateTimeOffset?)null, (int?)25),
(updated.Recurrence, updated.Priority, updated.Enabled, updated.NextRun, updated.Parameters.Percent));
}
// PATCH does the same: enabled again, it gets a next run.
using (var patch = await admin.PatchAsJsonAsync($"/api/v1/schedules/{created.Id}", new UpdateScheduleRequest(Enabled: true, Name: "Check"),
ProtocolJson.Default.UpdateScheduleRequest))
{
var enabled = await patch.ReadAsync(ProtocolJson.Default.ScheduleDto);
Assert.Equal(("Check", true), (enabled.Name, enabled.Enabled));
Assert.NotNull(enabled.NextRun);
}
using (var one = await admin.GetAsync($"/api/v1/schedules/{created.Id}")) Assert.Equal("Check", (await one.ReadAsync(ProtocolJson.Default.ScheduleDto)).Name);
using (var delete = await admin.DeleteAsync($"/api/v1/schedules/{created.Id}")) await delete.ExpectAsync(HttpStatusCode.NoContent);
using (var gone = await admin.GetAsync($"/api/v1/schedules/{created.Id}")) await gone.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found");
using (var again = await admin.DeleteAsync($"/api/v1/schedules/{created.Id}")) await again.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found");
Assert.SequenceEqual(SystemKeys, (await ListAsync(admin)).Items.Select(s => s.SystemKey));
// Every change is in the audit log.
using var audit = await admin.GetAsync("/api/v1/audit?action=schedule_created,schedule_updated,schedule_deleted");
var entries = (await audit.ReadAsync(ProtocolJson.Default.PageAuditEntryDto)).Items;
Assert.SequenceEqual(["schedule_deleted", "schedule_updated", "schedule_updated", "schedule_created"], entries.Select(e => e.Action));
Assert.True(entries.All(e => e.TargetId == created.Id.ToString() && e.Username == "admin"), "target and user");
Assert.Equal("Weekly check · verify · weekly:mon,thu@02:30", entries[^1].Details);
Assert.Equal(("Weekly check · verify · every:2h · off", "Check · verify · every:2h"), (entries[2].Details, entries[1].Details));
}
[Test]
public async Task OnlyArchiveAdministratorsSeeAndChangeSchedules()
{
await using var server = await TestServer.StartAsync(context);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
using (var list = await editor.GetAsync("/api/v1/schedules")) await list.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
using (var create = await editor.PostAsJsonAsync("/api/v1/schedules", new CreateScheduleRequest("Mine", "scan", new RecurrenceDto("daily", Time: "01:00")),
ProtocolJson.Default.CreateScheduleRequest))
await create.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
var scan = Assert.NotNull(await server.Database.Schedules.GetBySystemKeyAsync(ServerScheduleHost.ArchiveScan, Ct));
using (var run = await editor.PostAsync($"/api/v1/schedules/{scan.Id}/run", null)) await run.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
using (var anonymous = await server.Client.GetAsync("/api/v1/schedules")) Assert.Equal(HttpStatusCode.Unauthorized, anonymous.StatusCode);
}
[Test]
public async Task InvalidFieldsAreNamed()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var admin = await server.AdminAsync();
var daily = new RecurrenceDto("daily", Time: "03:00");
var cases = new (CreateScheduleRequest Request, string Field)[]
{
(new(" ", "scan", daily), "name"),
(new(new string('x', Scheduler.MaxNameLength + 1), "scan", daily), "name"),
(new("Render", "render", daily), "action"),
(new("Often", "scan", new RecurrenceDto("every", 1, "minutes")), "recurrence"),
(new("No days", "scan", new RecurrenceDto("weekly", Time: "03:00", Days: [])), "recurrence"),
(new("Late", "scan", new RecurrenceDto("daily", Time: "25:00")), "recurrence"),
(new("Day 31", "scan", new RecurrenceDto("monthly", Time: "03:00", DayOfMonth: 31)), "recurrence"),
(new("None", "verify", daily, new ScheduleParametersDto(Percent: 0)), "parameters.percent"),
(new("Old", "verify", daily, new ScheduleParametersDto(OlderThanDays: -1)), "parameters.olderThanDays"),
(new("Copies", "backup", daily, new ScheduleParametersDto(Keep: 0)), "parameters.keep"),
(new("Unknown step", "cleanup", daily, new ScheduleParametersDto(Steps: ["photos"])), "parameters.steps"),
(new("No step", "cleanup", daily, new ScheduleParametersDto(Steps: [])), "parameters.steps"),
(new("Urgent", "scan", daily, Priority: "urgent"), "priority"),
(new("Elsewhere", "scan", daily, Target: "another-archive"), "target"),
};
foreach (var (request, field) in cases)
{
using var response = await admin.PostAsJsonAsync("/api/v1/schedules", request, ProtocolJson.Default.CreateScheduleRequest);
var error = await response.ExpectErrorAsync(HttpStatusCode.BadRequest, "invalid_schedule");
Assert.Equal(field, error.Details, request.Name);
Assert.Contains(field, error.Message);
}
Assert.Count(SystemKeys.Length, (await ListAsync(admin)).Items);
}
[Test]
public async Task SystemSchedulesAreEditedAndResetButNeverDeleted()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var admin = await server.AdminAsync();
var housekeeping = SystemRow(await ListAsync(admin), ServerScheduleHost.Housekeeping);
using (var delete = await admin.DeleteAsync($"/api/v1/schedules/{housekeeping.Id}")) await delete.ExpectErrorAsync(HttpStatusCode.Conflict, "schedule_system");
foreach (var (request, field) in new[] { (new UpdateScheduleRequest(Name: "Mine"), "name"), (new UpdateScheduleRequest(Action: "verify"), "action") })
{
using var refused = await admin.PutAsJsonAsync($"/api/v1/schedules/{housekeeping.Id}", request, ProtocolJson.Default.UpdateScheduleRequest);
Assert.Equal(field, (await refused.ExpectErrorAsync(HttpStatusCode.BadRequest, "invalid_schedule")).Details);
}
// Its timing, its parameters and its switch change like any schedule's; the same name and action may be sent back.
using (var edit = await admin.PutAsJsonAsync($"/api/v1/schedules/{housekeeping.Id}", new UpdateScheduleRequest(housekeeping.Name, "cleanup",
new RecurrenceDto("weekly", Time: "05:15", Days: ["sun"]), new ScheduleParametersDto(Steps: ["job-history", "trash-report"], OlderThanDays: 30),
CatchUp: false, Enabled: false), ProtocolJson.Default.UpdateScheduleRequest))
{
var edited = await edit.ReadAsync(ProtocolJson.Default.ScheduleDto);
Assert.Equal(("weekly", (int?)30, false, false), (edited.Recurrence.Kind, edited.Parameters.OlderThanDays, edited.CatchUp, edited.Enabled));
}
// Reset puts the server's defaults back, with a new next run.
using (var reset = await admin.PostAsync($"/api/v1/schedules/{housekeeping.Id}/reset", null))
{
var back = await reset.ReadAsync(ProtocolJson.Default.ScheduleDto);
Assert.Equal((new RecurrenceDto("daily", Time: "04:00"), (int?)90, true, true), (back.Recurrence, back.Parameters.OlderThanDays, back.CatchUp, back.Enabled));
Assert.SequenceEqual(["job-history"], back.Parameters.Steps!);
Assert.NotNull(back.NextRun);
}
var custom = await CreateAsync(admin, new CreateScheduleRequest("Mine", "duplicates", new RecurrenceDto("monthly", Time: "01:00", DayOfMonth: 1)));
using (var notSystem = await admin.PostAsync($"/api/v1/schedules/{custom.Id}/reset", null))
await notSystem.ExpectErrorAsync(HttpStatusCode.Conflict, "schedule_no_defaults");
using (var unknown = await admin.PostAsync($"/api/v1/schedules/{Guid.CreateVersion7()}/reset", null)) await unknown.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found");
Assert.Equal("queued", (await RunAsync(admin, housekeeping.Id)).Outcome);
// A system schedule is audited after its key, so the admin panel names it in its language, as its Scheduled tab does.
using var audit = await admin.GetAsync("/api/v1/audit?action=schedule_updated,schedule_reset,schedule_run");
var entries = (await audit.ReadAsync(ProtocolJson.Default.PageAuditEntryDto)).Items;
Assert.SequenceEqual(["schedule_run", "schedule_reset", "schedule_updated"], entries.Select(e => e.Action));
Assert.True(entries.All(e => e.TargetId == housekeeping.Id.ToString()), "the target is the schedule");
Assert.SequenceEqual(["system:housekeeping · Housekeeping · queued", "system:housekeeping · Housekeeping · cleanup · daily:04:00",
"system:housekeeping · Housekeeping · cleanup · weekly:sun@05:15 · off"], entries.Select(e => e.Details));
}
///
/// Every change and run of a schedule goes out on the event socket, also a run that queues no job, so the admin panel's
/// Scheduled tab follows what the scheduler does.
///
[Test]
public async Task ScheduleChangesAndRunsAreEvents()
{
await using var server = await TestServer.StartAsync(context);
var admin = await server.AdminAsync();
using var events = server.Services.GetRequiredService().Subscribe();
// Off, so the scheduler's own passes leave it alone; without a duplicates handler its run queues nothing.
var created = await CreateAsync(admin, new CreateScheduleRequest("Duplicates", "duplicates", new RecurrenceDto("daily", Time: "01:00"), Enabled: false));
var run = await RunAsync(admin, created.Id);
Assert.Equal(("unavailable", (Guid?)null), (run.Outcome, run.JobId));
using (var delete = await admin.DeleteAsync($"/api/v1/schedules/{created.Id}")) await delete.ExpectAsync(HttpStatusCode.NoContent);
var changes = new List();
while (events.Reader.TryRead(out var e))
{
if (e is ScheduleChangedEvent changed && changed.ScheduleId == created.Id) changes.Add(changed.Deleted);
}
Assert.SequenceEqual([false, false, true], changes);
Assert.Equal($$"""{"type":"scheduleChanged","scheduleId":"{{created.Id}}","deleted":true}""",
JsonSerializer.Serialize(new ScheduleChangedEvent(created.Id, true), ProtocolJson.Default.ServerEvent));
}
///
/// Settings a desktop put into archive.json reach the index every 10 minutes whatever the archive scan does, as with the
/// former polling: the scan's runs read them too, but they are skipped while a long scan is pending, and the schedule may
/// be rare or off.
///
[Test]
public async Task ArchiveJsonIsReadEveryTenMinutesWhateverTheScanScheduleDoes()
{
var clock = new SteppedTimeProvider();
await using var server = await TestServer.StartAsync(context, services: s =>
{
ArchiveModule.Register(s);
s.AddSingleton(clock);
});
var host = server.Services.GetRequiredService();
var archive = server.Services.GetRequiredService();
// The scan at start is over, then the scan schedule is turned off.
await ScansAsync(server, 1);
var scanId = Assert.NotNull(await server.Database.Schedules.GetBySystemKeyAsync(ServerScheduleHost.ArchiveScan, Ct)).Id;
Assert.Null((await host.Scheduler.UpdateAsync(scanId, new UpdateScheduleRequest(Enabled: false), Ct)).Problem);
// A desktop sharing the archive changes its folder template; nothing has read it yet.
const string template = "{camera}/{date}";
var path = new ArchivePaths(archive.Root).ManifestPath;
var manifest = ManifestStore.ReadArchive(path);
Assert.NotEqual(template, manifest.LayoutTemplates.Folder);
ManifestStore.WriteArchive(path, manifest with { LayoutTemplates = manifest.LayoutTemplates with { Folder = template } });
Assert.Equal(manifest.LayoutTemplates.Folder, Assert.NotNull(await server.Database.Archives.GetAsync(archive.Id, Ct)).FolderTemplate);
// Ten minutes on, uploads (the index row) and the admin panel (the served archive) follow it, with no scan.
clock.Advance(TimeSpan.FromMinutes(ServerScheduleHost.ScanMinutes));
var until = DateTime.UtcNow + TimeSpan.FromSeconds(TestTiming.Budget(30));
while (Assert.NotNull(await server.Database.Archives.GetAsync(archive.Id, Ct)).FolderTemplate != template)
{
Assert.True(DateTime.UtcNow < until, "archive.json was not read again");
await Task.Delay(20);
}
Assert.Equal(template, archive.Current.FolderTemplate);
Assert.Count(1, (await server.Database.Jobs.ListAsync(null, JobKind.Scan, 50, null, Ct)).Items);
}
[Test]
public async Task TheJobHistoryOfTheSettingsIsWhatTheHousekeepingScheduleIsResetTo()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var admin = await server.AdminAsync();
using (var read = await admin.GetAsync("/api/v1/server/settings"))
{
var settings = await read.ReadAsync(ProtocolJson.Default.ServerSettingsDto);
// The test server listens on ports the OS chose, which the settings refuse: real ones are sent with the edit.
using var saved = await admin.PutAsJsonAsync("/api/v1/server/settings", settings with { JobHistoryDays = 30, HttpPort = 18080, HttpsPort = 18443 },
ProtocolJson.Default.ServerSettingsDto);
await saved.ExpectAsync(HttpStatusCode.OK);
}
// The schedule keeps its own value; a reset takes the new one, without a restart.
Assert.Equal(30, server.Options.JobHistoryDays);
var housekeeping = SystemRow(await ListAsync(admin), ServerScheduleHost.Housekeeping);
Assert.Equal((int?)90, housekeeping.Parameters.OlderThanDays);
using var reset = await admin.PostAsync($"/api/v1/schedules/{housekeeping.Id}/reset", null);
Assert.Equal((int?)30, (await reset.ReadAsync(ProtocolJson.Default.ScheduleDto)).Parameters.OlderThanDays);
}
///
/// Owner review 2, R10 and R11: the audit retention has one source of truth, the audit clean-up schedule's age limit.
/// Settings › Logs shows it wherever it was set (0 while the schedule is off or without its audit step) and sets it
/// (turning the schedule on with its audit step, or off at 0); Reset goes back to the value last saved there, and at 0
/// the audit step deletes nothing.
///
[Test]
public async Task TheAuditRetentionOfTheSettingsIsTheAgeLimitOfTheAuditCleanUp()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var admin = await server.AdminAsync();
var audit = SystemRow(await ListAsync(admin), ServerScheduleHost.AuditCleanup);
Assert.Equal(((int?)0, false), (audit.Parameters.OlderThanDays, audit.Enabled));
// At 0 a run (also of the disabled schedule, on request) keeps the whole audit log.
var entries = (await server.Database.Audit.ListByActionsAsync(null, [], 1000, null, Ct)).Items.Select(e => e.Id).ToList();
Assert.True(entries.Count > 0, "the sign-in left audit entries");
var run = await RunAsync(admin, audit.Id);
var job = await JobApiTests.WaitForAsync(admin, run.JobId!.Value, "done");
Assert.Contains("Audit log kept in full", job.LogTail);
var kept = (await server.Database.Audit.ListByActionsAsync(null, [], 1000, null, Ct)).Items.Select(e => e.Id).ToHashSet();
Assert.True(entries.All(kept.Contains), "no audit entry was deleted");
// Saved in Settings › Logs, a retention becomes the schedule's age limit and turns it on, without a restart.
await SaveAuditRetentionAsync(admin, 30);
Assert.Equal(30, server.Options.AuditRetentionDays);
audit = SystemRow(await ListAsync(admin), ServerScheduleHost.AuditCleanup);
Assert.Equal(((int?)30, true), (audit.Parameters.OlderThanDays, audit.Enabled));
// Without an environment variable the schedule stays editable, and Settings shows its limit.
await EditAsync(admin, audit.Id, new UpdateScheduleRequest(Parameters: new ScheduleParametersDto(Steps: ["audit"], OlderThanDays: 60)));
Assert.Equal(60, (await SettingsAsync(admin)).Logs.AuditRetentionDays);
// Saved back as shown, it changes neither the schedule nor the value Reset goes back to.
await SaveAuditRetentionAsync(admin, 60);
Assert.Equal(30, server.Options.AuditRetentionDays);
Assert.Equal((int?)60, SystemRow(await ListAsync(admin), ServerScheduleHost.AuditCleanup).Parameters.OlderThanDays);
using (var reset = await admin.PostAsync($"/api/v1/schedules/{audit.Id}/reset", null))
Assert.Equal((int?)30, (await reset.ReadAsync(ProtocolJson.Default.ScheduleDto)).Parameters.OlderThanDays);
Assert.Equal(30, (await SettingsAsync(admin)).Logs.AuditRetentionDays);
// Turned off on the schedule, nothing is deleted: Settings shows 0, and saving 30 there turns it on again.
await EditAsync(admin, audit.Id, new UpdateScheduleRequest(Enabled: false));
Assert.Equal(0, (await SettingsAsync(admin)).Logs.AuditRetentionDays);
await SaveAuditRetentionAsync(admin, 30);
audit = SystemRow(await ListAsync(admin), ServerScheduleHost.AuditCleanup);
Assert.Equal(((int?)30, true), (audit.Parameters.OlderThanDays, audit.Enabled));
// Without its audit step it deletes nothing either: Settings shows 0, and saving 30 there ticks the step again.
await EditAsync(admin, audit.Id, new UpdateScheduleRequest(Parameters: new ScheduleParametersDto(Steps: ["job-history"], OlderThanDays: 30)));
Assert.Equal(0, (await SettingsAsync(admin)).Logs.AuditRetentionDays);
await SaveAuditRetentionAsync(admin, 30);
audit = SystemRow(await ListAsync(admin), ServerScheduleHost.AuditCleanup);
Assert.Equal(((int?)30, true), (audit.Parameters.OlderThanDays, audit.Enabled));
Assert.SequenceEqual(["job-history", "audit"], audit.Parameters.Steps!);
// Back to 0: kept forever, the schedule off.
await SaveAuditRetentionAsync(admin, 0);
audit = SystemRow(await ListAsync(admin), ServerScheduleHost.AuditCleanup);
Assert.Equal(((int?)0, false), (audit.Parameters.OlderThanDays, audit.Enabled));
// A limit set on the schedule while the setting is 0 is shown, and saving 0 turns the schedule off.
await EditAsync(admin, audit.Id, new UpdateScheduleRequest(Parameters: new ScheduleParametersDto(Steps: ["audit"], OlderThanDays: 45), Enabled: true));
Assert.Equal(45, (await SettingsAsync(admin)).Logs.AuditRetentionDays);
await SaveAuditRetentionAsync(admin, 0);
audit = SystemRow(await ListAsync(admin), ServerScheduleHost.AuditCleanup);
Assert.Equal(((int?)0, false), (audit.Parameters.OlderThanDays, audit.Enabled));
}
private static async Task SettingsAsync(HttpClient admin)
{
using var read = await admin.GetAsync("/api/v1/server/settings");
return await read.ReadAsync(ProtocolJson.Default.ServerSettingsDto);
}
/// Saves Settings › Logs › Keep audit (days) as the admin panel does; the answer shows the value saved.
private static async Task SaveAuditRetentionAsync(HttpClient admin, int days)
{
var settings = await SettingsAsync(admin);
// The test server listens on ports the OS chose, which the settings refuse: real ones are sent with the edit.
using var saved = await admin.PutAsJsonAsync("/api/v1/server/settings",
settings with { HttpPort = 18080, HttpsPort = 18443, Logs = settings.Logs with { AuditRetentionDays = days } }, ProtocolJson.Default.ServerSettingsDto);
await saved.ExpectAsync(HttpStatusCode.OK);
Assert.Equal(days, (await saved.ReadAsync(ProtocolJson.Default.ServerSettingsDto)).Logs.AuditRetentionDays);
}
private static async Task EditAsync(HttpClient admin, Guid id, UpdateScheduleRequest request)
{
using var edit = await admin.PutAsJsonAsync($"/api/v1/schedules/{id}", request, ProtocolJson.Default.UpdateScheduleRequest);
await edit.ExpectAsync(HttpStatusCode.OK);
}
[Test]
public async Task RunNowQueuesTheJobsOnceAndListsThemAsRuns()
{
var handler = new BlockingJobHandler(JobKind.Duplicates);
await using var server = await TestServer.StartAsync(context, services: s => s.AddSingleton(handler));
var admin = await server.AdminAsync();
// Also a disabled schedule runs now.
var schedule = await CreateAsync(admin, new CreateScheduleRequest("Duplicates", "duplicates", new RecurrenceDto("daily", Time: "01:00"), Enabled: false));
var first = await RunAsync(admin, schedule.Id);
Assert.Equal("queued", first.Outcome);
var job = await JobApiTests.WaitForAsync(admin, first.JobId!.Value, "running");
Assert.Equal(("duplicates", (Guid?)schedule.Id, "background"), (job.Kind, job.ScheduleId, job.Priority));
Assert.Equal((first.JobId, "queued", (DateTimeOffset?)null), (first.Schedule.LastJobId, first.Schedule.LastOutcome, first.Schedule.NextRun));
// While its last job has not ended, it is not run again.
using (var busy = await admin.PostAsync($"/api/v1/schedules/{schedule.Id}/run", null))
Assert.Equal(first.JobId.ToString(), (await busy.ExpectErrorAsync(HttpStatusCode.Conflict, "schedule_pending")).Details);
handler.Release();
await JobApiTests.WaitForAsync(admin, first.JobId!.Value, "done");
var second = await RunAsync(admin, schedule.Id);
Assert.Equal("queued", second.Outcome);
await JobApiTests.WaitForAsync(admin, second.JobId!.Value, "running");
handler.Release();
await JobApiTests.WaitForAsync(admin, second.JobId!.Value, "done");
// Its runs, newest first, a page at a time.
using (var page1 = await admin.GetAsync($"/api/v1/schedules/{schedule.Id}/runs?limit=1"))
{
var runs = await page1.ReadAsync(ProtocolJson.Default.PageJobDto);
Assert.Equal(second.JobId, runs.Items.Single().Id);
using var page2 = await admin.GetAsync($"/api/v1/schedules/{schedule.Id}/runs?limit=1&cursor={Uri.EscapeDataString(Assert.NotNull(runs.NextCursor))}");
var rest = await page2.ReadAsync(ProtocolJson.Default.PageJobDto);
Assert.Equal((first.JobId, (string?)null), (rest.Items.Single().Id, rest.NextCursor));
}
using (var bad = await admin.GetAsync($"/api/v1/schedules/{schedule.Id}/runs?cursor=nonsense")) await bad.ExpectErrorAsync(HttpStatusCode.BadRequest, "invalid_cursor");
using (var unknown = await admin.GetAsync($"/api/v1/schedules/{Guid.CreateVersion7()}/runs")) await unknown.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found");
using (var other = await admin.PostAsync($"/api/v1/schedules/{Guid.CreateVersion7()}/run", null)) await other.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found");
// The queue shows which jobs a schedule queued.
using (var queue = await admin.GetAsync("/api/v1/jobs?kind=duplicates"))
Assert.True((await queue.ReadAsync(ProtocolJson.Default.PageJobDto)).Items.All(j => j.ScheduleId == schedule.Id), "scheduleId on the job rows");
using var audit = await admin.GetAsync("/api/v1/audit?action=schedule_run");
Assert.SequenceEqual(["Duplicates · queued", "Duplicates · queued"], (await audit.ReadAsync(ProtocolJson.Default.PageAuditEntryDto)).Items.Select(e => e.Details));
}
[Test]
public async Task SystemSchedulesAreSeededOnceAndKeepTheirEditsAcrossRestarts()
{
var root = context.CreateTempDirectory("restart");
void Paths(ServerOptions o)
{
o.DataPath = Path.Combine(root, "data");
o.ArchivePath = Path.Combine(root, "archive");
Directory.CreateDirectory(o.ArchivePath);
}
Guid scanId;
DateTimeOffset? lastRun;
await using (var first = await TestServer.StartAsync(context, Paths))
{
var admin = await first.AdminAsync();
scanId = SystemRow(await ListAsync(admin), ServerScheduleHost.ArchiveScan).Id;
// Its run at start (no scan handler here: "unavailable") is recorded before the edit.
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
while ((lastRun = (await first.Database.Schedules.GetAsync(scanId, Ct))!.LastRun) is null)
{
Assert.True(DateTime.UtcNow < until, "no run at start");
await Task.Delay(20);
}
using var put = await admin.PutAsJsonAsync($"/api/v1/schedules/{scanId}", new UpdateScheduleRequest(Recurrence: new RecurrenceDto("every", 20, "minutes"), Enabled: false),
ProtocolJson.Default.UpdateScheduleRequest);
await put.ExpectAsync(HttpStatusCode.OK);
}
await using var second = await TestServer.StartAsync(context, Paths);
var rows = await second.Database.Schedules.ListAsync(Ct);
Assert.SequenceEqual(SystemKeys, rows.Select(s => s.SystemKey));
var scan = rows.Single(s => s.SystemKey == ServerScheduleHost.ArchiveScan);
Assert.Equal((scanId, "every:20m", false, (DateTimeOffset?)null), (scan.Id, scan.Recurrence, scan.Enabled, scan.NextRun));
// Disabled, the scan does not run at start either.
Assert.Equal(lastRun, scan.LastRun);
}
[Test]
public async Task TheScanScheduleQueuesAScanWhenDueAndNoTimerScansBesideIt()
{
var clock = new ManualTimeProvider();
await using var server = await TestServer.StartAsync(context, services: s =>
{
ArchiveModule.Register(s);
s.AddSingleton(clock);
});
var hosted = server.Services.GetServices().Select(h => h.GetType().Name).ToList();
Assert.Contains(nameof(SchedulerHost), hosted);
Assert.True(hosted.IndexOf(nameof(JobEngineHost)) < hosted.IndexOf(nameof(SchedulerHost)), "the scheduler starts after the engine");
Assert.True(!hosted.Contains("ArchivePolling") && !hosted.Contains("DatabaseBackupService"), string.Join(", ", hosted));
var scheduleId = Assert.NotNull(await server.Database.Schedules.GetBySystemKeyAsync(ServerScheduleHost.ArchiveScan, Ct)).Id;
// At start the scan schedule runs once, as the polling did; no other scan is queued.
var scans = await ScansAsync(server, 1);
Assert.Equal(scheduleId, scans.Single().ScheduleId);
// Ten minutes later it is due: the next pass queues the second scan.
clock.Advance(TimeSpan.FromMinutes(11));
await server.Services.GetRequiredService().Scheduler.RunDueAsync(Ct);
scans = await ScansAsync(server, 2);
Assert.True(scans.All(j => j.ScheduleId == scheduleId && j.Priority == JobPriority.Background), "scans of the schedule, in the background");
var row = Assert.NotNull(await server.Database.Schedules.GetAsync(scheduleId, Ct));
Assert.Equal(("queued", (Guid?)scans[^1].Id), (row.LastOutcome, row.LastJobId));
Assert.True(row.NextRun > clock.GetUtcNow(), "the next run moved on");
}
[Test]
public async Task TheUploadCleanupScheduleSweepsAbandonedUploadsLikeTheFormerTimer()
{
var clock = new ManualTimeProvider();
await using var server = await TestServer.StartAsync(context, o => o.UploadRetentionHours = 48, s =>
{
ArchiveModule.Register(s);
s.AddSingleton(clock);
});
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
var id = (await Uploads.StartAsync(uploader, Uploads.Photo(1000, 10))).UploadId!.Value;
var store = server.Services.GetRequiredService();
Assert.True(store.Exists(id));
var scheduleId = Assert.NotNull(await server.Database.Schedules.GetBySystemKeyAsync(ServerScheduleHost.UploadCleanup, Ct)).Id;
// Two days later its run (every 6 hours, with catch-up) expires the upload and deletes its partial file.
clock.Advance(TimeSpan.FromHours(49));
await server.Services.GetRequiredService().Scheduler.RunDueAsync(Ct);
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
Job? sweep = null;
while (sweep is not { State: JobState.Done })
{
Assert.True(DateTime.UtcNow < until, $"the clean-up did not finish: {sweep?.State} {sweep?.Error}");
await Task.Delay(50);
sweep = (await server.Database.Jobs.ListByScheduleAsync(scheduleId, cancellationToken: Ct)).FirstOrDefault();
}
Assert.Equal(JobKind.Cleanup, sweep.Kind);
Assert.Contains("Uploads expired: 1", sweep.LogTail);
Assert.False(store.Exists(id));
using var state = await uploader.GetAsync($"/api/v1/uploads/{id}");
Assert.Equal("cancelled", (await state.ReadAsync(ProtocolJson.Default.UploadState)).State);
}
[Test]
public async Task AnAuditRetentionFromTheEnvironmentIsAppliedAtStartAndCannotBeChanged()
{
var root = context.CreateTempDirectory("locked");
void Paths(ServerOptions o)
{
o.DataPath = Path.Combine(root, "data");
o.ArchivePath = Path.Combine(root, "archive");
Directory.CreateDirectory(o.ArchivePath);
}
await using (var first = await TestServer.StartAsync(context, Paths, ArchiveModule.Register))
{
var seeded = Assert.NotNull(await first.Database.Schedules.GetBySystemKeyAsync(ServerScheduleHost.AuditCleanup, Ct));
Assert.Equal(((int?)0, false), (ScheduleMapping.Parameters(seeded).OlderThanDays, seeded.Enabled));
// Its audit step unticked before the variable is set.
var unticked = await first.Services.GetRequiredService().Scheduler.UpdateAsync(seeded.Id,
new UpdateScheduleRequest(Parameters: new ScheduleParametersDto(Steps: ["job-history"], OlderThanDays: 0)), Ct);
Assert.Null(unticked.Problem);
}
// CHRONOS_AUDIT_RETENTION_DAYS=30 at the next start: the limit, the switch and the audit step it deletes with.
await using var second = await TestServer.StartAsync(context, o =>
{
Paths(o);
o.AuditRetentionDays = 30;
o.Locked = [nameof(ServerOptions.AuditRetentionDays)];
}, ArchiveModule.Register);
var admin = await second.AdminAsync();
var audit = SystemRow(await ListAsync(admin), ServerScheduleHost.AuditCleanup);
Assert.Equal(((int?)30, true), (audit.Parameters.OlderThanDays, audit.Enabled));
Assert.SequenceEqual(["job-history", "audit"], audit.Parameters.Steps!);
using (var changed = await admin.PutAsJsonAsync($"/api/v1/schedules/{audit.Id}", new UpdateScheduleRequest(Parameters: new ScheduleParametersDto(Steps: ["audit"], OlderThanDays: 60)),
ProtocolJson.Default.UpdateScheduleRequest))
Assert.Equal("AuditRetentionDays", (await changed.ExpectErrorAsync(HttpStatusCode.Conflict, "setting_locked")).Details);
// Turned off, or without its audit step, it would keep what the variable says to delete.
using (var off = await admin.PutAsJsonAsync($"/api/v1/schedules/{audit.Id}", new UpdateScheduleRequest(Enabled: false), ProtocolJson.Default.UpdateScheduleRequest))
Assert.Equal("AuditRetentionDays", (await off.ExpectErrorAsync(HttpStatusCode.Conflict, "setting_locked")).Details);
using (var unticked = await admin.PutAsJsonAsync($"/api/v1/schedules/{audit.Id}",
new UpdateScheduleRequest(Parameters: new ScheduleParametersDto(Steps: ["job-history"], OlderThanDays: 30)), ProtocolJson.Default.UpdateScheduleRequest))
Assert.Equal("AuditRetentionDays", (await unticked.ExpectErrorAsync(HttpStatusCode.Conflict, "setting_locked")).Details);
Assert.SequenceEqual(["job-history", "audit"], SystemRow(await ListAsync(admin), ServerScheduleHost.AuditCleanup).Parameters.Steps!);
// Everything else stays editable.
using var timing = await admin.PutAsJsonAsync($"/api/v1/schedules/{audit.Id}", new UpdateScheduleRequest(Recurrence: new RecurrenceDto("daily", Time: "05:00"),
Parameters: new ScheduleParametersDto(Steps: ["audit", "job-history"], OlderThanDays: 30)), ProtocolJson.Default.UpdateScheduleRequest);
await timing.ExpectAsync(HttpStatusCode.OK);
// Settings shows the fixed value, and the other settings still save with it.
Assert.Equal(30, (await SettingsAsync(admin)).Logs.AuditRetentionDays);
await SaveAuditRetentionAsync(admin, 30);
}
///
/// A missed audit clean-up caught up at start runs with the retention CHRONOS_AUDIT_RETENTION_DAYS fixes, not with the
/// age limit the schedule held before: the variable is applied before the first pass.
///
[Test]
public async Task ALockedAuditRetentionAppliesToTheRunCaughtUpAtStart()
{
foreach (var days in new[] { 365, 0 })
{
var root = context.CreateTempDirectory($"catch-up-{days}");
void Paths(ServerOptions o)
{
o.DataPath = Path.Combine(root, "data");
o.ArchivePath = Path.Combine(root, "archive");
Directory.CreateDirectory(o.ArchivePath);
}
DatabaseOptions database;
await using (var first = await TestServer.StartAsync(context, Paths, ArchiveModule.Register))
database = first.Options.ToDatabaseOptions();
// While the server was off: a 90-day audit clean-up whose run passed an hour ago, and an entry 200 days old.
var old = Guid.CreateVersion7();
await using (var db = await ChronosDatabase.OpenAsync(database, Ct))
{
var row = Assert.NotNull(await db.Schedules.GetBySystemKeyAsync(ServerScheduleHost.AuditCleanup, Ct));
var parameters = ScheduleMapping.Parameters(row) with { OlderThanDays = 90 };
Assert.True(await db.Schedules.UpdateAsync(row with
{
ParametersJson = ScheduleMapping.ParametersJson(row.Action, parameters),
Enabled = true,
NextRun = DateTimeOffset.UtcNow.AddHours(-1),
}, setNextRun: true, Ct));
await db.Audit.AppendAsync(new AuditEntry { Id = old, At = DateTimeOffset.UtcNow.AddDays(-200), Action = "user.login" }, Ct);
}
// CHRONOS_AUDIT_RETENTION_DAYS at the next start.
await using var second = await TestServer.StartAsync(context, o =>
{
Paths(o);
o.AuditRetentionDays = days;
o.Locked = [nameof(ServerOptions.AuditRetentionDays)];
}, ArchiveModule.Register);
var schedule = Assert.NotNull(await second.Database.Schedules.GetBySystemKeyAsync(ServerScheduleHost.AuditCleanup, Ct));
var queued = await second.Database.Jobs.ListByScheduleAsync(schedule.Id, cancellationToken: Ct);
if (days == 0)
{
// Kept forever: the schedule is off before its missed run could fire.
Assert.False(schedule.Enabled);
Assert.Empty(queued);
}
else
{
Assert.Count(1, queued);
var job = queued[0];
using (var json = JsonDocument.Parse(job.ParametersJson!))
Assert.Equal(days, json.RootElement.GetProperty("olderThanDays").GetInt32());
await JobApiTests.WaitForAsync(await second.AdminAsync(), job.Id, "done");
}
var kept = (await second.Database.Audit.ListByActionsAsync(null, [], 1000, null, Ct)).Items.Select(e => e.Id);
Assert.Contains(old, kept);
}
}
[Test]
public void AnUnknownTimeZoneFallsBackToUtc()
{
var local = ServerScheduleHost.ResolveTimeZone(null, NullLogger.Instance);
Assert.Equal(TimeZoneInfo.Local.BaseUtcOffset, local.BaseUtcOffset);
Assert.Equal(TimeZoneInfo.Local.SupportsDaylightSavingTime, local.SupportsDaylightSavingTime);
// The runtime ignores TZ on Windows; elsewhere a zone it cannot find would leave the schedules in an unknown zone.
if (!OperatingSystem.IsWindows())
{
Assert.Equal(TimeZoneInfo.Utc, ServerScheduleHost.ResolveTimeZone("Not/AZone", NullLogger.Instance));
Assert.Equal(TimeZoneInfo.Local.Id, ServerScheduleHost.ResolveTimeZone(":" + TimeZoneInfo.Local.Id, NullLogger.Instance).Id);
}
}
[Test]
public void TheAuditRetentionVariableIsReadAndChecked()
{
var data = context.CreateTempDirectory("env");
var options = ServerOptions.Load(new Dictionary { ["CHRONOS_DATA_PATH"] = data, ["CHRONOS_DB"] = "postgres", [ServerOptions.AuditRetentionVariable] = "0" });
Assert.Equal(0, options.AuditRetentionDays);
Assert.Contains(nameof(ServerOptions.AuditRetentionDays), options.Locked);
Assert.Equal(0, new ServerOptions().AuditRetentionDays);
Assert.Equal(3650, ServerOptions.Load(new Dictionary { ["CHRONOS_DATA_PATH"] = data, ["CHRONOS_DB"] = "postgres", [ServerOptions.AuditRetentionVariable] = "3650" })
.AuditRetentionDays);
foreach (var invalid in new[] { "-1", "3651", "a year" })
{
Assert.Throws(() =>
ServerOptions.Load(new Dictionary { ["CHRONOS_DATA_PATH"] = data, ["CHRONOS_DB"] = "postgres", [ServerOptions.AuditRetentionVariable] = invalid }));
}
}
/// Scan jobs, oldest first, once exist and none is waiting or running.
private async Task> ScansAsync(TestServer server, int count)
{
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
while (true)
{
var scans = (await server.Database.Jobs.ListAsync(null, JobKind.Scan, 50, null, Ct)).Items.OrderBy(j => j.Created).ToList();
if (scans.Count >= count && scans.All(j => j.State is JobState.Done or JobState.Failed or JobState.Cancelled))
{
Assert.Count(count, scans);
return scans;
}
Assert.True(DateTime.UtcNow < until, $"{scans.Count} scan(s): {string.Join(", ", scans.Select(j => j.State))}");
await Task.Delay(50);
}
}
private static ScheduleDto SystemRow(SchedulesDto list, string key) => list.Items.Single(s => s.SystemKey == key);
private static async Task ListAsync(HttpClient admin)
{
using var response = await admin.GetAsync("/api/v1/schedules");
return await response.ReadAsync(ProtocolJson.Default.SchedulesDto);
}
private static async Task CreateAsync(HttpClient admin, CreateScheduleRequest request)
{
using var response = await admin.PostAsJsonAsync("/api/v1/schedules", request, ProtocolJson.Default.CreateScheduleRequest);
await response.ExpectAsync(HttpStatusCode.Created);
return await response.ReadAsync(ProtocolJson.Default.ScheduleDto);
}
private static async Task RunAsync(HttpClient admin, Guid id)
{
using var response = await admin.PostAsync($"/api/v1/schedules/{id}/run", null);
return await response.ReadAsync(ProtocolJson.Default.ScheduleRunDto);
}
}