From e5508e7e693e5b39dda2798f78775c66564c17ee Mon Sep 17 00:00:00 2001 From: Alberto Balbo Date: Tue, 27 Oct 2026 00:41:48 +0100 Subject: [PATCH] feat(data): scheduler engine, schedules table and housekeeping jobs Owner review 2, R10: every timer that queues work becomes a visible schedule, so the server and the desktop need one shared engine to run them. - Migration 0007: schedules table (system rows keyed by system_key) and jobs.schedule_id; Job.ScheduleId, JobDto.ScheduleId, JobRepository HasPendingAsync / ListPendingTargetsAsync / ListByScheduleAsync. - ScheduleRepository: seeding that keeps user edits, reset, compare-and-set claim of a run (one winner across processes), outcomes. - Scheduling: in-house recurrence (every/daily/weekly/monthly, DST gap moves forward, ambiguous time runs once), ScheduleActions resolving scan, verify, thumbnails, proxies, analysis, maintenance, backup, cleanup, duplicates and orphans into archive jobs, IScheduleHost, and the Scheduler loop (catch-up, clock going back, run now, CRUD as API v1 DTOs). - New job kinds Backup (16) and Cleanup (17) with handlers, shared clean-up steps (job history, audit log, trash report that never deletes) and host steps through ArchiveServiceOptions; read-only Duplicates and Orphans report handlers, so the server can run those kinds too. - JobEngine no longer purges history at start: the housekeeping schedule's job-history step does; JobEngine.CanRun and an EnqueueAsync with scheduleId. - Protocol DTOs for schedules; job kind names in en and it. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit eb0edd2483a493db7c6945d5b494461cabb2a4ad) --- docs/ARCHITECTURE.md | 14 +- i18n/en/Admin.resx | 2 + i18n/en/Archive.resx | 16 + i18n/en/Jobs.resx | 2 + i18n/it/Admin.it.resx | 2 + i18n/it/Archive.it.resx | 16 + i18n/it/Jobs.it.resx | 2 + src/Chronos.App/DesignTime/SampleData.cs | 3 +- src/Chronos.App/Jobs/JobNotices.cs | 7 +- src/Chronos.App/Sections/JobsViewModel.cs | 4 +- src/Chronos.App/Sections/SettingsViewModel.cs | 1 + src/Chronos.Core/Model/Entities.cs | 57 +++ src/Chronos.Core/Model/Enums.cs | 6 + src/Chronos.Data/Archiving/ArchiveService.cs | 35 +- .../Archiving/Jobs/ArchiveJobParameters.cs | 11 + .../Archiving/Jobs/ReportJobHandlers.cs | 58 +++ src/Chronos.Data/Archiving/SequenceWriter.cs | 13 + src/Chronos.Data/ChronosDatabase.cs | 4 + src/Chronos.Data/DatabaseBackups.cs | 22 + src/Chronos.Data/Housekeeping/CleanupSteps.cs | 116 +++++ .../Housekeeping/HousekeepingJobHandlers.cs | 78 ++++ src/Chronos.Data/Jobs/JobEngine.cs | 27 +- src/Chronos.Data/Jobs/JobPriorityQueue.cs | 7 +- src/Chronos.Data/Jobs/JobWire.cs | 1 + .../Migrations/postgres/0007_schedules.sql | 27 ++ .../Migrations/sqlite/0007_schedules.sql | 27 ++ .../Repositories/AuditRepository.cs | 6 +- .../Repositories/JobRepository.cs | 50 +- .../Repositories/ScheduleRepository.cs | 167 +++++++ .../Repositories/SequenceRepository.cs | 15 + src/Chronos.Data/Scheduling/IScheduleHost.cs | 52 +++ .../Scheduling/ScheduleActions.cs | 291 ++++++++++++ .../Scheduling/ScheduleMapping.cs | 89 ++++ .../Scheduling/ScheduleOutcome.cs | 67 +++ .../Scheduling/ScheduleRecurrence.cs | 322 +++++++++++++ src/Chronos.Data/Scheduling/Scheduler.cs | 429 +++++++++++++++++ src/Chronos.Protocol/Jobs.cs | 4 +- src/Chronos.Protocol/ProtocolJson.cs | 9 + src/Chronos.Protocol/Schedules.cs | 97 ++++ src/Chronos.Server/Api/Dto.cs | 3 +- .../Configuration/ServerSettingsStore.cs | 2 +- src/Chronos.Server/Jobs/JobEngineHost.cs | 1 - .../Chronos.App.Tests/Jobs/JobNoticesTests.cs | 2 + .../Chronos.Data.Tests/Jobs/JobEngineTests.cs | 26 ++ .../Migrations/MigrationTests.cs | 2 +- .../Repositories/JobRepositoryTests.cs | 46 ++ .../Repositories/ScheduleRepositoryTests.cs | 192 ++++++++ .../Repositories/ServerRepositoryTests.cs | 11 + .../Scheduling/HousekeepingJobTests.cs | 194 ++++++++ .../Scheduling/ScheduleRecurrenceTests.cs | 174 +++++++ .../Scheduling/SchedulerTests.cs | 440 ++++++++++++++++++ 51 files changed, 3219 insertions(+), 30 deletions(-) create mode 100644 src/Chronos.Data/Archiving/Jobs/ReportJobHandlers.cs create mode 100644 src/Chronos.Data/Housekeeping/CleanupSteps.cs create mode 100644 src/Chronos.Data/Housekeeping/HousekeepingJobHandlers.cs create mode 100644 src/Chronos.Data/Migrations/postgres/0007_schedules.sql create mode 100644 src/Chronos.Data/Migrations/sqlite/0007_schedules.sql create mode 100644 src/Chronos.Data/Repositories/ScheduleRepository.cs create mode 100644 src/Chronos.Data/Scheduling/IScheduleHost.cs create mode 100644 src/Chronos.Data/Scheduling/ScheduleActions.cs create mode 100644 src/Chronos.Data/Scheduling/ScheduleMapping.cs create mode 100644 src/Chronos.Data/Scheduling/ScheduleOutcome.cs create mode 100644 src/Chronos.Data/Scheduling/ScheduleRecurrence.cs create mode 100644 src/Chronos.Data/Scheduling/Scheduler.cs create mode 100644 src/Chronos.Protocol/Schedules.cs create mode 100644 tests/Chronos.Data.Tests/Repositories/ScheduleRepositoryTests.cs create mode 100644 tests/Chronos.Data.Tests/Scheduling/HousekeepingJobTests.cs create mode 100644 tests/Chronos.Data.Tests/Scheduling/ScheduleRecurrenceTests.cs create mode 100644 tests/Chronos.Data.Tests/Scheduling/SchedulerTests.cs diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index e4a73041..3822ffa8 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -54,8 +54,10 @@ Dependencies point one way: `Core` ← `Core.Windows`, `Data`, `Protocol` ← `S |---|---| | `Sqlite`, `Postgres` | SQLite P/Invoke binding (winsqlite3 / libsqlite3), in-house PostgreSQL wire protocol v3 client with pool and LISTEN | | `Migrations/{sqlite,postgres}/NNNN_name.sql` | Embedded, applied in one transaction at open (`MigrationRunner`) | -| `Repositories` | `ChronosDatabase` facade: archives, sequences, frames, tags, renders, jobs, cards, users, tokens, uploads, render nodes, settings, audit; `DatabaseBackups` (daily `VACUUM INTO` copies of an SQLite index kept ×3, restore staged for the next start and carried out before the index opens, so it also recovers one that no longer opens) | -| `Jobs` | `JobEngine` (one job at a time; reviews the priority queue and makes a lower running job yield, requeued with its checkpoint), `JobPriorityQueue` (a level per job kind, the claim order, settings round trip), `IJobHandler`, `IJobHandlerProvider`, `JobContext` (progress, checkpoints, follow-up jobs that never interrupt their parent), `JobThroughput`/`JobEta` (rate and bounded time left) | +| `Repositories` | `ChronosDatabase` facade: archives, sequences, frames, tags, renders, jobs, cards, users, tokens, uploads, render nodes, settings, audit, schedules (`ScheduleRepository`: system rows seeded without overwriting edits and never deleted, the compare-and-set claim of a run); `DatabaseBackups` (`VACUUM INTO` copies of an SQLite index into `backups` next to it, restore staged for the next start and carried out before the index opens, so it also recovers one that no longer opens) | +| `Jobs` | `JobEngine` (one job at a time; reviews the priority queue and makes a lower running job yield, requeued with its checkpoint; no history purge, the housekeeping schedule's clean-up does it), `JobPriorityQueue` (a level per job kind, the claim order, settings round trip), `IJobHandler`, `IJobHandlerProvider`, `JobContext` (progress, checkpoints, follow-up jobs that never interrupt their parent), `JobThroughput`/`JobEta` (rate and bounded time left) | +| `Scheduling` | `Scheduler` (one per host: seeds the host's system schedules, claims due runs by compare-and-set, catch-up, clock changes, create/edit/reset/delete/run now as API v1 DTOs), `ScheduleRecurrence` (the in-house `every:`/`daily:`/`weekly:`/`monthly:` form, wall-clock times across DST), `ScheduleActions` (action names, parameters, availability, the archive jobs a run queues), `IScheduleHost` and `SystemSchedule` (what the server and the desktop supply), `ScheduleOutcome`, `ScheduleMapping` (model ↔ DTO) | +| `Housekeeping` | `BackupJobHandler`, `CleanupJobHandler`, `ICleanupStep` with the shared steps (`CleanupSteps`: job history, audit log, trash report that never deletes); hosts add their own steps through `ArchiveServiceOptions.CleanupSteps` | | `Archiving` | `ArchiveService` (create/adopt/open, archive settings written to `archive.json` then the index), `ArchiveIndexer` (scan, rebuild), `ArchiveIngest` (uploads and imports), `ImportService`, `ArchiveVerifier`, `ArchiveImages` (thumbnail and proxy passes, several photos at once through `OrderedWork`, which hands results back in frame order), `FrameTiers` (decode tier, `PreviewOnly`/`Undecodable` flags and preview size of frames, written to sidecar and index by the light analysis, deep analysis, renders and image passes), `GprCompanions` (the JPEG of each GoPro GPR, found in the index when an import kept it in another sequence), `ArchiveMaintenance`, `MassRenamer`, `SequenceTrash` (delete into the archive's hidden trash with a dry run, restore; the folder moves back when the index step fails), job handlers | | `Processing` | `ProcessingService`, `SequenceAnalyser`, `DeepReports` (a saved deep report while it still describes the frames), `SequenceRender`, `ProjectStore`, `ProjectBundles` (a `.chronos` file for a sequence, and the sequence a file names), `RenderNodeClient`, job handlers | | `Sync` | `SyncService` (sync table of one server: compare, link by content, settle conflicts, enqueue transfers), `SyncRules` (status, settings merge and differences, no I/O), `SyncPositions` (the settings' frame positions translated by photo to the frames of the copy they go to), `SyncLinkStore` (`sync_links`, `sync_state`), `SyncTransfers` and `SyncJobHandler` (resumable, hash-verified uploads and downloads as jobs; downloads commit through the archive ingest), `SyncAgent` (background comparison, server events, backoff, offline proxies), `ProxyCache` and `RemoteSequence` (server sequences opened with proxies), `ISyncRemote` (the server calls; `ChronosSyncRemote` over `ChronosClient`) | @@ -109,10 +111,14 @@ Dependencies point one way: `Core` ← `Core.Windows`, `Data`, `Protocol` ← `S | `LightAnalyse`, `DeepAnalyse` | `Processing/Jobs/AnalyseJobHandler` (via `ProcessingService.CreateJobHandlers`) | yes | yes | | `Render` | `Processing/Jobs/RenderJobHandler` | yes | managed formats, and MP4 jobs no render node can take (`RenderNodeAvailability`); render nodes claim the others (`Server/Jobs/RenderNodeService`) | | `UploadCommit`, `Download` | `Sync/SyncJobHandler` (via `SyncService.CreateJobHandlers`): this PC's uploads to and downloads from its server | yes | no: an upload completes synchronously on the server | -| `Duplicates`, `Orphans`, `Adopt`, `Move` | none: done synchronously (`ArchiveService` queries and actions) | — | — | +| `Duplicates`, `Orphans` | `Archiving/Jobs/ReportJobHandlers` (read-only reports into the job log, queued by schedules); the Archive screen still queries `ArchiveService` directly | yes | yes | +| `Backup` | `Housekeeping/BackupJobHandler` (via `ArchiveService.CreateJobHandlers`; an SQLite index file only) | yes | SQLite only | +| `Cleanup` | `Housekeeping/CleanupJobHandler` (shared steps, plus the host's from `ArchiveServiceOptions.CleanupSteps`) | yes | yes | +| `Adopt`, `Move` | none: done synchronously (`ArchiveService` actions) | — | — | A job engine claims only kinds it has handlers for. Light analysis is queued by the engine (`JobContext` follow-up) -after an import or a scan that indexed something; screens never queue it again. +after an import or a scan that indexed something; screens never queue it again. Recurring work is queued by the host's +`Scheduler` (`Data/Scheduling`): each run queues the action's jobs with the schedule's priority and `jobs.schedule_id`. ## 3. Data flow diff --git a/i18n/en/Admin.resx b/i18n/en/Admin.resx index d449f679..b202ef1f 100644 --- a/i18n/en/Admin.resx +++ b/i18n/en/Admin.resx @@ -116,6 +116,8 @@ Free space of the archive's own volume, where uploads are written first: on Unraid the share's primary storage, often the cache The server did not answer; try again.long Adopt + Backup + Clean-up Deep analysis Duplicates Import diff --git a/i18n/en/Archive.resx b/i18n/en/Archive.resx index b5fa7856..0ef01512 100644 --- a/i18n/en/Archive.resx +++ b/i18n/en/Archive.resx @@ -123,22 +123,38 @@ {0:N0} sequence unverified {0:N0} sequences unverified Last scan {0} · {1}long + Audit log kept in fulllong: job log line; a clean-up step whose age limit is 0 + Audit entries removed: {0}long: job log line + Index copied to {0} ({1}); copies kept: {2}long: job log line; {0} file path; {1} size, e.g. "12.4 MB" Dry run: nothing was changed.long: job log line + Extra copies of {0}: {1}long: job log line; {0} path of the first photo of a group sharing one hash + Duplicate groups: {0}, extra copies: {1}long: job log line Not indexed: {0} ({1})long: job log line + Folder missing: {0}long: job log line; an indexed sequence whose folder holds no sidecar any more + Folder not indexed: {0}long: job log line; a folder with photos the index does not know yet Changed: {0}long: job log line Damaged: {0}long: job log line Missing: {0}long: job log line + Job history kept in fulllong: job log line; a clean-up step whose age limit is 0 + Finished jobs removed: {0}long: job log line {0}: made {1}, reused {2}, unreadable {3}long: job log line Copied: {0}, skipped: {1}, failed: {2}long: job log line Not copied: {0} ({1})long: job log line Index: {0} reclaimablelong: job log line Index compacted: {0} freedlong: job log line Stale locks: {0}, temporary files: {1}, packs compacted: {2}, duplicate flags changed: {3}long: job log line + Sequences without their folder: {0}, folders not indexed: {1}long: job log line Unindexed photos: {0}, missing photos: {1}long: job log line {0}: made {1}, reused {2}, unreadable {3}, waiting {4}long: job log line; {4} frames whose proxy this host waits for from a client or render node Proxies for the server: {0} sent, {1} not sentlong: job log line; {0} proxies sent; {1} proxies this PC could not make or the server refused + Report only: nothing was changed.long: job log line Folders: {0}, indexed: {1}, adopted: {2}, removed: {3}long: job log line Sequence not found: {0}long: job log line + Clean-up step {0} failed: {1}long: job log line; {0} step name (job-history, audit…); {1} error message + Clean-up step unknown here: {0}long: job log line; {0} step name + In the trash since {0}: {1} ({2})long: job log line; {0} date; {1} sequence title; {2} size + Trash not checked: the archive cannot be reachedlong: job log line + Old trash entries: {0}, size: {1}; nothing was deletedlong: job log line; entries older than the clean-up's age limit Photos checked: {0}, intact: {1}, hashed: {2}, with problems: {3}long: job log line Back up database Dry run diff --git a/i18n/en/Jobs.resx b/i18n/en/Jobs.resx index 06bbcd5e..47ca6e4e 100644 --- a/i18n/en/Jobs.resx +++ b/i18n/en/Jobs.resx @@ -33,6 +33,8 @@ History keeps {0} day History keeps {0} days Adopt folder + Index backup + Clean-up Deep analysis Download Duplicates diff --git a/i18n/it/Admin.it.resx b/i18n/it/Admin.it.resx index 2bce5b32..153530bf 100644 --- a/i18n/it/Admin.it.resx +++ b/i18n/it/Admin.it.resx @@ -116,6 +116,8 @@ Spazio libero del volume dell'archivio, dove i caricamenti vengono scritti prima: su Unraid lo storage primario della condivisione, spesso la cache Il server non ha risposto; riprova. Adozione + Copia dell'indice + Pulizia Analisi completa Duplicati Importazione diff --git a/i18n/it/Archive.it.resx b/i18n/it/Archive.it.resx index cb7951ff..c84b9981 100644 --- a/i18n/it/Archive.it.resx +++ b/i18n/it/Archive.it.resx @@ -118,22 +118,38 @@ {0:N0} sequenza non verificata {0:N0} sequenze non verificate Ultima scansione {0} · {1} + Registro di controllo conservato per intero + Voci di controllo rimosse: {0} + Indice copiato in {0} ({1}); copie conservate: {2} Simulazione: nulla è stato modificato. + Copie in più di {0}: {1} + Gruppi di duplicati: {0}, copie in più: {1} Non indicizzata: {0} ({1}) + Cartella mancante: {0} + Cartella non indicizzata: {0} Modificato: {0} Danneggiato: {0} Mancante: {0} + Cronologia delle attività conservata per intero + Attività concluse rimosse: {0} {0}: create {1}, riusate {2}, illeggibili {3} Copiate: {0}, saltate: {1}, non riuscite: {2} Non copiata: {0} ({1}) Indice: {0} recuperabili Indice compattato: {0} liberati Blocchi scaduti: {0}, file temporanei: {1}, pacchetti compattati: {2}, segni di duplicato cambiati: {3} + Sequenze senza cartella: {0}, cartelle non indicizzate: {1} Foto non indicizzate: {0}, foto mancanti: {1} {0}: create {1}, riusate {2}, illeggibili {3}, in attesa {4} Proxy per il server: {0} inviati, {1} non inviati + Solo resoconto: nulla è stato modificato. Cartelle: {0}, indicizzate: {1}, adottate: {2}, rimosse: {3} Sequenza non trovata: {0} + Passo di pulizia {0} non riuscito: {1} + Passo di pulizia sconosciuto qui: {0} + Nel cestino dal {0}: {1} ({2}) + Cestino non controllato: l'archivio non è raggiungibile + Voci vecchie nel cestino: {0}, dimensione: {1}; nulla è stato eliminato Foto controllate: {0}, integre: {1}, con impronta nuova: {2}, con problemi: {3} Copia del database Simulazione diff --git a/i18n/it/Jobs.it.resx b/i18n/it/Jobs.it.resx index 55bf20cc..c815449b 100644 --- a/i18n/it/Jobs.it.resx +++ b/i18n/it/Jobs.it.resx @@ -33,6 +33,8 @@ Cronologia di {0} giorno Cronologia di {0} giorni Adozione cartella + Copia dell'indice + Pulizia Analisi approfondita Scaricamento Duplicati diff --git a/src/Chronos.App/DesignTime/SampleData.cs b/src/Chronos.App/DesignTime/SampleData.cs index c26d85f3..b75eff95 100644 --- a/src/Chronos.App/DesignTime/SampleData.cs +++ b/src/Chronos.App/DesignTime/SampleData.cs @@ -185,7 +185,7 @@ public sealed class SampleData { var db = _db ?? throw new InvalidOperationException("FillAsync first."); var service = _services.Sync.Service ?? throw new InvalidOperationException("The index is not open."); - // The history is kept whatever the date: the sample's finished jobs are dated from Now, not from today. + // The engine purges no history (the housekeeping schedule does): the sample's finished jobs, dated from Now, stay. await using var engine = new JobEngine(db, service.CreateJobHandlers(_services.Sync), new JobEngineOptions { @@ -193,7 +193,6 @@ public sealed class SampleData WorkerId = "sample-proxies", Kinds = [JobKind.Download], PollInterval = TimeSpan.FromMilliseconds(50), - HistoryRetention = TimeSpan.FromDays(36_500), }); await engine.StartAsync(); var job = await service.EnqueueDownloadAsync(engine, ServerAddress, _sunrise, "Sunrise over Superga", SyncDownloadMode.Proxies, null); diff --git a/src/Chronos.App/Jobs/JobNotices.cs b/src/Chronos.App/Jobs/JobNotices.cs index aea62f53..a01c623b 100644 --- a/src/Chronos.App/Jobs/JobNotices.cs +++ b/src/Chronos.App/Jobs/JobNotices.cs @@ -39,9 +39,12 @@ public static class JobNotices kind is JobKind.Render or JobKind.DeepAnalyse or JobKind.Verify or JobKind.Rebuild or JobKind.Maintenance or JobKind.UploadCommit or JobKind.Move or JobKind.Adopt; - /// Work that keeps the index and its previews current: the next run repeats it, the Archive screen shows what is missing. + /// + /// Work that keeps the index and its previews current: the next run repeats it, the Archive screen shows what is missing. + /// Clean-ups too: the housekeeping schedule runs them again (a failed index backup is said). + /// private static bool IsUpkeep(JobKind kind) => - kind is JobKind.Scan or JobKind.Thumbs or JobKind.Proxies or JobKind.LightAnalyse; + kind is JobKind.Scan or JobKind.Thumbs or JobKind.Proxies or JobKind.LightAnalyse or JobKind.Cleanup; } /// Endings said in one toast: one job, or several of a batch. diff --git a/src/Chronos.App/Sections/JobsViewModel.cs b/src/Chronos.App/Sections/JobsViewModel.cs index fba9c31b..5fa9cc0f 100644 --- a/src/Chronos.App/Sections/JobsViewModel.cs +++ b/src/Chronos.App/Sections/JobsViewModel.cs @@ -141,8 +141,8 @@ public sealed class JobsViewModel : SectionViewModel /// The priority queue of Settings › Rendering is on: higher jobs pause the running one. public bool IsPriorityQueueOn => HasEngine && Shell.Services.Settings.PriorityQueue; - /// "History keeps 90 days". - public string HistoryText => Loc.Plural("Jobs.History", (long)new JobEngineOptions().HistoryRetention.TotalDays); + /// "History keeps 90 days": the job-history step of the housekeeping schedule. + public string HistoryText => Loc.Plural("Jobs.History", (long)Chronos.Data.Housekeeping.CleanupSteps.DefaultHistoryDays); public bool IsEmpty => _hasLoaded && Rows.Count == 0; diff --git a/src/Chronos.App/Sections/SettingsViewModel.cs b/src/Chronos.App/Sections/SettingsViewModel.cs index 2f95402b..77cb6824 100644 --- a/src/Chronos.App/Sections/SettingsViewModel.cs +++ b/src/Chronos.App/Sections/SettingsViewModel.cs @@ -1147,6 +1147,7 @@ public sealed class SettingsViewModel : SectionViewModel [ JobKind.Render, JobKind.Import, JobKind.Download, JobKind.UploadCommit, JobKind.Scan, JobKind.Rebuild, JobKind.Verify, JobKind.LightAnalyse, JobKind.DeepAnalyse, JobKind.Thumbs, JobKind.Proxies, JobKind.Maintenance, JobKind.Duplicates, JobKind.Orphans, JobKind.Adopt, JobKind.Move, + JobKind.Backup, JobKind.Cleanup, ]; /// Stores for , keeping only the levels that differ from the defaults. diff --git a/src/Chronos.Core/Model/Entities.cs b/src/Chronos.Core/Model/Entities.cs index 75a56344..a904a782 100644 --- a/src/Chronos.Core/Model/Entities.cs +++ b/src/Chronos.Core/Model/Entities.cs @@ -182,6 +182,63 @@ public sealed record Job public DateTimeOffset Created { get; init; } public DateTimeOffset? Started { get; init; } public DateTimeOffset? Finished { get; init; } + + /// The schedule that queued this job; null for jobs queued by a user or by other jobs (owner review 2, R10). + public Guid? ScheduleId { get; init; } +} + +/// +/// A recurring job (owner review 2, R10): an action (scan, verify, backup, …) with its parameters, run on a recurrence +/// in the host's time zone. System schedules ( set) stand for the timers a host used to run out of +/// sight; they are seeded once, can be edited and reset, never deleted. +/// +public sealed record Schedule +{ + public required Guid Id { get; init; } + + /// Stable key of a host's system schedule (archive-scan); null for schedules a user created. + public string? SystemKey { get; init; } + + public required string Name { get; init; } + + /// Wire name of the action (scan, verify, backup…). + public required string Action { get; init; } + + /// What the action works on: current, the host's archive. + public string Target { get; init; } = "current"; + + /// The action's parameters as JSON (camelCase, only the fields the action reads). + public string? ParametersJson { get; init; } + + /// The recurrence in its text form (every:10m, daily:03:00, weekly:mon,thu@02:30, monthly:1@04:00). + public required string Recurrence { get; init; } + + /// A run missed while the host was off happens once when it comes back; otherwise it is skipped. + public bool CatchUp { get; init; } = true; + + /// Priority of the jobs it queues. + public JobPriority Priority { get; init; } = JobPriority.Background; + + public bool Enabled { get; init; } = true; + + /// When it fires next (UTC); null while disabled or not computed yet. + public DateTimeOffset? NextRun { get; init; } + + public DateTimeOffset? LastRun { get; init; } + + /// The first job the last run queued. + public Guid? LastJobId { get; init; } + + /// What the last run did: queued, skipped-pending, skipped-offline, skipped-nothing, + /// unavailable or failed:<reason>. + public string? LastOutcome { get; init; } + + public Guid? CreatedBy { get; init; } + public DateTimeOffset Created { get; init; } + public DateTimeOffset Updated { get; init; } + + /// True for a host's system schedule. + public bool IsSystem => SystemKey is not null; } /// A memory card known to this client (BRIEF §13). diff --git a/src/Chronos.Core/Model/Enums.cs b/src/Chronos.Core/Model/Enums.cs index 16bccf87..9491aeac 100644 --- a/src/Chronos.Core/Model/Enums.cs +++ b/src/Chronos.Core/Model/Enums.cs @@ -81,6 +81,12 @@ public enum JobKind /// Client: originals or proxies of a server sequence fetched into this PC (BRIEF §14.5). Download, + + /// A VACUUM INTO copy of an SQLite index into the backups folder (owner review 2, R10); stored as 16. + Backup = 16, + + /// Housekeeping steps: job history, audit log, trash report and the host's own (owner review 2, R10); stored as 17. + Cleanup = 17, } public enum JobState diff --git a/src/Chronos.Data/Archiving/ArchiveService.cs b/src/Chronos.Data/Archiving/ArchiveService.cs index 83235feb..e35ff8e3 100644 --- a/src/Chronos.Data/Archiving/ArchiveService.cs +++ b/src/Chronos.Data/Archiving/ArchiveService.cs @@ -40,6 +40,8 @@ public sealed class ArchiveService : IJobHandlerProvider Images = new ArchiveImages(db, _writer, Indexer); Imports = new ImportService(db, Ingest); Trash = new SequenceTrash(db, _writer, Indexer); + var shared = Housekeeping.CleanupSteps.Shared(Trash); + CleanupSteps = [.. shared, .. _writer.Options.CleanupSteps.Where(s => !shared.Any(o => o.Name == s.Name))]; } public ArchiveIndexer Indexer { get; } @@ -56,7 +58,16 @@ public sealed class ArchiveService : IJobHandlerProvider /// Deletes sequences into the archive's trash and restores them. public SequenceTrash Trash { get; } - /// Handlers for scan, rebuild, verify, import, thumbs, proxies and maintenance jobs; register them on the . + /// + /// The steps this host's jobs can run: the shared ones (), + /// then the host's (). A schedule host lists their names. + /// + public IReadOnlyList CleanupSteps { get; } + + /// + /// Handlers for scan, rebuild, verify, import, thumbs, proxies and maintenance jobs, the read-only duplicates and orphans + /// reports, index backups and clean-ups (owner review 2, R10); register them on the . + /// public IReadOnlyList CreateJobHandlers() => [ new ScanJobHandler(_db, Indexer, JobKind.Scan, _writer.Options.ImagesAfterScan), @@ -66,6 +77,10 @@ public sealed class ArchiveService : IJobHandlerProvider new ImageJobHandler(_db, Images, JobKind.Thumbs), new ImageJobHandler(_db, Images, JobKind.Proxies), new MaintenanceJobHandler(_db, Maintenance), + new DuplicatesJobHandler(_db, Maintenance), + new OrphansJobHandler(_db, Maintenance), + new Housekeeping.BackupJobHandler(_db, _writer.Options.BackupDirectory), + new Housekeeping.CleanupJobHandler(_db, CleanupSteps, TimeProvider.System), ]; /// Makes a new, empty archive and indexes it. @@ -482,6 +497,24 @@ public static class ArchiveJobs new(JobKind.Maintenance, [archive.Id], archive.Name, Json(new MaintenanceParameters(dryRun), ArchiveJobJson.Default.MaintenanceParameters), JobPriority.Normal); + /// : a read-only report of the archive's duplicate photos into the job log. + public static ArchiveJobRequest Duplicates(Archive archive) => new(JobKind.Duplicates, [archive.Id], archive.Name, null, JobPriority.Normal); + + /// : a read-only report of where the index and the disk disagree into the job log. + public static ArchiveJobRequest Orphans(Archive archive) => new(JobKind.Orphans, [archive.Id], archive.Name, null, JobPriority.Normal); + + /// : a copy of the SQLite index into the backups folder, keeping the newest . + public static ArchiveJobRequest Backup(int keep = DatabaseBackups.Keep) => + new(JobKind.Backup, [], null, Json(new BackupParameters(keep), ArchiveJobJson.Default.BackupParameters), JobPriority.Normal); + + /// + /// : the clean-up (by name, in order) with their age limit; the + /// archive, when given, is what archive steps (the trash report) look at. + /// + public static ArchiveJobRequest Cleanup(Archive? archive, IReadOnlyList steps, int olderThanDays) => + new(JobKind.Cleanup, archive is null ? [] : [archive.Id], archive?.Name, + Json(new CleanupParameters(steps, olderThanDays), ArchiveJobJson.Default.CleanupParameters), JobPriority.Normal); + public static Task EnqueueAsync(this JobEngine engine, ArchiveJobRequest request, Guid? createdBy = null, CancellationToken cancellationToken = default) => engine.EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, createdBy, cancellationToken); diff --git a/src/Chronos.Data/Archiving/Jobs/ArchiveJobParameters.cs b/src/Chronos.Data/Archiving/Jobs/ArchiveJobParameters.cs index 90eb0355..eaba7cc8 100644 --- a/src/Chronos.Data/Archiving/Jobs/ArchiveJobParameters.cs +++ b/src/Chronos.Data/Archiving/Jobs/ArchiveJobParameters.cs @@ -22,6 +22,15 @@ public sealed record MaintenanceParameters(bool DryRun); /// : the exact plan the user confirmed. public sealed record ImportJobParameters(Guid ArchiveId, ImportPlan Plan, Guid? CardId, bool Verify, Guid? UserId); +/// : no target; the newest copies are kept. +public sealed record BackupParameters(int Keep); + +/// +/// : target = the archive (or none); the clean-up steps to run, by name and in order, and +/// the age in days from which what they clean goes (0 keeps everything). +/// +public sealed record CleanupParameters(IReadOnlyList Steps, int OlderThanDays); + /// A job ready to enqueue. public sealed record ArchiveJobRequest(JobKind Kind, IReadOnlyList Targets, string? Label, string? ParametersJson, JobPriority Priority); @@ -31,6 +40,8 @@ public sealed record ArchiveJobRequest(JobKind Kind, IReadOnlyList Targets [JsonSerializable(typeof(ImageJobParameters))] [JsonSerializable(typeof(MaintenanceParameters))] [JsonSerializable(typeof(ImportJobParameters))] +[JsonSerializable(typeof(BackupParameters))] +[JsonSerializable(typeof(CleanupParameters))] [JsonSerializable(typeof(CardProfile))] internal sealed partial class ArchiveJobJson : JsonSerializerContext { diff --git a/src/Chronos.Data/Archiving/Jobs/ReportJobHandlers.cs b/src/Chronos.Data/Archiving/Jobs/ReportJobHandlers.cs new file mode 100644 index 00000000..dfa3c8fe --- /dev/null +++ b/src/Chronos.Data/Archiving/Jobs/ReportJobHandlers.cs @@ -0,0 +1,58 @@ +using Chronos.Core.Localization; +using Chronos.Core.Model; +using Chronos.Data.Jobs; + +namespace Chronos.Data.Archiving.Jobs; + +/// +/// as a job (owner review 2, R10): the archive's groups of photos sharing a hash, counted and +/// the largest listed in the job log. Read-only: the Archive screen's Duplicates tab and the maintenance job's duplicate flags +/// stay what acts on them. This makes the kind runnable on the server too (scheduled reports). +/// +internal sealed class DuplicatesJobHandler(ChronosDatabase db, ArchiveMaintenance maintenance) : IJobHandler +{ + /// Groups listed one per log line, the largest first; the summary counts them all. + private const int ListedGroups = 20; + + public JobKind Kind => JobKind.Duplicates; + + public async Task RunAsync(JobContext context) + { + var archive = await JobArchive.LoadAsync(db, context.Job, context.Cancellation).ConfigureAwait(false); + await context.ReportAsync(0, 1).ConfigureAwait(false); + var groups = await maintenance.FindDuplicatesAsync(archive.Id, context.Cancellation).ConfigureAwait(false); + foreach (var group in groups.OrderByDescending(g => g.Frames.Count).ThenBy(g => g.Frames[0].RelativePath, StringComparer.Ordinal).Take(ListedGroups)) + context.Log(Loc.L("Archive.Log.DuplicateGroup", group.Frames[0].RelativePath, group.Frames.Count - 1)); + context.Log(Loc.L("Archive.Log.DuplicatesDone", groups.Count, groups.Sum(g => g.Frames.Count - 1))); + context.Log(Loc.L("Archive.Log.ReportOnly")); + await context.ReportAsync(1, 1).ConfigureAwait(false); + } +} + +/// +/// as a job (owner review 2, R10): where the index and the disk disagree — photos the index +/// does not list, frames whose file is gone, sequences without their folder, folders not indexed yet — counted and listed in +/// the job log. Read-only: the Archive screen's fixes and a scan stay what acts on them. The archive must be reachable. +/// +internal sealed class OrphansJobHandler(ChronosDatabase db, ArchiveMaintenance maintenance) : IJobHandler +{ + /// Folders listed one per log line; the summary counts them all. + private const int ListedFolders = 20; + + public JobKind Kind => JobKind.Orphans; + + public async Task RunAsync(JobContext context) + { + var archive = await JobArchive.LoadAsync(db, context.Job, context.Cancellation).ConfigureAwait(false); + await context.ReportAsync(0, 1).ConfigureAwait(false); + var report = await maintenance.FindOrphansAsync(archive, context.Cancellation).ConfigureAwait(false); + foreach (var missing in report.MissingSequences.Take(ListedFolders)) + context.Log(Loc.L("Archive.Log.FolderMissing", missing.Folder)); + foreach (var folder in report.UnindexedFolders.Take(ListedFolders)) + context.Log(Loc.L("Archive.Log.FolderUnindexed", folder)); + context.Log(Loc.L("Archive.Log.Orphans", report.UnindexedFiles, report.MissingFiles)); + context.Log(Loc.L("Archive.Log.OrphanFolders", report.MissingSequences.Count, report.UnindexedFolders.Count)); + context.Log(Loc.L("Archive.Log.ReportOnly")); + await context.ReportAsync(1, 1).ConfigureAwait(false); + } +} diff --git a/src/Chronos.Data/Archiving/SequenceWriter.cs b/src/Chronos.Data/Archiving/SequenceWriter.cs index 16a6daee..2e391e65 100644 --- a/src/Chronos.Data/Archiving/SequenceWriter.cs +++ b/src/Chronos.Data/Archiving/SequenceWriter.cs @@ -38,6 +38,19 @@ public sealed record ArchiveServiceOptions /// public Func MemoryBudget { get; init; } = () => DefaultMemoryBudget; + /// + /// The host's own clean-up steps (the server's uploads sweep, the desktop's cache trim), run by + /// jobs after the shared ones of when a + /// schedule names them (owner review 2, R10). A name the shared steps use already is ignored. + /// + public IReadOnlyList CleanupSteps { get; init; } = []; + + /// + /// Folder of the copies jobs make; null: the backups folder next to the + /// SQLite index (), where the desktop and the server keep them. + /// + public string? BackupDirectory { get; init; } + /// Half the logical cores, at least one: the parallelism when the host sets none. public static int DefaultParallelism => Math.Max(1, Environment.ProcessorCount / 2); diff --git a/src/Chronos.Data/ChronosDatabase.cs b/src/Chronos.Data/ChronosDatabase.cs index 4c107b84..9f6e7172 100644 --- a/src/Chronos.Data/ChronosDatabase.cs +++ b/src/Chronos.Data/ChronosDatabase.cs @@ -22,6 +22,7 @@ public abstract class ChronosRepositories Tags = new TagRepository(access); Audit = new AuditRepository(access); Uploads = new UploadRepository(access); + Schedules = new ScheduleRepository(access); } public ArchiveRepository Archives { get; } @@ -37,6 +38,9 @@ public abstract class ChronosRepositories public TagRepository Tags { get; } public AuditRepository Audit { get; } public UploadRepository Uploads { get; } + + /// Scheduled jobs (owner review 2, R10). + public ScheduleRepository Schedules { get; } } /// diff --git a/src/Chronos.Data/DatabaseBackups.cs b/src/Chronos.Data/DatabaseBackups.cs index 7c7112a2..91f50c34 100644 --- a/src/Chronos.Data/DatabaseBackups.cs +++ b/src/Chronos.Data/DatabaseBackups.cs @@ -30,6 +30,28 @@ public static partial class DatabaseBackups /// How often a running host checks whether a copy is due. public static readonly TimeSpan CheckInterval = TimeSpan.FromHours(1); + /// Name of the folder next to the index that holds its copies (the desktop's and the server's). + public const string DirectoryName = "backups"; + + /// The backups folder of the index at : backups next to it. + public static string DefaultDirectory(string databasePath) => + Path.Combine(Path.GetDirectoryName(Path.GetFullPath(databasePath)) ?? ".", DirectoryName); + + /// + /// Why cannot be copied by (a + /// code: postgres, or no-index-file for an index in memory); null for an SQLite index file. + /// + public static string? Unavailable(ChronosDatabase db) + { + ArgumentNullException.ThrowIfNull(db); + return db.Database switch + { + SqliteDatabase { Path: not null } => null, + SqliteDatabase => Scheduling.ScheduleUnavailable.NoIndexFile, + _ => Scheduling.ScheduleUnavailable.Postgres, + }; + } + /// /// The copies of the index at in , newest first. Their /// names are <index name>-yyyyMMdd-HHmmss-fff.db in UTC, as writes them. diff --git a/src/Chronos.Data/Housekeeping/CleanupSteps.cs b/src/Chronos.Data/Housekeeping/CleanupSteps.cs new file mode 100644 index 00000000..ffd54a03 --- /dev/null +++ b/src/Chronos.Data/Housekeeping/CleanupSteps.cs @@ -0,0 +1,116 @@ +using Chronos.Core.Archiving; +using Chronos.Core.Localization; +using Chronos.Core.Model; +using Chronos.Data.Archiving; +using Chronos.Data.Archiving.Jobs; +using Chronos.Data.Jobs; + +namespace Chronos.Data.Housekeeping; + +/// +/// One step of a job (owner review 2, R10). The shared steps are in ; +/// a host adds its own (the server's uploads sweep, the desktop's cache trim) through +/// . A step must be safe to run again at any time, honour +/// , and never delete user data (BRIEF §0.2 rule 8): only what the product made +/// and no longer needs (history, temporary files), never originals or the trash. +/// +public interface ICleanupStep +{ + /// Wire name, kebab-case (job-history): what a schedule's steps list. + string Name { get; } + + /// Runs the step and returns the line the job log shows for it (localized). + Task RunAsync(CleanupContext context, CleanupParameters parameters); +} + +/// What a clean-up step works with: the index, the job's archive (if any), the running job and the time it started. +public sealed class CleanupContext(ChronosDatabase database, Archive? archive, JobContext job, DateTimeOffset now) +{ + public ChronosDatabase Database { get; } = database; + + /// The archive the job targets; null when it targets none (no archive open). + public Archive? Archive { get; } = archive; + + /// The running clean-up job: its log and cancellation. + public JobContext Job { get; } = job; + + /// When the job started its steps (UTC); ages count from it. + public DateTimeOffset Now { get; } = now; + + public CancellationToken Cancellation => Job.Cancellation; + + /// The instant before which things are old; null for 0, which keeps everything. + public DateTimeOffset? Before(int olderThanDays) => olderThanDays > 0 ? Now - TimeSpan.FromDays(olderThanDays) : null; +} + +/// +/// The clean-up steps every host has (owner review 2, R10): (finished jobs, which the job engine +/// purged at start before), (the audit log; 0 days keeps it) and (lists the +/// archive's trash entries older than the limit with their sizes; it never deletes: emptying the trash stays a user action). +/// +public static class CleanupSteps +{ + public const string JobHistory = "job-history"; + public const string Audit = "audit"; + public const string TrashReport = "trash-report"; + + /// Days of job history the housekeeping schedule keeps unless the host says otherwise (BRIEF §14.6: 90). + public const int DefaultHistoryDays = 90; + + /// Age in days from which the trash report lists an entry unless the schedule says otherwise. + public const int DefaultTrashDays = 30; + + /// Trash entries listed one per log line; the report counts them all. + internal const int ListedTrashEntries = 20; + + /// The shared steps, in the order a host lists them: job history, audit log, trash report. + public static IReadOnlyList Shared(SequenceTrash trash) => [new JobHistoryStep(), new AuditStep(), new TrashReportStep(trash)]; + + /// "12.4 MB" in the language of the log. + internal static string Megabytes(long bytes) => Loc.L("Common.Size.MB", (bytes / 1e6).ToString("0.#", Loc.Culture)); + + private sealed class JobHistoryStep : ICleanupStep + { + public string Name => JobHistory; + + public async Task RunAsync(CleanupContext context, CleanupParameters parameters) + { + if (context.Before(parameters.OlderThanDays) is not { } before) return Loc.L("Archive.Log.HistoryKept"); + var removed = await context.Database.Jobs.PurgeHistoryAsync(before, context.Cancellation).ConfigureAwait(false); + return Loc.L("Archive.Log.HistoryPurged", removed); + } + } + + private sealed class AuditStep : ICleanupStep + { + public string Name => Audit; + + public async Task RunAsync(CleanupContext context, CleanupParameters parameters) + { + if (context.Before(parameters.OlderThanDays) is not { } before) return Loc.L("Archive.Log.AuditKept"); + var removed = await context.Database.Audit.PurgeAsync(before, context.Cancellation).ConfigureAwait(false); + return Loc.L("Archive.Log.AuditPurged", removed); + } + } + + /// Reads the trash and reports; nothing is moved or deleted (BRIEF §0.2 rule 8). + private sealed class TrashReportStep(SequenceTrash trash) : ICleanupStep + { + public string Name => TrashReport; + + public async Task RunAsync(CleanupContext context, CleanupParameters parameters) + { + if (context.Archive is not { } archive || ArchiveVolume.Probe(new ArchivePaths(archive.RootPath)) == ArchiveAvailability.Offline) + return Loc.L("Archive.Log.TrashOffline"); + var before = context.Before(parameters.OlderThanDays); + var old = (await trash.ListAsync(archive, context.Cancellation).ConfigureAwait(false)) + .Where(e => before is null || e.DeletedAt < before).OrderBy(e => e.DeletedAt).ToList(); + foreach (var entry in old.Take(ListedTrashEntries)) + { + context.Job.Log(Loc.L("Archive.Log.TrashEntry", entry.DeletedAt.ToString("yyyy-MM-dd", System.Globalization.CultureInfo.InvariantCulture), + entry.Title, Megabytes(entry.Bytes))); + } + return Loc.L("Archive.Log.TrashReport", old.Count, Megabytes(old.Sum(e => e.Bytes))); + } + } +} diff --git a/src/Chronos.Data/Housekeeping/HousekeepingJobHandlers.cs b/src/Chronos.Data/Housekeeping/HousekeepingJobHandlers.cs new file mode 100644 index 00000000..38a3a187 --- /dev/null +++ b/src/Chronos.Data/Housekeeping/HousekeepingJobHandlers.cs @@ -0,0 +1,78 @@ +using Chronos.Core.Diagnostics; +using Chronos.Core.Localization; +using Chronos.Core.Model; +using Chronos.Data.Archiving.Jobs; +using Chronos.Data.Jobs; +using Chronos.Data.Sqlite; + +namespace Chronos.Data.Housekeeping; + +/// +/// (owner review 2, R10): a VACUUM INTO copy of the SQLite index into +/// (by default the backups folder next to the index, ), +/// keeping the newest . A PostgreSQL index, or one without a file, cannot be copied this +/// way (): the job fails with that reason, and schedules do not queue it. +/// +internal sealed class BackupJobHandler(ChronosDatabase db, string? directory) : IJobHandler +{ + /// Most copies a schedule may keep. + public const int MaxKeep = 100; + + public JobKind Kind => JobKind.Backup; + + public async Task RunAsync(JobContext context) + { + if (db.Database is not SqliteDatabase { Path: { } path }) + throw new NotSupportedException(DatabaseBackups.Unavailable(db) == Scheduling.ScheduleUnavailable.Postgres + ? "Backups of a PostgreSQL database are made with the server's own tools." + : "An index without a file cannot be backed up."); + var keep = Math.Clamp(ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.BackupParameters, new BackupParameters(DatabaseBackups.Keep)).Keep, 1, MaxKeep); + await context.ReportAsync(0, 1).ConfigureAwait(false); + var copy = await db.BackupAsync(directory ?? DatabaseBackups.DefaultDirectory(path), keep, context.Cancellation).ConfigureAwait(false); + context.Log(Loc.L("Archive.Log.BackupDone", copy, CleanupSteps.Megabytes(new FileInfo(copy).Length), keep)); + await context.ReportAsync(1, 1).ConfigureAwait(false); + } +} + +/// +/// (owner review 2, R10): runs the named steps of in order, +/// each with the job's age limit, and logs one line per step. A step that fails is logged and the others still run; the job +/// then fails naming it. Steps are quick and safe to repeat, so a resumed job runs them all again (no checkpoint). An +/// unknown name (a step of another host, or of a newer build) is logged and skipped. +/// +internal sealed class CleanupJobHandler(ChronosDatabase db, IReadOnlyList steps, TimeProvider time) : IJobHandler +{ + public JobKind Kind => JobKind.Cleanup; + + public async Task RunAsync(JobContext context) + { + var parameters = ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.CleanupParameters, new CleanupParameters([], 0)); + var archive = context.Job.TargetIds.Count > 0 ? await db.Archives.GetAsync(context.Job.TargetIds[0], context.Cancellation).ConfigureAwait(false) : null; + var cleanup = new CleanupContext(db, archive, context, time.GetUtcNow()); + var names = parameters.Steps; + var failed = new List(); + for (var i = 0; i < names.Count; i++) + { + context.Cancellation.ThrowIfCancellationRequested(); + await context.ReportAsync(i, names.Count).ConfigureAwait(false); + var step = steps.FirstOrDefault(s => string.Equals(s.Name, names[i], StringComparison.Ordinal)); + if (step is null) + { + context.Log(Loc.L("Archive.Log.StepUnknown", names[i])); + continue; + } + try + { + context.Log(await step.RunAsync(cleanup, parameters).ConfigureAwait(false)); + } + catch (Exception ex) when (!(ex is OperationCanceledException && context.Cancellation.IsCancellationRequested)) + { + Log.Warning("jobs", $"Clean-up step {step.Name} of job {context.Job.Id} failed", ex); + context.Log(Loc.L("Archive.Log.StepFailed", step.Name, ex.Message)); + failed.Add(step.Name); + } + } + await context.ReportAsync(names.Count, names.Count).ConfigureAwait(false); + if (failed.Count > 0) throw new InvalidOperationException($"Clean-up steps failed: {string.Join(", ", failed)}."); + } +} diff --git a/src/Chronos.Data/Jobs/JobEngine.cs b/src/Chronos.Data/Jobs/JobEngine.cs index 1af2cf28..e84eac82 100644 --- a/src/Chronos.Data/Jobs/JobEngine.cs +++ b/src/Chronos.Data/Jobs/JobEngine.cs @@ -25,9 +25,6 @@ public sealed record JobEngineOptions /// Fallback poll for jobs enqueued by other processes; also how often the priority queue is reviewed. public TimeSpan PollInterval { get; init; } = TimeSpan.FromSeconds(3); - /// Finished jobs older than this are purged (BRIEF §14.6: 90 days). - public TimeSpan HistoryRetention { get; init; } = TimeSpan.FromDays(90); - /// Clock of the throughput windows and the checkpoint interval (tests pass their own). public TimeProvider Time { get; init; } = TimeProvider.System; @@ -38,7 +35,8 @@ public sealed record JobEngineOptions /// /// Persistent job queue runner (BRIEF §14.6, amended by owner review 1): one worker claims queued jobs from the database /// in claim order, runs the registered handler, checkpoints progress, and survives crashes (running jobs are requeued at -/// start with their checkpoints). Used by the desktop client (local mode) and the server. Jobs an +/// start with their checkpoints). Used by the desktop client (local mode) and the server. The history of finished jobs is +/// purged by the housekeeping schedule's job-history clean-up step, not here (owner review 2, R10). Jobs an /// declines stay queued for other claimers. With the priority queue on /// () a queued job of a higher level makes the running job of a lower level /// yield: it goes back to the queue with its checkpoint and resumes by itself afterwards, never interrupted by the @@ -87,12 +85,11 @@ public sealed class JobEngine : IAsyncDisposable /// The priority queue as it is now (). public JobPriorityQueue PriorityQueue => Options.PriorityQueue?.Invoke() ?? JobPriorityQueue.Off; - /// Requeues this worker's interrupted jobs, purges old history and starts the worker and the queue's review. + /// Requeues this worker's interrupted jobs and starts the worker and the queue's review. public async Task StartAsync(CancellationToken cancellationToken = default) { var requeued = await _db.Jobs.RequeueRunningAsync(Options.WorkerId, cancellationToken).ConfigureAwait(false); if (requeued > 0) Log.Info("jobs", $"Requeued {requeued} interrupted job(s)"); - await _db.Jobs.PurgeHistoryAsync(DateTimeOffset.UtcNow - Options.HistoryRetention, cancellationToken).ConfigureAwait(false); // Only kinds this engine can run: an empty filter would claim (and fail) every job, render-node work included. _kinds = [.. (Options.Kinds ?? _handlers.Keys).Where(_handlers.ContainsKey).Distinct()]; _selective = _kinds.Select(k => _handlers[k]).OfType().ToDictionary(h => h.Kind); @@ -118,7 +115,18 @@ public sealed class JobEngine : IAsyncDisposable /// Queues a job and wakes a worker. public Task EnqueueAsync(JobKind kind, IReadOnlyList targets, string? targetLabel = null, string? parametersJson = null, JobPriority priority = JobPriority.Normal, Guid? createdBy = null, CancellationToken cancellationToken = default) => - EnqueueAsync(Guid.CreateVersion7(), kind, targets, targetLabel, parametersJson, priority, createdBy, cancellationToken); + EnqueueAsync(Guid.CreateVersion7(), kind, targets, targetLabel, parametersJson, priority, createdBy, null, cancellationToken); + + /// Queues a job that the schedule fired () and wakes a worker. + public Task EnqueueAsync(JobKind kind, IReadOnlyList targets, string? targetLabel, string? parametersJson, JobPriority priority, + Guid? createdBy, Guid? scheduleId, CancellationToken cancellationToken = default) => + EnqueueAsync(Guid.CreateVersion7(), kind, targets, targetLabel, parametersJson, priority, createdBy, scheduleId, cancellationToken); + + /// + /// True when this engine runs jobs of : it has a handler for it and + /// does not leave it out. Known before , so a host that does not start the engine still answers. + /// + public bool CanRun(JobKind kind) => _handlers.ContainsKey(kind) && (Options.Kinds is null || Options.Kinds.Contains(kind)); /// Cancels a queued or running job; a running handler observes its token. public Task CancelAsync(Guid id, CancellationToken cancellationToken = default) => TransitionAsync(id, JobState.Cancelled, cancellationToken); @@ -212,7 +220,7 @@ public sealed class JobEngine : IAsyncDisposable var id = Guid.CreateVersion7(); // Known before the job is queued, so the review its queueing starts already leaves the parent running. if (_running.TryGetValue(parent, out var run)) run.FollowUps.TryAdd(id, 0); - return await EnqueueAsync(id, kind, targets, label, parametersJson, priority, createdBy, cancellationToken).ConfigureAwait(false); + return await EnqueueAsync(id, kind, targets, label, parametersJson, priority, createdBy, null, cancellationToken).ConfigureAwait(false); } internal void StopJob(Guid id) @@ -242,7 +250,7 @@ public sealed class JobEngine : IAsyncDisposable } private async Task EnqueueAsync(Guid id, JobKind kind, IReadOnlyList targets, string? targetLabel, string? parametersJson, - JobPriority priority, Guid? createdBy, CancellationToken cancellationToken) + JobPriority priority, Guid? createdBy, Guid? scheduleId, CancellationToken cancellationToken) { var job = new Job { @@ -255,6 +263,7 @@ public sealed class JobEngine : IAsyncDisposable State = JobState.Queued, CreatedBy = createdBy, Created = DateTimeOffset.UtcNow, + ScheduleId = scheduleId, }; await _db.Jobs.EnqueueAsync(job, cancellationToken).ConfigureAwait(false); JobChanged?.Invoke(job); diff --git a/src/Chronos.Data/Jobs/JobPriorityQueue.cs b/src/Chronos.Data/Jobs/JobPriorityQueue.cs index 350a5bfd..2b0c9156 100644 --- a/src/Chronos.Data/Jobs/JobPriorityQueue.cs +++ b/src/Chronos.Data/Jobs/JobPriorityQueue.cs @@ -21,7 +21,10 @@ public enum JobKindPriority /// public sealed record JobPriorityQueue { - /// The levels a kind has unless the user changed it: renders, imports and downloads first; the long passes that resume anywhere last. + /// + /// The levels a kind has unless the user changed it: renders, imports and downloads first; the long passes that resume + /// anywhere, and the housekeeping of the schedules (backups, clean-ups), last. + /// public static IReadOnlyDictionary DefaultLevels { get; } = new Dictionary { [JobKind.Render] = JobKindPriority.High, @@ -29,6 +32,8 @@ public sealed record JobPriorityQueue [JobKind.Download] = JobKindPriority.High, [JobKind.Verify] = JobKindPriority.Low, [JobKind.Proxies] = JobKindPriority.Low, + [JobKind.Backup] = JobKindPriority.Low, + [JobKind.Cleanup] = JobKindPriority.Low, }; /// The strict queue: no levels, no interruptions. diff --git a/src/Chronos.Data/Jobs/JobWire.cs b/src/Chronos.Data/Jobs/JobWire.cs index d22b3b0c..63158120 100644 --- a/src/Chronos.Data/Jobs/JobWire.cs +++ b/src/Chronos.Data/Jobs/JobWire.cs @@ -56,6 +56,7 @@ public static class JobWire Created = dto.Created, Started = dto.Started, Finished = dto.Finished, + ScheduleId = dto.ScheduleId, }; } diff --git a/src/Chronos.Data/Migrations/postgres/0007_schedules.sql b/src/Chronos.Data/Migrations/postgres/0007_schedules.sql new file mode 100644 index 00000000..17620598 --- /dev/null +++ b/src/Chronos.Data/Migrations/postgres/0007_schedules.sql @@ -0,0 +1,27 @@ +-- Scheduled jobs (owner review 2, R10): one row per schedule, the timers a host used to run out of sight included as +-- system schedules (system_key set, seeded by the host at start, never here). next_run is claimed by compare-and-set, +-- so one process fires each run. Jobs remember the schedule that queued them. Mirrors sqlite/0007_schedules.sql. + +CREATE TABLE schedules ( + id uuid NOT NULL PRIMARY KEY, + system_key text COLLATE "C" UNIQUE, + name text NOT NULL, + action text NOT NULL, + target text NOT NULL DEFAULT 'current', + parameters_json jsonb, + recurrence text NOT NULL, + catch_up boolean NOT NULL DEFAULT TRUE, + priority integer NOT NULL DEFAULT 0, + enabled boolean NOT NULL DEFAULT TRUE, + next_run timestamptz, + last_run timestamptz, + last_job_id uuid, + last_outcome text, + created_by uuid, + created timestamptz NOT NULL, + updated timestamptz NOT NULL +); +CREATE INDEX ix_schedules_due ON schedules (enabled, next_run); + +ALTER TABLE jobs ADD COLUMN schedule_id uuid; +CREATE INDEX ix_jobs_schedule ON jobs (schedule_id, created); diff --git a/src/Chronos.Data/Migrations/sqlite/0007_schedules.sql b/src/Chronos.Data/Migrations/sqlite/0007_schedules.sql new file mode 100644 index 00000000..45064c78 --- /dev/null +++ b/src/Chronos.Data/Migrations/sqlite/0007_schedules.sql @@ -0,0 +1,27 @@ +-- Scheduled jobs (owner review 2, R10): one row per schedule, the timers a host used to run out of sight included as +-- system schedules (system_key set, seeded by the host at start, never here). next_run is claimed by compare-and-set, +-- so one process fires each run. Jobs remember the schedule that queued them. Mirrors postgres/0007_schedules.sql. + +CREATE TABLE schedules ( + id TEXT NOT NULL PRIMARY KEY, + system_key TEXT UNIQUE, + name TEXT NOT NULL, + action TEXT NOT NULL, + target TEXT NOT NULL DEFAULT 'current', + parameters_json TEXT, + recurrence TEXT NOT NULL, + catch_up INTEGER NOT NULL DEFAULT 1, + priority INTEGER NOT NULL DEFAULT 0, + enabled INTEGER NOT NULL DEFAULT 1, + next_run TEXT, + last_run TEXT, + last_job_id TEXT, + last_outcome TEXT, + created_by TEXT, + created TEXT NOT NULL, + updated TEXT NOT NULL +); +CREATE INDEX ix_schedules_due ON schedules (enabled, next_run); + +ALTER TABLE jobs ADD COLUMN schedule_id TEXT; +CREATE INDEX ix_jobs_schedule ON jobs (schedule_id, created); diff --git a/src/Chronos.Data/Repositories/AuditRepository.cs b/src/Chronos.Data/Repositories/AuditRepository.cs index 1744f3c7..32de63a7 100644 --- a/src/Chronos.Data/Repositories/AuditRepository.cs +++ b/src/Chronos.Data/Repositories/AuditRepository.cs @@ -2,7 +2,7 @@ using Chronos.Core.Model; namespace Chronos.Data.Repositories; -/// Append-only record of who did what (BRIEF §5 AuditLog, §14.2). +/// Append-only record of who did what (BRIEF §5 AuditLog, §14.2); old entries leave only through a retention (). public sealed class AuditRepository { private const string Columns = "id, at, user_id, username, action, target_kind, target_id, details, remote_address"; @@ -16,6 +16,10 @@ public sealed class AuditRepository [entry.Id, entry.At, entry.UserId, entry.Username, entry.Action, entry.TargetKind, entry.TargetId, entry.Details, entry.RemoteAddress], cancellationToken).ConfigureAwait(false); + /// Deletes the entries written before (the audit retention's clean-up step); returns how many. + public Task PurgeAsync(DateTimeOffset before, CancellationToken cancellationToken = default) => + _db.ExecuteAsync("DELETE FROM audit_log WHERE at < $1", [before], cancellationToken); + /// Entries newest first, optionally of one user. public Task> ListAsync(Guid? userId = null, int limit = 100, string? cursor = null, CancellationToken cancellationToken = default) => ListAsync(userId, action: null, limit, cursor, cancellationToken); diff --git a/src/Chronos.Data/Repositories/JobRepository.cs b/src/Chronos.Data/Repositories/JobRepository.cs index 2d7f7add..e415fbbe 100644 --- a/src/Chronos.Data/Repositories/JobRepository.cs +++ b/src/Chronos.Data/Repositories/JobRepository.cs @@ -15,7 +15,11 @@ public sealed class JobRepository { private const string Columns = "id, kind, target_ids, target_label, parameters_json, state, priority, progress_done, progress_total, items_per_second, " + - "eta_seconds, checkpoint, log_tail, created_by, node_id, attempts, error, created, started, finished"; + "eta_seconds, checkpoint, log_tail, created_by, node_id, attempts, error, created, started, finished, schedule_id"; + + /// The states of a job that has not ended: queued, running, paused. + private static readonly string PendingStates = + string.Join(", ", new[] { JobState.Queued, JobState.Running, JobState.Paused }.Select(s => ((int)s).ToString(CultureInfo.InvariantCulture))); private readonly DbAccess _db; @@ -23,11 +27,11 @@ public sealed class JobRepository public async Task EnqueueAsync(Job job, CancellationToken cancellationToken = default) => await _db.ExecuteAsync( - $"INSERT INTO jobs ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20)", + $"INSERT INTO jobs ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21)", [ job.Id, (int)job.Kind, string.Join(',', job.TargetIds), job.TargetLabel, DbParam.Json(job.ParametersJson), (int)job.State, (int)job.Priority, job.ProgressDone, job.ProgressTotal, job.ItemsPerSecond, job.Eta?.TotalSeconds, job.Checkpoint, job.LogTail, - job.CreatedBy, job.NodeId, job.Attempts, job.Error, job.Created, job.Started, job.Finished, + job.CreatedBy, job.NodeId, job.Attempts, job.Error, job.Created, job.Started, job.Finished, job.ScheduleId, ], cancellationToken).ConfigureAwait(false); public Task GetAsync(Guid id, CancellationToken cancellationToken = default) => @@ -173,6 +177,45 @@ public sealed class JobRepository $"SELECT count(*) FROM jobs WHERE state = {(int)JobState.Queued} AND kind = $1 AND target_ids = $2", [(int)kind, targetId.ToString()], cancellationToken).ConfigureAwait(false) > 0; + /// + /// True when a job of one of is queued, running or paused; with , + /// only such a job that targets at least one of them (an archive, sequences) counts. A schedule does not queue its + /// action again while one waits (owner review 2, R10). + /// + public async Task HasPendingAsync(IReadOnlyCollection kinds, IReadOnlyCollection? targetIds = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(kinds); + if (kinds.Count == 0 || targetIds is { Count: 0 }) return false; + var p = new SqlParameters(); + var sql = $"SELECT count(*) FROM jobs WHERE state IN ({PendingStates}) AND kind IN ({KindList(kinds)})"; + // Target ids are stored comma-joined as lower-case text, which never holds a LIKE wildcard. + if (targetIds is not null) + sql += $" AND ({string.Join(" OR ", targetIds.Distinct().Select(id => $"target_ids LIKE {p.Add(SqlDialect.ContainsPattern(id.ToString()))} ESCAPE '\\'"))})"; + return await _db.ScalarAsync(sql, p.ToArray(), cancellationToken).ConfigureAwait(false) > 0; + } + + /// The targets of the queued, running and paused jobs of (the newest 5000 such jobs). + public Task> ListPendingTargetsAsync(JobKind kind, CancellationToken cancellationToken = default) => + _db.RunAsync(async session => + { + var rows = await session.QueryListAsync( + $"SELECT target_ids FROM jobs WHERE kind = $1 AND state IN ({PendingStates}) ORDER BY created DESC, id DESC LIMIT 5000", + [(int)kind], r => r.GetString(0), cancellationToken).ConfigureAwait(false); + var ids = new HashSet(); + foreach (var text in rows) + { + if (text.Length == 0) continue; + foreach (var id in text.Split(',')) ids.Add(Guid.Parse(id)); + } + return ids; + }, cancellationToken); + + /// The jobs the schedule queued, newest first. + public Task> ListByScheduleAsync(Guid scheduleId, int limit = 20, CancellationToken cancellationToken = default) => + _db.ListAsync($"SELECT {Columns} FROM jobs WHERE schedule_id = $1 ORDER BY created DESC, id DESC LIMIT $2", + [scheduleId, Cursor.ClampLimit(limit)], Map, cancellationToken); + /// /// Claims one specific queued job for (render nodes pick by capability); null when another /// claimer was faster or the job is no longer queued. @@ -245,5 +288,6 @@ public sealed class JobRepository Created = r.GetDateTimeOffset(17), Started = r.GetDateTimeOffsetOrNull(18), Finished = r.GetDateTimeOffsetOrNull(19), + ScheduleId = r.GetGuidOrNull(20), }; } diff --git a/src/Chronos.Data/Repositories/ScheduleRepository.cs b/src/Chronos.Data/Repositories/ScheduleRepository.cs new file mode 100644 index 00000000..aaf989b7 --- /dev/null +++ b/src/Chronos.Data/Repositories/ScheduleRepository.cs @@ -0,0 +1,167 @@ +using Chronos.Core.Model; +using Chronos.Data.Scheduling; + +namespace Chronos.Data.Repositories; + +/// +/// Scheduled jobs (owner review 2, R10). A run is claimed by moving next_run with a compare-and-set +/// (), so when several processes share one database (server instances on one PostgreSQL) +/// exactly one fires it. System schedules are seeded without touching rows that exist (), so +/// what the user changed stays, and they are never deleted (). +/// +public sealed class ScheduleRepository +{ + private const string Columns = + "id, system_key, name, action, target, parameters_json, recurrence, catch_up, priority, enabled, next_run, last_run, last_job_id, " + + "last_outcome, created_by, created, updated"; + + private const string InsertSql = + $"INSERT INTO schedules ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)"; + + /// System schedules first, then the others, oldest first. + private const string Order = "CASE WHEN system_key IS NULL THEN 1 ELSE 0 END, created, id"; + + private readonly DbAccess _db; + + internal ScheduleRepository(DbAccess db) => _db = db; + + /// Every schedule: the system ones first, then the others, oldest first. + public Task> ListAsync(CancellationToken cancellationToken = default) => + _db.ListAsync($"SELECT {Columns} FROM schedules ORDER BY {Order}", [], Map, cancellationToken); + + public Task GetAsync(Guid id, CancellationToken cancellationToken = default) => + _db.SingleAsync($"SELECT {Columns} FROM schedules WHERE id = $1", [id], Map, cancellationToken); + + public Task GetBySystemKeyAsync(string systemKey, CancellationToken cancellationToken = default) => + _db.SingleAsync($"SELECT {Columns} FROM schedules WHERE system_key = $1", [systemKey], Map, cancellationToken); + + /// Adds a schedule; a another row has already fails (unique). + public async Task InsertAsync(Schedule schedule, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(schedule); + await _db.ExecuteAsync(InsertSql, Values(schedule), cancellationToken).ConfigureAwait(false); + } + + /// + /// Stores what a user edits — name, action, target, parameters, recurrence, catch-up, priority, enabled — and the update + /// time; with the next run too, otherwise it stays as stored (a run the scheduler claims + /// meanwhile is not undone). The key, the creation and the last run stay as stored. False when the schedule is gone. + /// + public async Task UpdateAsync(Schedule schedule, bool setNextRun = true, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(schedule); + return await _db.ExecuteAsync( + "UPDATE schedules SET name = $2, action = $3, target = $4, parameters_json = $5, recurrence = $6, catch_up = $7, priority = $8, " + + $"enabled = $9, updated = $10{(setNextRun ? ", next_run = $11" : string.Empty)} WHERE id = $1", + [ + schedule.Id, schedule.Name, schedule.Action, schedule.Target, DbParam.Json(schedule.ParametersJson), schedule.Recurrence, + schedule.CatchUp, (int)schedule.Priority, schedule.Enabled, schedule.Updated, .. setNextRun ? new DbParam[] { schedule.NextRun } : [], + ], cancellationToken).ConfigureAwait(false) > 0; + } + + /// Deletes a schedule a user created; false when it is unknown or a system schedule (those are disabled, never deleted). + public async Task DeleteAsync(Guid id, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync("DELETE FROM schedules WHERE id = $1 AND system_key IS NULL", [id], cancellationToken).ConfigureAwait(false) > 0; + + /// + /// Inserts each system schedule whose key no row has yet (INSERT … ON CONFLICT (system_key) DO NOTHING), with + /// no next run (the scheduler computes it); rows that exist keep whatever the user changed. Rows added together are + /// stamped a microsecond apart, so listings keep the host's order. Returns how many were added. + /// + public Task EnsureSystemAsync(IEnumerable schedules, DateTimeOffset now, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(schedules); + var rows = schedules.Select((s, i) => FromSystem(s, now.AddTicks(i * TimeSpan.TicksPerMicrosecond))).ToList(); + return rows.Count == 0 ? Task.FromResult(0) : _db.InTransactionAsync(async session => + { + var added = 0; + foreach (var row in rows) + added += await session.ExecuteAsync(InsertSql + " ON CONFLICT (system_key) DO NOTHING", Values(row), cancellationToken).ConfigureAwait(false); + return added; + }, cancellationToken); + } + + /// + /// Puts the system schedule back to (name, action, parameters, + /// recurrence, catch-up, priority, enabled) with no next run, so the scheduler computes it again; its history stays. + /// False when no row has that key. + /// + public async Task ResetSystemAsync(string systemKey, SystemSchedule defaults, DateTimeOffset now, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(defaults); + var row = FromSystem(defaults, now); + return await _db.ExecuteAsync( + "UPDATE schedules SET name = $2, action = $3, target = $4, parameters_json = $5, recurrence = $6, catch_up = $7, priority = $8, " + + "enabled = $9, next_run = NULL, updated = $10 WHERE system_key = $1", + [ + systemKey, row.Name, row.Action, row.Target, DbParam.Json(row.ParametersJson), row.Recurrence, row.CatchUp, (int)row.Priority, row.Enabled, + now, + ], cancellationToken).ConfigureAwait(false) > 0; + } + + /// Enabled schedules whose next run is at or before , the longest due first. + public Task> ListDueAsync(DateTimeOffset nowUtc, CancellationToken cancellationToken = default) => + _db.ListAsync($"SELECT {Columns} FROM schedules WHERE enabled = TRUE AND next_run IS NOT NULL AND next_run <= $1 ORDER BY next_run, id", + [nowUtc], Map, cancellationToken); + + /// + /// Moves an enabled schedule's next run from (null: none yet) to + /// , only if it still is : true for the one caller that won. + /// Claiming a due run this way is what keeps two processes from both firing it. + /// + public async Task TryAdvanceAsync(Guid id, DateTimeOffset? expectedNextRun, DateTimeOffset? nextRun, CancellationToken cancellationToken = default) => + await (expectedNextRun is { } expected + ? _db.ExecuteAsync("UPDATE schedules SET next_run = $2 WHERE id = $1 AND enabled = TRUE AND next_run = $3", [id, nextRun, expected], cancellationToken) + : _db.ExecuteAsync("UPDATE schedules SET next_run = $2 WHERE id = $1 AND enabled = TRUE AND next_run IS NULL", [id, nextRun], cancellationToken)) + .ConfigureAwait(false) > 0; + + /// + /// Records a run: when (), what it did () and the first job it + /// queued; a run that queued nothing keeps the previous job, so the next run still sees whether it is pending. + /// + public async Task RecordOutcomeAsync(Guid id, Guid? jobId, string outcome, DateTimeOffset atUtc, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync("UPDATE schedules SET last_run = $2, last_job_id = COALESCE($3, last_job_id), last_outcome = $4 WHERE id = $1", + [id, atUtc, jobId, outcome], cancellationToken).ConfigureAwait(false) > 0; + + private static Schedule FromSystem(SystemSchedule system, DateTimeOffset now) => new() + { + Id = Guid.CreateVersion7(), + SystemKey = system.SystemKey, + Name = system.Name, + Action = system.Action, + ParametersJson = ScheduleMapping.ParametersJson(system.Action, system.Parameters), + Recurrence = system.Recurrence.Format(), + CatchUp = system.CatchUp, + Priority = system.Priority, + Enabled = system.Enabled, + Created = now, + Updated = now, + }; + + private static DbParam[] Values(Schedule s) => + [ + s.Id, s.SystemKey, s.Name, s.Action, s.Target, DbParam.Json(s.ParametersJson), s.Recurrence, s.CatchUp, (int)s.Priority, s.Enabled, s.NextRun, + s.LastRun, s.LastJobId, s.LastOutcome, s.CreatedBy, s.Created, s.Updated, + ]; + + private static Schedule Map(DbRow r) => new() + { + Id = r.GetGuid(0), + SystemKey = r.GetStringOrNull(1), + Name = r.GetString(2), + Action = r.GetString(3), + Target = r.GetString(4), + ParametersJson = r.GetStringOrNull(5), + Recurrence = r.GetString(6), + CatchUp = r.GetBool(7), + Priority = (JobPriority)r.GetInt32(8), + Enabled = r.GetBool(9), + NextRun = r.GetDateTimeOffsetOrNull(10), + LastRun = r.GetDateTimeOffsetOrNull(11), + LastJobId = r.GetGuidOrNull(12), + LastOutcome = r.GetStringOrNull(13), + CreatedBy = r.GetGuidOrNull(14), + Created = r.GetDateTimeOffset(15), + Updated = r.GetDateTimeOffset(16), + }; +} diff --git a/src/Chronos.Data/Repositories/SequenceRepository.cs b/src/Chronos.Data/Repositories/SequenceRepository.cs index b507718e..5ca128fe 100644 --- a/src/Chronos.Data/Repositories/SequenceRepository.cs +++ b/src/Chronos.Data/Repositories/SequenceRepository.cs @@ -389,6 +389,21 @@ public sealed class SequenceRepository }, cancellationToken); } + /// + /// The sequences of a rolling verification checks next (owner review 2, R10): those never + /// verified first, then the longest unverified, at most . With + /// only those never verified or last verified before it are listed. Sequences without frames are left out. + /// + public Task> ListVerifyCandidatesAsync(Guid archiveId, DateTimeOffset? verifiedBefore, int limit, + CancellationToken cancellationToken = default) + { + var p = new SqlParameters(); + var where = $"archive_id = {p.Add(archiveId)} AND frame_count > 0"; + if (verifiedBefore is { } before) where += $" AND (verified_at IS NULL OR verified_at < {p.Add(before)})"; + return _db.ListAsync($"SELECT id FROM sequences WHERE {where} ORDER BY verified_at ASC NULLS FIRST, id LIMIT {p.Add(Math.Max(1, limit))}", + p.ToArray(), r => r.GetGuid(0), cancellationToken); + } + /// Folder and title of every sequence of an archive, by folder. public Task> ListLocationsAsync(Guid archiveId, CancellationToken cancellationToken = default) => _db.ListAsync("SELECT id, folder, title FROM sequences WHERE archive_id = $1 ORDER BY folder, id", [archiveId], diff --git a/src/Chronos.Data/Scheduling/IScheduleHost.cs b/src/Chronos.Data/Scheduling/IScheduleHost.cs new file mode 100644 index 00000000..e01f27bd --- /dev/null +++ b/src/Chronos.Data/Scheduling/IScheduleHost.cs @@ -0,0 +1,52 @@ +using Chronos.Core.Model; +using Chronos.Protocol; + +namespace Chronos.Data.Scheduling; + +/// +/// What a host (the server, the desktop) tells its (owner review 2, R10): its time zone, the archive +/// schedules work on, the system schedules standing for its timers, the clean-up steps its jobs run, and the actions it +/// cannot run. Called from the scheduler's loop, off any UI thread; members must be thread-safe. +/// +public interface IScheduleHost +{ + /// The zone of daily, weekly and monthly times: the PC's on the desktop, the container's (TZ) on the server. + TimeZoneInfo TimeZone { get; } + + /// + /// The archive schedules targeting current work on (the server's archive, the desktop's current one), with its + /// settings as fresh as the host keeps them; null when there is none yet or it cannot be reached. The scheduler probes + /// whether it is online itself. + /// + ValueTask GetArchiveAsync(CancellationToken cancellationToken); + + /// + /// The system schedules: every timer that queues work on this host, as a visible schedule. Seeded at start by key and + /// never overwritten (the user's edits stay); puts one back to these values. + /// + IReadOnlyList SystemSchedules { get; } + + /// The names of the clean-up steps this host's jobs run (normally those of ArchiveService.CleanupSteps). + IReadOnlyList CleanupSteps { get; } + + /// + /// Why this host does not run (a short code, e.g. not-here), or null when it may. Asked + /// before the scheduler's own checks (), for the action list and at every run. + /// + string? Unavailable(string action); +} + +/// +/// A host's system schedule: identifies it across starts (archive-scan, +/// index-backup, housekeeping…); the rest is what it is seeded with and reset to. is +/// stored as given; clients show system schedules by their key in the user's language. +/// +public sealed record SystemSchedule( + string SystemKey, + string Name, + string Action, + ScheduleRecurrence Recurrence, + ScheduleParametersDto? Parameters = null, + bool Enabled = true, + bool CatchUp = true, + JobPriority Priority = JobPriority.Background); diff --git a/src/Chronos.Data/Scheduling/ScheduleActions.cs b/src/Chronos.Data/Scheduling/ScheduleActions.cs new file mode 100644 index 00000000..7151ce2b --- /dev/null +++ b/src/Chronos.Data/Scheduling/ScheduleActions.cs @@ -0,0 +1,291 @@ +using Chronos.Core.Archiving; +using Chronos.Core.Model; +using Chronos.Data.Archiving; +using Chronos.Data.Archiving.Jobs; +using Chronos.Data.Housekeeping; +using Chronos.Data.Processing; +using Chronos.Data.Repositories; +using Chronos.Protocol; + +namespace Chronos.Data.Scheduling; + +/// Whether a host runs an action, and why not ( or the host's own code). +public sealed record ScheduleActionInfo(string Name, bool Available, string? Reason); + +/// +/// What a schedule's run comes to: the jobs to queue (), or none and why +/// (, , +/// , ). +/// +public sealed record ScheduleResolution(IReadOnlyList Requests, string Outcome) +{ + public static ScheduleResolution Skip(string outcome) => new([], outcome); + + public static ScheduleResolution Queue(IReadOnlyList requests) => new(requests, ScheduleOutcome.Queued); +} + +/// What resolving an action needs: the index, the host, which job kinds the host's engine runs, and the time of the run. +public sealed record ScheduleActionContext(ChronosDatabase Database, IScheduleHost Host, Func CanRun, DateTimeOffset Now); + +/// +/// The actions a schedule can take (owner review 2, R10), shared by the server and the desktop: their names, parameters +/// (, normalised to the fields each reads), availability, and the archive jobs a run +/// queues for the host's current archive. Rebuild and render are not schedulable. Every action that works on an archive +/// skips a run while the archive is missing or offline (or read-only, for one that writes), and while a job doing the same +/// work on it is pending; a run that finds nothing to do queues nothing. +/// +public static class ScheduleActions +{ + /// Incremental scan of the archive (force: a full one; deep: deep analysis of what it indexes). + public const string Scan = "scan"; + + /// Integrity check of the originals, the least recently verified first (olderThanDays, percent). + public const string Verify = "verify"; + + /// Thumbnails of the sequences without them (force: of every sequence). + public const string Thumbnails = "thumbnails"; + + /// Proxies of the sequences without them, when the archive makes proxies (force: of every sequence). + public const string Proxies = "proxies"; + + /// Light analysis of the sequences without a current one (deep: deep analysis of those without one). + public const string Analysis = "analysis"; + + /// The archive's maintenance job (dryRun: report only). + public const string Maintenance = "maintenance"; + + /// A copy of the SQLite index (keep: copies kept). Not on PostgreSQL. + public const string Backup = "backup"; + + /// Clean-up steps (steps, olderThanDays): job history, audit log, trash report, the host's own. + public const string Cleanup = "cleanup"; + + /// Report of the duplicate photos into the job log. + public const string Duplicates = "duplicates"; + + /// Report of where the index and the disk disagree into the job log. + public const string Orphans = "orphans"; + + /// Most days a schedule's age limits accept (ten years). + public const int MaxDays = 3650; + + /// Sequences per job a run queues: a larger set becomes several jobs. + public const int SequencesPerJob = 500; + + /// Every action, in the order clients list them. + public static IReadOnlyList All { get; } = [Scan, Verify, Thumbnails, Proxies, Analysis, Maintenance, Backup, Cleanup, Duplicates, Orphans]; + + public static bool IsKnown(string? action) => action is not null && All.Contains(action, StringComparer.Ordinal); + + /// The job kinds queues (analysis: light, or deep with deep); empty for an unknown one. + public static IReadOnlyList KindsOf(string action) => action switch + { + Scan => [JobKind.Scan], + Verify => [JobKind.Verify], + Thumbnails => [JobKind.Thumbs], + Proxies => [JobKind.Proxies], + Analysis => [JobKind.LightAnalyse, JobKind.DeepAnalyse], + Maintenance => [JobKind.Maintenance], + Backup => [JobKind.Backup], + Cleanup => [JobKind.Cleanup], + Duplicates => [JobKind.Duplicates], + Orphans => [JobKind.Orphans], + _ => [], + }; + + /// The parameters a new schedule of starts with. + public static ScheduleParametersDto Defaults(string action) => action switch + { + Scan => new(Force: false, Deep: false), + Verify => new(OlderThanDays: 30, Percent: 100), + Thumbnails or Proxies => new(Force: false), + Analysis => new(Deep: false), + Maintenance => new(DryRun: false), + Backup => new(Keep: DatabaseBackups.Keep), + Cleanup => new(Steps: [CleanupSteps.JobHistory], OlderThanDays: CleanupSteps.DefaultHistoryDays), + _ => new(), + }; + + /// + /// reduced to the fields reads, each missing one taken from + /// ; clean-up steps keep their order without repeats. + /// + public static ScheduleParametersDto Normalize(string action, ScheduleParametersDto? parameters) + { + var p = parameters ?? new ScheduleParametersDto(); + var d = Defaults(action); + return action switch + { + Scan => new(Force: p.Force ?? d.Force, Deep: p.Deep ?? d.Deep), + Verify => new(OlderThanDays: p.OlderThanDays ?? d.OlderThanDays, Percent: p.Percent ?? d.Percent), + Thumbnails or Proxies => new(Force: p.Force ?? d.Force), + Analysis => new(Deep: p.Deep ?? d.Deep), + Maintenance => new(DryRun: p.DryRun ?? d.DryRun), + Backup => new(Keep: p.Keep ?? d.Keep), + Cleanup => new(Steps: p.Steps is { } steps ? [.. steps.Distinct(StringComparer.Ordinal)] : d.Steps, OlderThanDays: p.OlderThanDays ?? d.OlderThanDays), + _ => new(), + }; + } + + /// + /// The first invalid field of a schedule's action and parameters, by its API name (action, + /// parameters.percent, parameters.steps…); null when they are valid. are the + /// clean-up steps the host runs. + /// + public static string? Problem(string action, ScheduleParametersDto? parameters, IReadOnlyCollection steps) + { + ArgumentNullException.ThrowIfNull(steps); + if (!IsKnown(action)) return "action"; + var p = Normalize(action, parameters); + if (p.OlderThanDays is < 0 or > MaxDays) return "parameters.olderThanDays"; + if (p.Percent is < 1 or > 100) return "parameters.percent"; + if (p.Keep is < 1 or > BackupJobHandler.MaxKeep) return "parameters.keep"; + if (action == Cleanup && (p.Steps is not { Count: > 0 } named || named.Any(s => !steps.Contains(s)))) return "parameters.steps"; + return null; + } + + /// + /// Every action with whether this host runs it: not when the host says so (), its + /// engine has no handler for the kind (), the index cannot be copied (backup: + /// , ), or the current archive makes + /// no proxies (). A missing or offline archive leaves actions available: + /// their runs are skipped until it is back. + /// + public static async Task> AvailabilityAsync(ScheduleActionContext context, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(context); + var archive = await context.Host.GetArchiveAsync(cancellationToken).ConfigureAwait(false); + return [.. All.Select(a => Unavailable(context, a, null, archive) is { } reason ? new ScheduleActionInfo(a, false, reason) : new ScheduleActionInfo(a, true, null))]; + } + + /// + /// What a run of with queues now for the host's current archive, + /// or why it queues nothing (see the type's remarks). Read-only: nothing is queued here. + /// + public static async Task ResolveAsync(ScheduleActionContext context, string action, ScheduleParametersDto? parameters, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(context); + if (!IsKnown(action)) return ScheduleResolution.Skip(ScheduleOutcome.Unavailable); + var p = Normalize(action, parameters); + var db = context.Database; + var archive = await context.Host.GetArchiveAsync(cancellationToken).ConfigureAwait(false); + if (Unavailable(context, action, p, archive) is not null) return ScheduleResolution.Skip(ScheduleOutcome.Unavailable); + switch (action) + { + case Backup: + return ScheduleResolution.Queue([ArchiveJobs.Backup(p.Keep!.Value)]); + case Cleanup: + return ScheduleResolution.Queue([ArchiveJobs.Cleanup(archive, p.Steps!, p.OlderThanDays!.Value)]); + } + + if (archive is null) return ScheduleResolution.Skip(ScheduleOutcome.SkippedOffline); + var availability = ArchiveService.GetAvailability(archive); + var writes = action is not (Verify or Duplicates or Orphans) && !(action == Maintenance && p.DryRun == true); + if (availability == ArchiveAvailability.Offline || (writes && availability != ArchiveAvailability.Online)) + return ScheduleResolution.Skip(ScheduleOutcome.SkippedOffline); + + switch (action) + { + case Scan: + return await PendingAsync(db, [JobKind.Scan, JobKind.Rebuild], archive, cancellationToken).ConfigureAwait(false) + ?? ScheduleResolution.Queue([ArchiveJobs.Scan(archive, full: p.Force == true, deepAnalysis: p.Deep == true)]); + case Verify: + return await PendingAsync(db, [JobKind.Verify], archive, cancellationToken).ConfigureAwait(false) + ?? await VerifyAsync(context, archive, p, cancellationToken).ConfigureAwait(false); + case Thumbnails: + case Proxies: + { + var kind = action == Thumbnails ? JobKind.Thumbs : JobKind.Proxies; + var built = action == Thumbnails ? SequenceStatus.ThumbsBuilt : SequenceStatus.ProxiesBuilt; + var ids = await SequencesAsync(db, archive.Id, p.Force == true ? SequenceStatus.None : built, cancellationToken).ConfigureAwait(false); + return await ChunkedAsync(db, kind, ids, chunk => action == Thumbnails + ? ArchiveJobs.Thumbnails(chunk, archive.Name, p.Force == true) + : ArchiveJobs.Proxies(chunk, archive.Name, p.Force == true), cancellationToken).ConfigureAwait(false); + } + case Analysis: + { + var deep = p.Deep == true; + var ids = deep + ? await SequencesAsync(db, archive.Id, SequenceStatus.AnalyzedDeep, cancellationToken).ConfigureAwait(false) + : await AnalysisFollowUp.UnanalysedAsync(db, archive.Id, cancellationToken).ConfigureAwait(false); + return await ChunkedAsync(db, deep ? JobKind.DeepAnalyse : JobKind.LightAnalyse, ids, chunk => deep + ? ProcessingJobs.DeepAnalysis(chunk, archive.Name) + : ProcessingJobs.LightAnalysis(chunk, archive.Name), cancellationToken).ConfigureAwait(false); + } + case Maintenance: + return await PendingAsync(db, [JobKind.Maintenance], archive, cancellationToken).ConfigureAwait(false) + ?? ScheduleResolution.Queue([ArchiveJobs.Maintenance(archive, p.DryRun == true)]); + case Duplicates: + return await PendingAsync(db, [JobKind.Duplicates], archive, cancellationToken).ConfigureAwait(false) + ?? ScheduleResolution.Queue([ArchiveJobs.Duplicates(archive)]); + default: + return await PendingAsync(db, [JobKind.Orphans], archive, cancellationToken).ConfigureAwait(false) + ?? ScheduleResolution.Queue([ArchiveJobs.Orphans(archive)]); + } + } + + /// Why cannot run here; null when it can. With , the kind it would queue must run. + private static string? Unavailable(ScheduleActionContext context, string action, ScheduleParametersDto? parameters, Archive? archive) + { + if (context.Host.Unavailable(action) is { } reason) return reason; + var kinds = KindsOf(action); + var kind = action == Analysis && parameters?.Deep == true ? JobKind.DeepAnalyse : kinds[0]; + if (!context.CanRun(kind)) return ScheduleUnavailable.NoHandler; + if (action == Backup && DatabaseBackups.Unavailable(context.Database) is { } backup) return backup; + if (action == Proxies && archive is { ProxyLongEdge: ProxyPolicy.Disabled }) return ScheduleUnavailable.ProxiesDisabled; + return null; + } + + /// while a job of on the archive is queued, running or paused. + private static async Task PendingAsync(ChronosDatabase db, IReadOnlyCollection kinds, Archive archive, + CancellationToken cancellationToken) => + await db.Jobs.HasPendingAsync(kinds, [archive.Id], cancellationToken).ConfigureAwait(false) ? ScheduleResolution.Skip(ScheduleOutcome.SkippedPending) : null; + + /// + /// The least recently verified sequences: those not verified for olderThanDays (0: any), at most + /// percent of the archive's sequences (rounded up). All of them at 100 % and 0 days, as one archive-wide job. + /// + private static async Task VerifyAsync(ScheduleActionContext context, Archive archive, ScheduleParametersDto p, CancellationToken cancellationToken) + { + var db = context.Database; + var total = await db.Sequences.CountAsync(new SequenceQuery { ArchiveId = archive.Id }, cancellationToken).ConfigureAwait(false); + if (total == 0) return ScheduleResolution.Skip(ScheduleOutcome.SkippedNothing); + var days = p.OlderThanDays ?? 0; + var percent = p.Percent ?? 100; + if (days == 0 && percent >= 100) return ScheduleResolution.Queue([ArchiveJobs.Verify(archive)]); + var limit = (int)Math.Ceiling(total * percent / 100.0); + var ids = await db.Sequences.ListVerifyCandidatesAsync(archive.Id, days > 0 ? context.Now - TimeSpan.FromDays(days) : null, limit, cancellationToken) + .ConfigureAwait(false); + return ids.Count == 0 ? ScheduleResolution.Skip(ScheduleOutcome.SkippedNothing) : ScheduleResolution.Queue([ArchiveJobs.Verify(archive, ids)]); + } + + /// The sequences of an archive with frames and without (none: every one), oldest capture first. + private static async Task> SequencesAsync(ChronosDatabase db, Guid archiveId, SequenceStatus without, CancellationToken cancellationToken) + { + var ids = new List(); + var query = new SequenceQuery { ArchiveId = archiveId, WithoutStatus = without, Sort = SequenceSort.Oldest, Limit = 1000 }; + do + { + var page = await db.Sequences.ListAsync(query, cancellationToken).ConfigureAwait(false); + ids.AddRange(page.Items.Where(s => s.FrameCount > 0).Select(s => s.Id)); + query = query with { Cursor = page.NextCursor }; + } + while (query.Cursor is not null); + return ids; + } + + /// + /// The jobs for less those a pending job of already covers, at most + /// per job; when none is left. + /// + private static async Task ChunkedAsync(ChronosDatabase db, JobKind kind, IReadOnlyList ids, + Func, ArchiveJobRequest> request, CancellationToken cancellationToken) + { + if (ids.Count == 0) return ScheduleResolution.Skip(ScheduleOutcome.SkippedNothing); + var pending = await db.Jobs.ListPendingTargetsAsync(kind, cancellationToken).ConfigureAwait(false); + var left = ids.Where(id => !pending.Contains(id)).Distinct().ToList(); + if (left.Count == 0) return ScheduleResolution.Skip(pending.Count > 0 ? ScheduleOutcome.SkippedPending : ScheduleOutcome.SkippedNothing); + return ScheduleResolution.Queue([.. left.Chunk(SequencesPerJob).Select(chunk => request(chunk))]); + } +} diff --git a/src/Chronos.Data/Scheduling/ScheduleMapping.cs b/src/Chronos.Data/Scheduling/ScheduleMapping.cs new file mode 100644 index 00000000..3220df67 --- /dev/null +++ b/src/Chronos.Data/Scheduling/ScheduleMapping.cs @@ -0,0 +1,89 @@ +using System.Globalization; +using System.Text.Json; +using Chronos.Core.Model; +using Chronos.Data.Jobs; +using Chronos.Protocol; + +namespace Chronos.Data.Scheduling; + +/// +/// Schedules between the model and API v1 (, ), and the stored form of +/// their parameters ( as camelCase JSON, normalised to the fields the action reads). +/// +public static class ScheduleMapping +{ + /// Wire names of : every, daily, weekly, monthly. + public static string Kind(RecurrenceKind kind) => kind.ToString().ToLowerInvariant(); + + /// Wire names of : minutes, hours, days. + public static string Unit(RecurrenceUnit unit) => unit.ToString().ToLowerInvariant(); + + public static ScheduleDto ToDto(Schedule schedule) + { + ArgumentNullException.ThrowIfNull(schedule); + var recurrence = ScheduleRecurrence.TryParse(schedule.Recurrence, out var parsed) ? ToDto(parsed) : new RecurrenceDto(schedule.Recurrence); + return new ScheduleDto(schedule.Id, schedule.SystemKey, schedule.Name, schedule.Action, schedule.Target, Parameters(schedule), recurrence, + schedule.CatchUp, JobWire.Priority(schedule.Priority), schedule.Enabled, schedule.NextRun, schedule.LastRun, schedule.LastJobId, + schedule.LastOutcome, schedule.CreatedBy, schedule.Created, schedule.Updated); + } + + public static RecurrenceDto ToDto(ScheduleRecurrence recurrence) + { + ArgumentNullException.ThrowIfNull(recurrence); + var time = recurrence.Time.ToString("HH:mm", CultureInfo.InvariantCulture); + return recurrence.Kind switch + { + RecurrenceKind.Every => new RecurrenceDto(Kind(recurrence.Kind), Every: recurrence.Every, Unit: Unit(recurrence.Unit)), + RecurrenceKind.Daily => new RecurrenceDto(Kind(recurrence.Kind), Time: time), + RecurrenceKind.Weekly => new RecurrenceDto(Kind(recurrence.Kind), Time: time, Days: [.. recurrence.Days.Select(ScheduleRecurrence.DayName)]), + _ => new RecurrenceDto(Kind(recurrence.Kind), Time: time, DayOfMonth: recurrence.DayOfMonth), + }; + } + + /// The recurrence describes; null when it is missing or invalid (out of range, unknown names). + public static ScheduleRecurrence? FromDto(RecurrenceDto? dto) + { + if (dto is null) return null; + var text = dto.Kind?.ToLowerInvariant() switch + { + "every" when dto.Every is { } every && Enum.TryParse(dto.Unit, ignoreCase: true, out var unit) && Enum.IsDefined(unit) + => string.Create(CultureInfo.InvariantCulture, $"every:{every}{Unit(unit)[0]}"), + "daily" => "daily:" + dto.Time, + "weekly" when dto.Days is { Count: > 0 } days => $"weekly:{string.Join(',', days)}@{dto.Time}", + "monthly" when dto.DayOfMonth is { } day => string.Create(CultureInfo.InvariantCulture, $"monthly:{day}@{dto.Time}"), + _ => null, + }; + return ScheduleRecurrence.TryParse(text, out var recurrence) ? recurrence : null; + } + + public static ScheduleActionDto ToDto(ScheduleActionInfo info) + { + ArgumentNullException.ThrowIfNull(info); + return new ScheduleActionDto(info.Name, info.Available, info.Reason); + } + + /// The schedule's parameters, normalised for its action (stored JSON that cannot be read counts as none). + public static ScheduleParametersDto Parameters(Schedule schedule) + { + ArgumentNullException.ThrowIfNull(schedule); + ScheduleParametersDto? stored = null; + if (schedule.ParametersJson is { Length: > 0 } json) + { + try + { + stored = JsonSerializer.Deserialize(json, ProtocolJson.Default.ScheduleParametersDto); + } + catch (JsonException) + { + } + } + return ScheduleActions.Normalize(schedule.Action, stored); + } + + /// The stored form of for ; null for an action without parameters. + public static string? ParametersJson(string action, ScheduleParametersDto? parameters) + { + var normalized = ScheduleActions.Normalize(action, parameters); + return normalized == new ScheduleParametersDto() ? null : JsonSerializer.Serialize(normalized, ProtocolJson.Default.ScheduleParametersDto); + } +} diff --git a/src/Chronos.Data/Scheduling/ScheduleOutcome.cs b/src/Chronos.Data/Scheduling/ScheduleOutcome.cs new file mode 100644 index 00000000..11ae0da0 --- /dev/null +++ b/src/Chronos.Data/Scheduling/ScheduleOutcome.cs @@ -0,0 +1,67 @@ +namespace Chronos.Data.Scheduling; + +/// +/// What a schedule's run did (), as stored and sent over the API. Clients +/// show them in their own words; carries the reason as text. +/// +public static class ScheduleOutcome +{ + /// The action's jobs were queued (the schedule keeps the first one's id). + public const string Queued = "queued"; + + /// Not queued: the schedule's previous job, or a job doing the same work, is still queued, running or paused. + public const string SkippedPending = "skipped-pending"; + + /// Not queued: the archive the action works on is missing, offline, or read-only for an action that writes. + public const string SkippedOffline = "skipped-offline"; + + /// Not queued: nothing to do (every sequence verified recently, every thumbnail built…). + public const string SkippedNothing = "skipped-nothing"; + + /// Not queued: this host cannot run the action ( says why). + public const string Unavailable = "unavailable"; + + /// Prefix of . + public const string FailedPrefix = "failed:"; + + /// Longest reason kept after . + public const int MaxReasonLength = 200; + + /// failed:<reason>: resolving or queueing the action threw; the reason is the error's message, shortened. + public static string Failed(string reason) + { + var text = string.IsNullOrWhiteSpace(reason) ? "error" : reason.ReplaceLineEndings(" ").Trim(); + return FailedPrefix + (text.Length > MaxReasonLength ? text[..MaxReasonLength] : text); + } + + /// True for a failed: outcome, with its reason. + public static bool IsFailed(string? outcome, out string reason) + { + var failed = outcome?.StartsWith(FailedPrefix, StringComparison.Ordinal) == true; + reason = failed ? outcome![FailedPrefix.Length..] : string.Empty; + return failed; + } + + /// The outcomes without a reason, for clients that name each one. + public static IReadOnlyList Plain { get; } = [Queued, SkippedPending, SkippedOffline, SkippedNothing, Unavailable]; +} + +/// +/// Why an action cannot run on a host (, ScheduleActionDto.reason). A host's +/// own hook () may answer with codes of its own; clients show unknown codes as +/// "not available here". +/// +public static class ScheduleUnavailable +{ + /// This host's job engine has no handler for the action's job kind. + public const string NoHandler = "no-handler"; + + /// Index backups: the index is on PostgreSQL, which its own tools back up. + public const string Postgres = "postgres"; + + /// Index backups: the SQLite index has no file (in memory). + public const string NoIndexFile = "no-index-file"; + + /// Proxies: the archive's proxy policy makes none. + public const string ProxiesDisabled = "proxies-disabled"; +} diff --git a/src/Chronos.Data/Scheduling/ScheduleRecurrence.cs b/src/Chronos.Data/Scheduling/ScheduleRecurrence.cs new file mode 100644 index 00000000..71b719d6 --- /dev/null +++ b/src/Chronos.Data/Scheduling/ScheduleRecurrence.cs @@ -0,0 +1,322 @@ +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Text; + +namespace Chronos.Data.Scheduling; + +/// The shape of a . +public enum RecurrenceKind +{ + /// A fixed interval: every n minutes, hours or days. + Every, + + /// Every day at a wall-clock time. + Daily, + + /// On some days of the week at a wall-clock time. + Weekly, + + /// On one day of the month (1–28) at a wall-clock time. + Monthly, +} + +/// Unit of an interval. +public enum RecurrenceUnit +{ + Minutes, + Hours, + Days, +} + +/// +/// When a schedule fires (owner review 2, R10), in an in-house text form instead of cron: every:<n>m|h|d +/// (at least 5 minutes), daily:HH:MM, weekly:mon,tue,…@HH:MM and monthly:<1–28>@HH:MM. +/// Intervals are fixed durations anchored to the previous due time, so they never drift and the occurrences missed meanwhile +/// collapse into one; days of an interval are 24 hours, whatever the clocks do. Daily, weekly and monthly times are wall-clock +/// times of the host's zone: a time a spring-forward gap skips moves forward by the gap (02:30 becomes 03:30), and a time the +/// clocks go back over happens once, at its first occurrence. Instances compare by value. +/// +public sealed record ScheduleRecurrence +{ + /// The shortest interval: a schedule never fires more than every 5 minutes. + public const int MinMinutes = 5; + + public const int MaxMinutes = 24 * 60; + public const int MaxHours = 7 * 24; + public const int MaxDays = 366; + + /// The last day a monthly schedule can name: every month has it. + public const int MaxDayOfMonth = 28; + + /// Wire names of the days, Monday first, as the text form and the API write them. + public static IReadOnlyList DayNames { get; } = ["mon", "tue", "wed", "thu", "fri", "sat", "sun"]; + + private static readonly DayOfWeek[] WeekOrder = + [DayOfWeek.Monday, DayOfWeek.Tuesday, DayOfWeek.Wednesday, DayOfWeek.Thursday, DayOfWeek.Friday, DayOfWeek.Saturday, DayOfWeek.Sunday]; + + /// Bit 1 << (int)DayOfWeek per weekly day; a field, so equality compares the days. + private readonly int _days; + + private ScheduleRecurrence(RecurrenceKind kind, int every, RecurrenceUnit unit, TimeOnly time, int days, int dayOfMonth) + { + Kind = kind; + Every = every; + Unit = unit; + Time = time; + _days = days; + DayOfMonth = dayOfMonth; + } + + public RecurrenceKind Kind { get; } + + /// The interval's count of ; 0 for the other kinds. + public int Every { get; } + + /// The interval's unit ( only). + public RecurrenceUnit Unit { get; } + + /// Wall-clock time of a daily, weekly or monthly schedule, to the minute; midnight for an interval. + public TimeOnly Time { get; } + + /// The days of a weekly schedule, Monday first; empty for the other kinds. + public IReadOnlyList Days => [.. WeekOrder.Where(d => (_days & Bit(d)) != 0)]; + + /// Day of the month of a monthly schedule (1–28); 0 for the other kinds. + public int DayOfMonth { get; } + + /// + /// The longest time between two occurrences: the interval, a day, a week, or 31 days for a monthly schedule. A next run + /// further away than two of them means the clock went back, and the next run is computed again. + /// + public TimeSpan Period => Kind switch + { + RecurrenceKind.Every => Unit switch + { + RecurrenceUnit.Minutes => TimeSpan.FromMinutes(Every), + RecurrenceUnit.Hours => TimeSpan.FromHours(Every), + _ => TimeSpan.FromDays(Every), + }, + RecurrenceKind.Daily => TimeSpan.FromDays(1), + RecurrenceKind.Weekly => TimeSpan.FromDays(7), + _ => TimeSpan.FromDays(31), + }; + + /// Every (at least minutes). + /// The interval is shorter than 5 minutes or longer than its unit allows. + public static ScheduleRecurrence Interval(int every, RecurrenceUnit unit) + { + var (min, max) = unit switch + { + RecurrenceUnit.Minutes => (MinMinutes, MaxMinutes), + RecurrenceUnit.Hours => (1, MaxHours), + RecurrenceUnit.Days => (1, MaxDays), + _ => throw new ArgumentOutOfRangeException(nameof(unit), unit, "Unknown unit."), + }; + ArgumentOutOfRangeException.ThrowIfLessThan(every, min); + ArgumentOutOfRangeException.ThrowIfGreaterThan(every, max); + return new ScheduleRecurrence(RecurrenceKind.Every, every, unit, default, 0, 0); + } + + /// Every day at (seconds are dropped). + public static ScheduleRecurrence DailyAt(TimeOnly time) => new(RecurrenceKind.Daily, 0, default, Minute(time), 0, 0); + + /// On at . + /// No day given. + public static ScheduleRecurrence WeeklyAt(IEnumerable days, TimeOnly time) + { + ArgumentNullException.ThrowIfNull(days); + var mask = 0; + foreach (var day in days) + { + if (!Enum.IsDefined(day)) throw new ArgumentOutOfRangeException(nameof(days), day, "Unknown day."); + mask |= Bit(day); + } + if (mask == 0) throw new ArgumentException("A weekly schedule needs at least one day.", nameof(days)); + return new ScheduleRecurrence(RecurrenceKind.Weekly, 0, default, Minute(time), mask, 0); + } + + /// On day (1–28) of every month at . + public static ScheduleRecurrence MonthlyAt(int dayOfMonth, TimeOnly time) + { + ArgumentOutOfRangeException.ThrowIfLessThan(dayOfMonth, 1); + ArgumentOutOfRangeException.ThrowIfGreaterThan(dayOfMonth, MaxDayOfMonth); + return new ScheduleRecurrence(RecurrenceKind.Monthly, 0, default, Minute(time), 0, dayOfMonth); + } + + /// The recurrence written as (any case, spaces around parts allowed). + /// The text is not a valid recurrence. + public static ScheduleRecurrence Parse(string text) => + TryParse(text, out var recurrence) ? recurrence : throw new FormatException($"'{text}' is not a schedule recurrence."); + + /// Parses the text form; false for anything else, values out of range included. + public static bool TryParse([NotNullWhen(true)] string? text, [NotNullWhen(true)] out ScheduleRecurrence? recurrence) + { + recurrence = null; + if (string.IsNullOrWhiteSpace(text)) return false; + var colon = text.IndexOf(':', StringComparison.Ordinal); + if (colon <= 0) return false; + var kind = text[..colon].Trim().ToLowerInvariant(); + var rest = text[(colon + 1)..].Trim(); + try + { + recurrence = kind switch + { + "every" => ParseInterval(rest), + "daily" => TryTime(rest, out var time) ? DailyAt(time) : null, + "weekly" => ParseWeekly(rest), + "monthly" => ParseMonthly(rest), + _ => null, + }; + } + catch (ArgumentException) + { + // A value out of range (every:4m, monthly:29@…). + recurrence = null; + } + return recurrence is not null; + } + + /// The canonical text form: every:10m, daily:03:00, weekly:mon,thu@02:30, monthly:1@04:00. + public string Format() + { + var time = Time.ToString("HH:mm", CultureInfo.InvariantCulture); + return Kind switch + { + RecurrenceKind.Every => string.Create(CultureInfo.InvariantCulture, + $"every:{Every}{Unit switch { RecurrenceUnit.Minutes => 'm', RecurrenceUnit.Hours => 'h', _ => 'd' }}"), + RecurrenceKind.Daily => "daily:" + time, + RecurrenceKind.Weekly => new StringBuilder("weekly:").AppendJoin(',', Days.Select(DayName)).Append('@').Append(time).ToString(), + _ => string.Create(CultureInfo.InvariantCulture, $"monthly:{DayOfMonth}@{time}"), + }; + } + + public override string ToString() => Format(); + + /// The wire name of : mon … sun. + public static string DayName(DayOfWeek day) => DayNames[Array.IndexOf(WeekOrder, day)]; + + /// The day named (, any case); false for anything else. + public static bool TryParseDay(string? name, out DayOfWeek day) + { + var index = IndexOfDay(name); + day = index < 0 ? default : WeekOrder[index]; + return index >= 0; + } + + /// + /// The first occurrence strictly after , in UTC and to the minute. An interval is laid on the + /// grid of (the previous due time: no drift, missed occurrences collapse into the next one), + /// or starts one interval after 's minute without an anchor. Daily, weekly and monthly times + /// are read in (see the type's remarks for clock changes); they need no anchor. + /// + public DateTimeOffset Next(DateTimeOffset afterUtc, TimeZoneInfo zone, DateTimeOffset? anchorUtc = null) + { + ArgumentNullException.ThrowIfNull(zone); + var after = afterUtc.ToUniversalTime(); + if (Kind == RecurrenceKind.Every) + { + var period = Period.Ticks; + if (anchorUtc is not { } anchor) return Truncate(after) + Period; + var behind = after.UtcTicks - anchor.UtcTicks; + var steps = behind / period; + if (behind % period != 0 && behind < 0) steps--; + return new DateTimeOffset(anchor.UtcTicks + (steps + 1) * period, TimeSpan.Zero); + } + var local = DateOnly.FromDateTime(TimeZoneInfo.ConvertTime(after, zone).DateTime); + // From the day before: a time a gap moves forward can land after a later midnight. + for (var i = -1; i <= 400; i++) + { + var day = local.AddDays(i); + if (!Matches(day)) continue; + var at = ToUtc(day.ToDateTime(Time), zone); + if (at > after) return at; + } + throw new InvalidOperationException($"No occurrence of {Format()} within a year."); + } + + /// + /// The UTC instant of the wall-clock time in : a time inside a + /// spring-forward gap moves forward by the gap, an ambiguous one takes its first occurrence (still on the larger offset). + /// + public static DateTimeOffset ToUtc(DateTime local, TimeZoneInfo zone) + { + ArgumentNullException.ThrowIfNull(zone); + local = DateTime.SpecifyKind(local, DateTimeKind.Unspecified); + TimeSpan offset; + if (zone.IsInvalidTime(local)) offset = zone.GetUtcOffset(local.AddHours(-12)); + else if (zone.IsAmbiguousTime(local)) offset = zone.GetAmbiguousTimeOffsets(local).Max(); + else offset = zone.GetUtcOffset(local); + return new DateTimeOffset(local.Ticks - offset.Ticks, TimeSpan.Zero); + } + + /// 10m, 6h, 7d; null for anything else. + private static ScheduleRecurrence? ParseInterval(string text) + { + if (text.Length < 2) return null; + RecurrenceUnit? unit = char.ToLowerInvariant(text[^1]) switch + { + 'm' => RecurrenceUnit.Minutes, + 'h' => RecurrenceUnit.Hours, + 'd' => RecurrenceUnit.Days, + _ => null, + }; + return unit is { } u && TryNumber(text[..^1], out var every) ? Interval(every, u) : null; + } + + /// mon,thu@02:30; null for anything else. + private static ScheduleRecurrence? ParseWeekly(string text) + { + var at = text.LastIndexOf('@'); + if (at <= 0 || !TryTime(text[(at + 1)..], out var time)) return null; + var days = new List(); + foreach (var name in text[..at].Split(',')) + { + if (!TryParseDay(name.Trim(), out var day)) return null; + days.Add(day); + } + return WeeklyAt(days, time); + } + + /// 1@04:00; null for anything else. + private static ScheduleRecurrence? ParseMonthly(string text) + { + var at = text.LastIndexOf('@'); + return at > 0 && TryNumber(text[..at], out var day) && TryTime(text[(at + 1)..], out var time) ? MonthlyAt(day, time) : null; + } + + private bool Matches(DateOnly day) => Kind switch + { + RecurrenceKind.Daily => true, + RecurrenceKind.Weekly => (_days & Bit(day.DayOfWeek)) != 0, + _ => day.Day == DayOfMonth, + }; + + private static int Bit(DayOfWeek day) => 1 << (int)day; + + private static TimeOnly Minute(TimeOnly time) => new(time.Hour, time.Minute); + + private static DateTimeOffset Truncate(DateTimeOffset utc) => new(utc.UtcTicks - utc.UtcTicks % TimeSpan.TicksPerMinute, TimeSpan.Zero); + + private static int IndexOfDay(string? name) + { + for (var i = 0; i < DayNames.Count; i++) + { + if (string.Equals(DayNames[i], name, StringComparison.OrdinalIgnoreCase)) return i; + } + return -1; + } + + private static bool TryNumber(string text, out int value) => + int.TryParse(text.Trim(), NumberStyles.None, CultureInfo.InvariantCulture, out value); + + private static bool TryTime(string text, out TimeOnly time) + { + time = default; + var parts = text.Trim().Split(':'); + if (parts is not [var h, var m] || h.Length is < 1 or > 2 || m.Length != 2 + || !TryNumber(h, out var hour) || !TryNumber(m, out var minute) || hour > 23 || minute > 59) + return false; + time = new TimeOnly(hour, minute); + return true; + } +} diff --git a/src/Chronos.Data/Scheduling/Scheduler.cs b/src/Chronos.Data/Scheduling/Scheduler.cs new file mode 100644 index 00000000..f82c0c7a --- /dev/null +++ b/src/Chronos.Data/Scheduling/Scheduler.cs @@ -0,0 +1,429 @@ +using System.Threading.Channels; +using Chronos.Core.Diagnostics; +using Chronos.Core.Model; +using Chronos.Data.Jobs; +using Chronos.Protocol; + +namespace Chronos.Data.Scheduling; + +/// How a keeps time. +public sealed record SchedulerOptions +{ + /// Clock of the runs (tests pass their own); its timers pace the loop. + public TimeProvider Time { get; init; } = TimeProvider.System; + + /// How often the loop looks for due schedules (and for schedules other processes changed). + public TimeSpan Tick { get; init; } = TimeSpan.FromSeconds(30); + + /// + /// How late a run may be and still count as on time. A schedule further behind (the host was off, the PC slept) missed + /// its run: with catch-up it fires once, without it moves to its next occurrence without running. + /// + public TimeSpan CatchUpGrace { get; init; } = TimeSpan.FromMinutes(5); +} + +/// What a schedule's run did: the outcome () and the first job it queued. +internal sealed record ScheduleFire(Guid ScheduleId, string Outcome, Guid? JobId); + +/// A schedule created or edited, or the first invalid field by its API name (name, recurrence, parameters.steps…). +public sealed record ScheduleEditResult(ScheduleDto? Schedule, string? Problem) +{ + /// True when the schedule to edit does not exist. + public bool NotFound => Schedule is null && Problem is null; +} + +/// What did. +public enum ScheduleDeleteOutcome +{ + Deleted, + NotFound, + + /// A system schedule: disable it instead. + System, +} + +/// +/// Runs the schedules of one host (owner review 2, R10), in the style of the it queues jobs on. +/// seeds the host's system schedules (edits kept), gives enabled schedules without a next run one, +/// moves past the runs that schedules without catch-up missed, fires those with catch-up once, then looks every +/// (or at ). A due run is claimed by moving its next run with a +/// compare-and-set, so several processes on one database fire it once; the claimed run then queues the action's jobs (at the +/// schedule's priority, with its id) unless the schedule's previous job is still pending, and records the outcome. A crash +/// between the claim and the queueing loses that one run (it is logged as claimed first). A next run more than two periods +/// ahead means the clock went back: it is computed again. Edits go through this class so the next run follows them. +/// +public sealed class Scheduler : IAsyncDisposable +{ + /// Longest schedule name. + public const int MaxNameLength = 100; + + /// The only target so far: the host's archive. + public const string CurrentTarget = "current"; + + private readonly ChronosDatabase _db; + private readonly JobEngine _engine; + private readonly Channel _wake = Channel.CreateBounded(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.DropWrite }); + private readonly CancellationTokenSource _shutdown = new(); + private readonly Lock _gate = new(); + private IScheduleHost? _host; + private Task _loop = Task.CompletedTask; + + public Scheduler(ChronosDatabase db, JobEngine engine, SchedulerOptions? options = null) + { + _db = db ?? throw new ArgumentNullException(nameof(db)); + _engine = engine ?? throw new ArgumentNullException(nameof(engine)); + Options = options ?? new SchedulerOptions(); + } + + public SchedulerOptions Options { get; } + + /// The host given to ; null before. + public IScheduleHost? Host => Volatile.Read(ref _host); + + /// A schedule was created, edited, reset or ran (also by the loop). Raised on the caller's or the loop's thread. + public event Action? ScheduleChanged; + + /// A schedule was deleted. + public event Action? ScheduleDeleted; + + /// + /// Seeds 's system schedules, settles the runs missed while the host was off (see the type's + /// remarks) and starts the loop. The first pass runs before this returns. + /// + public async Task StartAsync(IScheduleHost host, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(host); + if (Host is not null) throw new InvalidOperationException("The scheduler has started already."); + var added = await _db.Schedules.EnsureSystemAsync(host.SystemSchedules, Now(), cancellationToken).ConfigureAwait(false); + if (added > 0) Log.Info("schedules", $"Added {added} system schedule(s)"); + if (Interlocked.CompareExchange(ref _host, host, null) is not null) throw new InvalidOperationException("The scheduler has started already."); + try + { + await RunDueAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + Log.Warning("schedules", "The schedules could not be run at start; the loop tries again", ex); + } + var shutdown = _shutdown.Token; + lock (_gate) _loop = Task.Run(() => LoopAsync(shutdown), CancellationToken.None); + } + + /// Makes the loop look for due schedules now (another process changed them, the archive came back). + public void Wake() => _wake.Writer.TryWrite(true); + + /// + /// One pass of the loop: corrects next runs (missing, clock gone back, missed without catch-up), then claims and fires + /// every due schedule. Returns how many fired here. + /// + public async Task RunDueAsync(CancellationToken cancellationToken = default) + { + var host = RequireHost(); + var now = Now(); + foreach (var schedule in await _db.Schedules.ListAsync(cancellationToken).ConfigureAwait(false)) + { + if (schedule.Enabled) await CorrectAsync(host, schedule, now, cancellationToken).ConfigureAwait(false); + } + var fired = 0; + foreach (var schedule in await _db.Schedules.ListDueAsync(now, cancellationToken).ConfigureAwait(false)) + { + DateTimeOffset? next = ScheduleRecurrence.TryParse(schedule.Recurrence, out var recurrence) + ? recurrence.Next(now, host.TimeZone, schedule.NextRun) + : null; + // The claim: one process wins the run; the others see the moved next run and leave it. + if (!await _db.Schedules.TryAdvanceAsync(schedule.Id, schedule.NextRun, next, cancellationToken).ConfigureAwait(false)) continue; + if (next is null) + { + Log.Warning("schedules", $"Schedule {schedule.Id} has an invalid recurrence '{schedule.Recurrence}'; it stops until edited"); + await RecordAsync(schedule.Id, null, ScheduleOutcome.Failed("recurrence"), cancellationToken).ConfigureAwait(false); + continue; + } + Log.Debug("schedules", $"{schedule.Name} ({schedule.Action}) due {schedule.NextRun:O} claimed; next {next:O}"); + await FireAsync(host, schedule, schedule.CreatedBy, manual: false, cancellationToken).ConfigureAwait(false); + fired++; + } + return fired; + } + + /// + /// Runs the schedule now, as its next run would (also a disabled one), without moving its next run; null when it does + /// not exist. is stamped on the jobs it queues. + /// + public async Task RunNowAsync(Guid id, Guid? userId, CancellationToken cancellationToken = default) + { + var host = RequireHost(); + if (await _db.Schedules.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } schedule) return null; + var fire = await FireAsync(host, schedule, userId ?? schedule.CreatedBy, manual: true, cancellationToken).ConfigureAwait(false); + var dto = await GetAsync(id, cancellationToken).ConfigureAwait(false) ?? ScheduleMapping.ToDto(schedule); + return new ScheduleRunDto(dto, fire.Outcome, fire.JobId); + } + + /// The schedules (system ones first), the actions with their availability here, the host's time zone and clean-up steps. + public async Task ListAsync(CancellationToken cancellationToken = default) + { + var host = RequireHost(); + var items = await _db.Schedules.ListAsync(cancellationToken).ConfigureAwait(false); + var actions = await ScheduleActions.AvailabilityAsync(Context(host), cancellationToken).ConfigureAwait(false); + return new SchedulesDto([.. items.Select(ScheduleMapping.ToDto)], [.. actions.Select(ScheduleMapping.ToDto)], host.TimeZone.Id, host.CleanupSteps); + } + + public async Task GetAsync(Guid id, CancellationToken cancellationToken = default) => + await _db.Schedules.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } schedule ? ScheduleMapping.ToDto(schedule) : null; + + /// Creates a schedule (enabled ones get their next run at once); the result names the first invalid field instead when there is one. + public async Task CreateAsync(CreateScheduleRequest request, Guid? createdBy, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + var host = RequireHost(); + if (Problem(host, request.Name, request.Action, request.Target, request.Recurrence, request.Parameters, request.Priority) is { } problem) + return new ScheduleEditResult(null, problem); + var recurrence = ScheduleMapping.FromDto(request.Recurrence)!; + var now = Now(); + var schedule = new Schedule + { + Id = Guid.CreateVersion7(), + Name = request.Name.Trim(), + Action = request.Action, + Target = CurrentTarget, + ParametersJson = ScheduleMapping.ParametersJson(request.Action, request.Parameters), + Recurrence = recurrence.Format(), + CatchUp = request.CatchUp, + Priority = Priority(request.Priority, JobPriority.Background), + Enabled = request.Enabled, + NextRun = request.Enabled ? recurrence.Next(now, host.TimeZone) : null, + CreatedBy = createdBy, + Created = now, + Updated = now, + }; + await _db.Schedules.InsertAsync(schedule, cancellationToken).ConfigureAwait(false); + Log.Info("schedules", $"Schedule {schedule.Name} ({schedule.Action}, {schedule.Recurrence}) created"); + return new ScheduleEditResult(Changed(schedule), null); + } + + /// + /// Changes the fields gives. The next run is computed again when the recurrence changes or + /// the schedule is enabled; a disabled schedule has none. A system schedule keeps its name and action (asking to change + /// them is a problem of field name or action). + /// + public async Task UpdateAsync(Guid id, UpdateScheduleRequest request, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + var host = RequireHost(); + if (await _db.Schedules.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } current) return new ScheduleEditResult(null, null); + if (current.IsSystem && request.Name is { } name && name.Trim() != current.Name) return new ScheduleEditResult(null, "name"); + if (current.IsSystem && request.Action is { } changed && changed != current.Action) return new ScheduleEditResult(null, "action"); + var action = request.Action ?? current.Action; + // Another action starts from its own defaults; the same one keeps the stored values the request leaves out. + var parameters = request.Parameters ?? (action == current.Action ? ScheduleMapping.Parameters(current) : null); + var recurrenceDto = request.Recurrence ?? (ScheduleRecurrence.TryParse(current.Recurrence, out var stored) ? ScheduleMapping.ToDto(stored) : null); + if (Problem(host, request.Name ?? current.Name, action, null, recurrenceDto, parameters, request.Priority) is { } problem) + return new ScheduleEditResult(null, problem); + var recurrence = ScheduleMapping.FromDto(recurrenceDto)!; + var now = Now(); + var enabled = request.Enabled ?? current.Enabled; + var reschedule = !enabled || !current.Enabled || current.NextRun is null || recurrence.Format() != current.Recurrence; + var schedule = current with + { + Name = (request.Name ?? current.Name).Trim(), + Action = action, + ParametersJson = ScheduleMapping.ParametersJson(action, parameters), + Recurrence = recurrence.Format(), + CatchUp = request.CatchUp ?? current.CatchUp, + Priority = Priority(request.Priority, current.Priority), + Enabled = enabled, + NextRun = !enabled ? null : reschedule ? recurrence.Next(now, host.TimeZone) : current.NextRun, + Updated = now, + }; + if (!await _db.Schedules.UpdateAsync(schedule, reschedule, cancellationToken).ConfigureAwait(false)) return new ScheduleEditResult(null, null); + Log.Info("schedules", $"Schedule {schedule.Name} ({schedule.Action}, {schedule.Recurrence}, {(enabled ? "enabled" : "disabled")}) changed"); + var saved = await _db.Schedules.GetAsync(id, cancellationToken).ConfigureAwait(false) ?? schedule; + return new ScheduleEditResult(Changed(saved), null); + } + + /// Deletes a schedule a user created; a system schedule is refused (). + public async Task DeleteAsync(Guid id, CancellationToken cancellationToken = default) + { + if (await _db.Schedules.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } schedule) return ScheduleDeleteOutcome.NotFound; + if (schedule.IsSystem) return ScheduleDeleteOutcome.System; + if (!await _db.Schedules.DeleteAsync(id, cancellationToken).ConfigureAwait(false)) return ScheduleDeleteOutcome.NotFound; + Log.Info("schedules", $"Schedule {schedule.Name} ({schedule.Action}) deleted"); + ScheduleDeleted?.Invoke(id); + return ScheduleDeleteOutcome.Deleted; + } + + /// + /// Puts a system schedule back to the host's defaults () with a new next + /// run. A schedule a user created, or one the host no longer defines, is a problem of field systemKey. + /// + public async Task ResetAsync(Guid id, CancellationToken cancellationToken = default) + { + var host = RequireHost(); + if (await _db.Schedules.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } current) return new ScheduleEditResult(null, null); + if (current.SystemKey is not { } key || host.SystemSchedules.FirstOrDefault(s => s.SystemKey == key) is not { } defaults) + return new ScheduleEditResult(null, "systemKey"); + var now = Now(); + await _db.Schedules.ResetSystemAsync(key, defaults, now, cancellationToken).ConfigureAwait(false); + if (defaults.Enabled) await _db.Schedules.TryAdvanceAsync(id, null, defaults.Recurrence.Next(now, host.TimeZone), cancellationToken).ConfigureAwait(false); + Log.Info("schedules", $"System schedule {key} reset"); + var saved = await _db.Schedules.GetAsync(id, cancellationToken).ConfigureAwait(false); + return saved is null ? new ScheduleEditResult(null, null) : new ScheduleEditResult(Changed(saved), null); + } + + public async ValueTask DisposeAsync() + { + Task loop; + lock (_gate) loop = _loop; + await _shutdown.CancelAsync().ConfigureAwait(false); + try + { + await loop.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + } + _shutdown.Dispose(); + } + + /// + /// Fills in a missing next run, computes it again after the clock went back, and moves a schedule without catch-up past + /// a run it missed. Compare-and-set like a claim, so concurrent passes agree. + /// + private async Task CorrectAsync(IScheduleHost host, Schedule schedule, DateTimeOffset now, CancellationToken cancellationToken) + { + if (!ScheduleRecurrence.TryParse(schedule.Recurrence, out var recurrence)) return; + DateTimeOffset next; + string why; + if (schedule.NextRun is not { } due) + { + next = recurrence.Next(now, host.TimeZone); + why = "first run"; + } + else if (due - now > recurrence.Period * 2) + { + next = recurrence.Next(now, host.TimeZone); + why = "the clock went back"; + } + else if (!schedule.CatchUp && now - due > Options.CatchUpGrace) + { + next = recurrence.Next(now, host.TimeZone, due); + why = $"missed the run of {due:O} (no catch-up)"; + } + else + { + return; + } + if (!await _db.Schedules.TryAdvanceAsync(schedule.Id, schedule.NextRun, next, cancellationToken).ConfigureAwait(false)) return; + Log.Info("schedules", $"{schedule.Name} ({schedule.Action}): next run {next:O} ({why})"); + Changed(schedule with { NextRun = next }); + } + + /// Queues the schedule's action unless its previous job is pending, records the outcome and logs one line. + private async Task FireAsync(IScheduleHost host, Schedule schedule, Guid? createdBy, bool manual, CancellationToken cancellationToken) + { + string outcome; + Guid? jobId = null; + try + { + if (schedule.LastJobId is { } last + && await _db.Jobs.GetAsync(last, cancellationToken).ConfigureAwait(false) is { State: JobState.Queued or JobState.Running or JobState.Paused }) + { + outcome = ScheduleOutcome.SkippedPending; + } + else + { + var resolution = await ScheduleActions.ResolveAsync(Context(host), schedule.Action, ScheduleMapping.Parameters(schedule), cancellationToken) + .ConfigureAwait(false); + outcome = resolution.Outcome; + foreach (var request in resolution.Requests) + { + var job = await _engine.EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, schedule.Priority, createdBy, + schedule.Id, cancellationToken).ConfigureAwait(false); + jobId ??= job.Id; + } + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + Log.Warning("schedules", $"{schedule.Name} ({schedule.Action}) could not run", ex); + outcome = ScheduleOutcome.Failed(ex.Message); + } + await RecordAsync(schedule.Id, jobId, outcome, CancellationToken.None).ConfigureAwait(false); + Log.Info("schedules", $"{schedule.Name} ({schedule.Action}{(manual ? ", run now" : string.Empty)}): {outcome}{(jobId is { } id ? $", job {id}" : string.Empty)}"); + return new ScheduleFire(schedule.Id, outcome, jobId); + } + + private async Task RecordAsync(Guid id, Guid? jobId, string outcome, CancellationToken cancellationToken) + { + await _db.Schedules.RecordOutcomeAsync(id, jobId, outcome, Now(), cancellationToken).ConfigureAwait(false); + if (await _db.Schedules.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } saved) Changed(saved); + } + + private async Task LoopAsync(CancellationToken shutdown) + { + while (!shutdown.IsCancellationRequested) + { + await WaitAsync(shutdown).ConfigureAwait(false); + if (shutdown.IsCancellationRequested) return; + try + { + await RunDueAsync(shutdown).ConfigureAwait(false); + } + catch (OperationCanceledException) when (shutdown.IsCancellationRequested) + { + return; + } + catch (Exception ex) + { + // Nothing a pass throws ends the loop (a database briefly away, a host hook that failed). + Log.Warning("schedules", "The schedules could not be run; the next tick tries again", ex); + } + } + } + + /// Waits for , the tick or shutdown, whichever comes first. + private async Task WaitAsync(CancellationToken shutdown) + { + using var tick = new CancellationTokenSource(Options.Tick, Options.Time); + using var either = CancellationTokenSource.CreateLinkedTokenSource(shutdown, tick.Token); + try + { + await _wake.Reader.ReadAsync(either.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // Tick elapsed or shutdown. + } + } + + private ScheduleDto Changed(Schedule schedule) + { + var dto = ScheduleMapping.ToDto(schedule); + ScheduleChanged?.Invoke(dto); + return dto; + } + + private ScheduleActionContext Context(IScheduleHost host) => new(_db, host, _engine.CanRun, Now()); + + private DateTimeOffset Now() => Options.Time.GetUtcNow(); + + private IScheduleHost RequireHost() => Host ?? throw new InvalidOperationException("Start the scheduler with its host first."); + + private static JobPriority Priority(string? wire, JobPriority fallback) => wire is not null && JobWire.TryParsePriority(wire, out var priority) ? priority : fallback; + + /// The first invalid field of a schedule as given, by its API name; null when it is valid. + private static string? Problem(IScheduleHost host, string? name, string? action, string? target, RecurrenceDto? recurrence, + ScheduleParametersDto? parameters, string? priority) + { + if (string.IsNullOrWhiteSpace(name) || name.Trim().Length > MaxNameLength) return "name"; + if (action is null) return "action"; + if (ScheduleActions.Problem(action, parameters, host.CleanupSteps) is { } invalid) return invalid; + if (target is not null && target != CurrentTarget) return "target"; + if (ScheduleMapping.FromDto(recurrence) is null) return "recurrence"; + if (priority is not null && !JobWire.TryParsePriority(priority, out _)) return "priority"; + return null; + } +} diff --git a/src/Chronos.Protocol/Jobs.cs b/src/Chronos.Protocol/Jobs.cs index 0713f193..a157f990 100644 --- a/src/Chronos.Protocol/Jobs.cs +++ b/src/Chronos.Protocol/Jobs.cs @@ -1,5 +1,6 @@ namespace Chronos.Protocol; +/// A job of the queue. : the schedule that queued it (owner review 2, R10), absent otherwise. public sealed record JobDto( Guid Id, string Kind, @@ -17,7 +18,8 @@ public sealed record JobDto( string? Error, DateTimeOffset Created, DateTimeOffset? Started, - DateTimeOffset? Finished); + DateTimeOffset? Finished, + Guid? ScheduleId = null); /// POST /api/v1/jobs: kind = render, deep-analyse, proxies, thumbs, verify, scan, rebuild, maintenance… public sealed record CreateJobRequest(string Kind, IReadOnlyList Target, string? ParametersJson); diff --git a/src/Chronos.Protocol/ProtocolJson.cs b/src/Chronos.Protocol/ProtocolJson.cs index 5fb3b6c4..8a927282 100644 --- a/src/Chronos.Protocol/ProtocolJson.cs +++ b/src/Chronos.Protocol/ProtocolJson.cs @@ -64,4 +64,13 @@ namespace Chronos.Protocol; [JsonSerializable(typeof(List))] [JsonSerializable(typeof(LogTailDto))] [JsonSerializable(typeof(DashboardDto))] +[JsonSerializable(typeof(ScheduleParametersDto))] +[JsonSerializable(typeof(RecurrenceDto))] +[JsonSerializable(typeof(ScheduleDto))] +[JsonSerializable(typeof(List))] +[JsonSerializable(typeof(ScheduleActionDto))] +[JsonSerializable(typeof(SchedulesDto))] +[JsonSerializable(typeof(CreateScheduleRequest))] +[JsonSerializable(typeof(UpdateScheduleRequest))] +[JsonSerializable(typeof(ScheduleRunDto))] public sealed partial class ProtocolJson : JsonSerializerContext; diff --git a/src/Chronos.Protocol/Schedules.cs b/src/Chronos.Protocol/Schedules.cs new file mode 100644 index 00000000..b9efc559 --- /dev/null +++ b/src/Chronos.Protocol/Schedules.cs @@ -0,0 +1,97 @@ +namespace Chronos.Protocol; + +/// +/// The parameters of a schedule's action (owner review 2, R10), flat: each action reads its own fields, the others stay +/// null (and are not stored). scan: (full scan), (background deep analysis of +/// what it indexes). verify: (only sequences not verified for that many days; 0: any), +/// (share of the archive's sequences per run, least recently verified first). thumbnails, +/// proxies: (make again those already made). analysis: (deep instead of +/// light). maintenance: . backup: (copies kept). cleanup: +/// (step names) and (what is older goes; 0 keeps everything). +/// duplicates, orphans: none. +/// +public sealed record ScheduleParametersDto( + int? OlderThanDays = null, + int? Percent = null, + bool? Deep = null, + bool? DryRun = null, + bool? Force = null, + IReadOnlyList? Steps = null, + int? Keep = null); + +/// +/// When a schedule fires, structured. = every ( and : +/// minutes, hours or days; at least 5 minutes), daily (), weekly +/// (: mon … sun; ) or monthly ( 1–28; +/// ). is HH:MM, wall-clock time of the host's time zone. +/// +public sealed record RecurrenceDto( + string Kind, + int? Every = null, + string? Unit = null, + string? Time = null, + IReadOnlyList? Days = null, + int? DayOfMonth = null); + +/// +/// A schedule (owner review 2, R10). is set on the host's system schedules (the timers it runs, +/// shown so nothing runs out of sight): they can be edited, disabled and reset, never deleted. is the +/// priority of the jobs it queues (background, normal, user). : +/// queued, skipped-pending, skipped-offline, skipped-nothing, unavailable or +/// failed:<reason>. is null while the schedule is disabled. +/// +public sealed record ScheduleDto( + Guid Id, + string? SystemKey, + string Name, + string Action, + string Target, + ScheduleParametersDto Parameters, + RecurrenceDto Recurrence, + bool CatchUp, + string Priority, + bool Enabled, + DateTimeOffset? NextRun, + DateTimeOffset? LastRun, + Guid? LastJobId, + string? LastOutcome, + Guid? CreatedBy, + DateTimeOffset Created, + DateTimeOffset Updated); + +/// An action a schedule can take and whether this host runs it; says why not (a code: no-handler, postgres, proxies-disabled…). +public sealed record ScheduleActionDto(string Name, bool Available, string? Reason); + +/// +/// GET /api/v1/schedules: the schedules, system ones first, the actions with their availability, the host's time +/// zone (IANA or Windows id, as the host names it) and the clean-up steps its cleanup action runs. +/// +public sealed record SchedulesDto(IReadOnlyList Items, IReadOnlyList Actions, string TimeZone, + IReadOnlyList? Steps = null); + +/// POST /api/v1/schedules: a new schedule; defaults to current, to background. +public sealed record CreateScheduleRequest( + string Name, + string Action, + RecurrenceDto Recurrence, + ScheduleParametersDto? Parameters = null, + bool CatchUp = true, + string? Priority = null, + bool Enabled = true, + string? Target = null); + +/// +/// PATCH /api/v1/schedules/{id}: the fields given replace the stored ones ( as a whole). A +/// system schedule keeps its name and action. +/// +public sealed record UpdateScheduleRequest( + string? Name = null, + string? Action = null, + RecurrenceDto? Recurrence = null, + ScheduleParametersDto? Parameters = null, + bool? CatchUp = null, + string? Priority = null, + bool? Enabled = null); + +/// POST /api/v1/schedules/{id}/run: what running the schedule now did, and the schedule afterwards. +public sealed record ScheduleRunDto(ScheduleDto Schedule, string Outcome, Guid? JobId); diff --git a/src/Chronos.Server/Api/Dto.cs b/src/Chronos.Server/Api/Dto.cs index 9a2bae7e..0a5b6720 100644 --- a/src/Chronos.Server/Api/Dto.cs +++ b/src/Chronos.Server/Api/Dto.cs @@ -27,7 +27,8 @@ public static class Dto public static JobDto Job(Job j) => new( j.Id, JobWire.Kind(j.Kind), j.TargetIds, j.TargetLabel, JobWire.State(j.State), JobWire.Priority(j.Priority), j.ProgressDone, - j.ProgressTotal, j.ItemsPerSecond, j.Eta?.TotalSeconds, j.LogTail, j.NodeId, j.Attempts, j.Error, j.Created, j.Started, j.Finished); + j.ProgressTotal, j.ItemsPerSecond, j.Eta?.TotalSeconds, j.LogTail, j.NodeId, j.Attempts, j.Error, j.Created, j.Started, j.Finished, + j.ScheduleId); public static AuditEntryDto Audit(AuditEntry a) => new(a.Id, a.At, a.UserId, a.Username, a.Action, a.TargetKind, a.TargetId, a.Details, a.RemoteAddress); diff --git a/src/Chronos.Server/Configuration/ServerSettingsStore.cs b/src/Chronos.Server/Configuration/ServerSettingsStore.cs index 0d59323a..e28ad982 100644 --- a/src/Chronos.Server/Configuration/ServerSettingsStore.cs +++ b/src/Chronos.Server/Configuration/ServerSettingsStore.cs @@ -22,7 +22,7 @@ public sealed class ServerSettingsStore(ServerOptions options, ServerLogging log public static IReadOnlyList Kinds { get; } = [ JobKind.Render, JobKind.Import, JobKind.Scan, JobKind.Rebuild, JobKind.Verify, JobKind.LightAnalyse, JobKind.DeepAnalyse, JobKind.Thumbs, - JobKind.Proxies, JobKind.Maintenance, JobKind.Duplicates, JobKind.Orphans, JobKind.Adopt, JobKind.Move, + JobKind.Proxies, JobKind.Maintenance, JobKind.Duplicates, JobKind.Orphans, JobKind.Adopt, JobKind.Move, JobKind.Backup, JobKind.Cleanup, ]; private readonly Lock _gate = new(); diff --git a/src/Chronos.Server/Jobs/JobEngineHost.cs b/src/Chronos.Server/Jobs/JobEngineHost.cs index fd0288ea..e77f3fc1 100644 --- a/src/Chronos.Server/Jobs/JobEngineHost.cs +++ b/src/Chronos.Server/Jobs/JobEngineHost.cs @@ -40,7 +40,6 @@ public sealed class JobEngineHost : IHostedService { WorkerId = WorkerId, Kinds = _kinds, - HistoryRetention = TimeSpan.FromDays(options.JobHistoryDays), PriorityQueue = () => ServerSettingsStore.PriorityQueue(options), }); Engine.JobChanged += OnJobChanged; diff --git a/tests/Chronos.App.Tests/Jobs/JobNoticesTests.cs b/tests/Chronos.App.Tests/Jobs/JobNoticesTests.cs index f225af0f..43d83fd8 100644 --- a/tests/Chronos.App.Tests/Jobs/JobNoticesTests.cs +++ b/tests/Chronos.App.Tests/Jobs/JobNoticesTests.cs @@ -30,6 +30,7 @@ public sealed class JobNoticesTests [Case(JobKind.LightAnalyse)] [Case(JobKind.Thumbs)] [Case(JobKind.Proxies)] + [Case(JobKind.Cleanup)] public void UpkeepAndScansEndSilently(JobKind kind) { Assert.Equal(JobNotice.None, JobNotices.For(JobState.Running, Job(kind, JobState.Done))); @@ -41,6 +42,7 @@ public sealed class JobNoticesTests [Case(JobKind.Download)] [Case(JobKind.Duplicates)] [Case(JobKind.Orphans)] + [Case(JobKind.Backup)] public void EveryOtherFailureIsSaid(JobKind kind) { // An import announces its result on the Import screen, a download on its Library card and Server row. diff --git a/tests/Chronos.Data.Tests/Jobs/JobEngineTests.cs b/tests/Chronos.Data.Tests/Jobs/JobEngineTests.cs index 28a5f1e8..ef48fbd7 100644 --- a/tests/Chronos.Data.Tests/Jobs/JobEngineTests.cs +++ b/tests/Chronos.Data.Tests/Jobs/JobEngineTests.cs @@ -446,4 +446,30 @@ public sealed class JobEngineTests(TestContext context) Assert.Equal("encoder lost", failed.Error); Assert.Contains("encoder lost", failed.LogTail); } + + [Test] + public async Task StartingKeepsTheHistoryAndKnowsItsKindsBefore() + { + // The housekeeping schedule's job-history step purges finished jobs now (owner review 2, R10), not the engine's start. + await using var db = await OpenAsync(); + var old = new Job + { + Id = Guid.CreateVersion7(), + Kind = JobKind.Scan, + State = JobState.Done, + Created = DateTimeOffset.UtcNow.AddDays(-400), + Finished = DateTimeOffset.UtcNow.AddDays(-400), + }; + await db.Jobs.EnqueueAsync(old); + await using var engine = new JobEngine(db, [new CountingHandler(JobKind.Scan, 1), new CountingHandler(JobKind.Verify, 1)], + new JobEngineOptions { WorkerId = "t", Kinds = [JobKind.Scan] }); + Assert.True(engine.CanRun(JobKind.Scan)); + Assert.False(engine.CanRun(JobKind.Verify), "filtered out"); + Assert.False(engine.CanRun(JobKind.Render), "no handler"); + await engine.StartAsync(); + Assert.NotNull(await db.Jobs.GetAsync(old.Id)); + + var scheduled = await engine.EnqueueAsync(JobKind.Scan, [], null, null, JobPriority.Background, null, Guid.CreateVersion7()); + Assert.Equal(scheduled.ScheduleId, (await db.Jobs.GetAsync(scheduled.Id))!.ScheduleId); + } } diff --git a/tests/Chronos.Data.Tests/Migrations/MigrationTests.cs b/tests/Chronos.Data.Tests/Migrations/MigrationTests.cs index 9f21bba8..1c576e6b 100644 --- a/tests/Chronos.Data.Tests/Migrations/MigrationTests.cs +++ b/tests/Chronos.Data.Tests/Migrations/MigrationTests.cs @@ -9,7 +9,7 @@ public abstract class MigrationTests(TestContext context) : RepositoryTest(conte private static readonly string[] Tables = [ "archives", "sequences", "frames", "renders", "jobs", "media_cards", "import_history", "users", "tokens", - "render_nodes", "settings", "tags", "sequence_tags", "audit_log", "uploads", + "render_nodes", "settings", "tags", "sequence_tags", "audit_log", "uploads", "schedules", ]; [Test] diff --git a/tests/Chronos.Data.Tests/Repositories/JobRepositoryTests.cs b/tests/Chronos.Data.Tests/Repositories/JobRepositoryTests.cs index 25c11acd..66477324 100644 --- a/tests/Chronos.Data.Tests/Repositories/JobRepositoryTests.cs +++ b/tests/Chronos.Data.Tests/Repositories/JobRepositoryTests.cs @@ -29,11 +29,57 @@ public abstract class JobRepositoryTests(TestContext context) : RepositoryTest(c State = JobState.Failed, Started = TestData.Now.AddMinutes(1), Finished = TestData.Now.AddMinutes(2), + ScheduleId = Guid.CreateVersion7(), }; await db.Jobs.EnqueueAsync(job, Ct); var read = Assert.NotNull(await db.Jobs.GetAsync(job.Id, Ct)); Assert.SequenceEqual(job.TargetIds, read.TargetIds); Assert.Equal(job with { TargetIds = read.TargetIds }, read); + Assert.Null((await db.Jobs.GetAsync((await EnqueueAsync(db, TestData.Job())).Id, Ct))!.ScheduleId); + } + + [Test] + public async Task JobsOfAScheduleAreListedNewestFirst() + { + var db = await OpenChronosAsync(); + var schedule = Guid.CreateVersion7(); + var older = await EnqueueAsync(db, TestData.Job(created: TestData.Now.AddMinutes(-5)) with { ScheduleId = schedule }); + var newer = await EnqueueAsync(db, TestData.Job() with { ScheduleId = schedule }); + await EnqueueAsync(db, TestData.Job()); + Assert.SequenceEqual([newer.Id, older.Id], (await db.Jobs.ListByScheduleAsync(schedule, cancellationToken: Ct)).Select(j => j.Id)); + Assert.Empty(await db.Jobs.ListByScheduleAsync(Guid.CreateVersion7(), cancellationToken: Ct)); + } + + [Test] + public async Task PendingJobsAreFoundByKindAndTarget() + { + // A schedule queues nothing while a job doing the same work waits, runs or is paused (owner review 2, R10). + var db = await OpenChronosAsync(); + Guid archive = Guid.CreateVersion7(), a = Guid.CreateVersion7(), b = Guid.CreateVersion7(), c = Guid.CreateVersion7(); + await EnqueueAsync(db, TestData.Job(JobKind.Scan) with { TargetIds = [archive] }); + await EnqueueAsync(db, TestData.Job(JobKind.Thumbs) with { TargetIds = [a, b], State = JobState.Paused }); + await EnqueueAsync(db, TestData.Job(JobKind.Thumbs) with { TargetIds = [c], State = JobState.Done, Finished = TestData.Now }); + await EnqueueAsync(db, TestData.Job(JobKind.Verify) with { TargetIds = [archive], State = JobState.Failed, Finished = TestData.Now }); + + Assert.True(await db.Jobs.HasPendingAsync([JobKind.Scan, JobKind.Rebuild], [archive], Ct)); + Assert.False(await db.Jobs.HasPendingAsync([JobKind.Rebuild], [archive], Ct)); + Assert.False(await db.Jobs.HasPendingAsync([JobKind.Verify], [archive], Ct), "a failed job is not pending"); + Assert.True(await db.Jobs.HasPendingAsync([JobKind.Thumbs], [b], Ct), "the second of a job's targets"); + Assert.True(await db.Jobs.HasPendingAsync([JobKind.Thumbs], null, Ct)); + Assert.False(await db.Jobs.HasPendingAsync([JobKind.Thumbs], [c], Ct), "a done job is not pending"); + Assert.False(await db.Jobs.HasPendingAsync([JobKind.Scan], [], Ct)); + + var claimed = await db.Jobs.ClaimNextAsync("node", [JobKind.Scan], Ct); + Assert.Equal(JobState.Running, claimed!.State); + Assert.True(await db.Jobs.HasPendingAsync([JobKind.Scan], [archive], Ct), "a running job is pending"); + Assert.SequenceEqual(new[] { a, b }.Order(), (await db.Jobs.ListPendingTargetsAsync(JobKind.Thumbs, Ct)).Order()); + Assert.Empty(await db.Jobs.ListPendingTargetsAsync(JobKind.Verify, Ct)); + } + + private async Task EnqueueAsync(ChronosDatabase db, Job job) + { + await db.Jobs.EnqueueAsync(job, Ct); + return job; } [Test] diff --git a/tests/Chronos.Data.Tests/Repositories/ScheduleRepositoryTests.cs b/tests/Chronos.Data.Tests/Repositories/ScheduleRepositoryTests.cs new file mode 100644 index 00000000..edba626a --- /dev/null +++ b/tests/Chronos.Data.Tests/Repositories/ScheduleRepositoryTests.cs @@ -0,0 +1,192 @@ +using Chronos.Core.Model; +using Chronos.Data.Repositories; +using Chronos.Data.Scheduling; +using Chronos.Data.Tests.Infrastructure; +using Chronos.Protocol; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Repositories; + +/// Schedules on both dialects (owner review 2, R10): round trip, system rows, the compare-and-set claim, due rows. +public abstract class ScheduleRepositoryTests(TestContext context) : RepositoryTest(context) +{ + private static Schedule NewSchedule(string name = "Nightly scan", string? systemKey = null) => new() + { + Id = Guid.CreateVersion7(), + SystemKey = systemKey, + Name = name, + Action = ScheduleActions.Scan, + Recurrence = "daily:03:00", + Created = TestData.Now, + Updated = TestData.Now, + }; + + private static SystemSchedule System(string key, string recurrence = "every:10m", bool enabled = true) => + new(key, "System " + key, ScheduleActions.Verify, ScheduleRecurrence.Parse(recurrence), new ScheduleParametersDto(Percent: 25), enabled, CatchUp: false); + + [Test] + public async Task SchedulesRoundTripEveryField() + { + var db = await OpenChronosAsync(); + var schedule = NewSchedule() with + { + SystemKey = "archive-scan", + Target = "current", + ParametersJson = """{"force": true}""", + CatchUp = false, + Priority = JobPriority.User, + Enabled = false, + NextRun = TestData.Now.AddHours(1), + LastRun = TestData.Now.AddHours(-1), + LastJobId = Guid.CreateVersion7(), + LastOutcome = ScheduleOutcome.SkippedPending, + CreatedBy = Guid.CreateVersion7(), + Updated = TestData.Now.AddMinutes(5), + }; + await db.Schedules.InsertAsync(schedule, Ct); + Assert.Equal(schedule, await db.Schedules.GetAsync(schedule.Id, Ct)); + Assert.Equal(schedule, await db.Schedules.GetBySystemKeyAsync("archive-scan", Ct)); + Assert.Null(await db.Schedules.GetAsync(Guid.CreateVersion7(), Ct)); + + var plain = NewSchedule("Weekly verify"); + await db.Schedules.InsertAsync(plain, Ct); + Assert.Equal(plain, await db.Schedules.GetAsync(plain.Id, Ct)); + Assert.SequenceEqual([schedule.Id, plain.Id], (await db.Schedules.ListAsync(Ct)).Select(s => s.Id)); + } + + [Test] + public async Task UpdatesKeepTheRunHistoryAndSystemRowsAreNeverDeleted() + { + var db = await OpenChronosAsync(); + var user = NewSchedule(); + var system = NewSchedule("Backup", "index-backup"); + await db.Schedules.InsertAsync(user, Ct); + await db.Schedules.InsertAsync(system, Ct); + await db.Schedules.RecordOutcomeAsync(user.Id, Guid.CreateVersion7(), ScheduleOutcome.Queued, TestData.Now, Ct); + var recorded = (await db.Schedules.GetAsync(user.Id, Ct))!; + + var edited = recorded with { Name = "Renamed", Recurrence = "every:6h", Enabled = false, NextRun = null, Updated = TestData.Now.AddDays(1), LastOutcome = "ignored" }; + Assert.True(await db.Schedules.UpdateAsync(edited, cancellationToken: Ct)); + Assert.Equal(edited with { LastOutcome = recorded.LastOutcome }, await db.Schedules.GetAsync(user.Id, Ct)); + // Without the next run: a run claimed meanwhile stays claimed. + Assert.True(await db.Schedules.TryAdvanceAsync(system.Id, null, TestData.Now.AddHours(2), Ct)); + Assert.True(await db.Schedules.UpdateAsync(system with { Name = "Backup 2", NextRun = null }, setNextRun: false, Ct)); + Assert.Equal(TestData.Now.AddHours(2), (await db.Schedules.GetAsync(system.Id, Ct))!.NextRun); + Assert.False(await db.Schedules.UpdateAsync(NewSchedule(), cancellationToken: Ct)); + + Assert.False(await db.Schedules.DeleteAsync(system.Id, Ct)); + Assert.NotNull(await db.Schedules.GetAsync(system.Id, Ct)); + Assert.True(await db.Schedules.DeleteAsync(user.Id, Ct)); + Assert.False(await db.Schedules.DeleteAsync(user.Id, Ct)); + } + + [Test] + public async Task SeedingSystemSchedulesKeepsTheUsersEdits() + { + var db = await OpenChronosAsync(); + Assert.Equal(2, await db.Schedules.EnsureSystemAsync([System("archive-scan"), System("index-backup", enabled: false)], TestData.Now, Ct)); + var seeded = (await db.Schedules.GetBySystemKeyAsync("archive-scan", Ct))!; + Assert.Equal(("System archive-scan", "verify", "every:10m", false, true, JobPriority.Background), (seeded.Name, seeded.Action, seeded.Recurrence, seeded.CatchUp, seeded.Enabled, seeded.Priority)); + // Stored normalised: the verify defaults fill what the host left out. + Assert.Equal(new ScheduleParametersDto(OlderThanDays: 30, Percent: 25), ScheduleMapping.Parameters(seeded)); + Assert.Null(seeded.NextRun); + Assert.False((await db.Schedules.GetBySystemKeyAsync("index-backup", Ct))!.Enabled); + + await db.Schedules.UpdateAsync(seeded with { Recurrence = "every:30m", Enabled = false }, cancellationToken: Ct); + // A later start brings other defaults and a new system schedule: the edited row stays as the user left it. + Assert.Equal(1, await db.Schedules.EnsureSystemAsync([System("archive-scan", "every:5m"), System("index-backup"), System("housekeeping")], TestData.Now, Ct)); + var kept = (await db.Schedules.GetBySystemKeyAsync("archive-scan", Ct))!; + Assert.Equal((seeded.Id, "every:30m", false), (kept.Id, kept.Recurrence, kept.Enabled)); + Assert.Count(3, await db.Schedules.ListAsync(Ct)); + Assert.Equal(0, await db.Schedules.EnsureSystemAsync([], TestData.Now, Ct)); + + await db.Schedules.TryAdvanceAsync(kept.Id, null, TestData.Now.AddHours(1), Ct); + Assert.True(await db.Schedules.ResetSystemAsync("archive-scan", System("archive-scan", "every:15m"), TestData.Now.AddDays(1), Ct)); + var reset = (await db.Schedules.GetBySystemKeyAsync("archive-scan", Ct))!; + Assert.Equal((seeded.Id, "every:15m", true, (DateTimeOffset?)null, TestData.Now.AddDays(1)), (reset.Id, reset.Recurrence, reset.Enabled, reset.NextRun, reset.Updated)); + Assert.False(await db.Schedules.ResetSystemAsync("unknown", System("unknown"), TestData.Now, Ct)); + } + + [Test] + public async Task OnlyOneClaimOfARunWins() + { + var db = await OpenChronosAsync(multiConnection: true); + var schedule = NewSchedule() with { NextRun = TestData.Now }; + await db.Schedules.InsertAsync(schedule, Ct); + for (var round = 0; round < 5; round++) + { + var due = (await db.Schedules.GetAsync(schedule.Id, Ct))!.NextRun; + var next = due!.Value.AddDays(1); + var claims = await Task.WhenAll(Enumerable.Range(0, 8).Select(_ => Task.Run(() => db.Schedules.TryAdvanceAsync(schedule.Id, due, next, Ct)))); + Assert.Equal(1, claims.Count(c => c)); + Assert.Equal(next, (await db.Schedules.GetAsync(schedule.Id, Ct))!.NextRun); + } + Assert.False(await db.Schedules.TryAdvanceAsync(schedule.Id, TestData.Now, TestData.Now.AddDays(2), Ct), "a stale expectation loses"); + + var unset = NewSchedule("Unset"); + await db.Schedules.InsertAsync(unset, Ct); + Assert.True(await db.Schedules.TryAdvanceAsync(unset.Id, null, TestData.Now, Ct)); + Assert.False(await db.Schedules.TryAdvanceAsync(unset.Id, null, TestData.Now.AddHours(1), Ct)); + var disabled = NewSchedule("Disabled") with { Enabled = false, NextRun = TestData.Now }; + await db.Schedules.InsertAsync(disabled, Ct); + Assert.False(await db.Schedules.TryAdvanceAsync(disabled.Id, TestData.Now, TestData.Now.AddHours(1), Ct), "a disabled schedule is never claimed"); + } + + [Test] + public async Task DueSchedulesAreEnabledOnesAtOrBeforeNow() + { + var db = await OpenChronosAsync(); + var late = NewSchedule("Late") with { NextRun = TestData.Now.AddHours(-2) }; + var onTime = NewSchedule("On time") with { NextRun = TestData.Now }; + var future = NewSchedule("Future") with { NextRun = TestData.Now.AddSeconds(1) }; + var disabled = NewSchedule("Disabled") with { NextRun = TestData.Now.AddHours(-3), Enabled = false }; + var unset = NewSchedule("Unset"); + foreach (var s in new[] { onTime, future, disabled, unset, late }) await db.Schedules.InsertAsync(s, Ct); + Assert.SequenceEqual([late.Id, onTime.Id], (await db.Schedules.ListDueAsync(TestData.Now, Ct)).Select(s => s.Id)); + } + + [Test] + public async Task ARunWithoutAJobKeepsThePreviousJob() + { + var db = await OpenChronosAsync(); + var schedule = NewSchedule(); + await db.Schedules.InsertAsync(schedule, Ct); + var job = Guid.CreateVersion7(); + Assert.True(await db.Schedules.RecordOutcomeAsync(schedule.Id, job, ScheduleOutcome.Queued, TestData.Now, Ct)); + Assert.True(await db.Schedules.RecordOutcomeAsync(schedule.Id, null, ScheduleOutcome.SkippedPending, TestData.Now.AddMinutes(10), Ct)); + var read = (await db.Schedules.GetAsync(schedule.Id, Ct))!; + Assert.Equal((job, ScheduleOutcome.SkippedPending, TestData.Now.AddMinutes(10)), (read.LastJobId!.Value, read.LastOutcome!, read.LastRun!.Value)); + Assert.False(await db.Schedules.RecordOutcomeAsync(Guid.CreateVersion7(), null, ScheduleOutcome.Queued, TestData.Now, Ct)); + } + + [Test] + public async Task VerifyCandidatesAreTheLeastRecentlyVerifiedFirst() + { + var db = await OpenChronosAsync(); + var archive = TestData.Archive(); + await db.Archives.UpsertAsync(archive, Ct); + var never = TestData.Sequence(archive.Id) with { FrameCount = 3 }; + var old = TestData.Sequence(archive.Id) with { FrameCount = 3, VerifiedAt = TestData.Now.AddDays(-60) }; + var recent = TestData.Sequence(archive.Id) with { FrameCount = 3, VerifiedAt = TestData.Now.AddDays(-2) }; + var empty = TestData.Sequence(archive.Id); + foreach (var s in new[] { recent, empty, old, never }) await db.Sequences.UpsertAsync(s, Ct); + var other = TestData.Archive("Other"); + await db.Archives.UpsertAsync(other, Ct); + await db.Sequences.UpsertAsync(TestData.Sequence(other.Id) with { FrameCount = 1 }, Ct); + + Assert.SequenceEqual([never.Id, old.Id, recent.Id], await db.Sequences.ListVerifyCandidatesAsync(archive.Id, null, 10, Ct)); + Assert.SequenceEqual([never.Id, old.Id], await db.Sequences.ListVerifyCandidatesAsync(archive.Id, TestData.Now.AddDays(-30), 10, Ct)); + Assert.SequenceEqual([never.Id], await db.Sequences.ListVerifyCandidatesAsync(archive.Id, null, 1, Ct)); + } +} + +public sealed class SqliteScheduleRepositoryTests(TestContext context) : ScheduleRepositoryTests(context) +{ + protected override Backend Backend => Backend.SqliteMemory; +} + +[RequiresEnvironment(PgTestServer.Variable)] +public sealed class PostgresScheduleRepositoryTests(TestContext context) : ScheduleRepositoryTests(context) +{ + protected override Backend Backend => Backend.Postgres; +} diff --git a/tests/Chronos.Data.Tests/Repositories/ServerRepositoryTests.cs b/tests/Chronos.Data.Tests/Repositories/ServerRepositoryTests.cs index ebaded73..8c06c59d 100644 --- a/tests/Chronos.Data.Tests/Repositories/ServerRepositoryTests.cs +++ b/tests/Chronos.Data.Tests/Repositories/ServerRepositoryTests.cs @@ -158,6 +158,17 @@ public abstract class ServerRepositoryTests(TestContext context) : RepositoryTes Assert.Equal(3, (await db.Audit.ListAsync(user, cancellationToken: Ct)).Items.Count); } + [Test] + public async Task TheAuditRetentionRemovesOnlyOlderEntries() + { + var db = await OpenChronosAsync(); + foreach (var days in new[] { 400, 366, 10 }) + await db.Audit.AppendAsync(new AuditEntry { Id = Guid.CreateVersion7(), At = TestData.Now.AddDays(-days), Action = "user.login" }, Ct); + Assert.Equal(2, await db.Audit.PurgeAsync(TestData.Now.AddDays(-365), Ct)); + Assert.Equal(TestData.Now.AddDays(-10), Assert.NotNull((await db.Audit.ListAsync(cancellationToken: Ct)).Items.SingleOrDefault()).At); + Assert.Equal(0, await db.Audit.PurgeAsync(TestData.Now.AddDays(-365), Ct)); + } + [Test] public async Task UploadsTrackChunksAndStates() { diff --git a/tests/Chronos.Data.Tests/Scheduling/HousekeepingJobTests.cs b/tests/Chronos.Data.Tests/Scheduling/HousekeepingJobTests.cs new file mode 100644 index 00000000..c48e9b24 --- /dev/null +++ b/tests/Chronos.Data.Tests/Scheduling/HousekeepingJobTests.cs @@ -0,0 +1,194 @@ +using Chronos.Core.Localization; +using Chronos.Core.Model; +using Chronos.Data.Archiving; +using Chronos.Data.Archiving.Jobs; +using Chronos.Data.Housekeeping; +using Chronos.Data.Jobs; +using Chronos.Data.Sqlite; +using Chronos.Data.Tests.Archiving; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Scheduling; + +/// The jobs schedules add (owner review 2, R10): index backups, clean-up steps, and the read-only duplicates and orphans reports. +[Category("schedules")] +public sealed class HousekeepingJobTests(TestContext context) +{ + /// A host step that counts its runs, or fails. + private sealed class TestStep(string name, bool fails = false) : ICleanupStep + { + public int Runs; + + public string Name => name; + + public Task RunAsync(CleanupContext context, CleanupParameters parameters) + { + Interlocked.Increment(ref Runs); + return fails ? throw new IOException("disk gone") : Task.FromResult($"{name} ran for {parameters.OlderThanDays} days"); + } + } + + private static async Task IngestAsync(ArchiveFixture fx, string key, int count, int seed) + { + Guid sequence = default; + for (var i = 0; i < count; i++) + { + var staged = fx.Stage(ArchiveFixture.Start.AddSeconds(10 * i), seed + i); + sequence = (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, key, staged.Path, $"IMG_{i:0000}.JPG", + staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId; + } + return sequence; + } + + private static async Task RunAsync(ArchiveFixture fx, JobEngine engine, ArchiveJobRequest request, JobState state = JobState.Done) + { + var job = await engine.EnqueueAsync(request, cancellationToken: fx.Ct); + return await fx.WaitForAsync(job.Id, state); + } + + [Test] + public async Task BackupsGoNextToTheIndexAndKeepTheNewest() + { + await using var fx = await ArchiveFixture.CreateAsync(context); + var engine = await fx.StartEngineAsync(); + var path = ((SqliteDatabase)fx.Db.Database).Path!; + Job done = null!; + for (var i = 0; i < 3; i++) done = await RunAsync(fx, engine, ArchiveJobs.Backup(keep: 2)); + var backups = DatabaseBackups.List(DatabaseBackups.DefaultDirectory(path), path); + Assert.Equal(2, backups.Count); + Assert.Equal(Path.Combine(Path.GetDirectoryName(path)!, "backups"), DatabaseBackups.DefaultDirectory(path)); + Assert.Contains(backups[0].Path, done.LogTail); + Assert.Equal((1L, 1L), (done.ProgressDone, done.ProgressTotal)); + Assert.Null(DatabaseBackups.Unavailable(fx.Db)); + } + + [Test] + public async Task ABackupOfAnIndexWithoutAFileFails() + { + await using var db = await ChronosDatabase.OpenAsync(DatabaseOptions.SqliteInMemory(), context.Cancellation); + Assert.Equal(Data.Scheduling.ScheduleUnavailable.NoIndexFile, DatabaseBackups.Unavailable(db)); + await using var engine = new JobEngine(db, new ArchiveService(db).CreateJobHandlers(), + new JobEngineOptions { WorkerId = "memory", PollInterval = TimeSpan.FromMilliseconds(50) }); + await engine.StartAsync(context.Cancellation); + var job = await engine.EnqueueAsync(ArchiveJobs.Backup(), cancellationToken: context.Cancellation); + for (var i = 0; i < 500 && (await db.Jobs.GetAsync(job.Id))!.State is JobState.Queued or JobState.Running; i++) await Task.Delay(20); + var failed = (await db.Jobs.GetAsync(job.Id))!; + Assert.Equal(JobState.Failed, failed.State); + Assert.Contains("without a file", failed.Error); + } + + [Test] + public async Task TheJobHistoryStepRemovesOnlyOldFinishedJobs() + { + await using var fx = await ArchiveFixture.CreateAsync(context); + var now = DateTimeOffset.UtcNow; + Job Old(JobState state, int days, JobKind kind = JobKind.Scan) => new() + { + Id = Guid.CreateVersion7(), + Kind = kind, + State = state, + Created = now.AddDays(-days), + Finished = state is JobState.Done or JobState.Failed or JobState.Cancelled ? now.AddDays(-days) : null, + }; + // A render waits: the archive engine has no render handler, so it stays queued however old. + var old = Old(JobState.Done, 100); + var oldFailed = Old(JobState.Failed, 91); + var recent = Old(JobState.Done, 10); + var waiting = Old(JobState.Queued, 200, JobKind.Render); + foreach (var job in new[] { old, oldFailed, recent, waiting }) await fx.Db.Jobs.EnqueueAsync(job, fx.Ct); + var engine = await fx.StartEngineAsync(); + + var kept = await RunAsync(fx, engine, ArchiveJobs.Cleanup(fx.Archive, [CleanupSteps.JobHistory], 0)); + Assert.Contains(Loc.L("Archive.Log.HistoryKept"), kept.LogTail); + Assert.NotNull(await fx.Db.Jobs.GetAsync(old.Id, fx.Ct)); + + var purged = await RunAsync(fx, engine, ArchiveJobs.Cleanup(fx.Archive, [CleanupSteps.JobHistory], 90)); + Assert.Contains(Loc.L("Archive.Log.HistoryPurged", 2), purged.LogTail); + Assert.Null(await fx.Db.Jobs.GetAsync(old.Id, fx.Ct)); + Assert.Null(await fx.Db.Jobs.GetAsync(oldFailed.Id, fx.Ct)); + Assert.NotNull(await fx.Db.Jobs.GetAsync(recent.Id, fx.Ct)); + Assert.NotNull(await fx.Db.Jobs.GetAsync(waiting.Id, fx.Ct)); + } + + [Test] + public async Task TheAuditStepKeepsEverythingAtZeroDays() + { + await using var fx = await ArchiveFixture.CreateAsync(context); + var now = DateTimeOffset.UtcNow; + foreach (var days in new[] { 400, 1 }) + await fx.Db.Audit.AppendAsync(new AuditEntry { Id = Guid.CreateVersion7(), At = now.AddDays(-days), Action = "user.login" }, fx.Ct); + var engine = await fx.StartEngineAsync(); + + var kept = await RunAsync(fx, engine, ArchiveJobs.Cleanup(null, [CleanupSteps.Audit], 0)); + Assert.Contains(Loc.L("Archive.Log.AuditKept"), kept.LogTail); + Assert.Count(2, (await fx.Db.Audit.ListAsync(cancellationToken: fx.Ct)).Items); + var purged = await RunAsync(fx, engine, ArchiveJobs.Cleanup(null, [CleanupSteps.Audit], 365)); + Assert.Contains(Loc.L("Archive.Log.AuditPurged", 1), purged.LogTail); + Assert.Count(1, (await fx.Db.Audit.ListAsync(cancellationToken: fx.Ct)).Items); + } + + [Test] + public async Task TheTrashReportListsOldEntriesAndNeverDeletes() + { + await using var fx = await ArchiveFixture.CreateAsync(context); + var id = await IngestAsync(fx, "a", 2, 1); + var entry = (await fx.Service.DeleteAsync(await fx.Service.PlanDeleteAsync([id], cancellationToken: fx.Ct), "test", cancellationToken: fx.Ct)).Single(); + var folder = Path.Combine(SequenceTrash.TrashDirectory(fx.Paths), entry.Name); + var files = Directory.GetFiles(folder, "*", SearchOption.AllDirectories).Length; + var engine = await fx.StartEngineAsync(); + + var all = await RunAsync(fx, engine, ArchiveJobs.Cleanup(fx.Archive, [CleanupSteps.TrashReport, "shred"], 0)); + Assert.Contains(Loc.L("Archive.Log.TrashReport", 1, CleanupSteps.Megabytes(entry.Bytes)), all.LogTail); + Assert.Contains(entry.Title, all.LogTail); + Assert.Contains(Loc.L("Archive.Log.StepUnknown", "shred"), all.LogTail); + var none = await RunAsync(fx, engine, ArchiveJobs.Cleanup(fx.Archive, [CleanupSteps.TrashReport], 30)); + Assert.Contains(Loc.L("Archive.Log.TrashReport", 0, CleanupSteps.Megabytes(0)), none.LogTail); + + // Reported, never deleted: emptying the trash stays the user's action. + Assert.Equal(files, Directory.GetFiles(folder, "*", SearchOption.AllDirectories).Length); + Assert.Equal(entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name); + var offline = await RunAsync(fx, engine, ArchiveJobs.Cleanup(null, [CleanupSteps.TrashReport], 0)); + Assert.Contains(Loc.L("Archive.Log.TrashOffline"), offline.LogTail); + } + + [Test] + public async Task AFailingStepFailsTheJobAfterTheOthersRan() + { + var count = new TestStep("count"); + var explode = new TestStep("explode", fails: true); + await using var db = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "chronos.db")), context.Cancellation); + var service = new ArchiveService(db, new ArchiveServiceOptions { CleanupSteps = [explode, count, new TestStep(CleanupSteps.Audit)] }); + Assert.SequenceEqual([CleanupSteps.JobHistory, CleanupSteps.Audit, CleanupSteps.TrashReport, "explode", "count"], service.CleanupSteps.Select(s => s.Name)); + await using var engine = new JobEngine(db, service.CreateJobHandlers(), new JobEngineOptions { WorkerId = "steps", PollInterval = TimeSpan.FromMilliseconds(50) }); + await engine.StartAsync(context.Cancellation); + var job = await engine.EnqueueAsync(ArchiveJobs.Cleanup(null, ["explode", "count"], 7), cancellationToken: context.Cancellation); + for (var i = 0; i < 500 && (await db.Jobs.GetAsync(job.Id))!.State is JobState.Queued or JobState.Running; i++) await Task.Delay(20); + var failed = (await db.Jobs.GetAsync(job.Id))!; + Assert.Equal(JobState.Failed, failed.State); + Assert.Contains("explode", failed.Error); + Assert.Contains(Loc.L("Archive.Log.StepFailed", "explode", "disk gone"), failed.LogTail); + Assert.Contains("count ran for 7 days", failed.LogTail); + Assert.Equal((1, 1), (explode.Runs, count.Runs)); + } + + [Test] + public async Task DuplicatesAndOrphansReportWithoutChangingAnything() + { + await using var fx = await ArchiveFixture.CreateAsync(context); + await IngestAsync(fx, "a", 3, 1); + await IngestAsync(fx, "b", 2, 1); + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "New folder"), ArchiveFixture.Start, 1, firstSeed: 60); + var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id); + var engine = await fx.StartEngineAsync(); + + var duplicates = await RunAsync(fx, engine, ArchiveJobs.Duplicates(fx.Archive)); + Assert.Contains(Loc.L("Archive.Log.DuplicatesDone", 2, 2), duplicates.LogTail); + Assert.Contains(Loc.L("Archive.Log.ReportOnly"), duplicates.LogTail); + var orphans = await RunAsync(fx, engine, ArchiveJobs.Orphans(fx.Archive)); + Assert.Contains(Loc.L("Archive.Log.FolderUnindexed", "New folder"), orphans.LogTail); + Assert.Contains(Loc.L("Archive.Log.OrphanFolders", 0, 1), orphans.LogTail); + + Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id)); + Assert.Equal(0L, (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id, fx.Ct)).Health.DuplicateFrames); + } +} diff --git a/tests/Chronos.Data.Tests/Scheduling/ScheduleRecurrenceTests.cs b/tests/Chronos.Data.Tests/Scheduling/ScheduleRecurrenceTests.cs new file mode 100644 index 00000000..c3bc1648 --- /dev/null +++ b/tests/Chronos.Data.Tests/Scheduling/ScheduleRecurrenceTests.cs @@ -0,0 +1,174 @@ +using Chronos.Data.Scheduling; +using Chronos.Protocol; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Scheduling; + +/// The in-house recurrence of schedules (owner review 2, R10): text form, intervals on their anchor, wall-clock times across clock changes. +[Category("schedules")] +public sealed class ScheduleRecurrenceTests +{ + /// Central European time, built here so the test does not depend on the machine's zone data: +1, +2 from the last Sunday of March 02:00 to the last Sunday of October 03:00. + internal static readonly TimeZoneInfo Cet = TimeZoneInfo.CreateCustomTimeZone("Test/CET", TimeSpan.FromHours(1), "Test CET", "Test CET", "Test CEST", + [ + TimeZoneInfo.AdjustmentRule.CreateAdjustmentRule(DateTime.MinValue.Date, DateTime.MaxValue.Date, TimeSpan.FromHours(1), + TimeZoneInfo.TransitionTime.CreateFloatingDateRule(new DateTime(1, 1, 1, 2, 0, 0), 3, 5, DayOfWeek.Sunday), + TimeZoneInfo.TransitionTime.CreateFloatingDateRule(new DateTime(1, 1, 1, 3, 0, 0), 10, 5, DayOfWeek.Sunday)), + ]); + + private static DateTimeOffset Utc(int month, int day, int hour, int minute = 0, int second = 0) => new(2026, month, day, hour, minute, second, TimeSpan.Zero); + + [Theory] + [Case("every:5m", "every:5m")] + [Case("every:1440m", "every:1440m")] + [Case(" Every : 6H ", "every:6h")] + [Case("every:7d", "every:7d")] + [Case("daily:3:05", "daily:03:05")] + [Case("daily:23:59", "daily:23:59")] + [Case("weekly:thu,mon,thu@2:30", "weekly:mon,thu@02:30")] + [Case("WEEKLY:sun@00:00", "weekly:sun@00:00")] + [Case("monthly:28@04:00", "monthly:28@04:00")] + [Case("monthly: 1 @ 04:00", "monthly:1@04:00")] + public void TheTextFormRoundTrips(string text, string canonical) + { + var recurrence = ScheduleRecurrence.Parse(text); + Assert.Equal(canonical, recurrence.Format()); + Assert.Equal(recurrence, ScheduleRecurrence.Parse(recurrence.Format())); + Assert.Equal(recurrence, ScheduleMapping.FromDto(ScheduleMapping.ToDto(recurrence))); + } + + [Theory] + [Case("")] + [Case("every")] + [Case("every:")] + [Case("every:4m")] + [Case("every:1441m")] + [Case("every:0h")] + [Case("every:169h")] + [Case("every:367d")] + [Case("every:10x")] + [Case("every:-5m")] + [Case("every:1.5h")] + [Case("hourly:1")] + [Case("daily:24:00")] + [Case("daily:12:60")] + [Case("daily:3:5")] + [Case("daily:")] + [Case("weekly:@03:00")] + [Case("weekly:mon")] + [Case("weekly:mon,xyz@03:00")] + [Case("monthly:0@03:00")] + [Case("monthly:29@03:00")] + [Case("monthly:@03:00")] + public void InvalidTextIsRefused(string text) + { + Assert.False(ScheduleRecurrence.TryParse(text, out _), text); + Assert.Throws(() => ScheduleRecurrence.Parse(text)); + } + + [Test] + public void TheFactoriesCheckTheirRanges() + { + Assert.False(ScheduleRecurrence.TryParse(null, out _)); + Assert.Throws(() => ScheduleRecurrence.Interval(4, RecurrenceUnit.Minutes)); + Assert.Throws(() => ScheduleRecurrence.MonthlyAt(29, new TimeOnly(3, 0))); + Assert.Throws(() => ScheduleRecurrence.WeeklyAt([], new TimeOnly(3, 0))); + // Seconds are dropped: schedules work to the minute. + Assert.Equal(ScheduleRecurrence.DailyAt(new TimeOnly(3, 0)), ScheduleRecurrence.DailyAt(new TimeOnly(3, 0, 42))); + Assert.Equal(ScheduleRecurrence.Parse("weekly:thu,mon@02:30"), ScheduleRecurrence.WeeklyAt([DayOfWeek.Monday, DayOfWeek.Thursday], new TimeOnly(2, 30))); + Assert.NotEqual(ScheduleRecurrence.Parse("weekly:mon@02:30"), ScheduleRecurrence.Parse("weekly:tue@02:30")); + Assert.SequenceEqual([DayOfWeek.Monday, DayOfWeek.Sunday], ScheduleRecurrence.Parse("weekly:sun,mon@02:30").Days); + } + + [Test] + public void PeriodsBoundTheGapBetweenRuns() + { + Assert.Equal(TimeSpan.FromMinutes(10), ScheduleRecurrence.Parse("every:10m").Period); + Assert.Equal(TimeSpan.FromHours(6), ScheduleRecurrence.Parse("every:6h").Period); + Assert.Equal(TimeSpan.FromDays(2), ScheduleRecurrence.Parse("every:2d").Period); + Assert.Equal(TimeSpan.FromDays(1), ScheduleRecurrence.Parse("daily:03:00").Period); + Assert.Equal(TimeSpan.FromDays(7), ScheduleRecurrence.Parse("weekly:mon@03:00").Period); + Assert.Equal(TimeSpan.FromDays(31), ScheduleRecurrence.Parse("monthly:1@03:00").Period); + } + + [Test] + public void IntervalsStayOnTheGridOfTheirAnchor() + { + var every10 = ScheduleRecurrence.Parse("every:10m"); + var anchor = Utc(10, 2, 12, 0); + Assert.Equal(Utc(10, 2, 12, 10), every10.Next(anchor, TimeZoneInfo.Utc, anchor)); + // Late by 35 minutes: the missed runs collapse into the next one on the grid, no drift. + Assert.Equal(Utc(10, 2, 12, 40), every10.Next(Utc(10, 2, 12, 35, 10), TimeZoneInfo.Utc, anchor)); + Assert.Equal(Utc(10, 2, 12, 40), every10.Next(Utc(10, 2, 12, 30), TimeZoneInfo.Utc, anchor)); + // An anchor ahead of the time asked about is itself on the grid. + Assert.Equal(Utc(10, 2, 11, 50), every10.Next(Utc(10, 2, 11, 45), TimeZoneInfo.Utc, anchor)); + Assert.Equal(Utc(10, 2, 12, 0), every10.Next(Utc(10, 2, 11, 59, 59), TimeZoneInfo.Utc, anchor)); + // Without an anchor: one interval after the minute. + Assert.Equal(Utc(10, 2, 12, 13), every10.Next(Utc(10, 2, 12, 3, 27), TimeZoneInfo.Utc)); + // Days of an interval are 24 hours whatever the clocks do. + Assert.Equal(Utc(3, 29, 1, 30), ScheduleRecurrence.Parse("every:1d").Next(Utc(3, 28, 1, 30), Cet, Utc(3, 28, 1, 30))); + } + + [Test] + public void DailyWeeklyAndMonthlyTimesAreWallClockTimes() + { + var daily = ScheduleRecurrence.Parse("daily:03:00"); + Assert.Equal(Utc(10, 2, 3, 0), daily.Next(Utc(10, 2, 2, 59), TimeZoneInfo.Utc)); + Assert.Equal(Utc(10, 3, 3, 0), daily.Next(Utc(10, 2, 3, 0), TimeZoneInfo.Utc)); + // 03:00 in Rome on 2 October (summer time) is 01:00 UTC; in January (winter time) 02:00 UTC. + Assert.Equal(Utc(10, 2, 1, 0), daily.Next(Utc(10, 1, 23, 0), Cet)); + Assert.Equal(new DateTimeOffset(2027, 1, 15, 2, 0, 0, TimeSpan.Zero), daily.Next(new DateTimeOffset(2027, 1, 15, 1, 0, 0, TimeSpan.Zero), Cet)); + + // 2 October 2026 is a Friday. + var weekly = ScheduleRecurrence.Parse("weekly:mon,thu@08:00"); + Assert.Equal(Utc(10, 5, 6, 0), weekly.Next(Utc(10, 2, 12, 0), Cet)); + Assert.Equal(Utc(10, 8, 6, 0), weekly.Next(Utc(10, 5, 6, 0), Cet)); + Assert.Equal(Utc(10, 12, 6, 0), weekly.Next(Utc(10, 8, 7, 0), Cet)); + + var monthly = ScheduleRecurrence.Parse("monthly:28@04:00"); + Assert.Equal(Utc(3, 28, 4, 0), monthly.Next(Utc(2, 28, 5, 0), TimeZoneInfo.Utc)); + Assert.Equal(Utc(2, 28, 4, 0), monthly.Next(Utc(1, 28, 4, 0), TimeZoneInfo.Utc)); + } + + [Test] + public void ATimeTheClocksSkipMovesForwardByTheGap() + { + // 29 March 2026: 02:00 CET becomes 03:00 CEST, so 02:30 does not exist that night and runs at 03:30 CEST. + var daily = ScheduleRecurrence.Parse("daily:02:30"); + var next = daily.Next(Utc(3, 28, 23, 0), Cet); + Assert.Equal(Utc(3, 29, 1, 30), next); + Assert.Equal(new DateTime(2026, 3, 29, 3, 30, 0), TimeZoneInfo.ConvertTime(next, Cet).DateTime); + // The next night is a normal one again. + Assert.Equal(Utc(3, 30, 0, 30), daily.Next(next, Cet)); + Assert.Equal(Utc(3, 29, 1, 30), ScheduleRecurrence.ToUtc(new DateTime(2026, 3, 29, 2, 30, 0), Cet)); + } + + [Test] + public void ATimeTheClocksGoBackOverRunsOnceAtItsFirstOccurrence() + { + // 25 October 2026: 03:00 CEST becomes 02:00 CET, so 02:30 happens twice; it runs at the first (00:30 UTC). + var daily = ScheduleRecurrence.Parse("daily:02:30"); + var first = daily.Next(Utc(10, 24, 23, 0), Cet); + Assert.Equal(Utc(10, 25, 0, 30), first); + // Not again an hour later at the second 02:30: the next run is the next day's. + Assert.Equal(Utc(10, 26, 1, 30), daily.Next(first, Cet)); + Assert.Equal(Utc(10, 26, 1, 30), daily.Next(Utc(10, 25, 1, 0), Cet)); + Assert.Equal(Utc(10, 25, 0, 30), ScheduleRecurrence.ToUtc(new DateTime(2026, 10, 25, 2, 30, 0), Cet)); + } + + [Test] + public void TheStructuredFormIsCheckedLikeTheText() + { + Assert.Equal("every:15m", ScheduleMapping.FromDto(new RecurrenceDto("every", Every: 15, Unit: "minutes"))?.Format()); + Assert.Equal("weekly:mon,fri@06:00", ScheduleMapping.FromDto(new RecurrenceDto("Weekly", Time: "06:00", Days: ["fri", "mon"]))?.Format()); + Assert.Equal("monthly:3@01:15", ScheduleMapping.FromDto(new RecurrenceDto("monthly", Time: "01:15", DayOfMonth: 3))?.Format()); + Assert.Equal(new RecurrenceDto("weekly", Time: "06:00", Days: null), ScheduleMapping.ToDto(ScheduleRecurrence.Parse("weekly:mon@06:00")) with { Days = null }); + Assert.Null(ScheduleMapping.FromDto(null)); + Assert.Null(ScheduleMapping.FromDto(new RecurrenceDto("every", Every: 2, Unit: "minutes"))); + Assert.Null(ScheduleMapping.FromDto(new RecurrenceDto("every", Every: 2, Unit: "weeks"))); + Assert.Null(ScheduleMapping.FromDto(new RecurrenceDto("daily"))); + Assert.Null(ScheduleMapping.FromDto(new RecurrenceDto("weekly", Time: "06:00", Days: []))); + Assert.Null(ScheduleMapping.FromDto(new RecurrenceDto("monthly", Time: "06:00", DayOfMonth: 31))); + Assert.Null(ScheduleMapping.FromDto(new RecurrenceDto("yearly", Time: "06:00"))); + } +} diff --git a/tests/Chronos.Data.Tests/Scheduling/SchedulerTests.cs b/tests/Chronos.Data.Tests/Scheduling/SchedulerTests.cs new file mode 100644 index 00000000..f2040553 --- /dev/null +++ b/tests/Chronos.Data.Tests/Scheduling/SchedulerTests.cs @@ -0,0 +1,440 @@ +using Chronos.Core.Archiving; +using Chronos.Core.Model; +using Chronos.Data.Archiving; +using Chronos.Data.Archiving.Jobs; +using Chronos.Data.Jobs; +using Chronos.Data.Processing; +using Chronos.Data.Scheduling; +using Chronos.Data.Sqlite; +using Chronos.Data.Tests.Archiving; +using Chronos.Protocol; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Scheduling; + +/// A clock the test sets by hand; its timers are real ones, so the scheduler's loop waits in real time (tests drive passes themselves). +internal sealed class ScheduleClock(DateTimeOffset start) : TimeProvider +{ + private long _ticks = start.UtcTicks; + + public DateTimeOffset Now => GetUtcNow(); + + public override DateTimeOffset GetUtcNow() => new(Interlocked.Read(ref _ticks), TimeSpan.Zero); + + public void Advance(TimeSpan by) => Interlocked.Add(ref _ticks, by.Ticks); + + public void Set(DateTimeOffset at) => Interlocked.Exchange(ref _ticks, at.UtcTicks); +} + +/// A host whose archive, system schedules and refusals the test sets. +internal sealed class TestScheduleHost(Archive? archive) : IScheduleHost +{ + public Archive? Archive { get; set; } = archive; + + public TimeZoneInfo TimeZone { get; set; } = TimeZoneInfo.Utc; + + public List Systems { get; } = []; + + public Dictionary Refused { get; } = []; + + public IReadOnlyList SystemSchedules => Systems; + + public IReadOnlyList CleanupSteps { get; set; } = ["job-history", "audit", "trash-report"]; + + public ValueTask GetArchiveAsync(CancellationToken cancellationToken) => ValueTask.FromResult(Archive); + + public string? Unavailable(string action) => Refused.GetValueOrDefault(action); +} + +/// The scheduler (owner review 2, R10): due runs fire once, pending work is not queued twice, catch-up, clock changes, edits. +[Category("schedules")] +public sealed class SchedulerTests(TestContext context) +{ + private static readonly DateTimeOffset T0 = new(2026, 10, 2, 9, 0, 0, TimeSpan.Zero); + + private static readonly RecurrenceDto Every10 = new("every", Every: 10, Unit: "minutes"); + + private CancellationToken Ct => context.Cancellation; + + /// An engine with the archive and analysis handlers that is never started, so queued jobs stay queued. + private static JobEngine Engine(ArchiveFixture fx) => + new(fx.Db, [.. fx.Service.CreateJobHandlers(), .. new ProcessingService(fx.Db).CreateJobHandlers()], new JobEngineOptions { WorkerId = "test" }); + + private async Task StartAsync(ChronosDatabase db, JobEngine engine, IScheduleHost host, ScheduleClock clock) + { + var scheduler = new Scheduler(db, engine, new SchedulerOptions { Time = clock, Tick = TimeSpan.FromHours(1) }); + await scheduler.StartAsync(host, Ct); + return scheduler; + } + + private async Task CreateAsync(Scheduler scheduler, string action, RecurrenceDto? recurrence = null, ScheduleParametersDto? parameters = null, + bool enabled = true, bool catchUp = true) + { + var result = await scheduler.CreateAsync(new CreateScheduleRequest("Test " + action, action, recurrence ?? Every10, parameters, catchUp, Enabled: enabled), null, Ct); + Assert.Null(result.Problem); + return result.Schedule!; + } + + [Test] + public async Task ADueScheduleFiresOnceAtItsPriorityWithItsId() + { + await using var fx = await ArchiveFixture.CreateAsync(context); + await using var engine = Engine(fx); + var clock = new ScheduleClock(T0); + await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock); + var schedule = await CreateAsync(scheduler, ScheduleActions.Scan); + Assert.Equal(T0.AddMinutes(10), schedule.NextRun); + + Assert.Equal(0, await scheduler.RunDueAsync(Ct)); + clock.Advance(TimeSpan.FromMinutes(10)); + Assert.Equal(1, await scheduler.RunDueAsync(Ct)); + Assert.Equal(0, await scheduler.RunDueAsync(Ct)); + + var job = (await fx.Db.Jobs.ListByScheduleAsync(schedule.Id, cancellationToken: Ct)).Single(); + Assert.Equal((JobKind.Scan, JobPriority.Background, JobState.Queued), (job.Kind, job.Priority, job.State)); + Assert.SequenceEqual([fx.Archive.Id], job.TargetIds); + var after = (await scheduler.GetAsync(schedule.Id, Ct))!; + Assert.Equal((ScheduleOutcome.Queued, job.Id, T0.AddMinutes(20), clock.Now), (after.LastOutcome!, after.LastJobId!.Value, after.NextRun!.Value, after.LastRun!.Value)); + } + + [Test] + public async Task ARunWhosePreviousJobIsPendingIsSkipped() + { + await using var fx = await ArchiveFixture.CreateAsync(context); + await using var engine = Engine(fx); + var clock = new ScheduleClock(T0); + await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock); + var schedule = await CreateAsync(scheduler, ScheduleActions.Duplicates); + clock.Advance(TimeSpan.FromMinutes(10)); + await scheduler.RunDueAsync(Ct); + var first = (await scheduler.GetAsync(schedule.Id, Ct))!.LastJobId; + + clock.Advance(TimeSpan.FromMinutes(10)); + Assert.Equal(1, await scheduler.RunDueAsync(Ct)); + var skipped = (await scheduler.GetAsync(schedule.Id, Ct))!; + Assert.Equal((ScheduleOutcome.SkippedPending, first), (skipped.LastOutcome!, skipped.LastJobId)); + Assert.Count(1, await fx.Db.Jobs.ListByScheduleAsync(schedule.Id, cancellationToken: Ct)); + + // Once it ends, the next run queues again; a scan another user queued counts as the same work. + await fx.Db.Jobs.SetStateAsync(first!.Value, JobState.Done, cancellationToken: Ct); + clock.Advance(TimeSpan.FromMinutes(10)); + await scheduler.RunDueAsync(Ct); + Assert.Equal(ScheduleOutcome.Queued, (await scheduler.GetAsync(schedule.Id, Ct))!.LastOutcome); + await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: Ct); + var scan = await CreateAsync(scheduler, ScheduleActions.Scan); + Assert.Equal(ScheduleOutcome.SkippedPending, (await scheduler.RunNowAsync(scan.Id, null, Ct))!.Outcome); + } + + [Test] + public async Task MissedRunsFireOnceWithCatchUpAndAreSkippedWithout() + { + await using var fx = await ArchiveFixture.CreateAsync(context); + await using var engine = Engine(fx); + var clock = new ScheduleClock(T0); + var host = new TestScheduleHost(fx.Archive); + ScheduleDto catchUp, skip; + await using (var before = await StartAsync(fx.Db, engine, host, clock)) + { + catchUp = await CreateAsync(before, ScheduleActions.Scan, new RecurrenceDto("every", Every: 1, Unit: "hours"), catchUp: true); + skip = await CreateAsync(before, ScheduleActions.Orphans, new RecurrenceDto("every", Every: 1, Unit: "hours"), catchUp: false); + } + // The host was off for 5 h 30: five runs of each were missed. + clock.Advance(TimeSpan.FromMinutes(330)); + await using var after = await StartAsync(fx.Db, engine, host, clock); + + Assert.Count(1, await fx.Db.Jobs.ListByScheduleAsync(catchUp.Id, cancellationToken: Ct)); + Assert.Empty(await fx.Db.Jobs.ListByScheduleAsync(skip.Id, cancellationToken: Ct)); + var caught = (await after.GetAsync(catchUp.Id, Ct))!; + var skipped = (await after.GetAsync(skip.Id, Ct))!; + Assert.Equal(ScheduleOutcome.Queued, caught.LastOutcome); + Assert.Null(skipped.LastOutcome); + // Both move to the next occurrence on their grid, in the future. + Assert.Equal(T0.AddHours(6), caught.NextRun); + Assert.Equal(T0.AddHours(6), skipped.NextRun); + Assert.Equal(0, await after.RunDueAsync(Ct)); + } + + [Test] + public async Task DisabledSchedulesNeverRunAndGetANewNextRunWhenEnabled() + { + await using var fx = await ArchiveFixture.CreateAsync(context); + await using var engine = Engine(fx); + var clock = new ScheduleClock(T0); + await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock); + var disabled = await CreateAsync(scheduler, ScheduleActions.Scan, enabled: false); + Assert.Null(disabled.NextRun); + var enabled = await CreateAsync(scheduler, ScheduleActions.Orphans); + + Assert.Null((await scheduler.UpdateAsync(enabled.Id, new UpdateScheduleRequest(Enabled: false), Ct)).Schedule!.NextRun); + clock.Advance(TimeSpan.FromDays(1)); + Assert.Equal(0, await scheduler.RunDueAsync(Ct)); + Assert.Empty((await fx.Db.Jobs.ListAsync(cancellationToken: Ct)).Items); + + var back = (await scheduler.UpdateAsync(enabled.Id, new UpdateScheduleRequest(Enabled: true), Ct)).Schedule!; + Assert.Equal(clock.Now.AddMinutes(10), back.NextRun); + Assert.Equal(0, await scheduler.RunDueAsync(Ct)); + // An edit that keeps the recurrence keeps the next run; a new recurrence moves it. + Assert.Equal(back.NextRun, (await scheduler.UpdateAsync(enabled.Id, new UpdateScheduleRequest(Name: "Renamed"), Ct)).Schedule!.NextRun); + var hourly = (await scheduler.UpdateAsync(enabled.Id, new UpdateScheduleRequest(Recurrence: new RecurrenceDto("daily", Time: "03:00")), Ct)).Schedule!; + Assert.Equal(new DateTimeOffset(2026, 10, 4, 3, 0, 0, TimeSpan.Zero), hourly.NextRun); + } + + [Test] + public async Task RunNowQueuesForTheUserWithoutMovingTheNextRun() + { + await using var fx = await ArchiveFixture.CreateAsync(context); + await using var engine = Engine(fx); + var clock = new ScheduleClock(T0); + await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock); + var user = Guid.CreateVersion7(); + var disabled = await CreateAsync(scheduler, ScheduleActions.Duplicates, enabled: false); + var run = (await scheduler.RunNowAsync(disabled.Id, user, Ct))!; + Assert.Equal(ScheduleOutcome.Queued, run.Outcome); + var job = (await fx.Db.Jobs.GetAsync(run.JobId!.Value, Ct))!; + Assert.Equal((JobKind.Duplicates, user, disabled.Id), (job.Kind, job.CreatedBy!.Value, job.ScheduleId!.Value)); + Assert.Null(run.Schedule.NextRun); + Assert.Equal(run.JobId, run.Schedule.LastJobId); + + var enabled = await CreateAsync(scheduler, ScheduleActions.Orphans); + Assert.Equal(ScheduleOutcome.Queued, (await scheduler.RunNowAsync(enabled.Id, null, Ct))!.Outcome); + Assert.Equal(enabled.NextRun, (await scheduler.GetAsync(enabled.Id, Ct))!.NextRun); + Assert.Equal(ScheduleOutcome.SkippedPending, (await scheduler.RunNowAsync(enabled.Id, null, Ct))!.Outcome); + Assert.Null(await scheduler.RunNowAsync(Guid.CreateVersion7(), null, Ct)); + } + + [Test] + public async Task TwoSchedulersOnOneDatabaseFireEachRunOnce() + { + await using var fx = await ArchiveFixture.CreateAsync(context); + var path = ((SqliteDatabase)fx.Db.Database).Path!; + await using var otherDb = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(path), Ct); + await using var engine = Engine(fx); + await using var otherEngine = new JobEngine(otherDb, fx.Service.CreateJobHandlers(), new JobEngineOptions { WorkerId = "other" }); + var clock = new ScheduleClock(T0); + await using var a = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock); + await using var b = await StartAsync(otherDb, otherEngine, new TestScheduleHost(fx.Archive), clock); + var schedule = await CreateAsync(a, ScheduleActions.Backup); + for (var round = 1; round <= 4; round++) + { + clock.Advance(TimeSpan.FromMinutes(10)); + var fired = await Task.WhenAll(Task.Run(() => a.RunDueAsync(Ct)), Task.Run(() => b.RunDueAsync(Ct))); + Assert.Equal(1, fired.Sum(), $"round {round}"); + // The backup jobs never run here: each round queues once, then the next is skipped while it waits. + Assert.Count(1, await fx.Db.Jobs.ListByScheduleAsync(schedule.Id, cancellationToken: Ct)); + } + } + + [Test] + public async Task ANextRunFarAheadAfterTheClockWentBackIsComputedAgain() + { + await using var fx = await ArchiveFixture.CreateAsync(context); + await using var engine = Engine(fx); + var clock = new ScheduleClock(T0); + await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock); + var schedule = await CreateAsync(scheduler, ScheduleActions.Scan); + clock.Set(T0.AddHours(-5).AddSeconds(20)); + Assert.Equal(0, await scheduler.RunDueAsync(Ct)); + Assert.Equal(T0.AddHours(-5).AddMinutes(10), (await scheduler.GetAsync(schedule.Id, Ct))!.NextRun); + // Within two periods it is left alone. + clock.Advance(TimeSpan.FromMinutes(-5)); + await scheduler.RunDueAsync(Ct); + Assert.Equal(T0.AddHours(-5).AddMinutes(10), (await scheduler.GetAsync(schedule.Id, Ct))!.NextRun); + } + + [Test] + public async Task SystemSchedulesAreSeededOnceEditableAndResetNeverDeleted() + { + await using var fx = await ArchiveFixture.CreateAsync(context); + await using var engine = Engine(fx); + var clock = new ScheduleClock(T0); + var host = new TestScheduleHost(fx.Archive); + host.Systems.Add(new SystemSchedule("archive-scan", "Archive scan", ScheduleActions.Scan, ScheduleRecurrence.Parse("every:10m"))); + host.Systems.Add(new SystemSchedule("index-backup", "Index backup", ScheduleActions.Backup, ScheduleRecurrence.Parse("daily:03:00"), Enabled: false)); + Guid id; + await using (var first = await StartAsync(fx.Db, engine, host, clock)) + { + var list = await first.ListAsync(Ct); + Assert.SequenceEqual(["archive-scan", "index-backup"], list.Items.Select(s => s.SystemKey!)); + var scan = list.Items[0]; + id = scan.Id; + Assert.Equal(T0.AddMinutes(10), scan.NextRun); + Assert.Null(list.Items[1].NextRun); + Assert.Equal("UTC", list.TimeZone); + Assert.SequenceEqual(host.CleanupSteps, list.Steps!); + + Assert.Equal("name", (await first.UpdateAsync(id, new UpdateScheduleRequest(Name: "Mine"), Ct)).Problem); + Assert.Equal("action", (await first.UpdateAsync(id, new UpdateScheduleRequest(Action: ScheduleActions.Verify), Ct)).Problem); + Assert.Null((await first.UpdateAsync(id, new UpdateScheduleRequest(Name: "Archive scan", Recurrence: new RecurrenceDto("every", Every: 30, Unit: "minutes")), Ct)).Problem); + Assert.Equal(ScheduleDeleteOutcome.System, await first.DeleteAsync(id, Ct)); + } + host.Systems[0] = host.Systems[0] with { Recurrence = ScheduleRecurrence.Parse("every:5m") }; + await using var second = await StartAsync(fx.Db, engine, host, clock); + Assert.Equal(new RecurrenceDto("every", Every: 30, Unit: "minutes"), (await second.GetAsync(id, Ct))!.Recurrence); + + var reset = (await second.ResetAsync(id, Ct)).Schedule!; + Assert.Equal((new RecurrenceDto("every", Every: 5, Unit: "minutes"), T0.AddMinutes(5)), (reset.Recurrence, reset.NextRun!.Value)); + var mine = await CreateAsync(second, ScheduleActions.Orphans); + Assert.Equal("systemKey", (await second.ResetAsync(mine.Id, Ct)).Problem); + Assert.Equal(ScheduleDeleteOutcome.Deleted, await second.DeleteAsync(mine.Id, Ct)); + Assert.Equal(ScheduleDeleteOutcome.NotFound, await second.DeleteAsync(mine.Id, Ct)); + Assert.True((await second.UpdateAsync(mine.Id, new UpdateScheduleRequest(Enabled: false), Ct)).NotFound); + } + + [Test] + public async Task InvalidSchedulesNameTheirField() + { + await using var fx = await ArchiveFixture.CreateAsync(context); + await using var engine = Engine(fx); + await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), new ScheduleClock(T0)); + async Task Problem(CreateScheduleRequest request) => (await scheduler.CreateAsync(request, null, Ct)).Problem; + + Assert.Equal("name", await Problem(new CreateScheduleRequest(" ", ScheduleActions.Scan, Every10))); + Assert.Equal("name", await Problem(new CreateScheduleRequest(new string('n', Scheduler.MaxNameLength + 1), ScheduleActions.Scan, Every10))); + Assert.Equal("action", await Problem(new CreateScheduleRequest("Rebuild", "rebuild", Every10))); + Assert.Equal("recurrence", await Problem(new CreateScheduleRequest("Fast", ScheduleActions.Scan, new RecurrenceDto("every", Every: 2, Unit: "minutes")))); + Assert.Equal("parameters.percent", await Problem(new CreateScheduleRequest("Verify", ScheduleActions.Verify, Every10, new ScheduleParametersDto(Percent: 0)))); + Assert.Equal("parameters.olderThanDays", await Problem(new CreateScheduleRequest("Verify", ScheduleActions.Verify, Every10, new ScheduleParametersDto(OlderThanDays: -1)))); + Assert.Equal("parameters.keep", await Problem(new CreateScheduleRequest("Backup", ScheduleActions.Backup, Every10, new ScheduleParametersDto(Keep: 0)))); + Assert.Equal("parameters.steps", await Problem(new CreateScheduleRequest("Clean", ScheduleActions.Cleanup, Every10, new ScheduleParametersDto(Steps: ["shred"])))); + Assert.Equal("parameters.steps", await Problem(new CreateScheduleRequest("Clean", ScheduleActions.Cleanup, Every10, new ScheduleParametersDto(Steps: [])))); + Assert.Equal("priority", await Problem(new CreateScheduleRequest("Scan", ScheduleActions.Scan, Every10, Priority: "urgent"))); + Assert.Equal("target", await Problem(new CreateScheduleRequest("Scan", ScheduleActions.Scan, Every10, Target: "elsewhere"))); + + // Parameters are stored normalised: the action's own fields, its defaults for those left out. + var verify = (await scheduler.CreateAsync(new CreateScheduleRequest("Verify", ScheduleActions.Verify, Every10, + new ScheduleParametersDto(Percent: 25, DryRun: true, Steps: ["audit"]), Priority: "user"), null, Ct)).Schedule!; + Assert.Equal((new ScheduleParametersDto(OlderThanDays: 30, Percent: 25), "user"), (verify.Parameters, verify.Priority)); + // Another action starts from its own defaults. + var changed = (await scheduler.UpdateAsync(verify.Id, new UpdateScheduleRequest(Action: ScheduleActions.Backup), Ct)).Schedule!; + Assert.Equal(new ScheduleParametersDto(Keep: 3), changed.Parameters); + } + + [Test] + public async Task UnavailableActionsAndAMissingArchiveQueueNothing() + { + await using var fx = await ArchiveFixture.CreateAsync(context); + await using var engine = Engine(fx); + var host = new TestScheduleHost(fx.Archive); + host.Refused[ScheduleActions.Thumbnails] = "not-here"; + await using var scheduler = await StartAsync(fx.Db, engine, host, new ScheduleClock(T0)); + + var actions = (await scheduler.ListAsync(Ct)).Actions.ToDictionary(a => a.Name); + Assert.SequenceEqual(ScheduleActions.All, actions.Keys); + Assert.Equal(new ScheduleActionDto(ScheduleActions.Thumbnails, false, "not-here"), actions[ScheduleActions.Thumbnails]); + Assert.True(actions[ScheduleActions.Backup].Available, "an SQLite index file is backed up"); + Assert.True(actions[ScheduleActions.Proxies].Available); + var thumbnails = await CreateAsync(scheduler, ScheduleActions.Thumbnails); + Assert.Equal(ScheduleOutcome.Unavailable, (await scheduler.RunNowAsync(thumbnails.Id, null, Ct))!.Outcome); + + var proxies = await CreateAsync(scheduler, ScheduleActions.Proxies); + host.Archive = await fx.Service.UpdateSettingsAsync(fx.Archive.Id, s => s with { Proxies = s.Proxies with { LongEdge = ProxyPolicy.Disabled } }, Ct); + Assert.Equal(new ScheduleActionDto(ScheduleActions.Proxies, false, ScheduleUnavailable.ProxiesDisabled), + (await scheduler.ListAsync(Ct)).Actions.Single(a => a.Name == ScheduleActions.Proxies)); + Assert.Equal(ScheduleOutcome.Unavailable, (await scheduler.RunNowAsync(proxies.Id, null, Ct))!.Outcome); + + // No engine handler for the kind: a desktop without the archive engine, a server without the processing module. + await using var bare = new JobEngine(fx.Db, [], new JobEngineOptions { WorkerId = "bare" }); + await using var other = new Scheduler(fx.Db, bare, new SchedulerOptions { Time = new ScheduleClock(T0), Tick = TimeSpan.FromHours(1) }); + await other.StartAsync(new TestScheduleHost(fx.Archive), Ct); + Assert.True((await other.ListAsync(Ct)).Actions.All(a => a is { Available: false, Reason: ScheduleUnavailable.NoHandler })); + + host.Archive = null; + var scan = await CreateAsync(scheduler, ScheduleActions.Scan); + Assert.Equal(ScheduleOutcome.SkippedOffline, (await scheduler.RunNowAsync(scan.Id, null, Ct))!.Outcome); + // The backup and the clean-up need no archive. + var backup = await CreateAsync(scheduler, ScheduleActions.Backup); + Assert.Equal(ScheduleOutcome.Queued, (await scheduler.RunNowAsync(backup.Id, null, Ct))!.Outcome); + var cleanup = await CreateAsync(scheduler, ScheduleActions.Cleanup); + var cleanupRun = (await scheduler.RunNowAsync(cleanup.Id, null, Ct))!; + Assert.Equal(ScheduleOutcome.Queued, cleanupRun.Outcome); + Assert.Empty((await fx.Db.Jobs.GetAsync(cleanupRun.JobId!.Value, Ct))!.TargetIds); + } + + [Test] + public async Task BackupsAreUnavailableWithoutAnIndexFile() + { + await using var db = await ChronosDatabase.OpenAsync(DatabaseOptions.SqliteInMemory(), Ct); + await using var engine = new JobEngine(db, new ArchiveService(db).CreateJobHandlers(), new JobEngineOptions { WorkerId = "memory" }); + await using var scheduler = new Scheduler(db, engine, new SchedulerOptions { Time = new ScheduleClock(T0) }); + await scheduler.StartAsync(new TestScheduleHost(null), Ct); + Assert.Equal(new ScheduleActionDto(ScheduleActions.Backup, false, ScheduleUnavailable.NoIndexFile), + (await scheduler.ListAsync(Ct)).Actions.Single(a => a.Name == ScheduleActions.Backup)); + Assert.Equal(ScheduleOutcome.Unavailable, + (await scheduler.RunNowAsync((await CreateAsync(scheduler, ScheduleActions.Backup)).Id, null, Ct))!.Outcome); + Assert.Equal(ScheduleOutcome.SkippedOffline, + (await scheduler.RunNowAsync((await CreateAsync(scheduler, ScheduleActions.Duplicates)).Id, null, Ct))!.Outcome); + } + + [Test] + public async Task VerifyTakesTheLeastRecentlyVerifiedShareOfTheArchive() + { + await using var fx = await ArchiveFixture.CreateAsync(context); + await using var engine = Engine(fx); + var clock = new ScheduleClock(DateTimeOffset.UtcNow); + await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock); + var ids = new List(); + for (var i = 0; i < 4; i++) ids.Add(await IngestAsync(fx, "s" + i, 10 * i + 1)); + await fx.Db.Sequences.RecordVerifiedAsync(ids[0], SequenceStatus.Verified, SequenceStatus.None, clock.Now.AddDays(-60), Ct); + await fx.Db.Sequences.RecordVerifiedAsync(ids[1], SequenceStatus.Verified, SequenceStatus.None, clock.Now.AddDays(-2), Ct); + + var half = await CreateAsync(scheduler, ScheduleActions.Verify, parameters: new ScheduleParametersDto(OlderThanDays: 30, Percent: 50)); + var job = (await fx.Db.Jobs.GetAsync((await scheduler.RunNowAsync(half.Id, null, Ct))!.JobId!.Value, Ct))!; + var never = ids.Skip(2).Order().ToList(); + Assert.SequenceEqual(never, ArchiveJobJson.Read(job, ArchiveJobJson.Default.VerifyParameters, new VerifyParameters(null)).SequenceIds!.Order()); + await fx.Db.Jobs.SetStateAsync(job.Id, JobState.Done, cancellationToken: Ct); + + var all = await CreateAsync(scheduler, ScheduleActions.Verify, parameters: new ScheduleParametersDto(OlderThanDays: 0, Percent: 100)); + var whole = (await fx.Db.Jobs.GetAsync((await scheduler.RunNowAsync(all.Id, null, Ct))!.JobId!.Value, Ct))!; + Assert.Null(ArchiveJobJson.Read(whole, ArchiveJobJson.Default.VerifyParameters, new VerifyParameters([])).SequenceIds); + await fx.Db.Jobs.SetStateAsync(whole.Id, JobState.Done, cancellationToken: Ct); + + foreach (var id in ids) await fx.Db.Sequences.RecordVerifiedAsync(id, SequenceStatus.Verified, SequenceStatus.None, clock.Now.AddDays(-1), Ct); + Assert.Equal(ScheduleOutcome.SkippedNothing, (await scheduler.RunNowAsync(half.Id, null, Ct))!.Outcome); + } + + [Test] + public async Task ImagePassesCoverTheSequencesWithoutThemOnce() + { + await using var fx = await ArchiveFixture.CreateAsync(context); + await using var engine = Engine(fx); + await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), new ScheduleClock(T0)); + var a = await IngestAsync(fx, "a", 1); + var b = await IngestAsync(fx, "b", 11); + await fx.Db.Sequences.UpdateStatusAsync(b, SequenceStatus.ThumbsBuilt, SequenceStatus.None, Ct); + + var thumbnails = await CreateAsync(scheduler, ScheduleActions.Thumbnails); + var run = (await scheduler.RunNowAsync(thumbnails.Id, null, Ct))!; + var job = (await fx.Db.Jobs.GetAsync(run.JobId!.Value, Ct))!; + Assert.Equal((JobKind.Thumbs, ScheduleOutcome.Queued), (job.Kind, run.Outcome)); + Assert.SequenceEqual([a], job.TargetIds); + // Another schedule finds that work pending. + var again = await CreateAsync(scheduler, ScheduleActions.Thumbnails); + Assert.Equal(ScheduleOutcome.SkippedPending, (await scheduler.RunNowAsync(again.Id, null, Ct))!.Outcome); + var forced = await CreateAsync(scheduler, ScheduleActions.Thumbnails, parameters: new ScheduleParametersDto(Force: true)); + var forcedRun = (await scheduler.RunNowAsync(forced.Id, null, Ct))!; + Assert.Equal(ScheduleOutcome.Queued, forcedRun.Outcome); + var forcedJob = (await fx.Db.Jobs.GetAsync(forcedRun.JobId!.Value, Ct))!; + Assert.SequenceEqual([b], forcedJob.TargetIds); + + var analysis = await CreateAsync(scheduler, ScheduleActions.Analysis); + var light = (await fx.Db.Jobs.GetAsync((await scheduler.RunNowAsync(analysis.Id, null, Ct))!.JobId!.Value, Ct))!; + Assert.Equal(JobKind.LightAnalyse, light.Kind); + Assert.SequenceEqual(new[] { a, b }.Order(), light.TargetIds.Order()); + var deep = await CreateAsync(scheduler, ScheduleActions.Analysis, parameters: new ScheduleParametersDto(Deep: true)); + var deepJob = (await fx.Db.Jobs.GetAsync((await scheduler.RunNowAsync(deep.Id, null, Ct))!.JobId!.Value, Ct))!; + Assert.Equal((JobKind.DeepAnalyse, JobPriority.Background), (deepJob.Kind, deepJob.Priority)); + + // Without the analysis engine (a host that registers only the archive's handlers) analysis is not available. + await using var archiveOnly = new JobEngine(fx.Db, fx.Service.CreateJobHandlers(), new JobEngineOptions { WorkerId = "archive" }); + await using var other = new Scheduler(fx.Db, archiveOnly, new SchedulerOptions { Time = new ScheduleClock(T0), Tick = TimeSpan.FromHours(1) }); + await other.StartAsync(new TestScheduleHost(fx.Archive), Ct); + Assert.Equal(ScheduleOutcome.Unavailable, (await other.RunNowAsync((await CreateAsync(other, ScheduleActions.Analysis)).Id, null, Ct))!.Outcome); + } + + private static async Task IngestAsync(ArchiveFixture fx, string key, int seed) + { + var staged = fx.Stage(ArchiveFixture.Start.AddMinutes(seed), seed); + return (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, key, staged.Path, "IMG_0001.JPG", + staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId; + } +}