From 03d32f18d371d50a035becbc9d12f1fb91a0434c Mon Sep 17 00:00:00 2001 From: Alberto Balbo Date: Tue, 27 Oct 2026 17:36:50 +0100 Subject: [PATCH] feat(projects): workspace projects with linear versions, the local archive as their home Owner review 2, phase 5a (R8): editing moves out of the library into a workspace on a fast disk, and a sequence gets several projects, each with a linear version history pushed and pulled like git. - Core: ProjectSettings keeps members a newer Chronos wrote (Extra) and reads a newer settings format read-only; ProjectContent is the portable form of a version (every frame position parked by photo, canonical text, SHA-256 content hash); ProjectBundle format 2 adds the workspace section, and a file without it is still written as format 1. JobKind.WorkspaceCopy = 20 (19 kept for Convert). - Data: migration 0009 (projects, project_versions, the render's project columns); ProjectService on ArchiveService.Projects keeps projects as sidecars first (.chronos/projects//), then the index, under the sequence lease: create, rename, delete into the projects' trash, history, push with fast-forward, idempotent repeat, divergence and hash checks. Main is made from the sequence's settings and mirrored both ways, so every older code path keeps working; outside writes become "external" versions, a mirror a crash cut short is completed or merged. Full scans and rebuilds take the projects back from their sidecars. - ProjectMerge moves from the App to Data/Projects, merges pending frames photo by photo and replays unpushed commits on a moved head. - Workspaces: WorkspaceStore, WorkspaceSync (checkout, save, commit, push, pull with replay and merge, conflicts settled by keep mine / take server / save both, restore, status) and the WorkspaceCopy job (proxies copied hash-verified and resumable; originals refused until phase 5c). - Renders of a project version place its positions on the frames and record the project and version. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit c4c1802884e403db8759ed0501f770a67a047f20) --- docs/ARCHITECTURE.md | 25 +- i18n/en/Archive.resx | 2 + i18n/en/Errors.resx | 6 + i18n/en/Jobs.resx | 1 + i18n/it/Archive.it.resx | 2 + i18n/it/Errors.it.resx | 6 + i18n/it/Jobs.it.resx | 1 + src/Chronos.App/Editor/EditorSession.cs | 1 + src/Chronos.App/Editor/ISequenceSource.cs | 1 + src/Chronos.App/Editor/LocalSequenceSource.cs | 1 + src/Chronos.App/Editor/ProjectHistory.cs | 1 + src/Chronos.App/Editor/ProjectMerge.cs | 82 --- .../Editor/RemoteSequenceSource.cs | 1 + src/Chronos.Core/Archiving/ArchiveLayout.cs | 35 + src/Chronos.Core/Model/Entities.cs | 121 ++++ src/Chronos.Core/Model/Enums.cs | 6 + src/Chronos.Core/Project/ProjectBundle.cs | 118 +++- src/Chronos.Core/Project/ProjectContent.cs | 145 ++++ src/Chronos.Core/Project/ProjectSettings.cs | 58 +- src/Chronos.Data/Archiving/ArchiveIndexer.cs | 20 + src/Chronos.Data/Archiving/ArchiveService.cs | 8 + src/Chronos.Data/Archiving/SequenceWriter.cs | 24 +- src/Chronos.Data/ChronosDatabase.cs | 4 + .../Migrations/postgres/0009_projects.sql | 46 ++ .../Migrations/sqlite/0009_projects.sql | 46 ++ .../Processing/Jobs/RenderJobHandler.cs | 38 +- src/Chronos.Data/Processing/ProcessingJobs.cs | 35 +- src/Chronos.Data/Projects/IProjectHome.cs | 129 ++++ src/Chronos.Data/Projects/ProjectIndexer.cs | 179 +++++ src/Chronos.Data/Projects/ProjectMerge.cs | 253 +++++++ src/Chronos.Data/Projects/ProjectService.cs | 588 ++++++++++++++++ src/Chronos.Data/Projects/ProjectSidecars.cs | 276 ++++++++ .../Repositories/ProjectRepository.cs | 149 +++++ .../Repositories/RenderRepository.cs | 10 +- .../Workspaces/WorkspaceCopyJobHandler.cs | 340 ++++++++++ src/Chronos.Data/Workspaces/WorkspaceMedia.cs | 137 ++++ src/Chronos.Data/Workspaces/WorkspaceStore.cs | 312 +++++++++ src/Chronos.Data/Workspaces/WorkspaceSync.cs | 625 ++++++++++++++++++ src/Chronos.Protocol/Jobs.cs | 8 +- .../Chronos.App.Tests/Editor/EditorFixture.cs | 1 + .../Editor/EditorTextTests.cs | 1 + .../Editor/ProjectStoreTests.cs | 1 + .../Editor/RemoteSequenceSourceTests.cs | 1 + .../Editor/SavedCopiesTests.cs | 1 + .../Project/ProjectBundleTests.cs | 58 +- .../Project/ProjectContentTests.cs | 165 +++++ .../Migrations/MigrationTests.cs | 64 +- .../Processing/RenderJobTests.cs | 46 ++ .../Projects/ProjectServiceTests.cs | 350 ++++++++++ .../Workspaces/WorkspaceFixture.cs | 108 +++ .../Workspaces/WorkspaceTests.cs | 389 +++++++++++ 51 files changed, 4917 insertions(+), 108 deletions(-) delete mode 100644 src/Chronos.App/Editor/ProjectMerge.cs create mode 100644 src/Chronos.Core/Project/ProjectContent.cs create mode 100644 src/Chronos.Data/Migrations/postgres/0009_projects.sql create mode 100644 src/Chronos.Data/Migrations/sqlite/0009_projects.sql create mode 100644 src/Chronos.Data/Projects/IProjectHome.cs create mode 100644 src/Chronos.Data/Projects/ProjectIndexer.cs create mode 100644 src/Chronos.Data/Projects/ProjectMerge.cs create mode 100644 src/Chronos.Data/Projects/ProjectService.cs create mode 100644 src/Chronos.Data/Projects/ProjectSidecars.cs create mode 100644 src/Chronos.Data/Repositories/ProjectRepository.cs create mode 100644 src/Chronos.Data/Workspaces/WorkspaceCopyJobHandler.cs create mode 100644 src/Chronos.Data/Workspaces/WorkspaceMedia.cs create mode 100644 src/Chronos.Data/Workspaces/WorkspaceStore.cs create mode 100644 src/Chronos.Data/Workspaces/WorkspaceSync.cs create mode 100644 tests/Chronos.Core.Tests/Project/ProjectContentTests.cs create mode 100644 tests/Chronos.Data.Tests/Projects/ProjectServiceTests.cs create mode 100644 tests/Chronos.Data.Tests/Workspaces/WorkspaceFixture.cs create mode 100644 tests/Chronos.Data.Tests/Workspaces/WorkspaceTests.cs diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index e85a506d..66e6243a 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -36,15 +36,15 @@ Dependencies point one way: `Core` ← `Core.Windows`, `Data`, `Protocol` ← `S | `Advice` | `Advisor`: recommendations with localisable reasons, apply/optimise, sequence presets | | `Motion` | FFT, phase correlation, pyramids, optical flow, motion blur, stabiliser, virtual camera, interpolation, stacking | | `Math` (namespace `Chronos.Core.Mathematics`) | Splines, LOWESS, statistics, solar position, 3×3 colour matrices | -| `Project` | `ProjectSettings` (Auto/manual values), export presets, `ProjectBundle` (the `.chronos` project file: sequence reference and settings) | +| `Project` | `ProjectSettings` (Auto/manual values; members a newer Chronos wrote kept in `Extra` and written back; one of a newer settings format read with `FromJson(json, allowNewer: true)`, `IsNewerFormat`: read-only), export presets, `ProjectBundle` (the `.chronos` project file: sequence reference and settings; format 2 adds the `workspace` section of a workspace's working state: project, name, Main, home, base version, media; a file without it is still written as format 1), `ProjectContent` (the portable form of project versions: every frame position parked by photo with `TranslateFramePositions`, placed back on any copy's frames; canonical text, content hash = SHA-256 of its UTF-8 bytes) | | `Render` | Render plan (timing modes), preparation pass, geometry, `RenderPipeline`, self-test suite | | `Export` | Preset resolution, encoder selection with fallbacks, output naming | | `Video` (+ `Mp4`, `Avi`, `Bitstream`, `Yuv`, `Managed`) | `IVideoEncoderProvider`, MP4 muxer and inspector, AVI-MJPEG, NV12/P010, managed encoders (MJPEG-AVI, GIF, APNG, image sequences) | -| `Archiving` | Archive layout and paths, sidecars (`sequence.json`, `project.json`), scanner, index snapshot, leases, thumbnail packs, verify, adopt, rename plans (`RenamePlanner`: folder, title and file templates, `RenameTemplates`, the plan's fingerprint, which pins every file rename by its digest and not the frames version; `FileRenames`: a sequence's new file names; `FileRenameJournal`: two-phase renames through temporary names, `.chronos/rename-files.json`, a recovery safe to run again from wherever it stopped, a new name taken meanwhile sending that photo and its companions back together to their old names or a free `_2`, those that took their new names already with them; `FrameCompanions`: the XMP files that follow a photo, one rule for the trash and Reorganize, matched whatever their case and each folder listed once by `FrameCompanions.Index`) | +| `Archiving` | Archive layout and paths (a sequence's projects: `.chronos/projects//project.json` and `versions/000001-.json`, deleted ones in `projects/.trash`), sidecars (`sequence.json`, `project.json`), scanner, index snapshot, leases, thumbnail packs, verify, adopt, rename plans (`RenamePlanner`: folder, title and file templates, `RenameTemplates`, the plan's fingerprint, which pins every file rename by its digest and not the frames version; `FileRenames`: a sequence's new file names; `FileRenameJournal`: two-phase renames through temporary names, `.chronos/rename-files.json`, a recovery safe to run again from wherever it stopped, a new name taken meanwhile sending that photo and its companions back together to their old names or a free `_2`, those that took their new names already with them; `FrameCompanions`: the XMP files that follow a photo, one rule for the trash and Reorganize, matched whatever their case and each folder listed once by `FrameCompanions.Index`) | | `Import` | Card scan, session detection, import plan (photos named through `FrameNaming` with the camera and lens of their session's first photo), hash-verified copy, card eraser (re-hash before delete) | | `Conversion` | Mass conversion after import (owner review 2, R7): `ConvertOptions` (target JPEG 4:4:4 / TIFF-16 / linear DNG, JPEG quality, the converted name: the archive's file template with the frame's `FrameNaming` context in its sequence, as filing names it, `{ext}` the target's), `FrameConverter` (native-tier decode only, in-house encoders, Exif from the sidecar and the original via `ExifWriter`, hashed as made, written atomically, verified: read back, metadata again, decoded by the server's decoders), `ConvertPlanner` (per sequence the frames converted and skipped with `ConvertSkip`, size estimates per pixel, `ConvertPlan` with its space and fingerprint) | | `Naming` | Folder/file templates and tokens (folder levels scans skip escaped with `_`; `ExpandText` for titles; `ExpandFile` shortens a long file name before its `.{ext}`); `FrameNaming` (the naming context and unique file name of a frame, shared by import, upload, Reorganize and conversion: the sequence's camera and lens, the first frame for folders; `UniqueNames`: `_2`, `_3`… in linear time); `TemplateRules` (what every place a template is typed or sent refuses, the archive engine's new or changed templates included: a folder scans skip, a lost extension, names that repeat) | -| `Model` | Domain entities and enums (§5) shared by Data, Server and clients | +| `Model` | Domain entities and enums (§5) shared by Data, Server and clients (`SequenceProject`, `ProjectVersion` and `ProjectVersionKinds`: owner review 2, R8) | | `Packaging` | TPK1 installer payload writer/reader, staged install transaction | | `Diagnostics` | `Log` (never throws: a line its sink cannot write is dropped), `FileLogSink` with `LogFileOptions` (level, daily or hourly files, numbered parts past a size, pruning by age and by the total size, never the file being written), synthetic sequences, `SecretScrubber`, `DiagnosticsArchive` (scrubbed diagnostics zip) | | `Localization`, `IO`, `Hashing`, `Branding` | `Loc`/plural rules, `AtomicFile`, `StartupCommand` (sign-in entries of the programs, moved or removed by setup), CRC-32/SHA helpers, brand mark geometry | @@ -55,12 +55,14 @@ Dependencies point one way: `Core` ← `Core.Windows`, `Data`, `Protocol` ← `S |---|---| | `Sqlite`, `Postgres` | SQLite P/Invoke binding (winsqlite3 / libsqlite3), in-house PostgreSQL wire protocol v3 client with pool and LISTEN | | `Migrations/{sqlite,postgres}/NNNN_name.sql` | Embedded, applied in one transaction at open (`MigrationRunner`) | -| `Repositories` | `ChronosDatabase` facade: archives, sequences, frames, tags, renders, jobs, cards, users, tokens, uploads, render nodes, settings, audit, schedules (`ScheduleRepository`: system rows seeded without overwriting edits and never deleted, the compare-and-set claim of a run), tombstones of removed photos (`FrameRemovalRepository`, `frame_removals`, written from the sidecar's `removed` list), `SequenceRepository.GetStampAsync` (a digest of what a Reorganize plans from in the index), `RenderRepository.RebaseFolderAsync` (the export paths that follow a moved folder); `DatabaseBackups` (`VACUUM INTO` copies of an SQLite index into `backups` next to it, every copy keeping as many as the backup schedules keep (`KeptAsync`), the newest copy that seeds the `index-backup` schedule (`Newest`, `SystemSchedule.LastDone`), restore staged for the next start and carried out before the index opens, so it also recovers one that no longer opens) | +| `Repositories` | `ChronosDatabase` facade: archives, sequences, frames, tags, renders, jobs, cards, users, tokens, uploads, render nodes, settings, audit, schedules (`ScheduleRepository`: system rows seeded without overwriting edits and never deleted, the compare-and-set claim of a run), tombstones of removed photos (`FrameRemovalRepository`, `frame_removals`, written from the sidecar's `removed` list), `SequenceRepository.GetStampAsync` (a digest of what a Reorganize plans from in the index), `RenderRepository.RebaseFolderAsync` (the export paths that follow a moved folder); projects and their versions (`ProjectRepository`, `projects`, `project_versions`: rows derived from the projects' sidecars, a head advanced by compare-and-set, Main's last mirror); `DatabaseBackups` (`VACUUM INTO` copies of an SQLite index into `backups` next to it, every copy keeping as many as the backup schedules keep (`KeptAsync`), the newest copy that seeds the `index-backup` schedule (`Newest`, `SystemSchedule.LastDone`), restore staged for the next start and carried out before the index opens, so it also recovers one that no longer opens) | | `Jobs` | `JobEngine` (one job at a time; reviews the priority queue and makes a lower running job yield, requeued with its checkpoint; no history purge, the housekeeping schedule's clean-up does it), `JobPriorityQueue` (a level per job kind, the claim order, settings round trip), `IJobHandler`, `IJobHandlerProvider`, `JobContext` (progress, checkpoints, follow-up jobs that never interrupt their parent), `JobThroughput`/`JobEta` (rate and bounded time left) | | `Scheduling` | `Scheduler` (one per host: seeds the host's system schedules, claims due runs by compare-and-set, catch-up, clock changes, create/edit/reset/delete/run now as API v1 DTOs), `ScheduleRecurrence` (the in-house `every:`/`daily:`/`weekly:`/`monthly:` form, wall-clock times across DST), `ScheduleActions` (action names, parameters, availability, the archive jobs a run queues), `IScheduleHost` and `SystemSchedule` (what the server and the desktop supply), `ScheduleOutcome`, `ScheduleMapping` (model ↔ DTO) | | `Housekeeping` | `BackupJobHandler`, `CleanupJobHandler`, `ICleanupStep` with the shared steps (`CleanupSteps`: job history, audit log, trash report that never deletes); hosts add their own steps through `ArchiveServiceOptions.CleanupSteps` | -| `Archiving` | `ArchiveService` (create/adopt/open, archive settings written to `archive.json` then the index, `ArchiveSettings.CheckTemplates` and `TemplateRuleException` holding a template given anew to the template rules; `UnderLeaseAsync`: a write under a sequence's lease with its row read under it), `ArchiveIndexer` (scan, rebuild; a sequence found in another folder takes its exports' paths along), `ArchiveIngest` (uploads and imports, named through `FrameNaming`), `ImportService`, `ArchiveVerifier`, `ArchiveImages` (thumbnail and proxy passes, several photos at once through `OrderedWork`, which hands results back in frame order), `FrameTiers` (decode tier, `PreviewOnly`/`Undecodable` flags and preview size of frames, written to sidecar and index by the light analysis, deep analysis, renders and image passes), `GprCompanions` (the JPEG of each GoPro GPR, found in the index when an import kept it in another sequence), `ArchiveMaintenance`, `MassRenamer` (mass rename and Reorganize: the dry run reads each sidecar for a file template; each step checked under its leases before anything of it moves (where its sequences are, what is inside and at its new folder, a sidecar of a newer format, the title it was planned from when it changes it or names the folder or files after it), then the folder, with the sequences of the plan an earlier step left as they were inside it (taken along, their title and file names kept), every index row (the frame paths and render paths of the exports inside it, a file or an image-sequence folder, its own unless the export is still at the old place and another sequence's when it moved with it, follow it in one transaction; a sidecar that cannot be read or written then still has its row follow, and fails the step), then the files under the plan's journal in `journal/` (a sequence whose photos changed since, such as one still receiving them, or whose files are found changed or in use or have their new names taken keeps its file names, the renames being pinned by digest, and the step and the plan go on); the journal keeps the tally of the whole plan, and a plan its job held (paused, cancelled or failed) is left to that job, only its half-done file renames completed at an archive open, which completes any other plan an interruption left and tells each sequence changed; the file renames of a step: sidecar, a sequence whose sidecar cannot be read staying as it is with every folder around it, index (`FrameRepository.RenameFilesAsync`) and the sequence's journal), `SequenceFolders` (`Holds`: the folder the index has for a sequence is there and its sidecar names no other sequence; every writer into a sequence's folder asks it), `SequenceTrash` (delete into the archive's hidden trash with a dry run, restore; the folder moves back when the index step fails; lists sequence and photo entries; `SequenceTrash.Empty`: the emptying's dry run, fingerprint and erase, each entry checked against its fingerprint again under its lease), `FrameTrash` (photos of a sequence into `trash/photos` with a dry run, a journal that undoes an interrupted deletion whose entry stays unlisted until its sidecar is written, tombstones (a photo never hashed is hashed as it goes; one whose original is missing goes without a file or tombstone; a frame that stays and may be a twin is hashed, its hash kept), positions parked and placed back, proxies and thumbnails moved to follow their frames; `FrameTrash.Restore`: restore re-hashes each file and says why each kept photo stayed (`KeptReason`), under a journal of its own into the sequence that another writer settles only under the entry's lease, into a new sequence through the ingest when theirs is gone, each original staged inside its own entry and the record rewritten after each photo filed; a replaced entry swaps back while its converted frames are unchanged; `FrameTrash.Replace`: the files of frames swapped in place, same ids and indices, the old files into one `Replaced` entry with their replacements' hashes and a copy of `analysis.json`, tombstones aliasing old content to new, under `.chronos/replace.pending.json`, which the next writer undoes or completes), `MassConverter` (conversion's dry run with busy, offline and unreadable sequences and the space against `ArchiveServiceOptions.FreeSpace`; the job's stage A converts into `staging/convert-/` with `OrderedWork`, each staged file appended to `journal/convert-.staged`; stage B per sequence, all or nothing: replace through `FrameTrash.ReplaceAsync` (the sequence's folder read again under its lease, as every writer does), or a new sequence through the ingest with the source's tags, rating, notes and project; each sequence replaced or made told as it completes), job handlers | +| `Archiving` | `ArchiveService` (create/adopt/open, archive settings written to `archive.json` then the index, `ArchiveSettings.CheckTemplates` and `TemplateRuleException` holding a template given anew to the template rules; `UnderLeaseAsync`: a write under a sequence's lease with its row read under it), `ArchiveIndexer` (scan, rebuild; a sequence found in another folder takes its exports' paths along; a full scan or a rebuild takes the projects' sidecars back too), `ArchiveIngest` (uploads and imports, named through `FrameNaming`), `ImportService`, `ArchiveVerifier`, `ArchiveImages` (thumbnail and proxy passes, several photos at once through `OrderedWork`, which hands results back in frame order), `FrameTiers` (decode tier, `PreviewOnly`/`Undecodable` flags and preview size of frames, written to sidecar and index by the light analysis, deep analysis, renders and image passes), `GprCompanions` (the JPEG of each GoPro GPR, found in the index when an import kept it in another sequence), `ArchiveMaintenance`, `MassRenamer` (mass rename and Reorganize: the dry run reads each sidecar for a file template; each step checked under its leases before anything of it moves (where its sequences are, what is inside and at its new folder, a sidecar of a newer format, the title it was planned from when it changes it or names the folder or files after it), then the folder, with the sequences of the plan an earlier step left as they were inside it (taken along, their title and file names kept), every index row (the frame paths and render paths of the exports inside it, a file or an image-sequence folder, its own unless the export is still at the old place and another sequence's when it moved with it, follow it in one transaction; a sidecar that cannot be read or written then still has its row follow, and fails the step), then the files under the plan's journal in `journal/` (a sequence whose photos changed since, such as one still receiving them, or whose files are found changed or in use or have their new names taken keeps its file names, the renames being pinned by digest, and the step and the plan go on); the journal keeps the tally of the whole plan, and a plan its job held (paused, cancelled or failed) is left to that job, only its half-done file renames completed at an archive open, which completes any other plan an interruption left and tells each sequence changed; the file renames of a step: sidecar, a sequence whose sidecar cannot be read staying as it is with every folder around it, index (`FrameRepository.RenameFilesAsync`) and the sequence's journal), `SequenceFolders` (`Holds`: the folder the index has for a sequence is there and its sidecar names no other sequence; every writer into a sequence's folder asks it), `SequenceTrash` (delete into the archive's hidden trash with a dry run, restore; the folder moves back when the index step fails; lists sequence and photo entries; `SequenceTrash.Empty`: the emptying's dry run, fingerprint and erase, each entry checked against its fingerprint again under its lease), `FrameTrash` (photos of a sequence into `trash/photos` with a dry run, a journal that undoes an interrupted deletion whose entry stays unlisted until its sidecar is written, tombstones (a photo never hashed is hashed as it goes; one whose original is missing goes without a file or tombstone; a frame that stays and may be a twin is hashed, its hash kept), positions parked and placed back, proxies and thumbnails moved to follow their frames; `FrameTrash.Restore`: restore re-hashes each file and says why each kept photo stayed (`KeptReason`), under a journal of its own into the sequence that another writer settles only under the entry's lease, into a new sequence through the ingest when theirs is gone, each original staged inside its own entry and the record rewritten after each photo filed; a replaced entry swaps back while its converted frames are unchanged; `FrameTrash.Replace`: the files of frames swapped in place, same ids and indices, the old files into one `Replaced` entry with their replacements' hashes and a copy of `analysis.json`, tombstones aliasing old content to new, under `.chronos/replace.pending.json`, which the next writer undoes or completes), `MassConverter` (conversion's dry run with busy, offline and unreadable sequences and the space against `ArchiveServiceOptions.FreeSpace`; the job's stage A converts into `staging/convert-/` with `OrderedWork`, each staged file appended to `journal/convert-.staged`; stage B per sequence, all or nothing: replace through `FrameTrash.ReplaceAsync` (the sequence's folder read again under its lease, as every writer does), or a new sequence through the ingest with the source's tags, rating, notes and project; each sequence replaced or made told as it completes), job handlers | | `Processing` | `ProcessingService`, `SequenceAnalyser`, `DeepReports` (a saved deep report while it still describes the frames), `SequenceRender`, `ProjectStore`, `ProjectBundles` (a `.chronos` file for a sequence, and the sequence a file names), `RenderNodeClient`, job handlers | +| `Projects` | `ProjectService` (`ArchiveService.Projects`: the projects of this PC's sequences as a home, under the sequence's lease: list, create, rename, delete into `projects/.trash`, history, push with fast-forward / idempotent repeat / divergence / hash and portability checks; each call first catches up: the sidecars into the index (`ProjectIndexer`, also run by full scans and rebuilds; a version file a push left past its record adopted, a stray one moved to `versions/.stray`), Main made from the sequence's settings, settings another path wrote appended to Main as an `external` version, a mirror a crash cut short completed or merged; a push to Main mirrored into `project.json` and the settings with the settings version check), `ProjectSidecars` (record and immutable version files, the settings as a JSON string checked against their hash), `ProjectMerge` (three-way merge member by member, pending frames merged photo by photo, conflicts, `Replay` of local commits on a moved head), `IProjectHome` and `ArchiveProjectHome` (the home a workspace pushes to and pulls from; the server's comes in phase 5b) | +| `Workspaces` | `WorkspaceStore` (a folder outside the library: `workspace.json`, one folder per project, no index), `WorkspaceProject` (`project.chronos` working state, `media/` with `media.json`, `proxies/.jpg` and `analysis.json`, `versions/`, `sync.json` for a restore or a conflict, `exports/`, the lease `.lock.lease`), `WorkspaceSync` (checkout, open, save, commit, push, pull with replay and merge, conflicts settled by keep mine, take server or save both, restore, status), `WorkspaceCopyJobHandler` (`WorkspaceJobs.Copy`, registered by the desktop only: proxies copied hash-verified and resumable, the working state's positions moved by photo onto the new frames) | | `Sync` | `SyncService` (sync table of one server: compare, link by content, settle conflicts, enqueue transfers), `SyncRules` (status, settings merge and differences, no I/O), `SyncTombstones` (photos either side removed are not offered back, a replaced photo counts as its replacement), `SyncPositions` (the settings' frame positions translated by photo to the frames of the copy they go to), `SyncLinkStore` (`sync_links`, `sync_state`), `SyncTransfers` and `SyncJobHandler` (resumable, hash-verified uploads and downloads as jobs; downloads commit through the archive ingest), `SyncAgent` (background comparison, server events, backoff, offline proxies), `ProxyCache` and `RemoteSequence` (server sequences opened with proxies), `ISyncRemote` (the server calls; `ChronosSyncRemote` over `ChronosClient`) | ### 2.3 Chronos.App areas @@ -70,7 +72,7 @@ Dependencies point one way: `Core` ← `Core.Windows`, `Data`, `Protocol` ← `S | `Shell` | `MainWindow` (rail and its shortcuts, taskbar progress), `ShellViewModel` (rail, sections, first run), `OsNotifications` (toasts as Windows notifications while the window is minimised or in the background), dialogs, `ErrorText` (the sentence of an error toast), `ScreenCapture` (`--capture` scenes), `SectionTemplates.xaml` (view model → view) | | `Sections` | One `XxxView.xaml` + `XxxViewModel` per screen: Welcome, Library, Sequence (the Edit section: the editor, and `SequencePickerView` choosing the sequence in place), Import, Archive, Jobs (two tabs, Queue and Scheduled: `SchedulesView` shows `Jobs/SchedulesViewModel`), Server, Settings (its last page is About) | | `Library` | Filter, facets, pager, timeline, saved views, quick and mass actions, remote cards and thumbnails, thumbnail cache, list export | -| `Editor` | `EditorSession` (project, undo/redo, saving, merges), `ISequenceSource` (`LocalSequenceSource`, `RemoteSequenceSource`), the steps of the Edit path (`EditPath`: their order, each step's state, the step of each finding, the memory of the step shown; tabs Analysis, Frames, Exposure, Motion, Camera, Time, Stack, Export), `PreviewRenderer`, `ExportPlanner`, `ExportSync`, `SavedCopies`, `SequenceJobs`, `ProjectFiles` (save and open `.chronos` project files) | +| `Editor` | `EditorSession` (project, undo/redo, saving, merges through `Data/Projects/ProjectMerge`), `ISequenceSource` (`LocalSequenceSource`, `RemoteSequenceSource`), the steps of the Edit path (`EditPath`: their order, each step's state, the step of each finding, the memory of the step shown; tabs Analysis, Frames, Exposure, Motion, Camera, Time, Stack, Export), `PreviewRenderer`, `ExportPlanner`, `ExportSync`, `SavedCopies`, `SequenceJobs`, `ProjectFiles` (save and open `.chronos` project files) | | `Import` | `CardPool` (inserted cards, remembered cards, folders), camera profiles, plans and choices, destinations (archive, folder, server through `ServerImportSink`), card actions (verified delete, format, eject) | | `Archive` | `CurrentArchive`, `ArchiveTree`, figures for the charts, maintenance reports, Reorganize (`ReorganizeDialogViewModel`: folder, file and title templates each with its switch, the dry run, Apply queues the `move` job of that plan; `ReorganizePlans`: each template's check, the plan as rows and a summary; `ReorganizeOffer`: per archive, the templates Settings › Archive's toast offers, joined while that archive's toast is up), Convert (`ConvertDialogViewModel`: format, JPEG quality and a mode chosen at every run, the dry run, Apply behind a typed confirmation for Replace queues the `convert` job of that plan; `ConvertPlans`: the plan as rows, a summary, the space line and the Replace confirmation; `ConvertUndo`: the Undo of a finished Replace, which restores its `Replaced` trash entries), retag, delete and trash dialogs (`SequenceDeletion`, `DeletePlans`, `TrashEmptyPlans`: Empty… behind a dry run and a typed confirmation), `ArchiveFailures` | | `Server` | `ServerSession`, `ConnectFlow` (trust on first use), `SyncHost` (sync engine, agent, metered hold, editor connections, Export Sync), sync table, `RemoteSequenceApi` | @@ -110,7 +112,7 @@ Dependencies point one way: `Core` ← `Core.Windows`, `Data`, `Protocol` ← `S | `Thumbs`, `Proxies` | `Archiving/Jobs/ImageJobHandler` | yes | yes | | `Maintenance` | `Archiving/Jobs/MaintenanceJobHandler` | yes | yes | | `LightAnalyse`, `DeepAnalyse` | `Processing/Jobs/AnalyseJobHandler` (via `ProcessingService.CreateJobHandlers`) | yes | yes | -| `Render` | `Processing/Jobs/RenderJobHandler` | yes | managed formats, and MP4 jobs no render node can take (`RenderNodeAvailability`); render nodes claim the others (`Server/Jobs/RenderNodeService`) | +| `Render` | `Processing/Jobs/RenderJobHandler` (a project version's portable settings placed on the frames first, the render recording the project and the version) | yes | managed formats, and MP4 jobs no render node can take (`RenderNodeAvailability`); render nodes claim the others (`Server/Jobs/RenderNodeService`) | | `UploadCommit`, `Download` | `Sync/SyncJobHandler` (via `SyncService.CreateJobHandlers`): this PC's uploads to and downloads from its server | yes | no: an upload completes synchronously on the server | | `Duplicates`, `Orphans` | `Archiving/Jobs/ReportJobHandlers` (read-only reports into the job log, queued by schedules); the Archive screen still queries `ArchiveService` directly | yes | yes | | `Backup` | `Housekeeping/BackupJobHandler` (via `ArchiveService.CreateJobHandlers`; an SQLite index file only) | yes | SQLite only | @@ -118,6 +120,7 @@ Dependencies point one way: `Core` ← `Core.Windows`, `Data`, `Protocol` ← `S | `EmptyTrash` | `Archiving/Jobs/EmptyTrashJobHandler` (erases exactly a confirmed plan's trash entries; refuses a stale fingerprint) | yes | yes | | `Move` | `Archiving/Jobs/MoveJobHandler` (Reorganize, `MoveJobParameters`: the templates, the sequences and the confirmed fingerprint; plans again from the templates, applies the plan only while it is the one confirmed, holds it when paused, cancelled or failed, keeps it while the archive is offline, tells each sequence changed as its step completes) | yes | yes | | `Convert` | `Archiving/Jobs/ConvertJobHandler` (mass conversion through `MassConverter`: targets are the sequences; plans again, refuses short space, converts each sequence all or nothing, resumes from its journal, queues the passes the frames need again) | yes | yes (formats the server cannot decode natively are skipped: convert on a PC) | +| `WorkspaceCopy` | `Workspaces/WorkspaceCopyJobHandler` (via `WorkspaceJobs.CreateHandlers`: a sequence's proxies and deep report into a workspace project's media; originals refused until phase 5c) | yes | no: it writes into the folder its parameters name | | `Adopt` | none: done synchronously (`ArchiveService` actions) | — | — | A job engine claims only kinds it has handlers for. Light analysis is queued by the engine (`JobContext` follow-up) @@ -163,6 +166,14 @@ folder gone or holding another sequence (`SequenceFolders.Holds`; the server's r upload), and a deep analysis that loses it records nothing. A render whose folder another index moves while it runs is not stopped: its output can land in a folder created again at the old place, where its row keeps finding it. +Projects (owner review 2, R8): a sequence's home (this PC's archive; a server in phase 5b) keeps its projects as sidecars, then +index rows. A workspace on a fast disk checks a project out (its head as the base, the proxies copied by a `WorkspaceCopy` +job), edits the working state, saves versions locally (portable settings, positions parked by photo) and pushes them in +order: a push whose parent is the head fast-forwards; otherwise the home's new versions are pulled, each local version is +replayed on them (`ProjectMerge.Replay`) and the working state merged, and settings both sides changed stop there as a +conflict for the user. Main's head is the sequence's own settings (mirrored both ways), so the editor, analyses, renders +and server sync keep reading `project.json` and the index's settings; other projects never leave their home. + Sync (desktop ↔ server, BRIEF §14.5) reuses the same paths: `SyncService` compares the local index with the server's listing (settings and tags carry version counters), uploads go through the chunked protocol into the server's `ArchiveIngest`, downloaded originals are verified and committed through the local `ArchiveIngest`, and proxies land diff --git a/i18n/en/Archive.resx b/i18n/en/Archive.resx index e456f81b..377ed77b 100644 --- a/i18n/en/Archive.resx +++ b/i18n/en/Archive.resx @@ -254,6 +254,8 @@ Trash not checked: the archive cannot be reachedlong: job log line Old trash entries: {0}, size: {1}; nothing was deletedlong: job log line; entries older than the clean-up's age limit Photos checked: {0}, intact: {1}, hashed: {2}, with problems: {3}long: job log line + {0}: copied {1}, made {2}, unchanged {3}, without a file {4}long: job log line; {1} proxies copied from the archive; {2} made from the originals; {3} already in the workspace; {4} frames with neither + {0}: the frames changed during the copy; copying againlong: job log line Back up database Dry run Rebuild proxies diff --git a/i18n/en/Errors.resx b/i18n/en/Errors.resx index 47e8adbe..5ef37c3f 100644 --- a/i18n/en/Errors.resx +++ b/i18n/en/Errors.resx @@ -21,6 +21,7 @@ Render nodes send back one file per job; post each preset of a batch as its own job.long Render nodes send back one file per job; image sequences render where the archive is. The render job has no valid export settings. + The project version to render is not a version of this sequence. The sequence to render is no longer in the archive. Enter a server address such as nas.local:8443 or https://chronos.example.com. This file does not contain Chronos settings. @@ -47,4 +48,9 @@ Unknown token “{0}” at position {1}. The closing brace at position {1} has no opening one; double it for a literal brace. Something went wrong. + {0} did not match its source after it was copied.{0} file path + The project is no longer in the workspace. + The workspace copies proxies for now; copying the originals comes in a later version. + The workspace project belongs to another sequence. + The workspace copy job has no valid parameters. diff --git a/i18n/en/Jobs.resx b/i18n/en/Jobs.resx index 9c2a5942..f44956db 100644 --- a/i18n/en/Jobs.resx +++ b/i18n/en/Jobs.resx @@ -86,6 +86,7 @@ Thumbnails Upload Verify + Workspace copy after {0} next Paused for a higher-priority job ({0}); resumes by itselflong: job log line diff --git a/i18n/it/Archive.it.resx b/i18n/it/Archive.it.resx index de757e06..bdbf1559 100644 --- a/i18n/it/Archive.it.resx +++ b/i18n/it/Archive.it.resx @@ -248,6 +248,8 @@ Cestino non controllato: l'archivio non è raggiungibile Voci vecchie nel cestino: {0}, dimensione: {1}; nulla è stato eliminato Foto controllate: {0}, integre: {1}, con impronta nuova: {2}, con problemi: {3} + {0}: copiati {1}, creati {2}, invariati {3}, senza file {4} + {0}: i fotogrammi sono cambiati durante la copia; nuova copia Copia del database Simulazione Ricrea proxy diff --git a/i18n/it/Errors.it.resx b/i18n/it/Errors.it.resx index 0574bc79..bf37db7c 100644 --- a/i18n/it/Errors.it.resx +++ b/i18n/it/Errors.it.resx @@ -21,6 +21,7 @@ I nodi di rendering restituiscono un file per job; invia ogni preset di un gruppo come job a sé. I nodi di rendering restituiscono un file per job; le sequenze di immagini si renderizzano dove si trova l'archivio. Il job di rendering non ha impostazioni di esportazione valide. + La versione di progetto da renderizzare non è una versione di questa sequenza. La sequenza da renderizzare non è più nell'archivio. Inserisci l'indirizzo di un server, ad esempio nas.local:8443 o https://chronos.example.com. Questo file non contiene impostazioni di Chronos. @@ -47,4 +48,9 @@ Segnaposto sconosciuto «{0}» alla posizione {1}. La parentesi graffa chiusa alla posizione {1} non ha un'apertura; raddoppiala per una graffa letterale. Si è verificato un problema. + {0} non corrispondeva all'originale dopo la copia. + Il progetto non è più nello spazio di lavoro. + Per ora lo spazio di lavoro copia i proxy; la copia degli originali arriverà in una versione successiva. + Il progetto dello spazio di lavoro appartiene a un'altra sequenza. + Il job di copia di lavoro non ha parametri validi. diff --git a/i18n/it/Jobs.it.resx b/i18n/it/Jobs.it.resx index 9bc5ce93..151e2e1b 100644 --- a/i18n/it/Jobs.it.resx +++ b/i18n/it/Jobs.it.resx @@ -86,6 +86,7 @@ Miniature Caricamento Verifica + Copia di lavoro dopo {0} prossima In pausa per un'attività con priorità più alta ({0}); riprende da sola diff --git a/src/Chronos.App/Editor/EditorSession.cs b/src/Chronos.App/Editor/EditorSession.cs index 05c57ba9..aab5ae05 100644 --- a/src/Chronos.App/Editor/EditorSession.cs +++ b/src/Chronos.App/Editor/EditorSession.cs @@ -5,6 +5,7 @@ using Chronos.Core.Project; using Chronos.Core.Video; using Chronos.Data.Archiving.Jobs; using Chronos.Data.Processing; +using Chronos.Data.Projects; namespace Chronos.App.Editor; diff --git a/src/Chronos.App/Editor/ISequenceSource.cs b/src/Chronos.App/Editor/ISequenceSource.cs index 2ecd873f..46be0566 100644 --- a/src/Chronos.App/Editor/ISequenceSource.cs +++ b/src/Chronos.App/Editor/ISequenceSource.cs @@ -4,6 +4,7 @@ using Chronos.Core.Imaging; using Chronos.Core.Model; using Chronos.Core.Project; using Chronos.Data.Archiving.Jobs; +using Chronos.Data.Projects; namespace Chronos.App.Editor; diff --git a/src/Chronos.App/Editor/LocalSequenceSource.cs b/src/Chronos.App/Editor/LocalSequenceSource.cs index aaa07f4b..8beecae7 100644 --- a/src/Chronos.App/Editor/LocalSequenceSource.cs +++ b/src/Chronos.App/Editor/LocalSequenceSource.cs @@ -1,3 +1,4 @@ +using Chronos.Data.Projects; using System.IO; using System.Text.Json; using Chronos.Core.Advice; diff --git a/src/Chronos.App/Editor/ProjectHistory.cs b/src/Chronos.App/Editor/ProjectHistory.cs index 1c7d2f9c..de7d5ca8 100644 --- a/src/Chronos.App/Editor/ProjectHistory.cs +++ b/src/Chronos.App/Editor/ProjectHistory.cs @@ -1,4 +1,5 @@ using Chronos.Core.Project; +using Chronos.Data.Projects; namespace Chronos.App.Editor; diff --git a/src/Chronos.App/Editor/ProjectMerge.cs b/src/Chronos.App/Editor/ProjectMerge.cs deleted file mode 100644 index 71424a48..00000000 --- a/src/Chronos.App/Editor/ProjectMerge.cs +++ /dev/null @@ -1,82 +0,0 @@ -using System.Text.Json.Nodes; -using Chronos.Core.Project; -using Chronos.Data.Sync; - -namespace Chronos.App.Editor; - -/// -/// Three-way merge of projects: the editor's edit (ours) and a change stored meanwhile by someone else -/// (theirs: an analysis filling Auto values, another client through the server), both made from basis. -/// Values are compared member by member down the JSON tree; a member only one side changed takes that side's value, a -/// member both changed takes the editor's (the user is looking at it). lists those members for -/// a source that asks the user instead (a server sequence, BRIEF §14.5). -/// -public static class ProjectMerge -{ - public static ProjectSettings Merge(ProjectSettings basis, ProjectSettings ours, ProjectSettings theirs) - { - ArgumentNullException.ThrowIfNull(basis); - ArgumentNullException.ThrowIfNull(ours); - ArgumentNullException.ThrowIfNull(theirs); - return new Rebase(basis, theirs).Apply(ours); - } - - /// - /// The members both sides changed to different values, in document order: the path below the project - /// ("exposure.deflickerWindow.manual"), the editor's value and theirs as JSON (null when absent). - /// - public static IReadOnlyList Conflicts(ProjectSettings basis, ProjectSettings ours, ProjectSettings theirs) - { - ArgumentNullException.ThrowIfNull(basis); - ArgumentNullException.ThrowIfNull(ours); - ArgumentNullException.ThrowIfNull(theirs); - var conflicts = new List(); - Collect(JsonNode.Parse(basis.ToJson()), JsonNode.Parse(ours.ToJson()), JsonNode.Parse(theirs.ToJson()), string.Empty, conflicts); - return conflicts; - } - - /// True when both projects store the same values. - public static bool SameValues(ProjectSettings a, ProjectSettings b) => a.ToJson() == b.ToJson(); - - private static JsonNode? Merge(JsonNode? basis, JsonNode? ours, JsonNode? theirs) - { - if (JsonNode.DeepEquals(ours, basis)) return theirs?.DeepClone(); - if (JsonNode.DeepEquals(theirs, basis) || JsonNode.DeepEquals(theirs, ours)) return ours?.DeepClone(); - if (ours is not JsonObject mine || theirs is not JsonObject other) return ours?.DeepClone(); - var common = basis as JsonObject; - var merged = new JsonObject(); - foreach (var name in mine.Select(p => p.Key).Union(other.Select(p => p.Key))) - { - var value = Merge(common?[name], mine[name], other[name]); - if (value is not null || mine.ContainsKey(name)) merged[name] = value; - } - return merged; - } - - /// The members settles for the editor although both sides changed them. - private static void Collect(JsonNode? basis, JsonNode? ours, JsonNode? theirs, string path, List conflicts) - { - if (JsonNode.DeepEquals(ours, basis) || JsonNode.DeepEquals(theirs, basis) || JsonNode.DeepEquals(theirs, ours)) return; - if (ours is JsonObject mine && theirs is JsonObject other) - { - var common = basis as JsonObject; - foreach (var name in mine.Select(p => p.Key).Union(other.Select(p => p.Key))) - Collect(common?[name], mine[name], other[name], path.Length == 0 ? name : path + "." + name, conflicts); - return; - } - conflicts.Add(new SettingsDifference(path.Length == 0 ? "settings" : path, ours?.ToJsonString(), theirs?.ToJsonString())); - } - - /// One outside change ( → ) applied to any number of projects made from the basis. - public sealed class Rebase(ProjectSettings basis, ProjectSettings theirs) - { - private readonly JsonNode? _basis = JsonNode.Parse(basis.ToJson()); - private readonly JsonNode? _theirs = JsonNode.Parse(theirs.ToJson()); - - public ProjectSettings Apply(ProjectSettings ours) - { - ArgumentNullException.ThrowIfNull(ours); - return ProjectSettings.FromJson(Merge(_basis, JsonNode.Parse(ours.ToJson()), _theirs)?.ToJsonString() ?? "{}"); - } - } -} diff --git a/src/Chronos.App/Editor/RemoteSequenceSource.cs b/src/Chronos.App/Editor/RemoteSequenceSource.cs index f8720a5f..48b63756 100644 --- a/src/Chronos.App/Editor/RemoteSequenceSource.cs +++ b/src/Chronos.App/Editor/RemoteSequenceSource.cs @@ -1,3 +1,4 @@ +using Chronos.Data.Projects; using System.Collections.Concurrent; using System.IO; using System.Net; diff --git a/src/Chronos.Core/Archiving/ArchiveLayout.cs b/src/Chronos.Core/Archiving/ArchiveLayout.cs index 5c1959a6..a5e23cc4 100644 --- a/src/Chronos.Core/Archiving/ArchiveLayout.cs +++ b/src/Chronos.Core/Archiving/ArchiveLayout.cs @@ -42,6 +42,22 @@ public static class ArchiveLayout /// Moves of a frame replacement (a conversion, or its undo) that were started but not yet settled in and the trash. public const string ReplaceJournalFile = "replace.pending.json"; + /// + /// The projects of a sequence (owner review 2, R8) inside its : one folder per project, named by + /// its id, with and the immutable versions in . + /// + public const string ProjectsFolder = "projects"; + + /// The versions of a project, one file each: 000001-<id>.json. + public const string ProjectVersionsFolder = "versions"; + + /// Deleted projects inside : kept, never erased by Chronos. + public const string ProjectsTrashFolder = ".trash"; + + /// The file name of version of a project: 000001-<id>.json. + public static string ProjectVersionFileName(int number, Guid versionId) => + number.ToString("000000", CultureInfo.InvariantCulture) + "-" + versionId.ToString("N") + ".json"; + /// Folder names inside a sequence folder that never hold a sequence of their own. public static bool IsReservedFolder(string name) => name.Equals(MetaFolder, StringComparison.OrdinalIgnoreCase) @@ -115,6 +131,25 @@ public sealed class SequencePaths public string Proxy(int index) => Path.Combine(Proxies, ArchiveLayout.ProxyFileName(index)); + /// The projects of the sequence (owner review 2, R8): .chronos/projects. + public string ProjectsDirectory => Path.Combine(MetaDirectory, ArchiveLayout.ProjectsFolder); + + /// Where deleted projects are kept: .chronos/projects/.trash. + public string ProjectsTrashDirectory => Path.Combine(ProjectsDirectory, ArchiveLayout.ProjectsTrashFolder); + + /// The folder of one project: .chronos/projects/<id>. + public string ProjectDirectory(Guid projectId) => Path.Combine(ProjectsDirectory, projectId.ToString("N")); + + /// The record of one project (its name, whether it is Main, its head): .chronos/projects/<id>/project.json. + public string ProjectRecordPath(Guid projectId) => Path.Combine(ProjectDirectory(projectId), ArchiveLayout.ProjectFile); + + /// The versions of one project: .chronos/projects/<id>/versions. + public string ProjectVersionsDirectory(Guid projectId) => Path.Combine(ProjectDirectory(projectId), ArchiveLayout.ProjectVersionsFolder); + + /// One version of a project: .chronos/projects/<id>/versions/000001-<version id>.json. + public string VersionPath(Guid projectId, int number, Guid versionId) => + Path.Combine(ProjectVersionsDirectory(projectId), ArchiveLayout.ProjectVersionFileName(number, versionId)); + /// Absolute path of a frame file given its sidecar path (relative to the sequence folder). public string File(string sequenceRelative) => LongPath.FromPortableRelative(Folder, sequenceRelative); diff --git a/src/Chronos.Core/Model/Entities.cs b/src/Chronos.Core/Model/Entities.cs index 5873e098..8416b7ce 100644 --- a/src/Chronos.Core/Model/Entities.cs +++ b/src/Chronos.Core/Model/Entities.cs @@ -158,6 +158,127 @@ public sealed record Render public long Bytes { get; init; } public DateTimeOffset Created { get; init; } public string? ProducedBy { get; init; } + + /// The project whose version was rendered (owner review 2, R8); null for a render of the sequence's settings. + public Guid? ProjectId { get; init; } + + /// The project version rendered; null for a render of the sequence's settings. + public Guid? ProjectVersionId { get; init; } +} + +/// +/// A project of a sequence (owner review 2, R8): a named set of render settings ("4K landscape", "Vertical social") with a +/// linear history of versions. Every sequence has one Main project, whose head is the sequence's own settings +/// (). Its sidecar (.chronos/projects/<id>/project.json) is the truth; the +/// index row is derived. +/// +public sealed record SequenceProject +{ + public required Guid Id { get; init; } + public required Guid SequenceId { get; init; } + public required string Name { get; init; } + + /// The sequence's Main project: its head is mirrored into the sequence's settings and drives its hero frame. + public bool IsMain { get; init; } + + /// The newest version; null before the first. + public Guid? HeadVersionId { get; init; } + + /// The newest version's number (1, 2, …); 0 before the first. + public int HeadNumber { get; init; } + + /// Main only: the sequence's settings version its head was last mirrored to or taken from. + public int? MirroredSettingsVersion { get; init; } + + /// + /// Main only: the version last mirrored to or taken from the sequence's settings; another than the head while a mirror + /// is to complete (a push a crash cut short), null when unknown (an index rebuilt from the sidecars). + /// + public Guid? MirroredVersionId { get; init; } + + public string? CreatedBy { get; init; } + public DateTimeOffset CreatedAt { get; init; } + public DateTimeOffset UpdatedAt { get; init; } + + /// When the project was deleted (its folder moved to .chronos/projects/.trash); null while it exists. + public DateTimeOffset? DeletedAt { get; init; } +} + +/// How a project version came to be (stored as its wire name, ). +public enum ProjectVersionKind +{ + /// Saved by a user in a workspace. + Commit, + + /// A working state merged with versions stored meanwhile. + Merge, + + /// An earlier version brought back (). + Restore, + + /// The first version of a project made from settings that were there (Main from the sequence's settings, a copy of another project). + Imported, + + /// Main only: settings written by a path outside the projects (an analysis, the server's API, an older Chronos). + External, +} + +/// Wire names of : commit, merge, restore, imported, external. +public static class ProjectVersionKinds +{ + public static string Name(ProjectVersionKind kind) => kind.ToString().ToLowerInvariant(); + + /// The kind a wire name stands for; for one this build does not know. + public static ProjectVersionKind Parse(string? name) + { + foreach (var kind in Enum.GetValues()) + { + if (string.Equals(Name(kind), name, StringComparison.OrdinalIgnoreCase)) return kind; + } + return ProjectVersionKind.Commit; + } +} + +/// +/// One version of a project (owner review 2, R8): immutable once stored. is the portable form +/// of the settings (frame positions parked by photo, ProjectContent) and the SHA-256 of +/// its UTF-8 bytes, checked wherever it is stored or read. +/// +public sealed record ProjectVersion +{ + public required Guid Id { get; init; } + public required Guid ProjectId { get; init; } + + /// The version it was made from; null for a project's first. + public Guid? ParentId { get; init; } + + /// 1 for the first version of a project, then one more for each. + public required int Number { get; init; } + + public ProjectVersionKind Kind { get; init; } + public string? Author { get; init; } + + /// The program and PC it was made on ("Chronos on STUDIO-PC"). + public string? ClientName { get; init; } + + public string? Note { get; init; } + public required string SettingsJson { get; init; } + public required string ContentHash { get; init; } + + /// The settings format (ProjectSettings.Version) of ; a newer one opens read-only. + public int SettingsFormat { get; init; } = 1; + + /// The frames version of the sequence the settings were made against. + public int FramesVersion { get; init; } + + /// For a : the version brought back. + public Guid? RestoredFrom { get; init; } + + /// When it was made (in a workspace). + public DateTimeOffset CreatedAt { get; init; } + + /// When its home stored it; null for a workspace's version not pushed yet. + public DateTimeOffset? StoredAt { get; init; } } /// Any long operation: resumable, cancellable, logged (BRIEF §5, §14.6). diff --git a/src/Chronos.Core/Model/Enums.cs b/src/Chronos.Core/Model/Enums.cs index 15e6e0f8..54612451 100644 --- a/src/Chronos.Core/Model/Enums.cs +++ b/src/Chronos.Core/Model/Enums.cs @@ -99,6 +99,12 @@ public enum JobKind /// confirmed (owner review 2, R7); stored as 19. /// Convert = 19, + + /// + /// Copies a sequence's proxies (and its deep report) from its home into a workspace project's media folder, hash-verified + /// and resumable (owner review 2, R8); stored as 20. + /// + WorkspaceCopy = 20, } /// Why a photo left a sequence (its tombstone, owner review 2, R6); stored as a number. diff --git a/src/Chronos.Core/Project/ProjectBundle.cs b/src/Chronos.Core/Project/ProjectBundle.cs index 0b387165..6782811f 100644 --- a/src/Chronos.Core/Project/ProjectBundle.cs +++ b/src/Chronos.Core/Project/ProjectBundle.cs @@ -12,18 +12,63 @@ public sealed record BundleArchive(Guid Id, string Name, string? Path); /// The sequence a project file applies to: its id, folder (relative to the archive), title and frame count when saved. public sealed record BundleSequence(Guid Id, string Folder, string? Title, int Frames); +/// +/// Where a workspace project's versions live (owner review 2, R8): this PC's archive (, with the +/// archive's id) or a server (, with its address), and the sequence there. +/// +public sealed record BundleHome(string Kind, Guid SequenceId, Guid? ArchiveId = null, string? Server = null) +{ + public const string ArchiveKind = "archive"; + public const string ServerKind = "server"; + + public bool IsArchive => Kind == ArchiveKind; +} + +/// The version of the home a workspace project was last in step with: its id, number and content hash. +public sealed record BundleBase(Guid VersionId, int Number, string ContentHash); + +/// What a workspace holds of the sequence's photos: (the default) or , at a frames version of the home. +public sealed record BundleMedia(string Kind, int FramesVersion) +{ + public const string Proxies = "proxies"; + public const string Originals = "originals"; +} + +/// +/// The workspace section of a project file (format 2, owner review 2, R8): the project of the home it belongs to, its +/// name, whether it is the sequence's Main project, the home, the version it was last in step with (none before its +/// first pull) and the media the workspace holds. +/// +public sealed record BundleWorkspace(Guid ProjectId, string Name, bool IsMain, BundleHome Home, BundleBase? Base, BundleMedia Media); + /// /// A .chronos project file (BRIEF §18, file associations): which sequence of which archive, and the project /// settings to open it with. The photos stay in the archive; opening the file opens that sequence in the editor with /// these settings. UTF-8 JSON with a format name and version, so a newer file is refused rather than misread: /// { "format": "chronos-project", "formatVersion": 1, "saved": …, "archive": { "id", "name", "path" }, /// "sequence": { "id", "folder", "title", "frames" }, "project": { …project.json… } } +/// Format 2 (owner review 2, R8) adds the section of a workspace's working state: +/// "workspace": { "projectId", "name", "isMain", "home": { "kind": "archive"|"server", "server"?, "archiveId"?, "sequenceId" }, +/// "base": { "versionId", "number", "contentHash" }?, "media": { "kind": "proxies"|"originals", "framesVersion" } } +/// A file without that section is still written as format 1, so earlier versions of Chronos open it. A format 2 file +/// opens a project of a newer settings format too, read-only (). /// public sealed record ProjectBundle(BundleArchive Archive, BundleSequence Sequence, ProjectSettings Project, DateTimeOffset Saved) { public const string Extension = ".chronos"; public const string FormatName = "chronos-project"; - public const int CurrentFormatVersion = 1; + + /// The newest format this Chronos reads and writes (2: with a workspace section). + public const int CurrentFormatVersion = 2; + + /// The format of a file without a workspace section. + public const int PlainFormatVersion = 1; + + /// The workspace section (format 2); null for a plain project file. + public BundleWorkspace? Workspace { get; init; } + + /// The settings come from a newer Chronos (): show them, never edit them. + public bool IsReadOnly => Project.IsNewerFormat; /// Project settings take a few kilobytes; a larger file is not a project file. public const int MaxBytes = 4 << 20; @@ -37,7 +82,7 @@ public sealed record ProjectBundle(BundleArchive Archive, BundleSequence Sequenc { w.WriteStartObject(); w.WriteString("format", FormatName); - w.WriteNumber("formatVersion", CurrentFormatVersion); + w.WriteNumber("formatVersion", Workspace is null ? PlainFormatVersion : CurrentFormatVersion); w.WriteString("saved", Saved); w.WriteStartObject("archive"); w.WriteString("id", Archive.Id); @@ -50,6 +95,7 @@ public sealed record ProjectBundle(BundleArchive Archive, BundleSequence Sequenc if (Sequence.Title is not null) w.WriteString("title", Sequence.Title); w.WriteNumber("frames", Sequence.Frames); w.WriteEndObject(); + if (Workspace is { } workspace) WriteWorkspace(w, workspace); w.WritePropertyName("project"); // The project as project.json holds it, so both are read by the same tolerant reader. using (var project = JsonDocument.Parse(Project.ToJson())) project.RootElement.WriteTo(w); @@ -89,7 +135,8 @@ public sealed record ProjectBundle(BundleArchive Archive, BundleSequence Sequenc ProjectSettings settings; try { - settings = ProjectSettings.FromJson(project.GetRawText()); + // A format 1 file refuses newer settings as it always did; a workspace's opens them read-only. + settings = ProjectSettings.FromJson(project.GetRawText(), allowNewer: version >= 2); } catch (JsonException ex) { @@ -100,7 +147,10 @@ public sealed record ProjectBundle(BundleArchive Archive, BundleSequence Sequenc new BundleArchive(Id(archive), Text(archive, "name") ?? string.Empty, Text(archive, "path")), new BundleSequence(Id(sequence), folder, Text(sequence, "title"), frames), settings, - saved); + saved) + { + Workspace = version >= 2 && root.TryGetProperty("workspace", out var workspace) ? ReadWorkspace(workspace) : null, + }; } } @@ -151,6 +201,66 @@ public sealed record ProjectBundle(BundleArchive Archive, BundleSequence Sequenc return path; } + private static void WriteWorkspace(Utf8JsonWriter w, BundleWorkspace workspace) + { + w.WriteStartObject("workspace"); + w.WriteString("projectId", workspace.ProjectId); + w.WriteString("name", workspace.Name); + w.WriteBoolean("isMain", workspace.IsMain); + w.WriteStartObject("home"); + w.WriteString("kind", workspace.Home.Kind); + if (workspace.Home.Server is { } server) w.WriteString("server", server); + if (workspace.Home.ArchiveId is { } archiveId) w.WriteString("archiveId", archiveId); + w.WriteString("sequenceId", workspace.Home.SequenceId); + w.WriteEndObject(); + if (workspace.Base is { } basis) + { + w.WriteStartObject("base"); + w.WriteString("versionId", basis.VersionId); + w.WriteNumber("number", basis.Number); + w.WriteString("contentHash", basis.ContentHash); + w.WriteEndObject(); + } + w.WriteStartObject("media"); + w.WriteString("kind", workspace.Media.Kind); + w.WriteNumber("framesVersion", workspace.Media.FramesVersion); + w.WriteEndObject(); + w.WriteEndObject(); + } + + /// The section is not a workspace section this Chronos can use. + private static BundleWorkspace ReadWorkspace(JsonElement section) + { + if (section.ValueKind != JsonValueKind.Object) throw new InvalidDataException("The project file's workspace section is not an object."); + var projectId = OptionalId(section, "projectId") ?? throw new InvalidDataException("The workspace section names no project."); + var home = Section(section, "home"); + var kind = Text(home, "kind"); + if (kind is not (BundleHome.ArchiveKind or BundleHome.ServerKind)) throw new InvalidDataException($"The workspace's home is of an unknown kind '{kind}'."); + var sequenceId = OptionalId(home, "sequenceId") ?? throw new InvalidDataException("The workspace's home names no sequence."); + BundleBase? basis = null; + if (section.TryGetProperty("base", out var b) && b.ValueKind == JsonValueKind.Object) + { + basis = new BundleBase( + OptionalId(b, "versionId") ?? throw new InvalidDataException("The workspace's base names no version."), + Number(b, "number"), + Text(b, "contentHash") ?? throw new InvalidDataException("The workspace's base has no content hash.")); + } + var media = section.TryGetProperty("media", out var m) && m.ValueKind == JsonValueKind.Object + ? new BundleMedia(Text(m, "kind") ?? BundleMedia.Proxies, Number(m, "framesVersion")) + : new BundleMedia(BundleMedia.Proxies, 0); + return new BundleWorkspace(projectId, Text(section, "name") ?? string.Empty, + section.TryGetProperty("isMain", out var main) && main.ValueKind == JsonValueKind.True, + new BundleHome(kind, sequenceId, OptionalId(home, "archiveId"), Text(home, "server")), basis, media); + } + + private static Guid? OptionalId(JsonElement section, string name) => + section.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.String && value.TryGetGuid(out var guid) && guid != Guid.Empty + ? guid + : null; + + private static int Number(JsonElement section, string name) => + section.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.Number && value.TryGetInt32(out var number) && number >= 0 ? number : 0; + private static JsonElement Section(JsonElement root, string name) { if (!root.TryGetProperty(name, out var section) || section.ValueKind != JsonValueKind.Object) diff --git a/src/Chronos.Core/Project/ProjectContent.cs b/src/Chronos.Core/Project/ProjectContent.cs new file mode 100644 index 00000000..02b654cb --- /dev/null +++ b/src/Chronos.Core/Project/ProjectContent.cs @@ -0,0 +1,145 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace Chronos.Core.Project; + +/// +/// The portable form of a project (owner review 2, R8): the settings with every frame position (exclusions, hero, camera +/// keyframes) parked by photo as , so the same text means the same thing on +/// every copy of the sequence (the archive, a workspace's proxies, a server) whatever frames were inserted or removed +/// since. Project versions store this form; a copy places the positions on its own frames when it opens one +/// (). The text is canonical (), so equal settings give equal text and an +/// equal . +/// +public static class ProjectContent +{ + private static readonly FramePhotos NoFrames = new([]); + + /// + /// The portable form of , whose positions name the frames + /// (by index). A position naming no frame of names nothing and goes. A project of a newer + /// Chronos is only made canonical: its positions are its own business. + /// + /// The text is not a project object. + public static string ToPortable(string settingsJson, FramePhotos frames) + { + ArgumentNullException.ThrowIfNull(frames); + var canonical = Canonical(settingsJson); + if (FormatOf(canonical) > ProjectSettings.CurrentVersion) return canonical; + // With no frames there is nothing to name: a translation to no frames would keep the positions as they are. + var translated = frames.Count == 0 ? WithoutPositions(canonical) : ProjectSettings.TranslateFramePositions(canonical, frames, NoFrames); + return translated is null ? canonical : Canonical(translated); + } + + /// + public static string ToPortable(ProjectSettings settings, FramePhotos frames) + { + ArgumentNullException.ThrowIfNull(settings); + return ToPortable(settings.ToJson(), frames); + } + + /// + /// The settings of a portable project with its positions placed on : each pending frame whose + /// photo a frame holds becomes a position again; those whose photo is not there keep waiting. + /// + /// The text is not a project object. + public static string Place(string portableJson, FramePhotos frames) + { + ArgumentNullException.ThrowIfNull(frames); + ArgumentNullException.ThrowIfNull(portableJson); + if (FormatOf(portableJson) > ProjectSettings.CurrentVersion) return portableJson; + return ProjectSettings.TranslateFramePositions(portableJson, null, frames) ?? portableJson; + } + + /// + /// The canonical text of a project: read (a newer Chronos's too, its unknown members kept) and written again, its + /// pending frames in a fixed order (by photo) and their capture times as wall-clock values. + /// + /// The text is not a project object. + public static string Canonical(string json) + { + ArgumentNullException.ThrowIfNull(json); + ProjectSettings settings; + try + { + settings = ProjectSettings.FromJson(json, allowNewer: true); + } + catch (InvalidDataException ex) + { + throw new JsonException(ex.Message, ex); + } + if (settings.PendingFrames is { Count: > 0 } pending) + { + settings = settings with { PendingFrames = [.. pending.Select(Normalized).OrderBy(Key, StringComparer.Ordinal)] }; + } + return settings.ToJson(); + } + + /// The SHA-256 (lower-case hex) of the UTF-8 bytes of : a project version's content hash. + public static string Hash(string text) + { + ArgumentNullException.ThrowIfNull(text); + return Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(text))); + } + + /// Whether is the content hash of (any case). + public static bool Verifies(string text, string? hash) => hash is { Length: 64 } && string.Equals(Hash(text), hash, StringComparison.OrdinalIgnoreCase); + + /// The settings format () a project's text declares; 1 when it says none or is not a project. + public static int FormatOf(string? json) + { + if (string.IsNullOrWhiteSpace(json)) return ProjectSettings.CurrentVersion; + try + { + return JsonNode.Parse(json, new JsonNodeOptions { PropertyNameCaseInsensitive = true }) is JsonObject project + && project["version"] is JsonValue value && value.TryGetValue(out var version) && version > 0 + ? version + : ProjectSettings.CurrentVersion; + } + catch (Exception ex) when (ex is JsonException or ArgumentException) + { + return ProjectSettings.CurrentVersion; + } + } + + /// + /// A pending frame as the canonical text holds it: its capture time as a wall-clock value, and the frame its keyframes + /// were on left out (0: it is set again when the photo takes its place). + /// + private static PendingFrame Normalized(PendingFrame frame) => frame with + { + CapturedAt = frame.CapturedAt is { } at ? DateTime.SpecifyKind(at, DateTimeKind.Unspecified) : null, + // Read from a hand-written entry without keyframes, the list is null. + Keyframes = frame.Keyframes is { Count: > 0 } keys ? [.. keys.Select(k => k with { Frame = 0 })] : [], + }; + + /// Order of pending frames in the canonical text: by hash, then capture time, size and which copy of the photo. + private static string Key(PendingFrame frame) => string.Create(CultureInfo.InvariantCulture, + $"{frame.Sha256?.ToLowerInvariant()}|{frame.CapturedAt?.Ticks:D20}|{frame.Size:D20}|{frame.Occurrence:D10}"); + + /// A project of a sequence without frames: its positions name nothing. + private static string? WithoutPositions(string json) + { + if (JsonNode.Parse(json) is not JsonObject project) return null; + var changed = false; + if (project["excludedFrames"] is JsonArray { Count: > 0 } excluded) + { + excluded.Clear(); + changed = true; + } + if (project["heroFrame"] is not null) + { + project.Remove("heroFrame"); + changed = true; + } + if (project["camera"] is JsonObject camera && camera["keyframes"] is JsonArray { Count: > 0 } keys) + { + keys.Clear(); + changed = true; + } + return changed ? project.ToJsonString() : null; + } +} diff --git a/src/Chronos.Core/Project/ProjectSettings.cs b/src/Chronos.Core/Project/ProjectSettings.cs index bb97d536..f6eec582 100644 --- a/src/Chronos.Core/Project/ProjectSettings.cs +++ b/src/Chronos.Core/Project/ProjectSettings.cs @@ -78,6 +78,10 @@ public sealed record PendingFrame [JsonIgnore] public FramePhoto Photo => new(Sha256, Size, CapturedAt); + + /// Members this Chronos does not know (written by a newer one): kept as they are, so they survive a round trip. + [JsonExtensionData] + public Dictionary? Extra { get; set; } } public sealed record TimingSettings @@ -91,6 +95,10 @@ public sealed record TimingSettings public double? FitToSeconds { get; init; } public IReadOnlyList RemapCurve { get; init; } = [new(0, 0), new(1, 1)]; + + /// Members this Chronos does not know (written by a newer one): kept as they are, so they survive a round trip. + [JsonExtensionData] + public Dictionary? Extra { get; set; } } public sealed record ExposureSettings @@ -119,6 +127,10 @@ public sealed record ExposureSettings /// Written even when null: a missing value reads back as the default knee, which would turn "off" on. [JsonIgnore(Condition = JsonIgnoreCondition.Never)] public double? HighlightKnee { get; init; } = 0.9; + + /// Members this Chronos does not know (written by a newer one): kept as they are, so they survive a round trip. + [JsonExtensionData] + public Dictionary? Extra { get; set; } } public sealed record MotionSettings @@ -134,6 +146,10 @@ public sealed record MotionSettings /// Path smoothing in frames (Gaussian sigma). public Auto StabilizeSmoothing { get; init; } = new(); + + /// Members this Chronos does not know (written by a newer one): kept as they are, so they survive a round trip. + [JsonExtensionData] + public Dictionary? Extra { get; set; } } public sealed record CameraSettings @@ -147,6 +163,10 @@ public sealed record CameraSettings public double OffsetX { get; init; } public double OffsetY { get; init; } public IReadOnlyList Keyframes { get; init; } = []; + + /// Members this Chronos does not know (written by a newer one): kept as they are, so they survive a round trip. + [JsonExtensionData] + public Dictionary? Extra { get; set; } } public sealed record StackSettings @@ -155,6 +175,10 @@ public sealed record StackSettings /// Window (median/mean) or trail length (star trails) in frames. public int Length { get; init; } = 9; + + /// Members this Chronos does not know (written by a newer one): kept as they are, so they survive a round trip. + [JsonExtensionData] + public Dictionary? Extra { get; set; } } /// Values of an export; null fields take the preset's value (BRIEF §11.4: editing any value makes it "Custom"). @@ -173,6 +197,10 @@ public sealed record ExportOverrides public int? Quality { get; init; } public string? RateControl { get; init; } public string? Encoder { get; init; } + + /// Members this Chronos does not know (written by a newer one): kept as they are, so they survive a round trip. + [JsonExtensionData] + public Dictionary? Extra { get; set; } } public sealed record ExportSettings @@ -196,6 +224,10 @@ public sealed record ExportSettings /// unless chosen, as renders travel on request (BRIEF §14.5). /// public bool SyncToServer { get; init; } + + /// Members this Chronos does not know (written by a newer one): kept as they are, so they survive a round trip. + [JsonExtensionData] + public Dictionary? Extra { get; set; } } /// @@ -236,6 +268,20 @@ public sealed record ProjectSettings [JsonIgnore] public bool HeroPending => PendingFrames?.Any(p => p.Hero) == true; + /// + /// Members this Chronos does not know (settings a newer Chronos added, such as filters): kept as they are, so a project + /// read and written again loses nothing (owner review 2, R8). + /// + [JsonExtensionData] + public Dictionary? Extra { get; set; } + + /// + /// Written by a newer Chronos ( above ), read with + /// allowing it: shown, never edited, as its known values may mean something else there. + /// + [JsonIgnore] + public bool IsNewerFormat => Version > CurrentVersion; + public string ToJson() => JsonSerializer.Serialize(this, ProjectJson.Default.ProjectSettings); /// @@ -245,7 +291,15 @@ public sealed record ProjectSettings /// /// The text is not a JSON object, or repeats a member. /// The project was written by a newer Chronos. - public static ProjectSettings FromJson(string json) + public static ProjectSettings FromJson(string json) => FromJson(json, allowNewer: false); + + /// + /// , reading a project of a newer Chronos too when : its + /// known values, its unknown members in , and set (open it read-only). + /// + /// The text is not a JSON object, repeats a member, or a known member has another type. + /// The project was written by a newer Chronos and is false. + public static ProjectSettings FromJson(string json, bool allowNewer) { var options = new JsonNodeOptions { PropertyNameCaseInsensitive = true }; var merged = JsonNode.Parse(new ProjectSettings().ToJson(), options)!.AsObject(); @@ -266,7 +320,7 @@ public sealed record ProjectSettings throw new JsonException("A project must be a JSON object."); } var settings = JsonSerializer.Deserialize(merged, ProjectJson.Default.ProjectSettings) ?? new ProjectSettings(); - if (settings.Version > CurrentVersion) + if (settings.IsNewerFormat && !allowNewer) throw new InvalidDataException($"project.json version {settings.Version} is newer than this Chronos ({CurrentVersion})."); return settings; } diff --git a/src/Chronos.Data/Archiving/ArchiveIndexer.cs b/src/Chronos.Data/Archiving/ArchiveIndexer.cs index f2232d8f..e160fde2 100644 --- a/src/Chronos.Data/Archiving/ArchiveIndexer.cs +++ b/src/Chronos.Data/Archiving/ArchiveIndexer.cs @@ -2,6 +2,7 @@ using Chronos.Core.Archiving; using Chronos.Core.Diagnostics; using Chronos.Core.IO; using Chronos.Core.Model; +using Chronos.Data.Projects; using Chronos.Data.Repositories; namespace Chronos.Data.Archiving; @@ -37,11 +38,13 @@ public sealed class ArchiveIndexer private readonly ChronosDatabase _db; private readonly SequenceWriter _writer; + private readonly ProjectIndexer _projects; internal ArchiveIndexer(ChronosDatabase db, SequenceWriter writer) { _db = db; _writer = writer; + _projects = new ProjectIndexer(db); } /// Last folder completed by an interrupted run (job checkpoint), or null. @@ -78,6 +81,8 @@ public sealed class ArchiveIndexer { var reconcile = mode == ScanMode.Full || (state is not null && state.Photos != folder.Photos); var (id, bytes) = await IndexFolderAsync(archive, paths, folder, reconcile, snapshot, locations, cancellationToken).ConfigureAwait(false); + // A full scan or a rebuild takes the projects' sidecars back too (owner review 2, R8). + if (mode != ScanMode.Incremental) await ReconcileProjectsAsync(paths, id, cancellationToken).ConfigureAwait(false); if (snapshotWriter is not null) { // Bytes go under the stamp they were read at (the listed one for the bytes indexed, else one taken before @@ -339,6 +344,21 @@ public sealed class ArchiveIndexer /// private SequenceStatus Uncovered => _writer.Options.ImagesAfterScan ? CoveringStatus : SequenceStatus.None; + /// + /// The projects of a sequence brought in line with their sidecars (), under its lease; nothing + /// for a sequence without a projects folder (its Main is made when a project is first asked for). + /// + private async Task ReconcileProjectsAsync(ArchivePaths paths, Guid sequenceId, CancellationToken cancellationToken) + { + if (await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) is not { } row + || !Directory.Exists(paths.Sequence(row.Folder).ProjectsDirectory)) return; + await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false)) + { + var folder = await SequenceWriter.FolderAsync(_db, paths, row, cancellationToken).ConfigureAwait(false); + await _projects.ReconcileAsync(folder, sequenceId, cancellationToken).ConfigureAwait(false); + } + } + /// Indexes one folder; returns its sequence and the sidecar bytes when they were read and not rewritten since (for the snapshot). private async Task<(Guid Id, byte[]? Bytes)> IndexFolderAsync(Archive archive, ArchivePaths paths, SequenceFolder folder, bool reconcile, IndexSnapshot? snapshot, Locations locations, CancellationToken cancellationToken) diff --git a/src/Chronos.Data/Archiving/ArchiveService.cs b/src/Chronos.Data/Archiving/ArchiveService.cs index 08864bb2..f128acea 100644 --- a/src/Chronos.Data/Archiving/ArchiveService.cs +++ b/src/Chronos.Data/Archiving/ArchiveService.cs @@ -9,6 +9,7 @@ using Chronos.Core.Naming; using Chronos.Data.Archiving.Jobs; using Chronos.Data.Jobs; using Chronos.Data.Processing; +using Chronos.Data.Projects; using Chronos.Data.Repositories; namespace Chronos.Data.Archiving; @@ -44,6 +45,7 @@ public sealed class ArchiveService : IJobHandlerProvider Trash = new SequenceTrash(db, _writer, Indexer); FrameTrash = new FrameTrash(db, _writer, Indexer, Ingest, Trash); Converter = new MassConverter(db, _writer, Ingest, FrameTrash, Trash, this); + Projects = new ProjectService(db, _writer); var shared = Housekeeping.CleanupSteps.Shared(Trash); CleanupSteps = [.. shared, .. _writer.Options.CleanupSteps.Where(s => !shared.Any(o => o.Name == s.Name))]; } @@ -68,6 +70,12 @@ public sealed class ArchiveService : IJobHandlerProvider /// Converts sequences to JPEG, TIFF-16 or linear DNG, replacing their frames or into new sequences (owner review 2, R7). public MassConverter Converter { get; } + /// The projects of the sequences and their versions: this PC's archive as a project's home (owner review 2, R8). + public ProjectService Projects { get; } + + /// The decoders of this engine's image passes (thumbnails, proxies). + public Core.Imaging.ImageDecoder Decoder => _writer.Options.Decoder; + /// /// The steps this host's jobs can run: the shared ones (), /// then the host's (). A schedule host lists their names. diff --git a/src/Chronos.Data/Archiving/SequenceWriter.cs b/src/Chronos.Data/Archiving/SequenceWriter.cs index d518a81c..40fe940d 100644 --- a/src/Chronos.Data/Archiving/SequenceWriter.cs +++ b/src/Chronos.Data/Archiving/SequenceWriter.cs @@ -90,14 +90,30 @@ internal sealed class SequenceWriter(ArchiveServiceOptions options) public ArchiveServiceOptions Options => options; /// Holds the sequence exclusively until disposed. - public async Task LockAsync(ArchivePaths archive, Guid sequenceId, CancellationToken cancellationToken) + public Task LockAsync(ArchivePaths archive, Guid sequenceId, CancellationToken cancellationToken) => + LockAsync(archive, sequenceId, null, cancellationToken); + + /// + /// Holds the sequence exclusively until disposed, waiting at most for this process's other writers + /// and for the lease (the options' lease wait when null). + /// + /// Another writer kept the sequence for longer than the wait. + public async Task LockAsync(ArchivePaths archive, Guid sequenceId, TimeSpan? wait, CancellationToken cancellationToken) { + var name = $"seq-{sequenceId:N}"; var gate = _gates.GetOrAdd(sequenceId, _ => new SemaphoreSlim(1, 1)); - await gate.WaitAsync(cancellationToken).ConfigureAwait(false); + if (wait is { } limit) + { + if (!await gate.WaitAsync(limit, cancellationToken).ConfigureAwait(false)) throw new LeaseUnavailableException(name, null); + } + else + { + await gate.WaitAsync(cancellationToken).ConfigureAwait(false); + } try { - var lease = await LeaseFile.AcquireAsync(archive.LocksDirectory, $"seq-{sequenceId:N}", options.Owner, options.LeaseWait, - options.LeaseDuration, cancellationToken).ConfigureAwait(false); + var lease = await LeaseFile.AcquireAsync(archive.LocksDirectory, name, options.Owner, wait ?? options.LeaseWait, options.LeaseDuration, + cancellationToken).ConfigureAwait(false); return new Held(gate, lease); } catch diff --git a/src/Chronos.Data/ChronosDatabase.cs b/src/Chronos.Data/ChronosDatabase.cs index ec4f1886..bb095929 100644 --- a/src/Chronos.Data/ChronosDatabase.cs +++ b/src/Chronos.Data/ChronosDatabase.cs @@ -24,6 +24,7 @@ public abstract class ChronosRepositories Uploads = new UploadRepository(access); Schedules = new ScheduleRepository(access); FrameRemovals = new FrameRemovalRepository(access); + Projects = new ProjectRepository(access); } public ArchiveRepository Archives { get; } @@ -45,6 +46,9 @@ public abstract class ChronosRepositories /// Tombstones of the photos removed from sequences (owner review 2, R6). public FrameRemovalRepository FrameRemovals { get; } + + /// Projects of sequences and their versions (owner review 2, R8). + public ProjectRepository Projects { get; } } /// diff --git a/src/Chronos.Data/Migrations/postgres/0009_projects.sql b/src/Chronos.Data/Migrations/postgres/0009_projects.sql new file mode 100644 index 00000000..f8b9c346 --- /dev/null +++ b/src/Chronos.Data/Migrations/postgres/0009_projects.sql @@ -0,0 +1,46 @@ +-- Projects of sequences (owner review 2, R8): named sets of render settings ("4K landscape", "Vertical social"), each +-- with a linear history of immutable versions. The sidecars in /.chronos/projects/ are the truth; these rows +-- are derived and rebuilt from them. Every sequence has one Main project, whose head is mirrored into +-- sequences.settings_json: mirrored_version_id and mirrored_settings_version are the version and the settings version of +-- the last mirror, so settings another path wrote are caught up as an "external" version, and a mirror a crash cut short +-- is completed. A version's settings are text, not jsonb, so the bytes its content hash covers stay exact. Renders +-- remember the project version they rendered. Mirrors sqlite/0009_projects.sql. + +CREATE TABLE projects ( + id uuid NOT NULL PRIMARY KEY, + sequence_id uuid NOT NULL REFERENCES sequences (id) ON DELETE CASCADE, + name text NOT NULL, + is_main boolean NOT NULL DEFAULT FALSE, + head_version_id uuid, + head_number integer NOT NULL DEFAULT 0, + mirrored_settings_version integer, + mirrored_version_id uuid, + created_by text, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + deleted_at timestamptz +); +CREATE UNIQUE INDEX ux_projects_name ON projects (sequence_id, name) WHERE deleted_at IS NULL; +CREATE UNIQUE INDEX ux_projects_main ON projects (sequence_id) WHERE is_main AND deleted_at IS NULL; + +CREATE TABLE project_versions ( + id uuid NOT NULL PRIMARY KEY, + project_id uuid NOT NULL REFERENCES projects (id) ON DELETE CASCADE, + parent_id uuid, + number integer NOT NULL, + kind text NOT NULL, + author text, + client_name text, + note text, + settings_json text NOT NULL, + content_hash text NOT NULL, + settings_format integer NOT NULL, + frames_version integer NOT NULL, + restored_from uuid, + created_at timestamptz NOT NULL, + stored_at timestamptz NOT NULL +); +CREATE UNIQUE INDEX ux_project_versions_number ON project_versions (project_id, number); + +ALTER TABLE renders ADD COLUMN project_id uuid; +ALTER TABLE renders ADD COLUMN project_version_id uuid; diff --git a/src/Chronos.Data/Migrations/sqlite/0009_projects.sql b/src/Chronos.Data/Migrations/sqlite/0009_projects.sql new file mode 100644 index 00000000..bf188d30 --- /dev/null +++ b/src/Chronos.Data/Migrations/sqlite/0009_projects.sql @@ -0,0 +1,46 @@ +-- Projects of sequences (owner review 2, R8): named sets of render settings ("4K landscape", "Vertical social"), each +-- with a linear history of immutable versions. The sidecars in /.chronos/projects/ are the truth; these rows +-- are derived and rebuilt from them. Every sequence has one Main project, whose head is mirrored into +-- sequences.settings_json: mirrored_version_id and mirrored_settings_version are the version and the settings version of +-- the last mirror, so settings another path wrote are caught up as an "external" version, and a mirror a crash cut short +-- is completed. A version's settings are text, never re-encoded, so the bytes its content hash covers stay exact. +-- Renders remember the project version they rendered. Mirrors postgres/0009_projects.sql. + +CREATE TABLE projects ( + id TEXT NOT NULL PRIMARY KEY, + sequence_id TEXT NOT NULL REFERENCES sequences (id) ON DELETE CASCADE, + name TEXT NOT NULL, + is_main INTEGER NOT NULL DEFAULT 0, + head_version_id TEXT, + head_number INTEGER NOT NULL DEFAULT 0, + mirrored_settings_version INTEGER, + mirrored_version_id TEXT, + created_by TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + deleted_at TEXT +); +CREATE UNIQUE INDEX ux_projects_name ON projects (sequence_id, name) WHERE deleted_at IS NULL; +CREATE UNIQUE INDEX ux_projects_main ON projects (sequence_id) WHERE is_main = 1 AND deleted_at IS NULL; + +CREATE TABLE project_versions ( + id TEXT NOT NULL PRIMARY KEY, + project_id TEXT NOT NULL REFERENCES projects (id) ON DELETE CASCADE, + parent_id TEXT, + number INTEGER NOT NULL, + kind TEXT NOT NULL, + author TEXT, + client_name TEXT, + note TEXT, + settings_json TEXT NOT NULL, + content_hash TEXT NOT NULL, + settings_format INTEGER NOT NULL, + frames_version INTEGER NOT NULL, + restored_from TEXT, + created_at TEXT NOT NULL, + stored_at TEXT NOT NULL +); +CREATE UNIQUE INDEX ux_project_versions_number ON project_versions (project_id, number); + +ALTER TABLE renders ADD COLUMN project_id TEXT; +ALTER TABLE renders ADD COLUMN project_version_id TEXT; diff --git a/src/Chronos.Data/Processing/Jobs/RenderJobHandler.cs b/src/Chronos.Data/Processing/Jobs/RenderJobHandler.cs index 70e9be93..6a6b10bb 100644 --- a/src/Chronos.Data/Processing/Jobs/RenderJobHandler.cs +++ b/src/Chronos.Data/Processing/Jobs/RenderJobHandler.cs @@ -48,7 +48,9 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt if (!SequenceFolders.Holds(folder, sequence.Id)) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved")); // A server writes only inside its archive (BRIEF §14.2): an export folder chosen on a client means nothing there. if (options.Host == EncoderHost.Server) export = export with { Folder = null }; - var project = (parameters.SettingsJson is { } embedded ? ProjectStore.Parse(embedded) : ProjectStore.Read(sequence, folder)) with { Export = export }; + var embedded = parameters.SettingsJson ?? await VersionSettingsAsync(parameters, sequence, cancellation).ConfigureAwait(false); + var frames = await SequenceFrames.LoadAsync(db, archive, sequence, parameters.UseProxies, options.Archive.Decoder, cancellation).ConfigureAwait(false); + var project = (embedded is not null ? ProjectStore.Parse(Placed(embedded, frames)) : ProjectStore.Read(sequence, folder)) with { Export = export }; var settingsJson = project.ToJson(); using var names = new OutputNames(); @@ -71,7 +73,6 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt var targets = all.Select((t, i) => (i, t)).Where(t => !done.Contains(t.i)).Select(t => (t.i, adapt(t.t))).ToList(); if (targets.Count == 0) return; - var frames = await SequenceFrames.LoadAsync(db, archive, sequence, parameters.UseProxies, options.Archive.Decoder, cancellation).ConfigureAwait(false); var companions = await GprCompanions.FindAsync(db, archive, frames.Frames, cancellation).ConfigureAwait(false); IReadOnlyList outcomes; await using (var render = new SequenceRender(frames, project, SequenceAnalyser.ReadReport(folder, frames), options, originals: !parameters.UseProxies, @@ -88,6 +89,8 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt { Id = RenderId(context.Job.Id, index), OutputPath = Stored(paths, result.OutputPath), + ProjectId = parameters.ProjectId, + ProjectVersionId = parameters.ProjectVersionId, }; await db.Renders.InsertAsync(entity, CancellationToken.None).ConfigureAwait(false); done.Add(index); @@ -120,6 +123,37 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt throw new InvalidOperationException(failed.Count == 1 ? failed[0].Error : Loc.L("Errors.Render.Failed", failed.Count, outcomes.Count)); } + /// + /// The settings of the project version a job names without embedding them (owner review 2, R8); null when it names none. + /// + /// The version is not one of the sequence's projects'. + private async Task VersionSettingsAsync(Protocol.RenderJobParameters parameters, Sequence sequence, CancellationToken cancellationToken) + { + if (parameters.ProjectVersionId is not { } versionId) return null; + return await db.Projects.GetVersionAsync(versionId, cancellationToken).ConfigureAwait(false) is { } version + && await db.Projects.GetAsync(version.ProjectId, cancellationToken).ConfigureAwait(false) is { } owner && owner.SequenceId == sequence.Id + ? version.SettingsJson + : throw new InvalidOperationException(Loc.L("Errors.Render.ProjectVersion")); + } + + /// + /// Settings whose positions wait for their photos (a project version's portable form, settings synced from another copy) + /// with them placed on the frames rendered; others as they are. + /// + internal static string Placed(string settingsJson, SequenceFrames frames) + { + if (!settingsJson.Contains("pendingFrames", StringComparison.OrdinalIgnoreCase)) return settingsJson; + try + { + return ProjectContent.Place(settingsJson, new FramePhotos(frames.Frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, f.CapturedAt))))); + } + catch (System.Text.Json.JsonException) + { + // Not a project: reading it says so. + return settingsJson; + } + } + /// The job's preset, then every other preset of the batch with its own values. internal static List Targets(ExportSettings export, string presetName) => [ diff --git a/src/Chronos.Data/Processing/ProcessingJobs.cs b/src/Chronos.Data/Processing/ProcessingJobs.cs index f361b269..e9632516 100644 --- a/src/Chronos.Data/Processing/ProcessingJobs.cs +++ b/src/Chronos.Data/Processing/ProcessingJobs.cs @@ -41,6 +41,35 @@ public static class ProcessingJobs JsonSerializer.Serialize(parameters, ProtocolJson.Default.RenderJobParameters), JobPriority.User); } + /// + /// of a project version (owner review 2, R8) on the sequence's home: the version's portable + /// settings, placed on the sequence's frames by the render, with as their export section; the + /// render records the project and the version. One job per preset, as . + /// + public static IReadOnlyList RenderVersion(Sequence sequence, ProjectVersion version, ExportSettings export, int sourceWidth, int sourceHeight, + bool useProxies = false) + { + ArgumentNullException.ThrowIfNull(sequence); + ArgumentNullException.ThrowIfNull(version); + ArgumentNullException.ThrowIfNull(export); + return + [ + VersionJob(sequence, version, export with { Batch = [] }, sourceWidth, sourceHeight, useProxies), + .. BatchPresets(export).Select(id => VersionJob(sequence, version, BatchExport(export, id), sourceWidth, sourceHeight, useProxies)), + ]; + } + + private static ArchiveJobRequest VersionJob(Sequence sequence, ProjectVersion version, ExportSettings export, int sourceWidth, int sourceHeight, bool useProxies) + { + var parameters = Parameters(export, sourceWidth, sourceHeight, version.SettingsJson, useProxies) with + { + ProjectId = version.ProjectId, + ProjectVersionId = version.Id, + }; + return new ArchiveJobRequest(JobKind.Render, [sequence.Id], RenderLabel(sequence.DisplayTitle, parameters.PresetName), + JsonSerializer.Serialize(parameters, ProtocolJson.Default.RenderJobParameters), JobPriority.User); + } + /// /// The label of a render job, "Harbour sunset → 4K HEVC". It is stored as written, in the language of the moment like /// the preset name: its arrow points from the sequence to the preset in that language's reading direction. @@ -64,7 +93,11 @@ public static class ProcessingJobs return [ parameters with { PresetJson = JsonSerializer.Serialize(export with { Batch = [] }, ProcessingJson.Default.ExportSettings) }, - .. batch.Select(id => Parameters(BatchExport(export, id), sourceWidth, sourceHeight, parameters.SettingsJson, parameters.UseProxies)), + .. batch.Select(id => Parameters(BatchExport(export, id), sourceWidth, sourceHeight, parameters.SettingsJson, parameters.UseProxies) with + { + ProjectId = parameters.ProjectId, + ProjectVersionId = parameters.ProjectVersionId, + }), ]; } diff --git a/src/Chronos.Data/Projects/IProjectHome.cs b/src/Chronos.Data/Projects/IProjectHome.cs new file mode 100644 index 00000000..bd78c22c --- /dev/null +++ b/src/Chronos.Data/Projects/IProjectHome.cs @@ -0,0 +1,129 @@ +using Chronos.Core.Archiving; +using Chronos.Core.Model; +using Chronos.Core.Project; + +namespace Chronos.Data.Projects; + +/// A project's home cannot be reached now (its archive offline or busy, its server out of reach): try again later. +public sealed class HomeUnavailableException(string message, Exception? inner = null) : IOException(message, inner); + +/// The sequence a home keeps projects for, as a workspace's project file names it. +public sealed record HomeSequence(BundleArchive Archive, BundleSequence Sequence, BundleHome Home); + +/// +/// Where a sequence's projects live (owner review 2, R8): the side a workspace pushes to and pulls from, "like git". This +/// PC's archive is one (); a server is the other (phase 5b). Every call may throw +/// when the home cannot be reached, and when it refuses. +/// +public interface IProjectHome +{ + /// The sequence and where it is. + Task SequenceAsync(CancellationToken cancellationToken); + + /// The sequence's live projects, Main first (made when the sequence had none). + Task> ListAsync(CancellationToken cancellationToken); + + /// A new project named from (a version of any project of the sequence), or from Main's head. + Task CreateAsync(string name, Guid? fromVersionId, string? author, CancellationToken cancellationToken); + + /// The project's newest version. + Task HeadAsync(Guid projectId, CancellationToken cancellationToken); + + /// A version of the project by id; null when the home has none. + Task GetVersionAsync(Guid projectId, Guid versionId, CancellationToken cancellationToken); + + /// The versions numbered above , oldest first, at most . + Task> VersionsAfterAsync(Guid projectId, int afterNumber, int limit, CancellationToken cancellationToken); + + /// One page of the history, newest first: the versions numbered below (from the head when null). + Task> HistoryAsync(Guid projectId, int? beforeNumber, int limit, CancellationToken cancellationToken); + + /// Stores a version made from the head (). + Task PushAsync(Guid projectId, ProjectVersion version, CancellationToken cancellationToken); + + /// The sequence's frames as the home holds them now. + Task FramesAsync(CancellationToken cancellationToken); +} + +/// A sequence of this PC's archive as its projects' home: the archive's . +public sealed class ArchiveProjectHome(ChronosDatabase db, ProjectService projects, Guid sequenceId) : IProjectHome +{ + public Guid SequenceId => sequenceId; + + public async Task SequenceAsync(CancellationToken cancellationToken) + { + var sequence = await db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) + ?? throw new ProjectException(ProjectError.SequenceNotFound, $"Sequence {sequenceId} is not in the index."); + var archive = await db.Archives.GetAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false); + return new HomeSequence(new BundleArchive(sequence.ArchiveId, archive?.Name ?? string.Empty, archive?.RootPath), + new BundleSequence(sequence.Id, sequence.Folder, sequence.DisplayTitle, sequence.FrameCount), + new BundleHome(BundleHome.ArchiveKind, sequence.Id, sequence.ArchiveId)); + } + + public Task> ListAsync(CancellationToken cancellationToken) => + Reach(async () => + { + // Made at once when the sequence has none, so every list has a Main. + await projects.MainAsync(sequenceId, cancellationToken).ConfigureAwait(false); + return await projects.ListAsync(sequenceId, cancellationToken).ConfigureAwait(false); + }); + + public Task CreateAsync(string name, Guid? fromVersionId, string? author, CancellationToken cancellationToken) => + Reach(() => projects.CreateAsync(sequenceId, name, fromVersionId, author, cancellationToken)); + + public Task HeadAsync(Guid projectId, CancellationToken cancellationToken) => + Reach(async () => + { + await Own(projectId, cancellationToken).ConfigureAwait(false); + return await projects.HeadAsync(projectId, cancellationToken).ConfigureAwait(false); + }); + + public async Task GetVersionAsync(Guid projectId, Guid versionId, CancellationToken cancellationToken) => + await projects.GetVersionAsync(versionId, cancellationToken).ConfigureAwait(false) is { } version && version.ProjectId == projectId ? version : null; + + public Task> VersionsAfterAsync(Guid projectId, int afterNumber, int limit, CancellationToken cancellationToken) => + Reach(async () => + { + await Own(projectId, cancellationToken).ConfigureAwait(false); + return await projects.VersionsAfterAsync(projectId, afterNumber, limit, cancellationToken).ConfigureAwait(false); + }); + + public Task> HistoryAsync(Guid projectId, int? beforeNumber, int limit, CancellationToken cancellationToken) => + Reach(async () => + { + await Own(projectId, cancellationToken).ConfigureAwait(false); + return await projects.HistoryAsync(projectId, beforeNumber, limit, cancellationToken).ConfigureAwait(false); + }); + + public Task PushAsync(Guid projectId, ProjectVersion version, CancellationToken cancellationToken) => + Reach(async () => + { + await Own(projectId, cancellationToken).ConfigureAwait(false); + return await projects.PushAsync(projectId, version, cancellationToken).ConfigureAwait(false); + }); + + public Task FramesAsync(CancellationToken cancellationToken) => projects.FramesAsync(sequenceId, cancellationToken); + + /// The project is one of this sequence's. + private async Task Own(Guid projectId, CancellationToken cancellationToken) => + await projects.GetAsync(projectId, cancellationToken).ConfigureAwait(false) is { } project && project.SequenceId == sequenceId + ? project + : throw new ProjectException(ProjectError.NotFound, $"Project {projectId} is not a project of sequence {sequenceId}."); + + /// An archive offline, read-only or busy is a home out of reach for now. + private static async Task Reach(Func> call) + { + try + { + return await call().ConfigureAwait(false); + } + catch (ArchiveOfflineException ex) + { + throw new HomeUnavailableException(ex.Message, ex); + } + catch (LeaseUnavailableException ex) + { + throw new HomeUnavailableException(ex.Message, ex); + } + } +} diff --git a/src/Chronos.Data/Projects/ProjectIndexer.cs b/src/Chronos.Data/Projects/ProjectIndexer.cs new file mode 100644 index 00000000..aaf75ab6 --- /dev/null +++ b/src/Chronos.Data/Projects/ProjectIndexer.cs @@ -0,0 +1,179 @@ +using System.Globalization; +using Chronos.Core.Archiving; +using Chronos.Core.Diagnostics; +using Chronos.Core.Model; + +namespace Chronos.Data.Projects; + +/// +/// Brings the index's projects of one sequence in line with their sidecars (owner review 2, R8), which are the truth: a +/// project or version the index lacks is added (an index rebuilt, a sequence restored from the trash), a head the index is +/// behind on moves forward, a name, Main flag or deletion recorded since is taken, and a live index row whose folder is gone +/// is marked deleted. A version file written past its record's head by a push a crash cut short is adopted when it +/// follows the head (its file is complete: versions are written atomically, then the record); any other file past the +/// head is moved aside to versions/.stray, never deleted. Runs under the sequence's lease. +/// +internal sealed class ProjectIndexer(ChronosDatabase db) +{ + private const string StrayFolder = ".stray"; + + /// Reconciles the projects of in ; the caller holds the sequence's lease. + public async Task ReconcileAsync(SequencePaths folder, Guid sequenceId, CancellationToken cancellationToken) + { + var seen = new HashSet(); + if (Directory.Exists(folder.ProjectsDirectory)) + { + foreach (var (directory, trashed) in ProjectFolders(folder)) + { + cancellationToken.ThrowIfCancellationRequested(); + ProjectRecord record; + try + { + record = ProjectSidecars.ReadRecord(Path.Combine(directory, ArchiveLayout.ProjectFile)); + } + catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) + { + Log.Warning("projects", $"The project in {directory} cannot be read", ex); + continue; + } + // A sequence folder copied by hand carries the projects of its source, which stay that sequence's. + if (record.SequenceId != sequenceId || !seen.Add(record.Id)) continue; + if (trashed && record.Deleted is null) record = record with { Deleted = Directory.GetLastWriteTimeUtc(directory) }; + if (record.Deleted is null) record = RollForward(folder, record, directory); + await IndexAsync(record, directory, cancellationToken).ConfigureAwait(false); + } + } + foreach (var gone in await db.Projects.ListBySequenceAsync(sequenceId, includeDeleted: false, cancellationToken).ConfigureAwait(false)) + { + if (seen.Contains(gone.Id)) continue; + Log.Warning("projects", $"Project {gone.Id} of sequence {sequenceId} has no folder any more; it is marked deleted"); + await db.Projects.UpdateAsync(gone with { DeletedAt = DateTimeOffset.UtcNow, UpdatedAt = DateTimeOffset.UtcNow }, cancellationToken).ConfigureAwait(false); + } + } + + /// The project folders of a sequence: the live ones, then those in the projects' trash (<id>-<time>). + private static IEnumerable<(string Directory, bool Trashed)> ProjectFolders(SequencePaths folder) + { + foreach (var directory in Directory.EnumerateDirectories(folder.ProjectsDirectory).Order(StringComparer.Ordinal)) + { + if (Guid.TryParseExact(Path.GetFileName(directory), "N", out _)) yield return (directory, false); + } + if (!Directory.Exists(folder.ProjectsTrashDirectory)) yield break; + foreach (var directory in Directory.EnumerateDirectories(folder.ProjectsTrashDirectory).Order(StringComparer.Ordinal)) + { + if (File.Exists(Path.Combine(directory, ArchiveLayout.ProjectFile))) yield return (directory, true); + } + } + + /// + /// The record with the version files that follow its head adopted (a push that wrote its version but not the record), + /// and the files past the head that do not follow it moved aside; the record is written again when its head moved. + /// + private static ProjectRecord RollForward(SequencePaths folder, ProjectRecord record, string directory) + { + var versions = Path.Combine(directory, ArchiveLayout.ProjectVersionsFolder); + var files = ProjectSidecars.ListVersions(versions); + var head = record.Head; + var moved = false; + while (true) + { + var next = (head?.Number ?? 0) + 1; + var candidates = files.Where(f => f.Number == next).ToList(); + ProjectVersion? follower = null; + foreach (var candidate in candidates) + { + try + { + var version = ProjectSidecars.ReadVersion(candidate.Path); + if (version.ProjectId == record.Id && version.ParentId == head?.Id) + { + follower = version; + break; + } + } + catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) + { + Log.Warning("projects", $"Version file {candidate.Path} cannot be read", ex); + } + } + if (follower is null) break; + head = new ProjectHead(follower.Id, follower.Number); + moved = true; + } + foreach (var stray in files.Where(f => f.Number > (head?.Number ?? 0))) + { + try + { + var aside = Path.Combine(versions, StrayFolder); + Directory.CreateDirectory(aside); + File.Move(stray.Path, Path.Combine(aside, Path.GetFileName(stray.Path) + "." + DateTime.UtcNow.ToString("yyyyMMddHHmmssfff", CultureInfo.InvariantCulture))); + Log.Warning("projects", $"Version file {stray.Path} does not follow the head of project {record.Id}; moved aside"); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warning("projects", $"Version file {stray.Path} could not be moved aside", ex); + } + } + if (!moved) return record; + var rolled = record with { Head = head, Updated = DateTimeOffset.UtcNow }; + ProjectSidecars.WriteRecord(folder.ProjectRecordPath(record.Id), rolled); + return rolled; + } + + /// Writes what the index lacks of one project's record and versions. + private async Task IndexAsync(ProjectRecord record, string directory, CancellationToken cancellationToken) + { + var row = await db.Projects.GetAsync(record.Id, cancellationToken).ConfigureAwait(false); + if (row is null) + { + var project = record.ToProject(); + var live = await db.Projects.ListBySequenceAsync(record.SequenceId, includeDeleted: false, cancellationToken).ConfigureAwait(false); + if (project.DeletedAt is null && project.IsMain && live.Any(p => p.IsMain)) + { + Log.Warning("projects", $"Project {record.Id} says it is Main, but sequence {record.SequenceId} has one; indexed as another project"); + project = project with { IsMain = false }; + } + if (project.DeletedAt is null && live.Any(p => string.Equals(p.Name, project.Name, StringComparison.OrdinalIgnoreCase))) + project = project with { Name = project.Name + " (" + project.Id.ToString("N")[..6] + ")" }; + // The head follows its versions below, so a version the index cannot take never leaves a head without its row. + await db.Projects.InsertAsync(project with { HeadVersionId = null, HeadNumber = 0 }, cancellationToken).ConfigureAwait(false); + row = project with { HeadVersionId = null, HeadNumber = 0 }; + } + var indexed = await db.Projects.ListVersionIdsAsync(record.Id, cancellationToken).ConfigureAwait(false); + var headNumber = row.HeadNumber; + Guid? headId = row.HeadVersionId; + var limit = record.Head?.Number ?? 0; + foreach (var file in ProjectSidecars.ListVersions(Path.Combine(directory, ArchiveLayout.ProjectVersionsFolder))) + { + if (file.Number > limit || indexed.ContainsKey(file.Number)) continue; + ProjectVersion version; + try + { + version = ProjectSidecars.ReadVersion(file.Path); + } + catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) + { + Log.Warning("projects", $"Version file {file.Path} cannot be read; it is not indexed", ex); + continue; + } + if (version.ProjectId != record.Id || version.Number != file.Number) continue; + await db.Projects.InsertVersionAsync(version, cancellationToken).ConfigureAwait(false); + } + if (record.Head is { } head && head.Number > headNumber && await db.Projects.GetVersionAsync(head.Id, cancellationToken).ConfigureAwait(false) is not null) + (headId, headNumber) = (head.Id, head.Number); + var updated = row with + { + Name = row.DeletedAt is null && record.Deleted is null && row.Name != record.Name && !await NameTakenAsync(record, cancellationToken).ConfigureAwait(false) + ? record.Name + : row.Name, + HeadVersionId = headId, + HeadNumber = headNumber, + DeletedAt = row.DeletedAt ?? record.Deleted, + }; + if (updated != row) await db.Projects.UpdateAsync(updated, cancellationToken).ConfigureAwait(false); + } + + private async Task NameTakenAsync(ProjectRecord record, CancellationToken cancellationToken) => + (await db.Projects.ListBySequenceAsync(record.SequenceId, includeDeleted: false, cancellationToken).ConfigureAwait(false)) + .Any(p => p.Id != record.Id && string.Equals(p.Name, record.Name, StringComparison.OrdinalIgnoreCase)); +} diff --git a/src/Chronos.Data/Projects/ProjectMerge.cs b/src/Chronos.Data/Projects/ProjectMerge.cs new file mode 100644 index 00000000..d59fd3d1 --- /dev/null +++ b/src/Chronos.Data/Projects/ProjectMerge.cs @@ -0,0 +1,253 @@ +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Nodes; +using Chronos.Core.Project; +using Chronos.Data.Sync; + +namespace Chronos.Data.Projects; + +/// The result of a three-way merge of projects: the merged settings and the members both sides changed to different values. +/// Each settled for the side asked (takeTheirs), else for ours; empty when the merge needed nobody. +public sealed record ProjectMergeResult(ProjectSettings Merged, IReadOnlyList Conflicts) +{ + public bool HasConflicts => Conflicts.Count > 0; +} + +/// +/// Commits not stored at their home yet, replayed on top of the version stored there meanwhile (owner review 2, R8): one +/// project per commit, in order, and the conflicts of the last one (the net change of all of them against theirs). +/// +public sealed record ProjectReplay(IReadOnlyList Commits, IReadOnlyList Conflicts) +{ + public bool HasConflicts => Conflicts.Count > 0; +} + +/// +/// Three-way merge of projects: the editor's edit (ours) and a change stored meanwhile by someone else +/// (theirs: an analysis filling Auto values, another client through the server, another workspace through a +/// project's home), both made from basis. Values are compared member by member down the JSON tree; a member only +/// one side changed takes that side's value, a member both changed takes the editor's (the user is looking at it) unless +/// its path is among those asked to take theirs. lists those members for a source that asks the +/// user instead (a server sequence, BRIEF §14.5; a workspace project, owner review 2, R8). Pending frames (frame +/// positions parked by photo, the portable form of ) merge photo by photo and flag by flag, so +/// two people excluding different photos never conflict; two heroes chosen on both sides do (path ). +/// Members this Chronos does not know merge like the others and are kept. +/// +public static class ProjectMerge +{ + /// The path of the conflict of two heroes chosen among pending frames on both sides. + public const string HeroPath = "hero"; + + private const string PendingPath = "pendingFrames"; + + public static ProjectSettings Merge(ProjectSettings basis, ProjectSettings ours, ProjectSettings theirs) => + MergeWithConflicts(basis, ours, theirs).Merged; + + /// + /// The merge of and made from , and its + /// conflicts. A conflict whose path is in takes theirs (the user chose "take server" + /// there), every other one ours. + /// + public static ProjectMergeResult MergeWithConflicts(ProjectSettings basis, ProjectSettings ours, ProjectSettings theirs, IReadOnlySet? takeTheirs = null) + { + ArgumentNullException.ThrowIfNull(basis); + ArgumentNullException.ThrowIfNull(ours); + ArgumentNullException.ThrowIfNull(theirs); + var conflicts = new List(); + var choice = takeTheirs ?? EmptySet; + var (b, o, t) = (Keyed(basis), Keyed(ours), Keyed(theirs)); + var merged = MergeNode(b, o, t, string.Empty, conflicts, choice) as JsonObject ?? new JsonObject(); + OneHero(merged, o, t, conflicts, choice); + return new ProjectMergeResult(FromKeyed(merged), conflicts); + } + + /// + /// The members both sides changed to different values, in document order: the path below the project + /// ("exposure.deflickerWindow.manual"; "pendingFrames.<photo>.excluded" for a pending frame), the editor's value and + /// theirs as JSON (null when absent). + /// + public static IReadOnlyList Conflicts(ProjectSettings basis, ProjectSettings ours, ProjectSettings theirs) => + MergeWithConflicts(basis, ours, theirs).Conflicts; + + /// + /// Replays commits made from (, oldest first, each including the ones + /// before it) on top of : each becomes theirs with that commit's changes since the basis, so the + /// history stays linear and the last one holds the net change. Conflicts are those of the last commit; their paths in + /// take theirs in every commit. + /// + public static ProjectReplay Replay(ProjectSettings basis, IReadOnlyList mine, ProjectSettings theirs, IReadOnlySet? takeTheirs = null) + { + ArgumentNullException.ThrowIfNull(mine); + var commits = new List(mine.Count); + IReadOnlyList conflicts = []; + foreach (var commit in mine) + { + var result = MergeWithConflicts(basis, commit, theirs, takeTheirs); + commits.Add(result.Merged); + conflicts = result.Conflicts; + } + return new ProjectReplay(commits, conflicts); + } + + /// True when both projects store the same values. + public static bool SameValues(ProjectSettings a, ProjectSettings b) => a.ToJson() == b.ToJson(); + + private static readonly IReadOnlySet EmptySet = new HashSet(); + + private static JsonNode? MergeNode(JsonNode? basis, JsonNode? ours, JsonNode? theirs, string path, List conflicts, + IReadOnlySet takeTheirs) + { + if (JsonNode.DeepEquals(ours, basis)) return theirs?.DeepClone(); + if (JsonNode.DeepEquals(theirs, basis) || JsonNode.DeepEquals(theirs, ours)) return ours?.DeepClone(); + if (path == PendingPath && ours is JsonObject or null && theirs is JsonObject or null) + return MergePending(basis as JsonObject, ours as JsonObject, theirs as JsonObject, conflicts, takeTheirs); + if (ours is JsonObject mine && theirs is JsonObject other) + { + var common = basis as JsonObject; + var merged = new JsonObject(); + foreach (var name in mine.Select(p => p.Key).Union(other.Select(p => p.Key))) + { + var child = path.Length == 0 ? name : path + "." + name; + var value = MergeNode(common?[name], mine[name], other[name], child, conflicts, takeTheirs); + if (value is not null || (takeTheirs.Contains(child) ? other.ContainsKey(name) : mine.ContainsKey(name))) merged[name] = value; + } + return merged; + } + var at = path.Length == 0 ? "settings" : path; + conflicts.Add(new SettingsDifference(at, ours?.ToJsonString(), theirs?.ToJsonString())); + return takeTheirs.Contains(at) ? theirs?.DeepClone() : ours?.DeepClone(); + } + + /// + /// Pending frames keyed by photo, merged entry by entry: an entry one side lacks counts as the photo with no position on + /// it, so excluding one photo here and another there both stay, and a flag both changed alike is no conflict. + /// + private static JsonObject MergePending(JsonObject? basis, JsonObject? ours, JsonObject? theirs, List conflicts, + IReadOnlySet takeTheirs) + { + var merged = new JsonObject(); + var keys = (ours?.Select(p => p.Key) ?? []).Union(theirs?.Select(p => p.Key) ?? []).Union(basis?.Select(p => p.Key) ?? []).ToList(); + foreach (var key in keys) + { + var template = ours?[key] ?? theirs?[key] ?? basis?[key]; + var value = MergeNode(basis?[key] ?? Unplaced(template), ours?[key] ?? Unplaced(template), theirs?[key] ?? Unplaced(template), + PendingPath + "." + key, conflicts, takeTheirs); + if (value is not null) merged[key] = value; + } + return merged; + } + + /// + /// Two heroes after a merge (each side chose one among photos that wait): one conflict, settled for ours or, when asked, + /// theirs; the other entry no longer holds the hero. + /// + private static void OneHero(JsonObject merged, JsonObject ours, JsonObject theirs, List conflicts, IReadOnlySet takeTheirs) + { + if (merged[PendingPath] is not JsonObject pending) return; + var heroes = pending.Where(p => p.Value is JsonObject entry && Flag(entry, "hero")).Select(p => p.Key).ToList(); + if (heroes.Count < 2) return; + var mine = HeroOf(ours); + var other = HeroOf(theirs); + var keep = takeTheirs.Contains(HeroPath) ? other ?? heroes[0] : mine ?? heroes[0]; + foreach (var key in heroes.Where(k => k != keep)) pending[key]!["hero"] = false; + conflicts.Add(new SettingsDifference(HeroPath, mine is null ? null : (ours[PendingPath] as JsonObject)?[mine]?.ToJsonString(), + other is null ? null : (theirs[PendingPath] as JsonObject)?[other]?.ToJsonString())); + } + + private static string? HeroOf(JsonObject project) => + project[PendingPath] is JsonObject pending ? pending.FirstOrDefault(p => p.Value is JsonObject entry && Flag(entry, "hero")).Key : null; + + /// The project as JSON with its pending frames as an object keyed by photo (). + private static JsonObject Keyed(ProjectSettings settings) + { + var project = JsonNode.Parse(settings.ToJson())!.AsObject(); + if (project[PendingPath] is JsonArray pending) + { + var keyed = new JsonObject(); + foreach (var entry in pending) keyed[KeyOf(entry)] = entry?.DeepClone(); + project[PendingPath] = keyed; + } + return project; + } + + /// A keyed project back as settings: pending frames as a list again, without those that no longer hold a position. + private static ProjectSettings FromKeyed(JsonObject project) + { + if (project[PendingPath] is JsonObject keyed) + { + var list = new JsonArray(); + foreach (var (_, entry) in keyed) + { + if (entry is JsonObject e && !Waits(e)) continue; + list.Add(entry?.DeepClone()); + } + if (list.Count == 0) project.Remove(PendingPath); + else project[PendingPath] = list; + } + return ProjectSettings.FromJson(project.ToJsonString(), allowNewer: true); + } + + /// + /// The key of a pending entry: its photo and which copy of it ("sha:…#0", or "photo:size@ticks#0" for a photo not hashed), + /// or its JSON for an entry naming no photo. + /// + private static string KeyOf(JsonNode? entry) + { + if (entry is JsonObject e) + { + var occurrence = e["occurrence"] is JsonValue o && o.TryGetValue(out var n) ? n : 0; + if (e["sha256"] is JsonValue s && s.TryGetValue(out var sha) && sha.Length > 0) + return string.Create(CultureInfo.InvariantCulture, $"sha:{sha.ToLowerInvariant()}#{occurrence}"); + var size = e["size"] is JsonValue z && z.TryGetValue(out var bytes) ? bytes : 0; + if (e["capturedAt"] is JsonValue c && c.TryGetValue(out var at)) + return string.Create(CultureInfo.InvariantCulture, $"photo:{size}@{at.Ticks}#{occurrence}"); + } + return "raw:" + (entry?.ToJsonString() ?? "null"); + } + + /// The entry of the photo of holding no position; null for an entry naming no photo. + private static JsonNode? Unplaced(JsonNode? template) + { + if (template is not JsonObject entry || KeyOf(entry).StartsWith("raw:", StringComparison.Ordinal)) return null; + var empty = new JsonObject(); + foreach (var name in new[] { "sha256", "size", "capturedAt", "occurrence" }) + { + if (entry[name] is { } value) empty[name] = value.DeepClone(); + } + empty["excluded"] = false; + empty["hero"] = false; + empty["keyframes"] = new JsonArray(); + return empty; + } + + private static bool Flag(JsonObject entry, string name) => entry[name] is JsonValue value && value.TryGetValue(out var set) && set; + + /// A pending entry still holds a position (its unknown members count too: they are not this Chronos's to drop). + private static bool Waits(JsonObject entry) => + Flag(entry, "excluded") || Flag(entry, "hero") || entry["keyframes"] is JsonArray { Count: > 0 } + || entry.Any(p => p.Key is not ("sha256" or "size" or "capturedAt" or "occurrence" or "excluded" or "hero" or "keyframes")); + + /// One outside change ( → ) applied to any number of projects made from the basis. + public sealed class Rebase(ProjectSettings basis, ProjectSettings theirs) + { + public ProjectSettings Apply(ProjectSettings ours) + { + ArgumentNullException.ThrowIfNull(ours); + return Merge(basis, ours, theirs); + } + } + + /// A project's JSON text as settings, a newer Chronos's too (read-only there). + /// The text is not a project. + internal static ProjectSettings Read(string json) + { + try + { + return ProjectSettings.FromJson(json, allowNewer: true); + } + catch (JsonException ex) + { + throw new InvalidDataException(ex.Message, ex); + } + } +} diff --git a/src/Chronos.Data/Projects/ProjectService.cs b/src/Chronos.Data/Projects/ProjectService.cs new file mode 100644 index 00000000..67f1c625 --- /dev/null +++ b/src/Chronos.Data/Projects/ProjectService.cs @@ -0,0 +1,588 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using Chronos.Core.Archiving; +using Chronos.Core.Diagnostics; +using Chronos.Core.IO; +using Chronos.Core.Model; +using Chronos.Core.Project; +using Chronos.Data.Archiving; +using Chronos.Data.Repositories; + +namespace Chronos.Data.Projects; + +/// Why a project operation was refused (). +public enum ProjectError +{ + /// The sequence is not in the index. + SequenceNotFound, + + /// The project (or the version named) does not exist, or was deleted. + NotFound, + + /// The name is blank or longer than . + InvalidName, + + /// Another project of the sequence has that name (case does not count). + NameTaken, + + /// The Main project cannot be deleted: it holds the sequence's own settings. + MainCannotBeDeleted, +} + +/// A project operation was refused; nothing was written. +public sealed class ProjectException(ProjectError reason, string message) : InvalidOperationException(message) +{ + public ProjectError Reason { get; } = reason; +} + +/// What became of a version pushed to its home (). +public enum PushOutcome +{ + /// Stored as the project's new head (its parent was the head: a fast-forward). + Stored, + + /// A version with that id is stored already (a push repeated after its answer was lost): nothing changed. + AlreadyStored, + + /// The head moved since the version's parent: pull the head, replay the version on it and push again. + Diverged, + + /// The settings do not match the content hash sent with them: nothing was stored. + HashMismatch, + + /// The settings are of a newer format than this home reads: nothing was stored. + NewerFormat, + + /// Not a version of this project: settings that are not portable (they name frames by position), or an id another project holds. + Invalid, +} + +/// The answer to a push: the outcome, the version as stored (, ) and the project's head. +public sealed record PushResult(PushOutcome Outcome, ProjectVersion? Version, ProjectVersion? Head); + +/// +/// The frames of a sequence at its home, as project settings name them: their frames version, rows in index order and +/// photos (, for ). +/// +public sealed record HomeFrames(int FramesVersion, IReadOnlyList Frames, FramePhotos Photos); + +/// +/// The projects of the sequences of this PC's archives (owner review 2, R8): the home side of a workspace's push and pull. +/// Every operation runs under the sequence's lease and first catches up: the sidecars into the index +/// (), the Main project made when the sequence has none (version 1, "imported" from the +/// sequence's settings), and settings another path wrote into the sequence since the last mirror appended to Main as an +/// "external" version (or merged with a push whose mirror a crash cut short). A version is written as a sidecar first +/// (immutable), then the project's record, then the index; a push to Main is also mirrored into the sequence's +/// project.json and settings (positions placed on its frames, through the settings version check), so the code +/// paths that read the sequence's settings keep working and only Main drives the hero frame the Library shows. Other +/// projects stay on this home: copies of a sequence linked to a server sync only its settings, i.e. Main. +/// +public sealed class ProjectService +{ + /// The name of the project made for every sequence (the desktop shows it in the UI language). + public const string MainName = "Main"; + + public const int MaxNameLength = 60; + + /// Frames read at once when a sequence's photos are gathered. + private const int PageSize = 1000; + + /// How long a read waits for a sequence another writer holds before it answers from the index alone. + private static readonly TimeSpan ReadWait = TimeSpan.FromSeconds(3); + + private readonly ChronosDatabase _db; + private readonly SequenceWriter _writer; + private readonly ProjectIndexer _indexer; + + internal ProjectService(ChronosDatabase db, SequenceWriter writer) + { + _db = db; + _writer = writer; + _indexer = new ProjectIndexer(db); + } + + /// The client name versions made here carry (the archive engine's lease owner). + public string ClientName => _writer.Options.Owner; + + /// The live projects of a sequence, Main first; read from the index alone while the archive is offline. + /// The sequence is not in the index. + public async Task> ListAsync(Guid sequenceId, CancellationToken cancellationToken = default) + { + await ReadAsync(sequenceId, cancellationToken).ConfigureAwait(false); + return await _db.Projects.ListBySequenceAsync(sequenceId, includeDeleted: false, cancellationToken).ConfigureAwait(false); + } + + /// The sequence's Main project, made now (from its settings) when it has none. + /// The sequence is not in the index. + /// The archive is offline and the index has no Main project yet. + public async Task MainAsync(Guid sequenceId, CancellationToken cancellationToken = default) + { + await ReadAsync(sequenceId, cancellationToken).ConfigureAwait(false); + if (await _db.Projects.GetMainAsync(sequenceId, cancellationToken).ConfigureAwait(false) is { } main) return main; + // The catch-up could not run (the archive offline, the sequence busy): it runs now, or says why it cannot. + return await UnderLeaseAsync(sequenceId, async (_, ct) => (await _db.Projects.GetMainAsync(sequenceId, ct).ConfigureAwait(false))!, cancellationToken) + .ConfigureAwait(false); + } + + public Task GetAsync(Guid projectId, CancellationToken cancellationToken = default) => _db.Projects.GetAsync(projectId, cancellationToken); + + /// + /// A new project of the sequence named , whose first version (kind "imported") holds the settings of + /// (a version of any project of the sequence), or of Main's head. + /// + /// Unknown sequence or version, or the name is invalid or taken. + public Task CreateAsync(Guid sequenceId, string name, Guid? fromVersionId = null, string? author = null, + CancellationToken cancellationToken = default) + { + var trimmed = ValidName(name); + return UnderLeaseAsync(sequenceId, async (context, ct) => + { + await EnsureNameFreeAsync(sequenceId, null, trimmed, ct).ConfigureAwait(false); + ProjectVersion source; + if (fromVersionId is { } from) + { + source = await _db.Projects.GetVersionAsync(from, ct).ConfigureAwait(false) is { } version + && await _db.Projects.GetAsync(version.ProjectId, ct).ConfigureAwait(false) is { SequenceId: var owner } && owner == sequenceId + ? version + : throw new ProjectException(ProjectError.NotFound, $"Version {from} is not a version of sequence {sequenceId}."); + } + else + { + var main = (await _db.Projects.GetMainAsync(sequenceId, ct).ConfigureAwait(false))!; + source = (await _db.Projects.GetVersionAsync(main.HeadVersionId!.Value, ct).ConfigureAwait(false))!; + } + var now = DateTimeOffset.UtcNow; + var project = new SequenceProject + { + Id = Guid.CreateVersion7(), + SequenceId = sequenceId, + Name = trimmed, + CreatedBy = author, + CreatedAt = now, + UpdatedAt = now, + }; + var first = new ProjectVersion + { + Id = Guid.CreateVersion7(), + ProjectId = project.Id, + Number = 1, + Kind = ProjectVersionKind.Imported, + Author = author, + ClientName = ClientName, + SettingsJson = source.SettingsJson, + ContentHash = source.ContentHash, + SettingsFormat = source.SettingsFormat, + FramesVersion = source.FramesVersion, + CreatedAt = now, + StoredAt = now, + }; + return await AddProjectAsync(context.Folder, project, first, ct).ConfigureAwait(false); + }, cancellationToken); + } + + /// Gives a project another name. + /// Unknown project, or the name is invalid or taken. + public async Task RenameAsync(Guid projectId, string name, CancellationToken cancellationToken = default) + { + var trimmed = ValidName(name); + var project = await LiveAsync(projectId, cancellationToken).ConfigureAwait(false); + return await UnderLeaseAsync(project.SequenceId, async (context, ct) => + { + var current = await LiveAsync(projectId, ct).ConfigureAwait(false); + if (current.Name == trimmed) return current; + await EnsureNameFreeAsync(current.SequenceId, projectId, trimmed, ct).ConfigureAwait(false); + var renamed = current with { Name = trimmed, UpdatedAt = DateTimeOffset.UtcNow }; + ProjectSidecars.WriteRecord(context.Folder.ProjectRecordPath(projectId), ProjectRecord.Of(renamed)); + await _db.Projects.UpdateAsync(renamed, ct).ConfigureAwait(false); + return renamed; + }, cancellationToken).ConfigureAwait(false); + } + + /// + /// Deletes a project: its record says so, its folder moves to .chronos/projects/.trash (kept, never erased by + /// Chronos) and it leaves the listing. + /// + /// Unknown project, or the Main project. + public async Task DeleteAsync(Guid projectId, CancellationToken cancellationToken = default) + { + var project = await LiveAsync(projectId, cancellationToken).ConfigureAwait(false); + if (project.IsMain) throw new ProjectException(ProjectError.MainCannotBeDeleted, "The Main project cannot be deleted."); + await UnderLeaseAsync(project.SequenceId, async (context, ct) => + { + var current = await LiveAsync(projectId, ct).ConfigureAwait(false); + var now = DateTimeOffset.UtcNow; + var deleted = current with { DeletedAt = now, UpdatedAt = now }; + ProjectSidecars.WriteRecord(context.Folder.ProjectRecordPath(projectId), ProjectRecord.Of(deleted)); + Directory.CreateDirectory(context.Folder.ProjectsTrashDirectory); + Directory.Move(context.Folder.ProjectDirectory(projectId), + Path.Combine(context.Folder.ProjectsTrashDirectory, projectId.ToString("N") + "-" + now.UtcTicks.ToString(System.Globalization.CultureInfo.InvariantCulture))); + await _db.Projects.UpdateAsync(deleted, ct).ConfigureAwait(false); + return true; + }, cancellationToken).ConfigureAwait(false); + } + + /// One page of a project's history, newest first: the versions numbered below (from the head when null). + /// Unknown project. + public async Task> HistoryAsync(Guid projectId, int? beforeNumber = null, int limit = 50, CancellationToken cancellationToken = default) + { + var project = await KnownAsync(projectId, cancellationToken).ConfigureAwait(false); + await ReadAsync(project.SequenceId, cancellationToken).ConfigureAwait(false); + return await _db.Projects.ListVersionsAsync(projectId, beforeNumber, limit, cancellationToken).ConfigureAwait(false); + } + + /// The versions numbered above , oldest first: what a workspace pulls. + /// Unknown project. + public async Task> VersionsAfterAsync(Guid projectId, int afterNumber, int limit = 100, CancellationToken cancellationToken = default) + { + var project = await KnownAsync(projectId, cancellationToken).ConfigureAwait(false); + await ReadAsync(project.SequenceId, cancellationToken).ConfigureAwait(false); + return await _db.Projects.ListVersionsAfterAsync(projectId, afterNumber, limit, cancellationToken).ConfigureAwait(false); + } + + /// A version by id; null when the index has none. + public Task GetVersionAsync(Guid versionId, CancellationToken cancellationToken = default) => + _db.Projects.GetVersionAsync(versionId, cancellationToken); + + /// The newest version of a project, after catching up. + /// Unknown project. + public async Task HeadAsync(Guid projectId, CancellationToken cancellationToken = default) + { + var project = await KnownAsync(projectId, cancellationToken).ConfigureAwait(false); + await ReadAsync(project.SequenceId, cancellationToken).ConfigureAwait(false); + project = await KnownAsync(projectId, cancellationToken).ConfigureAwait(false); + return await _db.Projects.GetVersionAsync(project.HeadVersionId ?? Guid.Empty, cancellationToken).ConfigureAwait(false) + ?? throw new ProjectException(ProjectError.NotFound, $"Project {projectId} has no version."); + } + + /// + /// Stores (made in a workspace from its ) as the + /// project's new head when its parent is the head: its number is the head's plus one, whatever it says. Repeated with + /// the id of a stored version, it answers as the first time. Its settings must be portable (positions parked by photo) + /// and match their content hash. A version of Main is mirrored into the sequence's settings. + /// + /// Unknown project. + public async Task PushAsync(Guid projectId, ProjectVersion version, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(version); + var project = await LiveAsync(projectId, cancellationToken).ConfigureAwait(false); + return await UnderLeaseAsync(project.SequenceId, async (context, ct) => + { + var current = await LiveAsync(projectId, ct).ConfigureAwait(false); + var head = await _db.Projects.GetVersionAsync(current.HeadVersionId ?? Guid.Empty, ct).ConfigureAwait(false); + if (await _db.Projects.GetVersionAsync(version.Id, ct).ConfigureAwait(false) is { } existing) + { + return existing.ProjectId == projectId && string.Equals(existing.ContentHash, version.ContentHash, StringComparison.OrdinalIgnoreCase) + ? new PushResult(PushOutcome.AlreadyStored, existing, head) + : new PushResult(PushOutcome.Invalid, null, head); + } + if (!ProjectContent.Verifies(version.SettingsJson, version.ContentHash)) return new PushResult(PushOutcome.HashMismatch, null, head); + var format = ProjectContent.FormatOf(version.SettingsJson); + if (format > ProjectSettings.CurrentVersion) return new PushResult(PushOutcome.NewerFormat, null, head); + if (!IsPortable(version.SettingsJson)) return new PushResult(PushOutcome.Invalid, null, head); + if (version.ParentId != current.HeadVersionId) return new PushResult(PushOutcome.Diverged, null, head); + var stored = version with + { + ProjectId = projectId, + Number = current.HeadNumber + 1, + ContentHash = version.ContentHash.ToLowerInvariant(), + SettingsFormat = format, + StoredAt = DateTimeOffset.UtcNow, + }; + current = await AppendAsync(context.Folder, current, stored, ct).ConfigureAwait(false); + if (current.IsMain) await MirrorAsync(context, current, stored, ct).ConfigureAwait(false); + return new PushResult(PushOutcome.Stored, stored, stored); + }, cancellationToken).ConfigureAwait(false); + } + + /// The frames of a sequence as they are indexed now, with its frames version. + /// The sequence is not in the index. + public async Task FramesAsync(Guid sequenceId, CancellationToken cancellationToken = default) + { + for (var attempt = 0; ; attempt++) + { + var before = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? throw SequenceMissing(sequenceId); + var frames = await LoadFramesAsync(sequenceId, cancellationToken).ConfigureAwait(false); + var after = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? throw SequenceMissing(sequenceId); + if (after.FramesVersion == before.FramesVersion || attempt == 2) return new HomeFrames(after.FramesVersion, frames.Frames, frames.Photos); + } + } + + /// + /// Catches the sequence's projects up under its lease (see the class remarks) before a read; the index alone answers while + /// the archive is offline or the sequence busy for longer than the lease wait. + /// + private async Task ReadAsync(Guid sequenceId, CancellationToken cancellationToken) + { + try + { + await UnderLeaseAsync(sequenceId, (_, _) => Task.FromResult(true), cancellationToken, ReadWait).ConfigureAwait(false); + } + catch (Exception ex) when (ex is ArchiveOfflineException or LeaseUnavailableException) + { + Log.Info("projects", $"The projects of sequence {sequenceId} are read from the index: {ex.Message}"); + } + } + + /// Runs under the sequence's lease, after catching its projects up. + private async Task UnderLeaseAsync(Guid sequenceId, Func> work, CancellationToken cancellationToken, + TimeSpan? wait = null) + { + var sequence = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? throw SequenceMissing(sequenceId); + var archive = await _db.Archives.GetAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false) ?? throw SequenceMissing(sequenceId); + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + await using (await _writer.LockAsync(paths, sequenceId, wait, cancellationToken).ConfigureAwait(false)) + { + var row = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? throw SequenceMissing(sequenceId); + var context = new LeaseContext(paths.Sequence(row.Folder), row); + await CatchUpAsync(context, cancellationToken).ConfigureAwait(false); + return await work(context, cancellationToken).ConfigureAwait(false); + } + } + + /// The sidecars into the index, the Main project made, and Main in step with the sequence's settings. + private async Task CatchUpAsync(LeaseContext context, CancellationToken cancellationToken) + { + await _indexer.ReconcileAsync(context.Folder, context.Row.Id, cancellationToken).ConfigureAwait(false); + var main = await _db.Projects.GetMainAsync(context.Row.Id, cancellationToken).ConfigureAwait(false); + if (main?.HeadVersionId is null) + { + await MakeMainAsync(context, main, cancellationToken).ConfigureAwait(false); + return; + } + if (main.MirroredVersionId == main.HeadVersionId && main.MirroredSettingsVersion == context.Row.SettingsVersion) return; + var head = (await _db.Projects.GetVersionAsync(main.HeadVersionId.Value, cancellationToken).ConfigureAwait(false))!; + var frames = await LoadFramesAsync(context.Row.Id, cancellationToken).ConfigureAwait(false); + if (CurrentPortable(context, frames.Photos) is not { } current) + { + // Settings no Chronos can read (or of a newer one): left as they are, recorded as seen. + await _db.Projects.SetMirroredAsync(main.Id, main.HeadVersionId, context.Row.SettingsVersion, cancellationToken).ConfigureAwait(false); + return; + } + var sameAsHead = ProjectContent.Hash(current) == head.ContentHash; + if (main.MirroredVersionId == main.HeadVersionId || main.MirroredVersionId is null) + { + // Only the sequence's settings version moved (settings another path wrote, or only its title or tags), or the index + // was rebuilt and knows no mirror: settings that differ from the head are its newest state. + var latest = sameAsHead ? head : await AppendDerivedAsync(context, main, head, current, ProjectVersionKind.External, cancellationToken).ConfigureAwait(false); + await _db.Projects.SetMirroredAsync(main.Id, latest.Id, context.Row.SettingsVersion, cancellationToken).ConfigureAwait(false); + return; + } + // The head is newer than the mirror: a push whose mirror a crash (or a settings edit at the same moment) cut short. + if (sameAsHead) + { + await _db.Projects.SetMirroredAsync(main.Id, head.Id, context.Row.SettingsVersion, cancellationToken).ConfigureAwait(false); + return; + } + var mirrored = await _db.Projects.GetVersionAsync(main.MirroredVersionId.Value, cancellationToken).ConfigureAwait(false); + if (main.MirroredSettingsVersion == context.Row.SettingsVersion || mirrored is null || ProjectContent.Hash(current) == mirrored.ContentHash) + { + // Nobody changed the settings since the last mirror: the mirror completes. + await MirrorAsync(context, main, head, cancellationToken).ConfigureAwait(false); + return; + } + // Both moved: the settings written meanwhile merge with the head (the version made in a workspace wins where both changed). + var merged = ProjectMerge.Merge(ProjectMerge.Read(mirrored.SettingsJson), ProjectMerge.Read(head.SettingsJson), ProjectMerge.Read(current)); + var version = await AppendDerivedAsync(context, main, head, ProjectContent.Canonical(merged.ToJson()), ProjectVersionKind.Merge, cancellationToken) + .ConfigureAwait(false); + main = (await _db.Projects.GetAsync(main.Id, cancellationToken).ConfigureAwait(false))!; + await MirrorAsync(context, main, version, cancellationToken).ConfigureAwait(false); + } + + /// The Main project of a sequence that has none (or one without versions): version 1 from the sequence's settings. + private async Task MakeMainAsync(LeaseContext context, SequenceProject? existing, CancellationToken cancellationToken) + { + var frames = await LoadFramesAsync(context.Row.Id, cancellationToken).ConfigureAwait(false); + var portable = CurrentPortable(context, frames.Photos) ?? ProjectContent.ToPortable(new ProjectSettings(), frames.Photos); + var now = DateTimeOffset.UtcNow; + var project = existing ?? new SequenceProject + { + Id = Guid.CreateVersion7(), + SequenceId = context.Row.Id, + Name = await FreeMainNameAsync(context.Row.Id, cancellationToken).ConfigureAwait(false), + IsMain = true, + CreatedAt = now, + UpdatedAt = now, + }; + var first = NewVersion(project.Id, null, project.HeadNumber + 1, ProjectVersionKind.Imported, portable, context.Row.FramesVersion, now); + if (existing is null) + { + await AddProjectAsync(context.Folder, project with { MirroredSettingsVersion = context.Row.SettingsVersion, MirroredVersionId = first.Id }, first, + cancellationToken).ConfigureAwait(false); + } + else + { + var updated = await AppendAsync(context.Folder, existing, first, cancellationToken).ConfigureAwait(false); + await _db.Projects.SetMirroredAsync(updated.Id, first.Id, context.Row.SettingsVersion, cancellationToken).ConfigureAwait(false); + } + } + + /// "Main", or "Main (2)"… when a project of that name exists already (one made before the sequence had its Main). + private async Task FreeMainNameAsync(Guid sequenceId, CancellationToken cancellationToken) + { + var names = (await _db.Projects.ListBySequenceAsync(sequenceId, includeDeleted: false, cancellationToken).ConfigureAwait(false)) + .Select(p => p.Name).ToHashSet(StringComparer.OrdinalIgnoreCase); + var name = MainName; + for (var n = 2; names.Contains(name); n++) name = $"{MainName} ({n})"; + return name; + } + + /// The portable form of the sequence's settings in force (index, else project.json, else defaults); null when unreadable or newer. + private static string? CurrentPortable(LeaseContext context, FramePhotos photos) + { + var json = context.Row.SettingsJson ?? (File.Exists(context.Folder.ProjectPath) ? AtomicFile.ReadAllText(context.Folder.ProjectPath) : null) + ?? new ProjectSettings().ToJson(); + try + { + if (ProjectContent.FormatOf(json) > ProjectSettings.CurrentVersion) return null; + return ProjectContent.ToPortable(json, photos); + } + catch (JsonException ex) + { + Log.Warning("projects", $"The settings of sequence {context.Row.Id} cannot be read", ex); + return null; + } + } + + /// A version made here (catch-up or merge) appended to Main. + private async Task AppendDerivedAsync(LeaseContext context, SequenceProject main, ProjectVersion head, string portable, ProjectVersionKind kind, + CancellationToken cancellationToken) + { + var version = NewVersion(main.Id, head.Id, main.HeadNumber + 1, kind, portable, context.Row.FramesVersion, DateTimeOffset.UtcNow); + await AppendAsync(context.Folder, main, version, cancellationToken).ConfigureAwait(false); + return version; + } + + private ProjectVersion NewVersion(Guid projectId, Guid? parentId, int number, ProjectVersionKind kind, string portable, int framesVersion, DateTimeOffset now) => new() + { + Id = Guid.CreateVersion7(), + ProjectId = projectId, + ParentId = parentId, + Number = number, + Kind = kind, + ClientName = ClientName, + SettingsJson = portable, + ContentHash = ProjectContent.Hash(portable), + SettingsFormat = ProjectContent.FormatOf(portable), + FramesVersion = framesVersion, + CreatedAt = now, + StoredAt = now, + }; + + /// A new project with its first version: sidecars (version, then record), then both rows in one transaction. + private async Task AddProjectAsync(SequencePaths folder, SequenceProject project, ProjectVersion first, CancellationToken cancellationToken) + { + var withHead = project with { HeadVersionId = first.Id, HeadNumber = first.Number }; + ProjectSidecars.WriteVersion(folder.VersionPath(project.Id, first.Number, first.Id), first); + ProjectSidecars.WriteRecord(folder.ProjectRecordPath(project.Id), ProjectRecord.Of(withHead)); + await using var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false); + await tx.Projects.InsertAsync(withHead with { HeadVersionId = null, HeadNumber = 0 }, cancellationToken).ConfigureAwait(false); + await tx.Projects.InsertVersionAsync(first, cancellationToken).ConfigureAwait(false); + await tx.Projects.UpdateAsync(withHead, cancellationToken).ConfigureAwait(false); + await tx.CommitAsync(cancellationToken).ConfigureAwait(false); + return withHead; + } + + /// A version appended to a project: its sidecar, the record's head, then the version row and the head in one transaction. + private async Task AppendAsync(SequencePaths folder, SequenceProject project, ProjectVersion version, CancellationToken cancellationToken) + { + var now = DateTimeOffset.UtcNow; + var advanced = project with { HeadVersionId = version.Id, HeadNumber = version.Number, UpdatedAt = now }; + ProjectSidecars.WriteVersion(folder.VersionPath(project.Id, version.Number, version.Id), version); + ProjectSidecars.WriteRecord(folder.ProjectRecordPath(project.Id), ProjectRecord.Of(advanced)); + await using var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false); + await tx.Projects.InsertVersionAsync(version, cancellationToken).ConfigureAwait(false); + if (!await tx.Projects.AdvanceHeadAsync(project.Id, project.HeadNumber, version.Id, version.Number, now, cancellationToken).ConfigureAwait(false)) + throw new IOException($"The head of project {project.Id} moved in the index while its lease was held."); + await tx.CommitAsync(cancellationToken).ConfigureAwait(false); + return advanced; + } + + /// + /// Writes Main's into the sequence: its positions placed on the sequence's frames, then + /// project.json and the index's settings (and hero frame) with the settings version as the check. Recorded as + /// mirrored when the index took it; a settings edit that won the race is caught up by the next operation (merged). + /// + private async Task MirrorAsync(LeaseContext context, SequenceProject main, ProjectVersion version, CancellationToken cancellationToken) + { + var frames = await LoadFramesAsync(context.Row.Id, cancellationToken).ConfigureAwait(false); + var placed = ProjectSettings.FromJson(ProjectContent.Place(version.SettingsJson, frames.Photos)); + var json = placed.ToJson(); + Directory.CreateDirectory(context.Folder.MetaDirectory); + AtomicFile.WriteAllText(context.Folder.ProjectPath, json); + var hero = placed.HeroFrame is { } index && (uint)index < (uint)frames.Frames.Count ? frames.Frames[index].Id : Guid.Empty; + var result = await _db.Sequences.EditAsync(context.Row.Id, new SequenceEdit { SettingsJson = json, HeroFrameId = hero }, context.Row.SettingsVersion, + context.Row.FramesVersion, cancellationToken).ConfigureAwait(false); + if (result is { Outcome: SequenceEditOutcome.Updated, Sequence: { } updated }) + { + context.Row = updated; + await _db.Projects.SetMirroredAsync(main.Id, version.Id, updated.SettingsVersion, cancellationToken).ConfigureAwait(false); + } + else + { + Log.Info("projects", $"The settings of sequence {context.Row.Id} changed while version {version.Number} of Main was mirrored; merged next time"); + } + } + + /// Whether settings name no frame by position (every position parked by photo). + private static bool IsPortable(string json) + { + try + { + return JsonNode.Parse(json, new JsonNodeOptions { PropertyNameCaseInsensitive = true }) is JsonObject project + && project["excludedFrames"] is null or JsonArray { Count: 0 } + && project["heroFrame"] is null + && (project["camera"] is not JsonObject camera || camera["keyframes"] is null or JsonArray { Count: 0 }); + } + catch (Exception ex) when (ex is JsonException or ArgumentException) + { + return false; + } + } + + private async Task<(IReadOnlyList Frames, FramePhotos Photos)> LoadFramesAsync(Guid sequenceId, CancellationToken cancellationToken) + { + var frames = new List(); + for (var after = -1; ;) + { + var page = await _db.Frames.ListBySequenceAsync(sequenceId, after, PageSize, cancellationToken).ConfigureAwait(false); + frames.AddRange(page); + if (page.Count < PageSize) break; + after = page[^1].Index; + } + return (frames, new FramePhotos(frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, f.CapturedAt))))); + } + + private async Task EnsureNameFreeAsync(Guid sequenceId, Guid? projectId, string name, CancellationToken cancellationToken) + { + var projects = await _db.Projects.ListBySequenceAsync(sequenceId, includeDeleted: false, cancellationToken).ConfigureAwait(false); + if (projects.Any(p => p.Id != projectId && string.Equals(p.Name, name, StringComparison.OrdinalIgnoreCase))) + throw new ProjectException(ProjectError.NameTaken, $"Sequence {sequenceId} has a project named '{name}'."); + } + + private static string ValidName(string name) + { + var trimmed = name?.Trim() ?? string.Empty; + return trimmed.Length is > 0 and <= MaxNameLength && !trimmed.Any(char.IsControl) + ? trimmed + : throw new ProjectException(ProjectError.InvalidName, "A project name has 1 to 60 characters."); + } + + private async Task KnownAsync(Guid projectId, CancellationToken cancellationToken) => + await _db.Projects.GetAsync(projectId, cancellationToken).ConfigureAwait(false) + ?? throw new ProjectException(ProjectError.NotFound, $"Project {projectId} does not exist."); + + private async Task LiveAsync(Guid projectId, CancellationToken cancellationToken) => + await KnownAsync(projectId, cancellationToken).ConfigureAwait(false) is { DeletedAt: null } project + ? project + : throw new ProjectException(ProjectError.NotFound, $"Project {projectId} was deleted."); + + private static ProjectException SequenceMissing(Guid sequenceId) => new(ProjectError.SequenceNotFound, $"Sequence {sequenceId} is not in the index."); + + /// The sequence as read under its lease (the mirror moves its settings version) and its folder. + private sealed class LeaseContext(SequencePaths folder, Sequence row) + { + public SequencePaths Folder { get; } = folder; + public Sequence Row { get; set; } = row; + } +} diff --git a/src/Chronos.Data/Projects/ProjectSidecars.cs b/src/Chronos.Data/Projects/ProjectSidecars.cs new file mode 100644 index 00000000..24d50d17 --- /dev/null +++ b/src/Chronos.Data/Projects/ProjectSidecars.cs @@ -0,0 +1,276 @@ +using System.Globalization; +using System.Text; +using System.Text.Json; +using Chronos.Core.Archiving; +using Chronos.Core.IO; +using Chronos.Core.Model; +using Chronos.Core.Project; + +namespace Chronos.Data.Projects; + +/// The newest version of a project as its record names it: id and number. +public sealed record ProjectHead(Guid Id, int Number); + +/// +/// The record of a project in its folder (.chronos/projects/<id>/project.json): what the index row is rebuilt +/// from. is set when the project was deleted, before its folder moves to the projects' trash. +/// +public sealed record ProjectRecord(Guid Id, Guid SequenceId, string Name, bool IsMain, ProjectHead? Head, DateTimeOffset Created, string? CreatedBy, + DateTimeOffset Updated, DateTimeOffset? Deleted = null) +{ + /// The record of an index row. + public static ProjectRecord Of(SequenceProject project) + { + ArgumentNullException.ThrowIfNull(project); + return new ProjectRecord(project.Id, project.SequenceId, project.Name, project.IsMain, + project.HeadVersionId is { } head ? new ProjectHead(head, project.HeadNumber) : null, project.CreatedAt, project.CreatedBy, project.UpdatedAt, + project.DeletedAt); + } + + /// The index row of this record (nothing mirrored known yet). + public SequenceProject ToProject() => new() + { + Id = Id, + SequenceId = SequenceId, + Name = Name, + IsMain = IsMain, + HeadVersionId = Head?.Id, + HeadNumber = Head?.Number ?? 0, + CreatedBy = CreatedBy, + CreatedAt = Created, + UpdatedAt = Updated, + DeletedAt = Deleted, + }; +} + +/// +/// The files of projects (owner review 2, R8), the same in an archive and in a workspace: the record of a project +/// () and one file per version, immutable once written: +/// { "format": "chronos-project-version", "formatVersion": 1, "id", "projectId", "parentId", "number", "kind", +/// "author", "clientName", "note", "settingsFormat", "framesVersion", "contentHash", "restoredFrom", "created", +/// "stored", "settings": "<the portable settings as text>" } +/// The settings are a JSON string, so the bytes the content hash covers come back exactly; a version whose text does not +/// match its hash is refused as damaged. Writes are atomic (temp, flush, rename). +/// +public static class ProjectSidecars +{ + public const string RecordFormat = "chronos-project-record"; + public const string VersionFormat = "chronos-project-version"; + public const int FormatVersion = 1; + + /// A version file is a few kilobytes; a larger one is not read. + public const int MaxVersionBytes = 8 << 20; + + private static readonly JsonWriterOptions Indented = new() { Indented = true }; + + /// + /// Writes a version file. A version is immutable: an existing file is left when it holds the same version (same id + /// and content hash) and refused otherwise, unless (a workspace's own copy of a version, + /// stored at its home since). + /// + /// Another version is at that path. + public static void WriteVersion(string path, ProjectVersion version, bool replace = false) + { + ArgumentNullException.ThrowIfNull(version); + if (!replace && File.Exists(path)) + { + ProjectVersion existing; + try + { + existing = ReadVersion(path); + } + catch (InvalidDataException) + { + // A damaged file is never overwritten: it is moved aside, kept. + File.Move(path, path + ".damaged-" + DateTime.UtcNow.ToString("yyyyMMddHHmmss", CultureInfo.InvariantCulture)); + existing = null!; + } + if (existing is not null) + { + if (existing.Id == version.Id && string.Equals(existing.ContentHash, version.ContentHash, StringComparison.OrdinalIgnoreCase)) return; + throw new IOException($"{path} holds another version ({existing.Id})."); + } + } + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + AtomicFile.WriteAllText(path, VersionJson(version)); + } + + /// Reads a version file and checks its content hash. + /// Not a version file, damaged, newer, or its settings do not match their hash. + /// The file cannot be read. + public static ProjectVersion ReadVersion(string path) + { + if (new FileInfo(path).Length > MaxVersionBytes) throw new InvalidDataException($"{path} is too large to be a project version."); + return ParseVersion(AtomicFile.ReadAllText(path), path); + } + + /// The text of a version file. + public static string VersionJson(ProjectVersion version) + { + ArgumentNullException.ThrowIfNull(version); + using var buffer = new MemoryStream(); + using (var w = new Utf8JsonWriter(buffer, Indented)) + { + w.WriteStartObject(); + w.WriteString("format", VersionFormat); + w.WriteNumber("formatVersion", FormatVersion); + w.WriteString("id", version.Id); + w.WriteString("projectId", version.ProjectId); + if (version.ParentId is { } parent) w.WriteString("parentId", parent); + w.WriteNumber("number", version.Number); + w.WriteString("kind", ProjectVersionKinds.Name(version.Kind)); + if (version.Author is { } author) w.WriteString("author", author); + if (version.ClientName is { } client) w.WriteString("clientName", client); + if (version.Note is { } note) w.WriteString("note", note); + w.WriteNumber("settingsFormat", version.SettingsFormat); + w.WriteNumber("framesVersion", version.FramesVersion); + w.WriteString("contentHash", version.ContentHash.ToLowerInvariant()); + if (version.RestoredFrom is { } restored) w.WriteString("restoredFrom", restored); + w.WriteString("created", version.CreatedAt); + if (version.StoredAt is { } stored) w.WriteString("stored", stored); + w.WriteString("settings", version.SettingsJson); + w.WriteEndObject(); + } + return Encoding.UTF8.GetString(buffer.GetBuffer(), 0, (int)buffer.Length); + } + + /// Reads a version file's text ( names it in errors) and checks its content hash. + /// Not a version file, damaged, newer, or its settings do not match their hash. + public static ProjectVersion ParseVersion(string json, string source) + { + using var document = Parse(json, source); + var root = document.RootElement; + Check(root, VersionFormat, source); + var settings = Text(root, "settings") ?? throw new InvalidDataException($"{source} holds no settings."); + var hash = Text(root, "contentHash") ?? throw new InvalidDataException($"{source} has no content hash."); + if (!ProjectContent.Verifies(settings, hash)) throw new InvalidDataException($"The settings in {source} do not match their content hash."); + return new ProjectVersion + { + Id = Id(root, "id") ?? throw new InvalidDataException($"{source} has no version id."), + ProjectId = Id(root, "projectId") ?? throw new InvalidDataException($"{source} names no project."), + ParentId = Id(root, "parentId"), + Number = Int(root, "number") is { } number and > 0 ? number : throw new InvalidDataException($"{source} has no version number."), + Kind = ProjectVersionKinds.Parse(Text(root, "kind")), + Author = Text(root, "author"), + ClientName = Text(root, "clientName"), + Note = Text(root, "note"), + SettingsJson = settings, + ContentHash = hash.ToLowerInvariant(), + SettingsFormat = Int(root, "settingsFormat") ?? ProjectContent.FormatOf(settings), + FramesVersion = Int(root, "framesVersion") ?? 0, + RestoredFrom = Id(root, "restoredFrom"), + CreatedAt = Time(root, "created") ?? default, + StoredAt = Time(root, "stored"), + }; + } + + /// Writes a project's record atomically. + public static void WriteRecord(string path, ProjectRecord record) + { + ArgumentNullException.ThrowIfNull(record); + using var buffer = new MemoryStream(); + using (var w = new Utf8JsonWriter(buffer, Indented)) + { + w.WriteStartObject(); + w.WriteString("format", RecordFormat); + w.WriteNumber("formatVersion", FormatVersion); + w.WriteString("id", record.Id); + w.WriteString("sequenceId", record.SequenceId); + w.WriteString("name", record.Name); + w.WriteBoolean("isMain", record.IsMain); + if (record.Head is { } head) + { + w.WriteStartObject("head"); + w.WriteString("id", head.Id); + w.WriteNumber("number", head.Number); + w.WriteEndObject(); + } + w.WriteString("created", record.Created); + if (record.CreatedBy is { } by) w.WriteString("createdBy", by); + w.WriteString("updated", record.Updated); + if (record.Deleted is { } deleted) w.WriteString("deleted", deleted); + w.WriteEndObject(); + } + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + AtomicFile.WriteAllBytes(path, buffer.GetBuffer().AsSpan(0, (int)buffer.Length)); + } + + /// Reads a project's record. + /// Not a project record, damaged or newer. + /// The file cannot be read. + public static ProjectRecord ReadRecord(string path) + { + using var document = Parse(AtomicFile.ReadAllText(path), path); + var root = document.RootElement; + Check(root, RecordFormat, path); + ProjectHead? head = null; + if (root.TryGetProperty("head", out var h) && h.ValueKind == JsonValueKind.Object && Id(h, "id") is { } headId && Int(h, "number") is { } number and > 0) + head = new ProjectHead(headId, number); + var name = Text(root, "name"); + return new ProjectRecord( + Id(root, "id") ?? throw new InvalidDataException($"{path} has no project id."), + Id(root, "sequenceId") ?? throw new InvalidDataException($"{path} names no sequence."), + string.IsNullOrWhiteSpace(name) ? throw new InvalidDataException($"{path} has no name.") : name, + root.TryGetProperty("isMain", out var main) && main.ValueKind == JsonValueKind.True, + head, + Time(root, "created") ?? default, + Text(root, "createdBy"), + Time(root, "updated") ?? Time(root, "created") ?? default, + Time(root, "deleted")); + } + + /// The version files of a folder by number and id (000001-<id>.json), in number order; other files are not listed. + public static IReadOnlyList<(int Number, Guid Id, string Path)> ListVersions(string directory) + { + if (!Directory.Exists(directory)) return []; + var found = new List<(int, Guid, string)>(); + foreach (var path in Directory.EnumerateFiles(directory, "*.json")) + { + if (TryParseName(Path.GetFileName(path), out var number, out var id)) found.Add((number, id, path)); + } + return [.. found.OrderBy(f => f.Item1).ThenBy(f => f.Item2)]; + } + + /// The number and id a version file name carries (). + public static bool TryParseName(string fileName, out int number, out Guid id) + { + number = 0; + id = Guid.Empty; + var name = Path.GetFileNameWithoutExtension(fileName); + var dash = name.IndexOf('-', StringComparison.Ordinal); + return dash > 0 && Path.GetExtension(fileName).Equals(".json", StringComparison.OrdinalIgnoreCase) + && int.TryParse(name.AsSpan(0, dash), NumberStyles.None, CultureInfo.InvariantCulture, out number) && number > 0 + && Guid.TryParseExact(name.AsSpan(dash + 1), "N", out id); + } + + private static JsonDocument Parse(string json, string source) + { + try + { + return JsonDocument.Parse(json, new JsonDocumentOptions { MaxDepth = 64 }); + } + catch (JsonException ex) + { + throw new InvalidDataException($"{source} is not valid JSON.", ex); + } + } + + private static void Check(JsonElement root, string format, string source) + { + if (root.ValueKind != JsonValueKind.Object || Text(root, "format") != format) throw new InvalidDataException($"{source} is not a {format} file."); + if (Int(root, "formatVersion") is not { } version || version < 1) throw new InvalidDataException($"{source} has no valid format version."); + if (version > FormatVersion) throw new InvalidDataException($"{source} uses format {version}, newer than this Chronos ({FormatVersion})."); + } + + private static string? Text(JsonElement element, string name) => + element.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.String ? value.GetString() : null; + + private static int? Int(JsonElement element, string name) => + element.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.Number && value.TryGetInt32(out var number) ? number : null; + + private static Guid? Id(JsonElement element, string name) => + element.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.String && value.TryGetGuid(out var id) && id != Guid.Empty ? id : null; + + private static DateTimeOffset? Time(JsonElement element, string name) => + element.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.String && value.TryGetDateTimeOffset(out var time) ? time : null; +} diff --git a/src/Chronos.Data/Repositories/ProjectRepository.cs b/src/Chronos.Data/Repositories/ProjectRepository.cs new file mode 100644 index 00000000..ea5f9a39 --- /dev/null +++ b/src/Chronos.Data/Repositories/ProjectRepository.cs @@ -0,0 +1,149 @@ +using Chronos.Core.Model; + +namespace Chronos.Data.Repositories; + +/// +/// Projects of sequences and their versions (owner review 2, R8; projects, project_versions). The sidecars +/// in .chronos/projects/ are the truth and are written first (Projects.ProjectService); these rows are +/// derived, rebuilt from the sidecars, and go with the sequence's row. +/// +public sealed class ProjectRepository +{ + private const string Columns = + "id, sequence_id, name, is_main, head_version_id, head_number, mirrored_settings_version, mirrored_version_id, created_by, created_at, " + + "updated_at, deleted_at"; + + private const string VersionColumns = + "id, project_id, parent_id, number, kind, author, client_name, note, settings_json, content_hash, settings_format, frames_version, " + + "restored_from, created_at, stored_at"; + + private readonly DbAccess _db; + + internal ProjectRepository(DbAccess db) => _db = db; + + /// The projects of a sequence: Main first, then the others oldest first; deleted ones only when asked. + public Task> ListBySequenceAsync(Guid sequenceId, bool includeDeleted = false, CancellationToken cancellationToken = default) => + _db.ListAsync($"SELECT {Columns} FROM projects WHERE sequence_id = $1{(includeDeleted ? string.Empty : " AND deleted_at IS NULL")} " + + "ORDER BY is_main DESC, created_at, id", [sequenceId], Map, cancellationToken); + + public Task GetAsync(Guid id, CancellationToken cancellationToken = default) => + _db.SingleAsync($"SELECT {Columns} FROM projects WHERE id = $1", [id], Map, cancellationToken); + + /// The Main project of a sequence; null until it is made. + public Task GetMainAsync(Guid sequenceId, CancellationToken cancellationToken = default) => + _db.SingleAsync($"SELECT {Columns} FROM projects WHERE sequence_id = $1 AND is_main = $2 AND deleted_at IS NULL", [sequenceId, true], Map, + cancellationToken); + + /// Adds a project; a name or a Main another live project of the sequence has already fails (unique). + public Task InsertAsync(SequenceProject project, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(project); + return _db.ExecuteAsync($"INSERT INTO projects ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)", + [ + project.Id, project.SequenceId, project.Name, project.IsMain, project.HeadVersionId, project.HeadNumber, project.MirroredSettingsVersion, + project.MirroredVersionId, project.CreatedBy, project.CreatedAt, project.UpdatedAt, project.DeletedAt, + ], cancellationToken); + } + + /// Stores every field of but its id, sequence and creation; false when it is gone. + public async Task UpdateAsync(SequenceProject project, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(project); + return await _db.ExecuteAsync( + "UPDATE projects SET name = $2, is_main = $3, head_version_id = $4, head_number = $5, mirrored_settings_version = $6, mirrored_version_id = $7, " + + "updated_at = $8, deleted_at = $9 WHERE id = $1", + [ + project.Id, project.Name, project.IsMain, project.HeadVersionId, project.HeadNumber, project.MirroredSettingsVersion, + project.MirroredVersionId, project.UpdatedAt, project.DeletedAt, + ], cancellationToken).ConfigureAwait(false) > 0; + } + + /// + /// Moves the head to (number ) only while it is still at + /// : two writers never both advance it from the same head. False when it moved. + /// + public async Task AdvanceHeadAsync(Guid projectId, int expectedNumber, Guid headId, int headNumber, DateTimeOffset at, + CancellationToken cancellationToken = default) => + await _db.ExecuteAsync("UPDATE projects SET head_version_id = $2, head_number = $3, updated_at = $4 WHERE id = $1 AND head_number = $5", + [projectId, headId, headNumber, at, expectedNumber], cancellationToken).ConfigureAwait(false) > 0; + + /// Records the version and the sequence's settings version a Main project's head was last mirrored with. + public async Task SetMirroredAsync(Guid projectId, Guid? versionId, int settingsVersion, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync("UPDATE projects SET mirrored_version_id = $2, mirrored_settings_version = $3 WHERE id = $1", + [projectId, versionId, settingsVersion], cancellationToken).ConfigureAwait(false) > 0; + + /// Adds a version; a number or id the project has already fails (unique). + public Task InsertVersionAsync(ProjectVersion version, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(version); + return _db.ExecuteAsync($"INSERT INTO project_versions ({VersionColumns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)", + [ + version.Id, version.ProjectId, version.ParentId, version.Number, ProjectVersionKinds.Name(version.Kind), version.Author, version.ClientName, + version.Note, DbParam.Text(version.SettingsJson), version.ContentHash, version.SettingsFormat, version.FramesVersion, version.RestoredFrom, + version.CreatedAt, version.StoredAt ?? version.CreatedAt, + ], cancellationToken); + } + + public Task GetVersionAsync(Guid id, CancellationToken cancellationToken = default) => + _db.SingleAsync($"SELECT {VersionColumns} FROM project_versions WHERE id = $1", [id], MapVersion, cancellationToken); + + public Task GetVersionByNumberAsync(Guid projectId, int number, CancellationToken cancellationToken = default) => + _db.SingleAsync($"SELECT {VersionColumns} FROM project_versions WHERE project_id = $1 AND number = $2", [projectId, number], MapVersion, + cancellationToken); + + /// + /// One page of a project's history, newest first: the versions numbered below (all + /// when null), at most (1–1000). + /// + public Task> ListVersionsAsync(Guid projectId, int? beforeNumber, int limit, CancellationToken cancellationToken = default) => + _db.ListAsync($"SELECT {VersionColumns} FROM project_versions WHERE project_id = $1 AND number < $2 ORDER BY number DESC LIMIT $3", + [projectId, beforeNumber ?? int.MaxValue, Cursor.ClampLimit(limit)], MapVersion, cancellationToken); + + /// The versions numbered above , oldest first, at most (1–1000): what a pull takes. + public Task> ListVersionsAfterAsync(Guid projectId, int afterNumber, int limit, CancellationToken cancellationToken = default) => + _db.ListAsync($"SELECT {VersionColumns} FROM project_versions WHERE project_id = $1 AND number > $2 ORDER BY number LIMIT $3", + [projectId, afterNumber, Cursor.ClampLimit(limit)], MapVersion, cancellationToken); + + /// The id of each version of a project by number (the index's side of a comparison with the sidecars). + public async Task> ListVersionIdsAsync(Guid projectId, CancellationToken cancellationToken = default) + { + var rows = await _db.ListAsync("SELECT number, id FROM project_versions WHERE project_id = $1", [projectId], r => (r.GetInt32(0), r.GetGuid(1)), + cancellationToken).ConfigureAwait(false); + return rows.ToDictionary(r => r.Item1, r => r.Item2); + } + + private static SequenceProject Map(DbRow r) => new() + { + Id = r.GetGuid(0), + SequenceId = r.GetGuid(1), + Name = r.GetString(2), + IsMain = r.GetBool(3), + HeadVersionId = r.GetGuidOrNull(4), + HeadNumber = r.GetInt32(5), + MirroredSettingsVersion = r.GetInt32OrNull(6), + MirroredVersionId = r.GetGuidOrNull(7), + CreatedBy = r.GetStringOrNull(8), + CreatedAt = r.GetDateTimeOffset(9), + UpdatedAt = r.GetDateTimeOffset(10), + DeletedAt = r.GetDateTimeOffsetOrNull(11), + }; + + private static ProjectVersion MapVersion(DbRow r) => new() + { + Id = r.GetGuid(0), + ProjectId = r.GetGuid(1), + ParentId = r.GetGuidOrNull(2), + Number = r.GetInt32(3), + Kind = ProjectVersionKinds.Parse(r.GetStringOrNull(4)), + Author = r.GetStringOrNull(5), + ClientName = r.GetStringOrNull(6), + Note = r.GetStringOrNull(7), + SettingsJson = r.GetString(8), + ContentHash = r.GetString(9), + SettingsFormat = r.GetInt32(10), + FramesVersion = r.GetInt32(11), + RestoredFrom = r.GetGuidOrNull(12), + CreatedAt = r.GetDateTimeOffset(13), + StoredAt = r.GetDateTimeOffset(14), + }; +} diff --git a/src/Chronos.Data/Repositories/RenderRepository.cs b/src/Chronos.Data/Repositories/RenderRepository.cs index 8d427c29..416c734d 100644 --- a/src/Chronos.Data/Repositories/RenderRepository.cs +++ b/src/Chronos.Data/Repositories/RenderRepository.cs @@ -6,7 +6,8 @@ namespace Chronos.Data.Repositories; public sealed class RenderRepository { private const string Columns = - "id, sequence_id, preset_name, settings_json, output_path, codec, width, height, fps, duration_seconds, bytes, created, produced_by"; + "id, sequence_id, preset_name, settings_json, output_path, codec, width, height, fps, duration_seconds, bytes, created, produced_by, project_id, " + + "project_version_id"; private const string RecountSql = "UPDATE sequences SET rendered_count = (SELECT count(*) FROM renders WHERE sequence_id = $1) WHERE id = $1"; @@ -19,10 +20,11 @@ public sealed class RenderRepository _db.InTransactionAsync(async session => { await LockSequenceAsync(session, render.SequenceId, cancellationToken).ConfigureAwait(false); - await session.ExecuteAsync($"INSERT INTO renders ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)", + await session.ExecuteAsync($"INSERT INTO renders ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)", [ render.Id, render.SequenceId, render.PresetName, DbParam.Json(render.SettingsJson), render.OutputPath, render.Codec, - render.Width, render.Height, render.Fps, render.DurationSeconds, render.Bytes, render.Created, render.ProducedBy, + render.Width, render.Height, render.Fps, render.DurationSeconds, render.Bytes, render.Created, render.ProducedBy, render.ProjectId, + render.ProjectVersionId, ], cancellationToken).ConfigureAwait(false); return await session.ExecuteAsync(RecountSql, [render.SequenceId], cancellationToken).ConfigureAwait(false); }, cancellationToken); @@ -106,5 +108,7 @@ public sealed class RenderRepository Bytes = r.GetInt64(10), Created = r.GetDateTimeOffset(11), ProducedBy = r.GetStringOrNull(12), + ProjectId = r.GetGuidOrNull(13), + ProjectVersionId = r.GetGuidOrNull(14), }; } diff --git a/src/Chronos.Data/Workspaces/WorkspaceCopyJobHandler.cs b/src/Chronos.Data/Workspaces/WorkspaceCopyJobHandler.cs new file mode 100644 index 00000000..ad780ac5 --- /dev/null +++ b/src/Chronos.Data/Workspaces/WorkspaceCopyJobHandler.cs @@ -0,0 +1,340 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text.Json; +using Chronos.Core.Archiving; +using Chronos.Core.Diagnostics; +using Chronos.Core.IO; +using Chronos.Core.Localization; +using Chronos.Core.Model; +using Chronos.Core.Project; +using Chronos.Data.Archiving; +using Chronos.Data.Archiving.Jobs; +using Chronos.Data.Jobs; + +namespace Chronos.Data.Workspaces; + +/// Builds the jobs of the workspace (owner review 2, R8), and their handlers for the desktop's engine. +public static class WorkspaceJobs +{ + /// + /// : the media of workspace project (checked out first, + /// ) copied from its home, the sequence of this PC's archive. + /// + public static ArchiveJobRequest Copy(Sequence sequence, string workspaceRoot, Guid projectId, string media = BundleMedia.Proxies) + { + ArgumentNullException.ThrowIfNull(sequence); + ArgumentException.ThrowIfNullOrWhiteSpace(workspaceRoot); + return new ArchiveJobRequest(JobKind.WorkspaceCopy, [sequence.Id], sequence.DisplayTitle, + JsonSerializer.Serialize(new WorkspaceCopyParameters(workspaceRoot, projectId, media), WorkspaceJson.Default.WorkspaceCopyParameters), JobPriority.User); + } + + /// + /// The handlers of the workspace's jobs for a desktop's engine (never a server's: the job writes into a folder its + /// parameters name). + /// + public static IReadOnlyList CreateHandlers(ChronosDatabase db, ArchiveService archives) => [new WorkspaceCopyJobHandler(db, archives)]; +} + +/// +/// for a sequence of this PC's archive (owner review 2, R8): its proxies into the +/// workspace project's media/proxies/<frame id>.jpg, the archive's own when current, the missing ones made +/// first by the archive's proxy pass (made here from the original when the archive makes none), each written atomically +/// and read back to check its SHA-256; then the deep report (media/analysis.json) and, last, media/media.json. +/// Frames changed at the home during the copy make it start again (three times at most). The working state's positions +/// follow the new frames by photo. Checkpoint: proxies:<frame> during the archive's pass, then +/// copy:<frame>. Originals come in phase 5c: asked for, the job fails saying so. +/// +internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService archives) : IJobHandler +{ + private const int PageSize = 1000; + private const int Passes = 3; + private static readonly TimeSpan LockWait = TimeSpan.FromMinutes(2); + private static readonly TimeSpan CheckpointEvery = TimeSpan.FromSeconds(5); + + public JobKind Kind => JobKind.WorkspaceCopy; + + public async Task RunAsync(JobContext context) + { + var ct = context.Cancellation; + var parameters = Parameters(context.Job); + if (parameters.Media == BundleMedia.Originals) throw new NotSupportedException(Loc.L("Errors.Workspace.OriginalsLater")); + if (parameters.Media != BundleMedia.Proxies) throw new InvalidDataException(Loc.L("Errors.Workspace.Parameters")); + if (context.Job.TargetIds is not [var sequenceId] || await db.Sequences.GetAsync(sequenceId, ct).ConfigureAwait(false) is not { } sequence + || await db.Archives.GetAsync(sequence.ArchiveId, ct).ConfigureAwait(false) is not { } archive) + throw new InvalidOperationException(Loc.L("Archive.Log.SequenceMissing", context.Job.TargetIds.FirstOrDefault())); + var store = new WorkspaceStore(parameters.WorkspaceRoot); + var project = store.Find(parameters.ProjectId) ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.NotCheckedOut")); + if (project.ReadBundle().Workspace is not { Home: { IsArchive: true } home } || home.SequenceId != sequenceId) + throw new InvalidOperationException(Loc.L("Errors.Workspace.OtherHome")); + + var paths = new ArchivePaths(archive.RootPath); + var (stage, next) = Parse(context.Checkpoint); + for (var pass = 1; ; pass++) + { + var (row, frames) = await FramesAsync(sequenceId, ct).ConfigureAwait(false); + var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy; + var makesProxies = policy.LongEdge != ProxyPolicy.Disabled && ArchiveVolume.Probe(paths) == ArchiveAvailability.Online; + var total = (long)frames.Count * (makesProxies ? 2 : 1); + long offset = 0; + if (makesProxies) + { + if (stage == Stage.Proxies && (row.Status & SequenceStatus.ProxiesBuilt) == 0) + { + var result = await archives.Images.BuildProxiesAsync(archive, row, force: false, next, + (frame, _) => context.CheckpointAsync(Checkpoint(Stage.Proxies, frame), frame, total), ct).ConfigureAwait(false); + context.Log(Loc.L("Archive.Log.ProxiesDone", row.DisplayTitle, result.Built, result.Reused, result.Undecodable, result.Waiting)); + next = 0; + } + offset = frames.Count; + } + // Frames before a copy checkpoint were copied by an earlier run. + var copied = stage == Stage.Copy ? next : 0; + (stage, next) = (Stage.Copy, 0); + var folder = paths.Sequence(row.Folder); + var tally = new Tally(); + var entries = new WorkspaceFrame[frames.Count]; + Directory.CreateDirectory(project.ProxiesDirectory); + var clock = System.Diagnostics.Stopwatch.StartNew(); + for (var i = 0; i < frames.Count; i++) + { + ct.ThrowIfCancellationRequested(); + entries[i] = Copy(project, folder, paths, frames[i], policy, done: i < copied, tally, ct); + if (clock.Elapsed >= CheckpointEvery) + { + await context.CheckpointAsync(Checkpoint(Stage.Copy, i + 1), offset + i + 1, total).ConfigureAwait(false); + clock.Restart(); + } + else + { + await context.ReportAsync(offset + i + 1, total).ConfigureAwait(false); + } + } + var analysis = CopyAnalysis(folder, project, frames.Count); + var after = await db.Sequences.GetAsync(sequenceId, ct).ConfigureAwait(false) + ?? throw new InvalidOperationException(Loc.L("Archive.Log.SequenceMissing", sequenceId)); + if (after.FramesVersion != row.FramesVersion && pass < Passes) + { + context.Log(Loc.L("Archive.Log.WorkspaceFramesChanged", row.DisplayTitle)); + (stage, next) = (Stage.Proxies, 0); + continue; + } + var media = new WorkspaceMedia(WorkspaceMedia.FormatName, WorkspaceMedia.CurrentFormatVersion, BundleMedia.Proxies, sequenceId, row.FramesVersion, entries, + DateTimeOffset.UtcNow, analysis); + // The working state is the editor's meanwhile: it is held only while the media index and the positions move. + using (await project.LockAsync(LockWait, ct).ConfigureAwait(false)) + { + Commit(project, project.ReadBundle(), media, row); + RemoveStale(project, media); + } + context.Log(Loc.L("Archive.Log.WorkspaceCopied", row.DisplayTitle, tally.Copied, tally.Made, tally.Kept, tally.Missing)); + await context.CheckpointAsync(Checkpoint(Stage.Done, frames.Count), total, total).ConfigureAwait(false); + return; + } + } + + /// + /// The frame's file in the workspace: kept when an earlier run copied it (before the checkpoint, or the same bytes as the + /// archive's proxy), else the archive's current proxy, else one made from the original; none for a frame without either. + /// + private WorkspaceFrame Copy(WorkspaceProject project, SequencePaths folder, ArchivePaths paths, Frame frame, ProxyPolicy policy, bool done, Tally tally, + CancellationToken cancellationToken) + { + var name = WorkspaceProject.ProxyName(frame.Id); + var target = project.MediaFile(name); + if (done && File.Exists(target)) + { + tally.Kept++; + var kept = AtomicFile.ReadAllBytes(target); + return WorkspaceFrame.Of(frame, name, Sha(kept), kept.Length); + } + var source = folder.Proxy(frame.Index); + byte[]? bytes = null; + if (File.Exists(source) && ArchiveImages.IsProxyOf(source, frame.CapturedAt, frame.Width, frame.Height, null)) bytes = AtomicFile.ReadAllBytes(source); + var made = false; + if (bytes is null && (frame.Flags & FrameFlags.Missing) == 0) + { + var original = paths.FromRelative(frame.RelativePath); + try + { + var edge = policy.LongEdge == ProxyPolicy.Disabled ? new ProxyPolicy().LongEdge : policy.LongEdge; + bytes = File.Exists(original) ? ArchiveImages.MakeProxy(archives.Decoder, original, frame, null, edge, policy.Quality, cancellationToken) : null; + made = bytes is not null; + } + catch (IOException ex) + { + Log.Warning("workspace", $"No proxy of {original} could be made", ex); + } + } + if (bytes is null) + { + tally.Missing++; + return WorkspaceFrame.Of(frame, null, null, 0); + } + var sha = Sha(bytes); + if (File.Exists(target) && Sha(AtomicFile.ReadAllBytes(target)) == sha) + { + tally.Kept++; + return WorkspaceFrame.Of(frame, name, sha, bytes.Length); + } + WriteVerified(target, bytes, sha); + if (made) tally.Made++; + else tally.Copied++; + return WorkspaceFrame.Of(frame, name, sha, bytes.Length); + } + + /// Writes atomically and reads back: a file that does not hash as its source is written again once, then refused. + private static void WriteVerified(string path, byte[] bytes, string sha) + { + for (var attempt = 1; ; attempt++) + { + AtomicFile.WriteAllBytes(path, bytes); + if (Sha(AtomicFile.ReadAllBytes(path)) == sha) return; + if (attempt == 2) throw new IOException(Loc.L("Errors.Workspace.CopyMismatch", path)); + } + } + + /// The home's deep report into the media folder when it describes these frames; its SHA-256, or null without one. + private static string? CopyAnalysis(SequencePaths folder, WorkspaceProject project, int frameCount) + { + if (!File.Exists(folder.AnalysisPath)) + { + if (File.Exists(project.AnalysisPath)) File.Delete(project.AnalysisPath); + return null; + } + var bytes = AtomicFile.ReadAllBytes(folder.AnalysisPath); + try + { + if (Core.Analysis.Deep.DeepAnalysisReport.FromJson(System.Text.Encoding.UTF8.GetString(bytes)).FrameCount != frameCount) return null; + } + catch (Exception ex) when (ex is InvalidDataException or JsonException or IOException) + { + Log.Warning("workspace", $"{folder.AnalysisPath} cannot be read; the workspace takes no deep report", ex); + return null; + } + var sha = Sha(bytes); + WriteVerified(project.AnalysisPath, bytes, sha); + return sha; + } + + /// + /// The media index and the working state moved onto it: first the working state with every position parked by photo + /// (valid whatever the frames), then the index, then the working state placed on the new frames. A crash in between + /// leaves positions that wait for their photos, which the next open places. + /// + private static void Commit(WorkspaceProject project, ProjectBundle bundle, WorkspaceMedia media, Sequence row) + { + var old = TryReadMedia(project); + var workspace = bundle.Workspace! with { Media = new BundleMedia(BundleMedia.Proxies, media.FramesVersion) }; + var sequence = bundle.Sequence with { Folder = row.Folder, Title = row.DisplayTitle, Frames = media.Frames.Count }; + var working = bundle.Project; + // Without the media index the positions were made against, they are taken as naming these frames. + if (!working.IsNewerFormat && old is not null) + { + var portable = ProjectSettings.FromJson(ProjectContent.ToPortable(working, old.Photos), allowNewer: true); + project.WriteBundle(bundle with { Project = portable, Workspace = workspace, Sequence = sequence }); + project.WriteMedia(media); + working = ProjectSettings.FromJson(ProjectContent.Place(portable.ToJson(), media.Photos), allowNewer: true); + } + else + { + project.WriteMedia(media); + } + project.WriteBundle(bundle with { Project = working, Workspace = workspace, Sequence = sequence, Saved = DateTimeOffset.UtcNow }); + } + + /// Proxies of frames the sequence no longer has: regenerable copies, removed so the media folder holds the frames listed. + private static void RemoveStale(WorkspaceProject project, WorkspaceMedia media) + { + var listed = media.Frames.Where(f => f.File is not null).Select(f => Path.GetFileName(f.File!)).ToHashSet(StringComparer.OrdinalIgnoreCase); + foreach (var file in Directory.EnumerateFiles(project.ProxiesDirectory, "*.jpg")) + { + if (listed.Contains(Path.GetFileName(file))) continue; + try + { + File.Delete(file); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warning("workspace", $"{file} could not be removed", ex); + } + } + } + + private static WorkspaceMedia? TryReadMedia(WorkspaceProject project) + { + try + { + return project.ReadMedia(); + } + catch (InvalidDataException ex) + { + Log.Warning("workspace", $"The media index of {project.Folder} cannot be read; positions are taken as they are", ex); + return null; + } + } + + /// The sequence row with its frames as one state (listed between two reads of the row at the same frames version). + private async Task<(Sequence Row, List Frames)> FramesAsync(Guid sequenceId, CancellationToken cancellationToken) + { + for (var attempt = 0; ; attempt++) + { + var before = await db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) + ?? throw new InvalidOperationException(Loc.L("Archive.Log.SequenceMissing", sequenceId)); + var frames = new List(); + for (var after = -1; ;) + { + var page = await db.Frames.ListBySequenceAsync(sequenceId, after, PageSize, cancellationToken).ConfigureAwait(false); + frames.AddRange(page); + if (page.Count < PageSize) break; + after = page[^1].Index; + } + var now = await db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? before; + if (now.FramesVersion == before.FramesVersion || attempt == 2) return (now, frames); + } + } + + private static WorkspaceCopyParameters Parameters(Job job) + { + try + { + if (job.ParametersJson is { Length: > 0 } json + && JsonSerializer.Deserialize(json, WorkspaceJson.Default.WorkspaceCopyParameters) is { WorkspaceRoot.Length: > 0 } parameters + && parameters.ProjectId != Guid.Empty) + return parameters with { Media = parameters.Media ?? BundleMedia.Proxies }; + } + catch (JsonException ex) + { + throw new InvalidDataException(Loc.L("Errors.Workspace.Parameters"), ex); + } + throw new InvalidDataException(Loc.L("Errors.Workspace.Parameters")); + } + + private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes)); + + private enum Stage + { + Proxies, + Copy, + Done, + } + + private static string Checkpoint(Stage stage, int frame) => + string.Create(CultureInfo.InvariantCulture, $"{stage.ToString().ToLowerInvariant()}:{frame}"); + + private static (Stage Stage, int Next) Parse(string? checkpoint) + { + var parts = checkpoint?.Split(':'); + return parts is [var name, var number] && Enum.TryParse(name, ignoreCase: true, out var stage) + && int.TryParse(number, NumberStyles.None, CultureInfo.InvariantCulture, out var next) + ? (stage == Stage.Done ? Stage.Copy : stage, stage == Stage.Done ? 0 : next) + : (Stage.Proxies, 0); + } + + private sealed class Tally + { + public int Copied { get; set; } + public int Made { get; set; } + public int Kept { get; set; } + public int Missing { get; set; } + } +} diff --git a/src/Chronos.Data/Workspaces/WorkspaceMedia.cs b/src/Chronos.Data/Workspaces/WorkspaceMedia.cs new file mode 100644 index 00000000..bca8e338 --- /dev/null +++ b/src/Chronos.Data/Workspaces/WorkspaceMedia.cs @@ -0,0 +1,137 @@ +using System.Text.Json.Serialization; +using Chronos.Core.Imaging; +using Chronos.Core.Model; +using Chronos.Core.Project; +using Chronos.Core.Render; +using Chronos.Data.Processing; + +namespace Chronos.Data.Workspaces; + +/// +/// One frame of a workspace's media (media/media.json): the frame's record at its home (what analyses and the +/// editor read) and the file the workspace holds for it, with its SHA-256 and size as copied. +/// +/// The file relative to the media folder ("proxies/<frame id>.jpg"); null when the workspace holds none. +public sealed record WorkspaceFrame( + Guid Id, + int Index, + string FileName, + string? OriginalFileName, + long Size, + string? Sha256, + DateTime? CapturedAt, + int? UtcOffsetMinutes, + TimestampSource TimestampSource, + double? ExposureTime, + double? FNumber, + int? Iso, + double? FocalLength, + double? ExposureBias, + int? WhiteBalance, + int Orientation, + int Width, + int Height, + ImageFileFormat Format, + int PreviewWidth, + int PreviewHeight, + double? MeanLogLuminance, + double? ClipLow, + double? ClipHigh, + FrameFlags Flags, + string? File, + string? FileSha256, + long FileBytes) +{ + /// The record of a home's frame, with the file the workspace holds for it. + public static WorkspaceFrame Of(Frame frame, string? file, string? fileSha256, long fileBytes) + { + ArgumentNullException.ThrowIfNull(frame); + return new WorkspaceFrame(frame.Id, frame.Index, frame.FileName, frame.OriginalFileName, frame.Size, frame.Sha256, frame.CapturedAt, frame.UtcOffsetMinutes, + frame.TimestampSource, frame.ExposureTime, frame.FNumber, frame.Iso, frame.FocalLength, frame.ExposureBias, frame.WhiteBalance, frame.Orientation, + frame.Width, frame.Height, frame.Format, frame.PreviewWidth, frame.PreviewHeight, frame.MeanLogLuminance, frame.ClipLow, frame.ClipHigh, frame.Flags, + file, fileSha256, fileBytes); + } + + /// The frame as the processing engine reads it; flagged missing while the workspace holds no file for it. + public Frame ToFrame(Guid sequenceId) => new() + { + Id = Id, + SequenceId = sequenceId, + Index = Index, + RelativePath = File ?? FileName, + FileName = FileName, + OriginalFileName = OriginalFileName, + Size = Size, + Sha256 = Sha256, + CapturedAt = CapturedAt, + UtcOffsetMinutes = UtcOffsetMinutes, + TimestampSource = TimestampSource, + ExposureTime = ExposureTime, + FNumber = FNumber, + Iso = Iso, + FocalLength = FocalLength, + ExposureBias = ExposureBias, + WhiteBalance = WhiteBalance, + Orientation = Orientation, + Width = Width, + Height = Height, + Format = Format, + PreviewWidth = PreviewWidth, + PreviewHeight = PreviewHeight, + MeanLogLuminance = MeanLogLuminance, + ClipLow = ClipLow, + ClipHigh = ClipHigh, + Flags = File is null ? Flags | FrameFlags.Missing : Flags, + }; +} + +/// +/// What a workspace project holds of its sequence's photos (media/media.json, owner review 2, R8): the kind +/// (; originals in phase 5c), the sequence and its frames version at the home when they +/// were copied, and every frame in index order. Written last by a copy, so a copy cut short leaves the previous index. +/// +public sealed record WorkspaceMedia(string Format, int FormatVersion, string Kind, Guid SequenceId, int FramesVersion, IReadOnlyList Frames, + DateTimeOffset Copied, string? AnalysisSha256 = null) +{ + public const string FormatName = "chronos-workspace-media"; + public const int CurrentFormatVersion = 1; + + /// The frames' photos in index order, as project positions name them. + [JsonIgnore] + public FramePhotos Photos => new(Frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, f.CapturedAt)))); + + /// The frames held, as the processing engine reads them. + public IReadOnlyList ToFrames() => [.. Frames.Select(f => f.ToFrame(SequenceId))]; +} + +/// +/// A workspace project's sequence opened for editing (like a server sequence opened with proxies): the home's sequence +/// row, the frames as the workspace holds them and the absolute path of each frame's file (null, and the frame flagged +/// missing, when it holds none). +/// +public sealed class WorkspaceSequence +{ + public WorkspaceSequence(Sequence sequence, IReadOnlyList frames, IReadOnlyList files) + { + if (files.Count != frames.Count) throw new ArgumentException("One file entry per frame is required.", nameof(files)); + Sequence = sequence; + Frames = frames; + Files = files; + } + + public Sequence Sequence { get; } + + public IReadOnlyList Frames { get; } + + public IReadOnlyList Files { get; } + + /// + /// The editor's frame source over the workspace's files, through the same read-ahead window as archive renders; it also + /// implements and (dispose it when done). + /// + public IRenderFrameSource OpenFrames(ImageDecoder decoder, FrameWindowOptions? window = null) + { + var frames = new SequenceFrames(Sequence, Frames, Files); + return new SequenceFrameSource(frames, frames.LightFrames(), decoder, window); + } +} diff --git a/src/Chronos.Data/Workspaces/WorkspaceStore.cs b/src/Chronos.Data/Workspaces/WorkspaceStore.cs new file mode 100644 index 00000000..b3448e14 --- /dev/null +++ b/src/Chronos.Data/Workspaces/WorkspaceStore.cs @@ -0,0 +1,312 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using Chronos.Core.Archiving; +using Chronos.Core.Diagnostics; +using Chronos.Core.IO; +using Chronos.Core.Model; +using Chronos.Core.Project; +using Chronos.Data.Projects; +using Chronos.Data.Sync; + +namespace Chronos.Data.Workspaces; + +/// The manifest of a workspace folder (workspace.json). +public sealed record WorkspaceManifest(string Format, int FormatVersion, Guid Id, DateTimeOffset Created); + +/// A workspace's sync state of one project besides its versions (sync.json): a restore to record, a conflict to settle. +/// The version a restore brought back; the next commit is a "restore" of it. +/// The conflict a pull or push stopped at, until the user settles it. +public sealed record WorkspaceSyncState(Guid? RestoredFrom, WorkspaceConflict? Conflict) +{ + public static WorkspaceSyncState None { get; } = new(null, null); +} + +/// +/// A pull or push that met real conflicts (owner review 2, R8): the home's head it met, the version the local commits were +/// made from, the local commits (oldest first) and the members both sides changed (mine, theirs). Settled by keeping mine, +/// taking the server's (each member or all), or saving both (). +/// +public sealed record WorkspaceConflict(Guid TheirsVersionId, int TheirsNumber, Guid BaseVersionId, IReadOnlyList Mine, + IReadOnlyList Differences, DateTimeOffset Found); + +/// +/// Parameters of a job: target = the home's sequence; the workspace folder, the +/// project whose media to copy and what to copy (; originals in phase 5c). +/// +public sealed record WorkspaceCopyParameters(string WorkspaceRoot, Guid ProjectId, string Media); + +[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + UseStringEnumConverter = true, WriteIndented = true)] +[JsonSerializable(typeof(WorkspaceManifest))] +[JsonSerializable(typeof(WorkspaceMedia))] +[JsonSerializable(typeof(WorkspaceSyncState))] +[JsonSerializable(typeof(WorkspaceCopyParameters))] +internal sealed partial class WorkspaceJson : JsonSerializerContext; + +/// +/// The workspace of this PC (owner review 2, R8): a folder on a fast disk where projects are edited, never inside the +/// library. <root>/workspace.json names it; each project has a folder named by its id +/// (). There is no index: the store lists the folders holding a project.chronos. The +/// desktop puts it in %LOCALAPPDATA%\Chronos\workspace unless set otherwise; the store takes any folder. +/// +public sealed class WorkspaceStore +{ + public const string ManifestFile = "workspace.json"; + public const string FormatName = "chronos-workspace"; + public const int FormatVersion = 1; + + /// The folder name of the workspace inside the program's data folder. + public const string DefaultFolderName = "workspace"; + + /// Opens the workspace at , making the folder and its manifest when there is none. + /// The folder's workspace.json is not a workspace manifest, or a newer one. + /// The folder cannot be made or read. + public WorkspaceStore(string root) + { + ArgumentException.ThrowIfNullOrWhiteSpace(root); + Root = LongPath.Normalize(Path.TrimEndingDirectorySeparator(Path.GetFullPath(root))); + Directory.CreateDirectory(Root); + var path = Path.Combine(Root, ManifestFile); + if (!File.Exists(path)) + { + var manifest = new WorkspaceManifest(FormatName, FormatVersion, Guid.CreateVersion7(), DateTimeOffset.UtcNow); + // Another program opening the same folder at the same moment: the first manifest wins. + try + { + using var stream = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.None); + JsonSerializer.Serialize(stream, manifest, WorkspaceJson.Default.WorkspaceManifest); + stream.Flush(flushToDisk: true); + } + catch (IOException) when (File.Exists(path)) + { + } + } + Manifest = Read(path); + } + + public string Root { get; } + + public WorkspaceManifest Manifest { get; } + + public Guid Id => Manifest.Id; + + /// The folder of a project, whether it exists or not. + public WorkspaceProject Project(Guid projectId) => new(this, projectId); + + /// The project's folder when it holds a working state; null otherwise. + public WorkspaceProject? Find(Guid projectId) + { + var project = Project(projectId); + return project.Exists ? project : null; + } + + /// The projects of the workspace: every folder named by a project id that holds a project.chronos. + public IReadOnlyList List() + { + var found = new List(); + foreach (var directory in Directory.EnumerateDirectories(Root).Order(StringComparer.Ordinal)) + { + if (Guid.TryParseExact(Path.GetFileName(directory), "N", out var id) && File.Exists(Path.Combine(directory, WorkspaceProject.BundleFile))) + found.Add(Project(id)); + } + return found; + } + + /// The working states of the workspace's projects (one unreadable is left out, and logged). + public IReadOnlyList<(WorkspaceProject Project, ProjectBundle Bundle)> ListBundles() + { + var bundles = new List<(WorkspaceProject, ProjectBundle)>(); + foreach (var project in List()) + { + try + { + bundles.Add((project, project.ReadBundle())); + } + catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) + { + Log.Warning("workspace", $"The project in {project.Folder} cannot be read", ex); + } + } + return bundles; + } + + private static WorkspaceManifest Read(string path) + { + WorkspaceManifest? manifest; + try + { + manifest = JsonSerializer.Deserialize(AtomicFile.ReadAllText(path), WorkspaceJson.Default.WorkspaceManifest); + } + catch (JsonException ex) + { + throw new InvalidDataException($"{path} is not a workspace manifest.", ex); + } + if (manifest is null || manifest.Format != FormatName || manifest.Id == Guid.Empty) throw new InvalidDataException($"{path} is not a workspace manifest."); + if (manifest.FormatVersion > FormatVersion) + throw new InvalidDataException($"{path} uses format {manifest.FormatVersion}, newer than this Chronos ({FormatVersion})."); + return manifest; + } +} + +/// +/// One project in the workspace (owner review 2, R8): <root>/<project id>/ with +/// project.chronos (the working state: a project file of format 2, its settings placed on the media's frames), +/// media/media.json with media/proxies/<frame id>.jpg (originals later) and media/analysis.json +/// (the home's deep report), versions/<number>-<id>.json (commits not pushed yet, and the versions pulled), +/// sync.json, exports/ and the lease .lock.lease that one writer at a time holds. +/// +public sealed class WorkspaceProject +{ + public const string BundleFile = "project.chronos"; + public const string MediaFolder = "media"; + public const string MediaIndexFile = "media.json"; + public const string ProxiesFolder = "proxies"; + public const string OriginalsFolder = "originals"; + public const string VersionsFolder = "versions"; + public const string ExportsFolder = "exports"; + public const string SyncFile = "sync.json"; + + /// Old local commits replaced by their replay on top of the home's head: kept, never erased by Chronos. + public const string ReplacedFolder = ".replaced"; + + private const string LeaseName = ".lock"; + + internal WorkspaceProject(WorkspaceStore store, Guid projectId) + { + Store = store; + ProjectId = projectId; + Folder = Path.Combine(store.Root, projectId.ToString("N")); + } + + public WorkspaceStore Store { get; } + public Guid ProjectId { get; } + public string Folder { get; } + public string BundlePath => Path.Combine(Folder, BundleFile); + public string MediaDirectory => Path.Combine(Folder, MediaFolder); + public string MediaIndexPath => Path.Combine(MediaDirectory, MediaIndexFile); + public string ProxiesDirectory => Path.Combine(MediaDirectory, ProxiesFolder); + public string OriginalsDirectory => Path.Combine(MediaDirectory, OriginalsFolder); + public string AnalysisPath => Path.Combine(MediaDirectory, ArchiveLayout.AnalysisFile); + public string VersionsDirectory => Path.Combine(Folder, VersionsFolder); + public string ExportsDirectory => Path.Combine(Folder, ExportsFolder); + public string SyncPath => Path.Combine(Folder, SyncFile); + + public bool Exists => File.Exists(BundlePath); + + /// The file a frame's proxy is copied to, relative to . + public static string ProxyName(Guid frameId) => ProxiesFolder + "/" + ProxyCache.FileName(frameId); + + /// The absolute path of a file named relative to . + public string MediaFile(string relative) => LongPath.FromPortableRelative(MediaDirectory, relative); + + /// Holds the project for one writer (this program's sync, a copy job) until disposed. + /// Another writer kept it for longer than . + public async Task LockAsync(TimeSpan wait, CancellationToken cancellationToken = default) + { + Directory.CreateDirectory(Folder); + return await LeaseFile.AcquireAsync(Folder, LeaseName, "workspace@" + Environment.MachineName, wait, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + /// Not a project file of a workspace, damaged or newer. + /// The file cannot be read. + public ProjectBundle ReadBundle() + { + var bundle = ProjectBundle.Load(BundlePath); + if (bundle.Workspace is not { } workspace || workspace.ProjectId != ProjectId) + throw new InvalidDataException($"{BundlePath} is not the working state of project {ProjectId}."); + return bundle; + } + + public void WriteBundle(ProjectBundle bundle) + { + ArgumentNullException.ThrowIfNull(bundle); + if (bundle.Workspace?.ProjectId != ProjectId) throw new ArgumentException("The project file is another project's.", nameof(bundle)); + Directory.CreateDirectory(Folder); + bundle.Save(BundlePath); + } + + /// The media index; null while the media were never copied, or were removed. + /// The index is damaged or newer. + public WorkspaceMedia? ReadMedia() + { + if (!File.Exists(MediaIndexPath)) return null; + WorkspaceMedia? media; + try + { + media = JsonSerializer.Deserialize(AtomicFile.ReadAllText(MediaIndexPath), WorkspaceJson.Default.WorkspaceMedia); + } + catch (JsonException ex) + { + throw new InvalidDataException($"{MediaIndexPath} cannot be read.", ex); + } + if (media is null || media.Format != WorkspaceMedia.FormatName) throw new InvalidDataException($"{MediaIndexPath} is not a media index."); + if (media.FormatVersion > WorkspaceMedia.CurrentFormatVersion) throw new InvalidDataException($"{MediaIndexPath} is newer than this Chronos."); + return media; + } + + internal void WriteMedia(WorkspaceMedia media) + { + Directory.CreateDirectory(MediaDirectory); + AtomicFile.WriteAllText(MediaIndexPath, JsonSerializer.Serialize(media, WorkspaceJson.Default.WorkspaceMedia)); + } + + /// The versions this workspace holds, oldest first; one that cannot be read is left out (and logged). + public IReadOnlyList Versions() + { + var versions = new List(); + foreach (var (_, _, path) in ProjectSidecars.ListVersions(VersionsDirectory)) + { + try + { + var version = ProjectSidecars.ReadVersion(path); + if (version.ProjectId == ProjectId) versions.Add(version); + } + catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) + { + Log.Warning("workspace", $"Version file {path} cannot be read", ex); + } + } + return versions; + } + + /// The path of a version in . + public string VersionPath(ProjectVersion version) => Path.Combine(VersionsDirectory, ArchiveLayout.ProjectVersionFileName(version.Number, version.Id)); + + internal void WriteVersion(ProjectVersion version, bool replace = false) => ProjectSidecars.WriteVersion(VersionPath(version), version, replace); + + /// Moves a version file aside into versions/.replaced (a local commit replayed under another id). + internal void SetAside(ProjectVersion version) + { + var path = VersionPath(version); + if (!File.Exists(path)) return; + var aside = Path.Combine(VersionsDirectory, ReplacedFolder); + Directory.CreateDirectory(aside); + var target = Path.Combine(aside, Path.GetFileName(path)); + if (File.Exists(target)) target += "." + DateTime.UtcNow.Ticks.ToString(System.Globalization.CultureInfo.InvariantCulture); + File.Move(path, target); + } + + public WorkspaceSyncState ReadSync() + { + if (!File.Exists(SyncPath)) return WorkspaceSyncState.None; + try + { + return JsonSerializer.Deserialize(AtomicFile.ReadAllText(SyncPath), WorkspaceJson.Default.WorkspaceSyncState) ?? WorkspaceSyncState.None; + } + catch (JsonException ex) + { + throw new InvalidDataException($"{SyncPath} cannot be read.", ex); + } + } + + internal void WriteSync(WorkspaceSyncState state) + { + if (state == WorkspaceSyncState.None || state is { RestoredFrom: null, Conflict: null }) + { + if (File.Exists(SyncPath)) File.Delete(SyncPath); + return; + } + Directory.CreateDirectory(Folder); + AtomicFile.WriteAllText(SyncPath, JsonSerializer.Serialize(state, WorkspaceJson.Default.WorkspaceSyncState)); + } +} diff --git a/src/Chronos.Data/Workspaces/WorkspaceSync.cs b/src/Chronos.Data/Workspaces/WorkspaceSync.cs new file mode 100644 index 00000000..4df7551d --- /dev/null +++ b/src/Chronos.Data/Workspaces/WorkspaceSync.cs @@ -0,0 +1,625 @@ +using System.Text.Json; +using Chronos.Core.Model; +using Chronos.Core.Project; +using Chronos.Data.Projects; +using Chronos.Data.Sync; + +namespace Chronos.Data.Workspaces; + +/// Where a workspace project stands against its home (), the most pressing first. +public enum WorkspaceState +{ + /// Nothing to save, push or pull. + InSync, + + /// The working state has changes no version holds yet (save a version). + Modified, + + /// Versions saved here wait to be pushed (). + Ahead, + + /// The home has versions this workspace has not pulled (). + Behind, + + /// A pull or push met changes both sides made to the same settings: keep mine, take the server's or save both. + Conflict, + + /// The home cannot be reached now; work goes on here and syncs later. + Offline, + + /// The workspace holds no media for the project (never copied, or removed): copy them again to edit. + MediaRemoved, +} + +/// +/// A workspace project's state (owner review 2, R8): the leading and every fact behind it. Without a +/// home to ask, is 0 and false. +/// +/// The home's version the project was last in step with. +/// The home's newest version, when it was asked. +public sealed record WorkspaceStatus(WorkspaceState State, bool Modified, int Ahead, int Behind, bool Offline, bool MediaRemoved, bool ReadOnly, + WorkspaceConflict? Conflict, BundleBase? Base, ProjectVersion? HomeHead); + +/// What a push, pull or resolve did: the versions it stored at the home and those it took from there, and where it left the project. +/// After : the new project at the home holding this workspace's versions. +public sealed record WorkspaceSyncResult(WorkspaceStatus Status, IReadOnlyList Pushed, IReadOnlyList Pulled, + SequenceProject? SavedAs = null); + +/// How a conflict is settled (owner review 2, R8: only on real conflicts). +public enum ConflictChoice +{ + /// This workspace's values where both changed (each member listed in takeServer takes the server's instead). + KeepMine, + + /// The server's values wherever both changed. + TakeServer, + + /// Both kept: this workspace's versions go to a new project at the home, and this project takes the server's. + SaveBoth, +} + +/// The working state of a workspace project opened for editing. +/// The settings, their positions placed on the media's frames. +/// The sequence over the media's files; null while the workspace holds no media. +public sealed record WorkspaceSession(WorkspaceProject Project, ProjectBundle Bundle, ProjectSettings Working, WorkspaceMedia? Media, WorkspaceSequence? Sequence) +{ + /// The settings come from a newer Chronos: shown, never saved. + public bool IsReadOnly => Bundle.IsReadOnly; +} + +/// A push the home refused for good (not a divergence: settings damaged on the way, of a newer format, or not this project's). +public sealed class WorkspaceSyncException(PushOutcome outcome, string message) : IOException(message) +{ + public PushOutcome Outcome { get; } = outcome; +} + +/// +/// Sync of workspace projects with their homes, "like git" with one linear history and no branches (owner review 2, R8). +/// A workspace edits the working state (), saves versions of it locally (, +/// parent = the last one), pushes them in order () and pulls the home's (). +/// When the home moved meanwhile, its new versions are pulled and each local commit is replayed on top of them +/// (, new ids, the old files kept in versions/.replaced), and the working state is +/// merged too (); settings both sides changed to different values stop +/// there as a conflict () for the user to settle (). Versions +/// travel in the portable form (), so frames inserted at the home never move a position. Every +/// operation holds the project's lease; nothing is ever deleted: a restore or save-both first saves uncommitted changes +/// as a version. +/// +public sealed class WorkspaceSync(WorkspaceStore store, string clientName) +{ + /// Versions asked for at once when pulling. + private const int PullPage = 100; + + /// Times a push pulls and replays when the home keeps moving under it. + private const int PushAttempts = 3; + + private static readonly FramePhotos NoFrames = new([]); + + public WorkspaceStore Store => store; + + /// How long an operation waits for the project's lease (another program working on it). + public TimeSpan LockWait { get; init; } = TimeSpan.FromSeconds(30); + + /// + /// The project of brought into the workspace: its folder, the home's + /// head as the base and the working state (positions waiting for the media, which a + /// job copies: ). A project already here is left as + /// it is. + /// + /// The home has no such project. + /// A project of that id from another home is in the workspace. + public async Task CheckoutAsync(IProjectHome home, Guid projectId, string media = BundleMedia.Proxies, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(home); + var project = store.Project(projectId); + using (await project.LockAsync(LockWait, cancellationToken).ConfigureAwait(false)) + { + var sequence = await home.SequenceAsync(cancellationToken).ConfigureAwait(false); + if (project.Exists) + { + var existing = project.ReadBundle(); + if (existing.Workspace!.Home.SequenceId != sequence.Home.SequenceId) + throw new InvalidOperationException($"Project {projectId} in the workspace belongs to another sequence."); + return project; + } + var listed = (await home.ListAsync(cancellationToken).ConfigureAwait(false)).FirstOrDefault(p => p.Id == projectId) + ?? throw new ProjectException(ProjectError.NotFound, $"The home has no project {projectId}."); + var head = await home.HeadAsync(projectId, cancellationToken).ConfigureAwait(false); + Verify(head); + project.WriteVersion(head, replace: true); + Directory.CreateDirectory(project.ExportsDirectory); + var bundle = new ProjectBundle(sequence.Archive, sequence.Sequence, ProjectSettings.FromJson(head.SettingsJson, allowNewer: true), DateTimeOffset.UtcNow) + { + Workspace = new BundleWorkspace(projectId, listed.Name, listed.IsMain, sequence.Home, Base(head), new BundleMedia(media, 0)), + }; + project.WriteBundle(bundle); + return project; + } + } + + /// The working state for the editor: its settings placed on the media's frames, and the media's sequence. + /// The project is not in the workspace. + public Task OpenAsync(Guid projectId, CancellationToken cancellationToken = default) => + Task.Run(() => + { + var project = Existing(projectId); + var bundle = project.ReadBundle(); + var media = project.ReadMedia(); + var working = bundle.Project; + if (media is not null && !working.IsNewerFormat) + working = ProjectSettings.FromJson(ProjectContent.Place(working.ToJson(), media.Photos), allowNewer: true); + return new WorkspaceSession(project, bundle, working, media, media is null ? null : SequenceOf(project, bundle, media)); + }, cancellationToken); + + /// Stores the working state (the editor's settings, positions on the media's frames); no version is made. + /// The project is read-only (a newer Chronos's settings) or not in the workspace. + public async Task SaveAsync(Guid projectId, ProjectSettings working, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(working); + var project = Existing(projectId); + using (await project.LockAsync(LockWait, cancellationToken).ConfigureAwait(false)) + { + var bundle = project.ReadBundle(); + if (bundle.IsReadOnly || working.IsNewerFormat) throw ReadOnly(projectId); + project.WriteBundle(bundle with { Project = working, Saved = DateTimeOffset.UtcNow }); + } + } + + /// + /// Saves the working state as a new local version (parent = the last one), unless it is the last one's already: null + /// then. The version waits here until pushed. After a restore it is a "restore" of the version brought back. + /// + /// The project is read-only, has a conflict to settle, or is not in the workspace. + public async Task CommitAsync(Guid projectId, string? note = null, string? author = null, CancellationToken cancellationToken = default) + { + var project = Existing(projectId); + using (await project.LockAsync(LockWait, cancellationToken).ConfigureAwait(false)) + { + if (project.ReadSync().Conflict is not null) throw new InvalidOperationException($"Project {projectId} has a conflict to settle first."); + return CommitLocked(project, project.ReadBundle(), note, author); + } + } + + /// + /// Pushes the local versions in order. When the home moved meanwhile, its versions are pulled and the local ones + /// replayed on top of them first; a real conflict stops the push (). + /// + /// The home cannot be reached; nothing was lost, push again later. + /// The home refused a version for good. + public async Task PushAsync(Guid projectId, IProjectHome home, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(home); + var project = Existing(projectId); + using (await project.LockAsync(LockWait, cancellationToken).ConfigureAwait(false)) + { + var pushed = new List(); + var pulled = new List(); + for (var attempt = 0; attempt < PushAttempts; attempt++) + { + var bundle = project.ReadBundle(); + if (project.ReadSync().Conflict is not null) break; + var diverged = false; + foreach (var local in Unpushed(project, bundle)) + { + var result = await home.PushAsync(projectId, local, cancellationToken).ConfigureAwait(false); + switch (result.Outcome) + { + case PushOutcome.Stored or PushOutcome.AlreadyStored when result.Version is { } stored: + Verify(stored); + if (stored.Number != local.Number) project.SetAside(local); + project.WriteVersion(stored, replace: true); + bundle = WithBase(bundle, stored); + project.WriteBundle(bundle); + pushed.Add(stored); + continue; + case PushOutcome.Diverged: + diverged = true; + break; + default: + throw new WorkspaceSyncException(result.Outcome, $"The home refused version {local.Number} of project {projectId}: {result.Outcome}."); + } + break; + } + if (!diverged) break; + var (_, conflict, taken) = await ReplayAsync(project, bundle, home, null, cancellationToken).ConfigureAwait(false); + pulled.AddRange(taken); + if (conflict is not null) break; + } + return new WorkspaceSyncResult(await StatusLockedAsync(project, home, cancellationToken).ConfigureAwait(false), pushed, pulled); + } + } + + /// + /// Takes the home's versions newer than the base: the working state (and any local versions, replayed) moves onto the + /// home's head; a real conflict stops it (). + /// + /// The home cannot be reached. + public async Task PullAsync(Guid projectId, IProjectHome home, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(home); + var project = Existing(projectId); + using (await project.LockAsync(LockWait, cancellationToken).ConfigureAwait(false)) + { + IReadOnlyList pulled = []; + if (project.ReadSync().Conflict is null) + { + var bundle = project.ReadBundle(); + var head = await home.HeadAsync(projectId, cancellationToken).ConfigureAwait(false); + if (head.Id != bundle.Workspace!.Base?.VersionId) (_, _, pulled) = await ReplayAsync(project, bundle, home, null, cancellationToken).ConfigureAwait(false); + } + return new WorkspaceSyncResult(await StatusLockedAsync(project, home, cancellationToken).ConfigureAwait(false), [], pulled); + } + } + + /// + /// Settles the conflict a pull or push stopped at. keeps this workspace's values + /// (but for the members in ), takes the server's + /// wherever both changed; either way the local versions are replayed on the home's head and wait to be pushed. + /// saves the working state as a version, sends this workspace's versions to a new + /// project at the home (, else this project's name with a number) made from the base, + /// and moves this project onto the home's head. + /// + /// The home cannot be reached; the conflict stays. + public async Task ResolveAsync(Guid projectId, IProjectHome home, ConflictChoice choice, IReadOnlySet? takeServer = null, + string? saveBothName = null, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(home); + var project = Existing(projectId); + using (await project.LockAsync(LockWait, cancellationToken).ConfigureAwait(false)) + { + var bundle = project.ReadBundle(); + var sync = project.ReadSync(); + if (sync.Conflict is null) return new WorkspaceSyncResult(await StatusLockedAsync(project, home, cancellationToken).ConfigureAwait(false), [], []); + if (choice == ConflictChoice.SaveBoth) + return await SaveBothAsync(project, bundle, home, saveBothName, cancellationToken).ConfigureAwait(false); + IReadOnlySet takeTheirs = choice == ConflictChoice.TakeServer ? AllPaths : takeServer ?? new HashSet(); + var (_, conflict, pulled) = await ReplayAsync(project, bundle, home, takeTheirs, cancellationToken).ConfigureAwait(false); + if (conflict is null) project.WriteSync(project.ReadSync() with { Conflict = null }); + return new WorkspaceSyncResult(await StatusLockedAsync(project, home, cancellationToken).ConfigureAwait(false), [], pulled); + } + } + + /// + /// Brings version back as the working state (from this workspace, else from + /// ); the next version saved is a "restore" of it. Uncommitted changes are saved as a version + /// first, so a restore loses nothing. + /// + /// Neither this workspace nor the home has the version. + public async Task RestoreAsync(Guid projectId, Guid versionId, IProjectHome? home = null, CancellationToken cancellationToken = default) + { + var project = Existing(projectId); + using (await project.LockAsync(LockWait, cancellationToken).ConfigureAwait(false)) + { + var version = project.Versions().FirstOrDefault(v => v.Id == versionId); + if (version is null && home is not null && await home.GetVersionAsync(projectId, versionId, cancellationToken).ConfigureAwait(false) is { } fetched) + { + Verify(fetched); + project.WriteVersion(fetched, replace: true); + version = fetched; + } + if (version is null) throw new ProjectException(ProjectError.NotFound, $"Version {versionId} of project {projectId} is not known here."); + var bundle = project.ReadBundle(); + if (bundle.IsReadOnly) throw ReadOnly(projectId); + if (project.ReadSync().Conflict is null) CommitLocked(project, bundle, null, null); + var media = project.ReadMedia(); + var working = ProjectSettings.FromJson(ProjectContent.Place(version.SettingsJson, media?.Photos ?? NoFrames), allowNewer: true); + project.WriteBundle(project.ReadBundle() with { Project = working, Saved = DateTimeOffset.UtcNow }); + project.WriteSync(project.ReadSync() with { RestoredFrom = versionId }); + return working; + } + } + + /// Where the project stands; with a , against the home's head too (offline when it cannot be reached). + public async Task StatusAsync(Guid projectId, IProjectHome? home = null, CancellationToken cancellationToken = default) + { + var project = Existing(projectId); + return await StatusLockedAsync(project, home, cancellationToken).ConfigureAwait(false); + } + + /// The versions this workspace saved and has not pushed yet, oldest first. + public IReadOnlyList Unpushed(Guid projectId) + { + var project = Existing(projectId); + return Unpushed(project, project.ReadBundle()); + } + + private static readonly IReadOnlySet AllPaths = new EveryPath(); + + /// + /// Pulls the home's versions after the base and replays the local ones and the working state on its head. With + /// null a conflict is recorded and nothing else moves (the versions pulled are kept as + /// files); with a choice the conflicts are settled by it. + /// + private async Task<(ProjectBundle Bundle, WorkspaceConflict? Conflict, IReadOnlyList Pulled)> ReplayAsync(WorkspaceProject project, + ProjectBundle bundle, IProjectHome home, IReadOnlySet? takeTheirs, CancellationToken cancellationToken) + { + var basis = await BaseVersionAsync(project, bundle, home, cancellationToken).ConfigureAwait(false); + // Read before the home's versions are written over them: a local commit the home stored although its answer was lost + // comes back among them, and the commits after it were made on it. + var locals = Unpushed(project, bundle); + var theirs = new List(); + for (var after = basis?.Number ?? 0; ;) + { + var page = await home.VersionsAfterAsync(project.ProjectId, after, PullPage, cancellationToken).ConfigureAwait(false); + foreach (var version in page) + { + Verify(version); + project.WriteVersion(version, replace: true); + } + theirs.AddRange(page); + if (page.Count < PullPage) break; + after = page[^1].Number; + } + if (theirs.Count == 0) return (bundle, null, theirs); + var head = theirs[^1]; + var atHome = theirs.Select(v => v.Id).ToHashSet(); + var ours = locals.TakeWhile(v => atHome.Contains(v.Id)).ToList(); + var tipVersion = locals.Count > 0 ? locals[^1] : basis; + if (ours.Count > 0) basis = theirs.First(v => v.Id == ours[^1].Id); + locals = [.. locals.Skip(ours.Count)]; + var media = project.ReadMedia(); + var photos = media?.Photos ?? NoFrames; + var basisSettings = basis is null ? new ProjectSettings() : ProjectMerge.Read(basis.SettingsJson); + var headSettings = ProjectMerge.Read(head.SettingsJson); + var tip = tipVersion is null ? basisSettings : ProjectMerge.Read(tipVersion.SettingsJson); + var working = ProjectMerge.Read(Portable(bundle.Project, media) ?? throw MediaMissing(project.ProjectId)); + // "Take the server's" everywhere: the paths are those the merge finds, so it runs once to find them. + var choice = takeTheirs is EveryPath ? null : takeTheirs; + List mine = [.. locals.Select(v => ProjectMerge.Read(v.SettingsJson))]; + var replay = ProjectMerge.Replay(basisSettings, mine, headSettings, choice); + if (takeTheirs is EveryPath) replay = ProjectMerge.Replay(basisSettings, mine, headSettings, Paths(replay.Conflicts)); + var newTip = replay.Commits.Count > 0 ? replay.Commits[^1] : headSettings; + var merge = ProjectMerge.MergeWithConflicts(tip, working, newTip, choice); + if (takeTheirs is EveryPath) merge = ProjectMerge.MergeWithConflicts(tip, working, newTip, Paths(merge.Conflicts)); + var differences = replay.Conflicts.Concat(merge.Conflicts).DistinctBy(d => d.Path).ToList(); + if (differences.Count > 0 && takeTheirs is null) + { + var conflict = new WorkspaceConflict(head.Id, head.Number, basis?.Id ?? Guid.Empty, [.. locals.Select(v => v.Id)], differences, DateTimeOffset.UtcNow); + project.WriteSync(project.ReadSync() with { Conflict = conflict }); + return (bundle, conflict, theirs); + } + var parent = head; + for (var i = 0; i < locals.Count; i++) + { + var local = locals[i]; + var portable = ProjectContent.Canonical(replay.Commits[i].ToJson()); + var rebased = local with + { + Id = Guid.CreateVersion7(), + ParentId = parent.Id, + Number = parent.Number + 1, + Kind = i == locals.Count - 1 && differences.Count > 0 ? ProjectVersionKind.Merge : local.Kind, + SettingsJson = portable, + ContentHash = ProjectContent.Hash(portable), + SettingsFormat = ProjectContent.FormatOf(portable), + StoredAt = null, + }; + project.SetAside(local); + project.WriteVersion(rebased); + parent = rebased; + } + var placed = ProjectSettings.FromJson(ProjectContent.Place(ProjectContent.Canonical(merge.Merged.ToJson()), photos), allowNewer: true); + var moved = WithBase(bundle, head) with { Project = placed, Saved = DateTimeOffset.UtcNow }; + project.WriteBundle(moved); + return (moved, null, theirs); + } + + /// Both sides kept: this workspace's versions to a new project at the home, this project onto the home's head. + private async Task SaveBothAsync(WorkspaceProject project, ProjectBundle bundle, IProjectHome home, string? name, CancellationToken cancellationToken) + { + var workspace = bundle.Workspace!; + if (!bundle.IsReadOnly) CommitLocked(project, bundle, null, null); + bundle = project.ReadBundle(); + var locals = Unpushed(project, bundle); + var basis = await BaseVersionAsync(project, bundle, home, cancellationToken).ConfigureAwait(false); + SequenceProject? copy = null; + var pushed = new List(); + if (locals.Count > 0) + { + copy = await CreateNamedAsync(home, name, workspace.Name, basis?.Id, cancellationToken).ConfigureAwait(false); + var parent = await home.HeadAsync(copy.Id, cancellationToken).ConfigureAwait(false); + foreach (var local in locals) + { + var moved = local with { Id = Guid.CreateVersion7(), ProjectId = copy.Id, ParentId = parent.Id, Number = parent.Number + 1, StoredAt = null }; + var result = await home.PushAsync(copy.Id, moved, cancellationToken).ConfigureAwait(false); + if (result is not { Outcome: PushOutcome.Stored or PushOutcome.AlreadyStored, Version: { } stored }) + throw new WorkspaceSyncException(result.Outcome, $"The home refused version {local.Number} saved as project '{copy.Name}': {result.Outcome}."); + pushed.Add(stored); + parent = stored; + } + } + // This project takes the home's head: the local versions now live in the copy (and their files in versions/.replaced). + var head = await home.HeadAsync(project.ProjectId, cancellationToken).ConfigureAwait(false); + Verify(head); + project.WriteVersion(head, replace: true); + foreach (var local in locals) project.SetAside(local); + var media = project.ReadMedia(); + var working = ProjectSettings.FromJson(ProjectContent.Place(head.SettingsJson, media?.Photos ?? NoFrames), allowNewer: true); + project.WriteBundle(WithBase(bundle, head) with { Project = working, Saved = DateTimeOffset.UtcNow }); + project.WriteSync(WorkspaceSyncState.None); + return new WorkspaceSyncResult(await StatusLockedAsync(project, home, cancellationToken).ConfigureAwait(false), pushed, [head], copy); + } + + /// A new project at the home named , else " (2)", "(3)"… whichever is free. + private static async Task CreateNamedAsync(IProjectHome home, string? name, string source, Guid? from, CancellationToken cancellationToken) + { + if (!string.IsNullOrWhiteSpace(name)) return await home.CreateAsync(name, from, null, cancellationToken).ConfigureAwait(false); + for (var n = 2; ; n++) + { + var candidate = $"{source} ({n})"; + if (candidate.Length > ProjectService.MaxNameLength) candidate = $"{source[..(ProjectService.MaxNameLength - 6)]} ({n})"; + try + { + return await home.CreateAsync(candidate, from, null, cancellationToken).ConfigureAwait(false); + } + catch (ProjectException ex) when (ex.Reason == ProjectError.NameTaken && n < 100) + { + } + } + } + + private ProjectVersion? CommitLocked(WorkspaceProject project, ProjectBundle bundle, string? note, string? author) + { + if (bundle.IsReadOnly) throw ReadOnly(project.ProjectId); + var workspace = bundle.Workspace!; + var media = project.ReadMedia(); + var portable = Portable(bundle.Project, media) ?? throw MediaMissing(project.ProjectId); + var hash = ProjectContent.Hash(portable); + var locals = Unpushed(project, bundle); + var (tipId, tipNumber, tipHash) = locals.Count > 0 + ? (locals[^1].Id, locals[^1].Number, locals[^1].ContentHash) + : (workspace.Base?.VersionId, workspace.Base?.Number ?? 0, workspace.Base?.ContentHash); + if (hash == tipHash) return null; + var sync = project.ReadSync(); + var version = new ProjectVersion + { + Id = Guid.CreateVersion7(), + ProjectId = project.ProjectId, + ParentId = tipId, + Number = tipNumber + 1, + Kind = sync.RestoredFrom is null ? ProjectVersionKind.Commit : ProjectVersionKind.Restore, + Author = author, + ClientName = clientName, + Note = string.IsNullOrWhiteSpace(note) ? null : note.Trim(), + SettingsJson = portable, + ContentHash = hash, + SettingsFormat = ProjectContent.FormatOf(portable), + FramesVersion = media?.FramesVersion ?? workspace.Media.FramesVersion, + RestoredFrom = sync.RestoredFrom, + CreatedAt = DateTimeOffset.UtcNow, + }; + project.WriteVersion(version); + if (sync.RestoredFrom is not null) project.WriteSync(sync with { RestoredFrom = null }); + return version; + } + + private async Task StatusLockedAsync(WorkspaceProject project, IProjectHome? home, CancellationToken cancellationToken) + { + var bundle = project.ReadBundle(); + var workspace = bundle.Workspace!; + WorkspaceMedia? media; + try + { + media = project.ReadMedia(); + } + catch (InvalidDataException) + { + media = null; + } + var conflict = project.ReadSync().Conflict; + var locals = Unpushed(project, bundle); + var tipHash = locals.Count > 0 ? locals[^1].ContentHash : workspace.Base?.ContentHash; + var modified = !bundle.IsReadOnly && (Portable(bundle.Project, media) is not { } portable || ProjectContent.Hash(portable) != tipHash); + ProjectVersion? head = null; + var offline = false; + if (home is not null) + { + try + { + head = await home.HeadAsync(project.ProjectId, cancellationToken).ConfigureAwait(false); + } + catch (HomeUnavailableException) + { + offline = true; + } + } + var behind = head is not null && head.Id != workspace.Base?.VersionId ? Math.Max(1, head.Number - (workspace.Base?.Number ?? 0)) : 0; + // A local commit the home stored although its answer was lost reads as behind by the commits it holds: they are ours. + if (head is not null && locals.Any(v => v.Id == head.Id)) behind = 0; + var state = media is null ? WorkspaceState.MediaRemoved + : conflict is not null ? WorkspaceState.Conflict + : offline ? WorkspaceState.Offline + : behind > 0 ? WorkspaceState.Behind + : locals.Count > 0 ? WorkspaceState.Ahead + : modified ? WorkspaceState.Modified + : WorkspaceState.InSync; + return new WorkspaceStatus(state, modified, locals.Count, behind, offline, media is null, bundle.IsReadOnly, conflict, workspace.Base, head); + } + + /// The local versions after the base, oldest first, each the child of the one before (the base's for the first). + private static List Unpushed(WorkspaceProject project, ProjectBundle bundle) + { + var basis = bundle.Workspace!.Base; + var candidates = project.Versions().Where(v => v.StoredAt is null && v.Number > (basis?.Number ?? 0)).ToList(); + var chain = new List(); + var parent = basis?.VersionId; + while (candidates.FirstOrDefault(v => v.ParentId == parent) is { } next) + { + chain.Add(next); + candidates.Remove(next); + parent = next.Id; + } + return chain; + } + + /// The base version's content: this workspace's file, else the home's (kept as a file). + private static async Task BaseVersionAsync(WorkspaceProject project, ProjectBundle bundle, IProjectHome home, CancellationToken cancellationToken) + { + if (bundle.Workspace!.Base is not { } basis) return null; + if (project.Versions().FirstOrDefault(v => v.Id == basis.VersionId) is { } local) return local; + var fetched = await home.GetVersionAsync(project.ProjectId, basis.VersionId, cancellationToken).ConfigureAwait(false) + ?? throw new ProjectException(ProjectError.NotFound, $"The home no longer has version {basis.Number} of project {project.ProjectId}."); + Verify(fetched); + project.WriteVersion(fetched, replace: true); + return fetched; + } + + private static ProjectBundle WithBase(ProjectBundle bundle, ProjectVersion version) => + bundle with { Workspace = bundle.Workspace! with { Base = Base(version) } }; + + private static BundleBase Base(ProjectVersion version) => new(version.Id, version.Number, version.ContentHash); + + /// A version as a home handed it: its settings match its hash. + private static void Verify(ProjectVersion version) + { + if (!ProjectContent.Verifies(version.SettingsJson, version.ContentHash)) + throw new InvalidDataException($"Version {version.Number} of project {version.ProjectId} does not match its content hash."); + } + + private static HashSet Paths(IEnumerable differences) => [.. differences.Select(d => d.Path)]; + + /// + /// The portable form of a working state, whose positions name the frames of ; null when the + /// workspace holds no media and the working state names frames by position (which frames, only the media say). + /// + private static string? Portable(ProjectSettings working, WorkspaceMedia? media) + { + if (media is not null) return ProjectContent.ToPortable(working, media.Photos); + var positions = working.ExcludedFrames.Count > 0 || working.HeroFrame is not null || working.Camera.Keyframes.Count > 0; + return positions ? null : ProjectContent.ToPortable(working, NoFrames); + } + + private static InvalidOperationException MediaMissing(Guid projectId) => + new($"Project {projectId} names frames the workspace holds no media of: copy the media again first."); + + private WorkspaceProject Existing(Guid projectId) => + store.Find(projectId) ?? throw new InvalidOperationException($"Project {projectId} is not in the workspace."); + + private static InvalidOperationException ReadOnly(Guid projectId) => + new($"Project {projectId} holds settings of a newer Chronos: it opens read-only."); + + /// The sequence of a workspace project over its media, for the editor. + private static WorkspaceSequence SequenceOf(WorkspaceProject project, ProjectBundle bundle, WorkspaceMedia media) + { + var frames = media.ToFrames(); + var files = media.Frames.Select(f => f.File is { } file && File.Exists(project.MediaFile(file)) ? project.MediaFile(file) : null).ToList(); + var times = frames.Where(f => f.CapturedAt is not null).Select(f => f.CapturedAt!.Value).Order().ToList(); + var sequence = new Sequence + { + Id = bundle.Workspace!.Home.SequenceId, + ArchiveId = bundle.Archive.Id, + Title = bundle.Sequence.Title, + Folder = bundle.Sequence.Folder, + FrameCount = frames.Count, + FirstCapture = times.Count > 0 ? times[0] : null, + LastCapture = times.Count > 0 ? times[^1] : null, + FramesVersion = media.FramesVersion, + SettingsJson = bundle.Project.ToJson(), + Origin = bundle.Workspace.Home.IsArchive ? SequenceOrigin.Local : SequenceOrigin.Server, + }; + return new WorkspaceSequence(sequence, frames, files.Select((f, i) => frames[i].Flags.HasFlag(FrameFlags.Missing) ? null : f).ToList()); + } + + /// The choice "take the server's wherever both changed": its paths are only known once the merge ran. + private sealed class EveryPath : HashSet; +} diff --git a/src/Chronos.Protocol/Jobs.cs b/src/Chronos.Protocol/Jobs.cs index 4ef5a995..ebd71a4d 100644 --- a/src/Chronos.Protocol/Jobs.cs +++ b/src/Chronos.Protocol/Jobs.cs @@ -54,7 +54,13 @@ public sealed record JobPriorityRequest(string Priority); /// parametersJson of a render job: what to encode. Render nodes claim the job when they have an encoder for /// at × (BRIEF §14.7). /// -public sealed record RenderJobParameters(string PresetName, string PresetJson, string Codec, int Width, int Height, bool UseProxies, string? SettingsJson); +/// The settings to render; null renders the sequence's own (or, with , +/// that version's). Settings in the portable form of a project version (positions parked by photo) are placed on the sequence's +/// frames before the render. +/// The project whose version is rendered (owner review 2, R8); recorded with the render. +/// The project version rendered; recorded with the render. +public sealed record RenderJobParameters(string PresetName, string PresetJson, string Codec, int Width, int Height, bool UseProxies, string? SettingsJson, + Guid? ProjectId = null, Guid? ProjectVersionId = null); /// /// POST /api/v1/nodes/{id}/jobs/{jobId}/result: the finished output a node is about to upload in chunks. A diff --git a/tests/Chronos.App.Tests/Editor/EditorFixture.cs b/tests/Chronos.App.Tests/Editor/EditorFixture.cs index c8248ac7..eef76342 100644 --- a/tests/Chronos.App.Tests/Editor/EditorFixture.cs +++ b/tests/Chronos.App.Tests/Editor/EditorFixture.cs @@ -1,3 +1,4 @@ +using Chronos.Data.Projects; using System.Globalization; using Chronos.App.Editor; using Chronos.Core.Analysis; diff --git a/tests/Chronos.App.Tests/Editor/EditorTextTests.cs b/tests/Chronos.App.Tests/Editor/EditorTextTests.cs index f10dcaef..1b291bbb 100644 --- a/tests/Chronos.App.Tests/Editor/EditorTextTests.cs +++ b/tests/Chronos.App.Tests/Editor/EditorTextTests.cs @@ -3,6 +3,7 @@ using Chronos.Core.Advice; using Chronos.Core.Imaging; using Chronos.Core.Localization; using Chronos.Core.Project; +using Chronos.Data.Projects; using Chronos.Testing; namespace Chronos.App.Tests.Editor; diff --git a/tests/Chronos.App.Tests/Editor/ProjectStoreTests.cs b/tests/Chronos.App.Tests/Editor/ProjectStoreTests.cs index 6539be00..1a9e1883 100644 --- a/tests/Chronos.App.Tests/Editor/ProjectStoreTests.cs +++ b/tests/Chronos.App.Tests/Editor/ProjectStoreTests.cs @@ -1,3 +1,4 @@ +using Chronos.Data.Projects; using System.IO; using Chronos.App.Editor; using Chronos.Core.Archiving; diff --git a/tests/Chronos.App.Tests/Editor/RemoteSequenceSourceTests.cs b/tests/Chronos.App.Tests/Editor/RemoteSequenceSourceTests.cs index 6bf6e59d..f9142d9f 100644 --- a/tests/Chronos.App.Tests/Editor/RemoteSequenceSourceTests.cs +++ b/tests/Chronos.App.Tests/Editor/RemoteSequenceSourceTests.cs @@ -1,3 +1,4 @@ +using Chronos.Data.Projects; using System.IO; using System.Net.Http; using Chronos.App.Editor; diff --git a/tests/Chronos.App.Tests/Editor/SavedCopiesTests.cs b/tests/Chronos.App.Tests/Editor/SavedCopiesTests.cs index acbe626d..cba02868 100644 --- a/tests/Chronos.App.Tests/Editor/SavedCopiesTests.cs +++ b/tests/Chronos.App.Tests/Editor/SavedCopiesTests.cs @@ -1,3 +1,4 @@ +using Chronos.Data.Projects; using System.IO; using Chronos.App.Editor; using Chronos.Core.Project; diff --git a/tests/Chronos.Core.Tests/Project/ProjectBundleTests.cs b/tests/Chronos.Core.Tests/Project/ProjectBundleTests.cs index 81d1a4ab..d9f8f049 100644 --- a/tests/Chronos.Core.Tests/Project/ProjectBundleTests.cs +++ b/tests/Chronos.Core.Tests/Project/ProjectBundleTests.cs @@ -37,12 +37,68 @@ public sealed class ProjectBundleTests(TestContext context) // The archive's own meta files are JSON too, but not project files. Assert.Throws(() => ProjectBundle.Parse("""{ "schemaVersion": 1, "id": "0192a4c1-7b3e-7cc0-8a51-2f9d0e6b1a01" }""")); var json = Sample().ToJson(); - Assert.Throws(() => ProjectBundle.Parse(json.Replace("\"formatVersion\": 1", "\"formatVersion\": 2", StringComparison.Ordinal))); + Assert.Throws(() => ProjectBundle.Parse(json.Replace("\"formatVersion\": 1", "\"formatVersion\": 3", StringComparison.Ordinal))); Assert.Throws(() => ProjectBundle.Parse(json.Replace(SequenceId.ToString(), "not-a-guid", StringComparison.Ordinal))); Assert.Throws(() => ProjectBundle.Parse(json.Replace("\"version\": 1", "\"version\": 99", StringComparison.Ordinal))); Assert.Throws(() => ProjectBundle.Parse(json.Replace("\"folder\"", "\"place\"", StringComparison.Ordinal))); } + [Test] + public void AWorkspaceProjectFileRoundTripsItsWorkspaceSection() + { + var basis = new BundleBase(Guid.Parse("0192a4c1-7b3e-7cc0-8a51-2f9d0e6b1a10"), 7, new string('a', 64)); + var workspace = new BundleWorkspace(Guid.Parse("0192a4c1-7b3e-7cc0-8a51-2f9d0e6b1a11"), "Vertical social", false, + new BundleHome(BundleHome.ArchiveKind, SequenceId, ArchiveId), basis, new BundleMedia(BundleMedia.Proxies, 12)); + var bundle = Sample() with { Workspace = workspace }; + var path = Path.Combine(context.CreateTempDirectory("bundle"), "project.chronos"); + bundle.Save(path); + + var json = File.ReadAllText(path); + Assert.Contains("\"formatVersion\": 2", json); + Assert.Contains("\"kind\": \"archive\"", json); + var back = ProjectBundle.Load(path); + Assert.Equal(workspace, back.Workspace); + Assert.Equal(bundle.Project.ToJson(), back.Project.ToJson()); + Assert.False(back.IsReadOnly); + + // A server home names its server; a project not pulled yet has no base. + var remote = workspace with { Home = new BundleHome(BundleHome.ServerKind, SequenceId, Server: "https://nas:8443"), Base = null, IsMain = true }; + Assert.Equal(remote, ProjectBundle.Parse((bundle with { Workspace = remote }).ToJson()).Workspace); + Assert.Throws(() => ProjectBundle.Parse(json.Replace("\"kind\": \"archive\"", "\"kind\": \"cloud\"", StringComparison.Ordinal))); + } + + [Test] + public void AFileOfTheFirstFormatStillOpens() + { + const string v1 = """ + { "format": "chronos-project", "formatVersion": 1, "saved": "2026-10-12T09:30:00+00:00", + "archive": { "id": "0192a4c1-7b3e-7cc0-8a51-2f9d0e6b1a01", "name": "Main archive" }, + "sequence": { "id": "0192a4c1-7b3e-7cc0-8a51-2f9d0e6b1a02", "folder": "2026/Harbour", "frames": 10 }, + "project": { "excludedFrames": [4] }, + "workspace": { "projectId": "0192a4c1-7b3e-7cc0-8a51-2f9d0e6b1a11" } } + """; + var bundle = ProjectBundle.Parse(v1); + Assert.SequenceEqual([4], bundle.Project.ExcludedFrames); + // A format 1 file has no workspace section, whatever it holds. + Assert.Null(bundle.Workspace); + Assert.Contains("\"formatVersion\": 1", bundle.ToJson()); + } + + [Test] + public void ANewerProjectOpensReadOnlyInAWorkspaceFileOnly() + { + var workspace = new BundleWorkspace(Guid.CreateVersion7(), "Main", true, new BundleHome(BundleHome.ArchiveKind, SequenceId, ArchiveId), null, + new BundleMedia(BundleMedia.Proxies, 0)); + var json = (Sample() with { Workspace = workspace }).ToJson().Replace("\"version\": 1", "\"version\": 2", StringComparison.Ordinal); + var bundle = ProjectBundle.Parse(json); + Assert.True(bundle.IsReadOnly); + Assert.True(bundle.Project.IsNewerFormat); + Assert.Equal(0.8, bundle.Project.Exposure.DeflickerStrength.Resolve(1.0)); + + var plain = Sample().ToJson().Replace("\"version\": 1", "\"version\": 2", StringComparison.Ordinal); + Assert.Throws(() => ProjectBundle.Parse(plain)); + } + [Test] public void AFileTooLargeToBeAProjectIsNotRead() { diff --git a/tests/Chronos.Core.Tests/Project/ProjectContentTests.cs b/tests/Chronos.Core.Tests/Project/ProjectContentTests.cs new file mode 100644 index 00000000..d0dd0ca7 --- /dev/null +++ b/tests/Chronos.Core.Tests/Project/ProjectContentTests.cs @@ -0,0 +1,165 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using Chronos.Core.Project; +using Chronos.Testing; + +namespace Chronos.Core.Tests.Project; + +/// The portable form of project versions (owner review 2, R8): positions parked by photo, canonical text, a stable hash. +public sealed class ProjectContentTests +{ + private static readonly DateTime Start = new(2026, 9, 28, 18, 0, 0); + + private static FramePhoto Photo(int n, bool hashed = true) => + new(hashed ? n.ToString("x64", System.Globalization.CultureInfo.InvariantCulture) : null, 1000 + n, Start.AddSeconds(10 * n)); + + private static FramePhotos Frames(params FramePhoto[] photos) => new(photos.Select((p, i) => (i, p))); + + private static ProjectSettings Positions(int[] excluded, int? hero, int keyframe) => new() + { + ExcludedFrames = excluded, + HeroFrame = hero, + Camera = new CameraSettings { Keyframes = [new CameraKey(keyframe, 0.4, 0.5, 1.2, 0)] }, + Exposure = new ExposureSettings { ExposureOffsetEv = 0.5 }, + }; + + [Test] + public void ThePortableFormParksEveryPositionByPhotoAndPlacesThemBack() + { + var frames = Frames(Photo(1), Photo(2), Photo(3), Photo(4), Photo(5)); + var settings = Positions([1, 3], 2, 0); + + var portable = ProjectContent.ToPortable(settings, frames); + var parked = ProjectSettings.FromJson(portable); + Assert.Empty(parked.ExcludedFrames); + Assert.Null(parked.HeroFrame); + Assert.Empty(parked.Camera.Keyframes); + Assert.Equal(4, parked.PendingFrames!.Count); + Assert.Equal(0.5, parked.Exposure.ExposureOffsetEv); + + var placed = ProjectSettings.FromJson(ProjectContent.Place(portable, frames)); + Assert.SequenceEqual([1, 3], placed.ExcludedFrames); + Assert.Equal(2, placed.HeroFrame); + Assert.Equal(0, Assert.NotNull(placed.Camera.Keyframes.SingleOrDefault()).Frame); + Assert.Null(placed.PendingFrames); + } + + [Test] + public void PositionsFollowTheirPhotosAcrossInsertionsAndRemovals() + { + var before = Frames(Photo(1), Photo(2), Photo(3), Photo(4)); + var portable = ProjectContent.ToPortable(Positions([1, 3], 2, 0), before); + + // Two photos inserted before the others, and photo 4 removed: each position names its photo, the one on 4 waits. + var after = Frames(Photo(10), Photo(11), Photo(1), Photo(2), Photo(3)); + var placed = ProjectSettings.FromJson(ProjectContent.Place(portable, after)); + Assert.SequenceEqual([3], placed.ExcludedFrames); + Assert.Equal(4, placed.HeroFrame); + Assert.Equal(2, placed.Camera.Keyframes.Single().Frame); + var waiting = Assert.NotNull(placed.PendingFrames).Single(); + Assert.True(waiting.Excluded && waiting.Photo.Matches(Photo(4))); + + // Photo 4 back: the exclusion takes its place again. + var back = ProjectSettings.FromJson(ProjectContent.Place(placed.ToJson(), Frames(Photo(10), Photo(11), Photo(1), Photo(2), Photo(3), Photo(4)))); + Assert.SequenceEqual([3, 5], back.ExcludedFrames); + Assert.Null(back.PendingFrames); + } + + [Test] + public void ACopyOfAPhotoHeldTwiceKeepsItsOwnPosition() + { + var frames = Frames(Photo(1), Photo(2), Photo(2), Photo(3)); + var portable = ProjectContent.ToPortable(new ProjectSettings { ExcludedFrames = [2] }, frames); + Assert.Equal(1, ProjectSettings.FromJson(portable).PendingFrames!.Single().Occurrence); + + var inserted = Frames(Photo(0), Photo(1), Photo(2), Photo(2), Photo(3)); + Assert.SequenceEqual([3], ProjectSettings.FromJson(ProjectContent.Place(portable, inserted)).ExcludedFrames); + } + + [Test] + public void PhotosNotHashedYetAreMatchedBySizeAndCaptureTime() + { + var unhashed = Frames(Photo(1, hashed: false), Photo(2, hashed: false), Photo(3, hashed: false)); + var portable = ProjectContent.ToPortable(new ProjectSettings { ExcludedFrames = [1] }, unhashed); + var parked = ProjectSettings.FromJson(portable).PendingFrames!.Single(); + Assert.Null(parked.Sha256); + Assert.Equal(Photo(2).Size, parked.Size); + + // The same photos hashed since, with one inserted before them. + var hashed = Frames(Photo(9), Photo(1), Photo(2), Photo(3)); + Assert.SequenceEqual([2], ProjectSettings.FromJson(ProjectContent.Place(portable, hashed)).ExcludedFrames); + } + + [Test] + public void TheSameSettingsGiveTheSameTextAndHashWhateverTheFrames() + { + var before = Frames(Photo(1), Photo(2), Photo(3)); + var after = Frames(Photo(7), Photo(1), Photo(2), Photo(3)); + var first = ProjectContent.ToPortable(Positions([0, 2], 1, 2), before); + // The same choices made against frames with one inserted before them: every position one further. + var second = ProjectContent.ToPortable(Positions([1, 3], 2, 3), after); + Assert.Equal(first, second); + Assert.Equal(ProjectContent.Hash(first), ProjectContent.Hash(second)); + Assert.Equal(64, ProjectContent.Hash(first).Length); + Assert.True(ProjectContent.Verifies(first, ProjectContent.Hash(first).ToUpperInvariant())); + Assert.False(ProjectContent.Verifies(first + " ", ProjectContent.Hash(first))); + + // Pending frames listed in another order, or written by hand with other spacing, read as the same text. + var parked = ProjectSettings.FromJson(first); + var shuffled = parked with { PendingFrames = [.. parked.PendingFrames!.Reverse()] }; + Assert.Equal(first, ProjectContent.Canonical(shuffled.ToJson())); + Assert.Equal(first, ProjectContent.Canonical(JsonNode.Parse(first)!.ToJsonString())); + Assert.Equal(first, ProjectContent.ToPortable(first, after)); + } + + [Test] + public void MembersOfANewerChronosSurviveEveryRoundTrip() + { + const string json = """ + { + "version": 1, + "filters": { "lut": "teal-orange", "strength": 0.4 }, + "exposure": { "exposureOffsetEv": 0.25, "toneCurve": [0, 0.5, 1] }, + "camera": { "aspect": { "isAuto": false, "manual": "9:16" }, "lensProfile": "rf-15-35" }, + "pendingFrames": [ { "sha256": "abc", "size": 10, "excluded": true, "rating": 5 } ] + } + """; + var settings = ProjectSettings.FromJson(json); + Assert.Equal(0.25, settings.Exposure.ExposureOffsetEv); + var written = JsonNode.Parse(settings.ToJson())!; + Assert.Equal("teal-orange", written["filters"]!["lut"]!.GetValue()); + Assert.Equal(3, written["exposure"]!["toneCurve"]!.AsArray().Count); + Assert.Equal("rf-15-35", written["camera"]!["lensProfile"]!.GetValue()); + Assert.Equal(5, written["pendingFrames"]![0]!["rating"]!.GetValue()); + + var portable = ProjectContent.ToPortable(json, Frames(Photo(1))); + Assert.Contains("teal-orange", portable); + Assert.Contains("lensProfile", ProjectContent.Place(portable, Frames(Photo(1)))); + Assert.Equal(portable, ProjectContent.Canonical(ProjectSettings.FromJson(portable).ToJson())); + } + + [Test] + public void SettingsOfANewerFormatAreReadOnlyAndKeptAsTheyAre() + { + var newer = """{ "version": 2, "excludedFrames": [1], "exposure": { "exposureOffsetEv": 1.5 }, "grade": { "look": "film" } }"""; + Assert.Throws(() => ProjectSettings.FromJson(newer)); + var read = ProjectSettings.FromJson(newer, allowNewer: true); + Assert.True(read.IsNewerFormat); + Assert.Equal(1.5, read.Exposure.ExposureOffsetEv); + Assert.Equal(2, ProjectContent.FormatOf(newer)); + Assert.Equal(1, ProjectContent.FormatOf("""{ "exposure": {} }""")); + + // Its positions are a newer Chronos's to translate: only the text is made canonical. + var portable = ProjectContent.ToPortable(newer, Frames(Photo(1), Photo(2))); + Assert.SequenceEqual([1], ProjectSettings.FromJson(portable, allowNewer: true).ExcludedFrames); + Assert.Contains("film", portable); + Assert.Equal(portable, ProjectContent.Place(portable, Frames(Photo(2)))); + } + + [Test] + public void TextThatIsNoProjectIsRefused() + { + Assert.Throws(() => ProjectContent.ToPortable("not json", Frames(Photo(1)))); + Assert.Throws(() => ProjectContent.Canonical("[1, 2]")); + } +} diff --git a/tests/Chronos.Data.Tests/Migrations/MigrationTests.cs b/tests/Chronos.Data.Tests/Migrations/MigrationTests.cs index 713bf6be..f9a1fadb 100644 --- a/tests/Chronos.Data.Tests/Migrations/MigrationTests.cs +++ b/tests/Chronos.Data.Tests/Migrations/MigrationTests.cs @@ -1,3 +1,4 @@ +using Chronos.Core.Model; using Chronos.Data.Migrations; using Chronos.Data.Tests.Infrastructure; using Chronos.Testing; @@ -9,7 +10,7 @@ public abstract class MigrationTests(TestContext context) : RepositoryTest(conte private static readonly string[] Tables = [ "archives", "sequences", "frames", "renders", "jobs", "media_cards", "import_history", "users", "tokens", - "render_nodes", "settings", "tags", "sequence_tags", "audit_log", "uploads", "schedules", "frame_removals", + "render_nodes", "settings", "tags", "sequence_tags", "audit_log", "uploads", "schedules", "frame_removals", "projects", "project_versions", ]; [Test] @@ -41,6 +42,67 @@ public abstract class MigrationTests(TestContext context) : RepositoryTest(conte Assert.True(recorded[0].GetDateTimeOffset(2) > DateTimeOffset.UtcNow.AddMinutes(-5)); } + [Test] + public async Task ProjectsKeepOneMainAndOneLiveNamePerSequence() + { + var chronos = await OpenChronosAsync(); + var archive = TestData.Archive("Main"); + await chronos.Archives.UpsertAsync(archive, Ct); + var sequence = TestData.Sequence(archive.Id); + await chronos.Sequences.UpsertAsync(sequence, Ct); + var now = DateTimeOffset.UtcNow; + SequenceProject Project(string name, bool main) => new() { Id = Guid.CreateVersion7(), SequenceId = sequence.Id, Name = name, IsMain = main, CreatedAt = now, UpdatedAt = now }; + + var main = Project("Main", true); + await chronos.Projects.InsertAsync(main, Ct); + await Assert.ThrowsAsync(() => chronos.Projects.InsertAsync(Project("Other main", true), Ct)); + var vertical = Project("Vertical", false); + await chronos.Projects.InsertAsync(vertical, Ct); + await Assert.ThrowsAsync(() => chronos.Projects.InsertAsync(Project("Vertical", false), Ct)); + // A deleted project gives its name back. + await chronos.Projects.UpdateAsync(vertical with { DeletedAt = now }, Ct); + await chronos.Projects.InsertAsync(Project("Vertical", false), Ct); + + // Versions: one number each, settings kept byte for byte, and they go with the sequence. + const string settings = "{\n \"version\": 1,\n \"exposure\": {}\n}"; + var version = new ProjectVersion + { + Id = Guid.CreateVersion7(), + ProjectId = main.Id, + Number = 1, + Kind = ProjectVersionKind.Imported, + SettingsJson = settings, + ContentHash = new string('0', 64), + FramesVersion = 3, + CreatedAt = now, + StoredAt = now, + }; + await chronos.Projects.InsertVersionAsync(version, Ct); + await Assert.ThrowsAsync(() => chronos.Projects.InsertVersionAsync(version with { Id = Guid.CreateVersion7() }, Ct)); + var stored = Assert.NotNull(await chronos.Projects.GetVersionAsync(version.Id, Ct)); + Assert.Equal(settings, stored.SettingsJson); + Assert.Equal((ProjectVersionKind.Imported, 3), (stored.Kind, stored.FramesVersion)); + Assert.True(await chronos.Projects.AdvanceHeadAsync(main.Id, 0, version.Id, 1, now, Ct)); + Assert.False(await chronos.Projects.AdvanceHeadAsync(main.Id, 0, version.Id, 1, now, Ct), "the head moved already"); + + var render = new Render + { + Id = Guid.CreateVersion7(), + SequenceId = sequence.Id, + PresetName = "MJPEG", + OutputPath = "x/exports/a.avi", + Created = now, + ProjectId = main.Id, + ProjectVersionId = version.Id, + }; + await chronos.Renders.InsertAsync(render, Ct); + Assert.Equal((main.Id, version.Id), ((await chronos.Renders.GetAsync(render.Id, Ct))!.ProjectId, (await chronos.Renders.GetAsync(render.Id, Ct))!.ProjectVersionId)); + + await chronos.Sequences.DeleteAsync(sequence.Id, Ct); + Assert.Empty(await chronos.Projects.ListBySequenceAsync(sequence.Id, includeDeleted: true, Ct)); + Assert.Null(await chronos.Projects.GetVersionAsync(version.Id, Ct)); + } + [Test] public async Task AppliesOnlyPendingMigrations() { diff --git a/tests/Chronos.Data.Tests/Processing/RenderJobTests.cs b/tests/Chronos.Data.Tests/Processing/RenderJobTests.cs index abacdad5..f30c947f 100644 --- a/tests/Chronos.Data.Tests/Processing/RenderJobTests.cs +++ b/tests/Chronos.Data.Tests/Processing/RenderJobTests.cs @@ -64,6 +64,52 @@ public sealed class RenderJobTests(TestContext context) Assert.Equal(3, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.RenderedCount); } + [Test] + public async Task AProjectVersionRendersWithItsPositionsPlacedOnTheFramesAndIsRecordedWithIt() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(Scene()); + var frames = await fx.Db.Frames.ListBySequenceAsync(sequence.Id, cancellationToken: fx.Ct); + var photos = new FramePhotos(frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, f.CapturedAt)))); + var projects = fx.Archive.Service.Projects; + var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); + var head = await projects.HeadAsync(vertical.Id, fx.Ct); + // The version holds its positions parked by photo (the adopted frames are not hashed: size and capture time name them). + var settings = new ProjectSettings { ExcludedFrames = [3, 5], Timing = new TimingSettings { Mode = new Auto().WithManual(TimingMode.Constant) } }; + var portable = ProjectContent.ToPortable(settings, photos); + Assert.Empty(ProjectSettings.FromJson(portable).ExcludedFrames); + var draft = head with + { + Id = Guid.CreateVersion7(), + ParentId = head.Id, + Number = 2, + Kind = ProjectVersionKind.Commit, + SettingsJson = portable, + ContentHash = ProjectContent.Hash(portable), + StoredAt = null, + }; + var version = (await projects.PushAsync(vertical.Id, draft, fx.Ct)).Version!; + await fx.StartEngineAsync(); + + var requests = ProcessingJobs.RenderVersion(sequence, version, Mjpeg with { Preset = new Auto().WithManual("png-sequence") }, 96, 64); + var job = await fx.EnqueueAsync(requests.Single()); + await fx.Archive.WaitForAsync(job.Id, JobState.Done, (int)TestTiming.Budget(60)); + + Assert.Equal(Frames - 2, Directory.GetFiles(Path.Combine(fx.Folder(sequence).Exports, "clip-PNG")).Length); + var render = (await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct)).Single(); + Assert.Equal((vertical.Id, version.Id), (render.ProjectId, render.ProjectVersionId)); + Assert.SequenceEqual([3, 5], ProjectSettings.FromJson(render.SettingsJson!).ExcludedFrames); + // A render of another project leaves the sequence's own settings alone. + Assert.Empty(ProjectSettings.FromJson((await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.SettingsJson ?? "{}").ExcludedFrames); + + // A job naming the version without its settings renders them too. + var named = Chronos.Protocol.ProtocolJson.Default.RenderJobParameters; + var parameters = System.Text.Json.JsonSerializer.Deserialize(requests[0].ParametersJson!, named)! with { SettingsJson = null }; + var second = await fx.EnqueueAsync(requests[0] with { ParametersJson = System.Text.Json.JsonSerializer.Serialize(parameters, named) }); + await fx.Archive.WaitForAsync(second.Id, JobState.Done, (int)TestTiming.Budget(60)); + Assert.True((await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct)).All(r => r.ProjectVersionId == version.Id)); + } + [Test] public async Task FramesThisHostCannotDecodeRenderFromTheirProxies() { diff --git a/tests/Chronos.Data.Tests/Projects/ProjectServiceTests.cs b/tests/Chronos.Data.Tests/Projects/ProjectServiceTests.cs new file mode 100644 index 00000000..ef149fe0 --- /dev/null +++ b/tests/Chronos.Data.Tests/Projects/ProjectServiceTests.cs @@ -0,0 +1,350 @@ +using Chronos.Core.Archiving; +using Chronos.Core.Model; +using Chronos.Core.Project; +using Chronos.Data.Archiving; +using Chronos.Data.Projects; +using Chronos.Data.Repositories; +using Chronos.Data.Tests.Archiving; +using Chronos.Data.Tests.Infrastructure; +using Chronos.Testing; +using static Chronos.Data.Tests.Archiving.ArchiveFixture; + +namespace Chronos.Data.Tests.Projects; + +/// Projects of a sequence with this PC's archive as their home (owner review 2, R8): sidecars, index, Main mirror, push. +[Category("projects")] +public abstract class ProjectServiceTests(TestContext context) : DatabaseTest(context) +{ + internal async Task FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true))); + + /// A sequence of uploaded photos (hashed), ten seconds apart. + internal static async Task<(Sequence Sequence, List Frames)> UploadedAsync(ArchiveFixture fx, int count, int seed = 100) + { + Guid? id = null; + for (var i = 0; i < count; i++) + { + var (path, size, sha) = fx.Stage(Start.AddSeconds(10 * i), seed + i); + var result = await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, id, null, id is null ? $"upload-{seed}" : null, path, + $"IMG_{i + 1:0000}.JPG", size, sha, null, null), fx.Ct); + id = result.SequenceId; + } + return ((await fx.Db.Sequences.GetAsync(id!.Value))!, [.. await fx.Db.Frames.ListBySequenceAsync(id.Value)]); + } + + internal static FramePhotos Photos(IEnumerable frames) => new(frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, f.CapturedAt)))); + + /// The sequence's settings written as an older code path writes them (the editor, an analysis): index first, with its version check. + internal static async Task WriteSettingsAsync(ArchiveFixture fx, Guid sequenceId, ProjectSettings settings) + { + var row = (await fx.Db.Sequences.GetAsync(sequenceId))!; + var result = await fx.Db.Sequences.EditAsync(sequenceId, new SequenceEdit { SettingsJson = settings.ToJson() }, row.SettingsVersion, fx.Ct); + Assert.Equal(SequenceEditOutcome.Updated, result.Outcome); + return result.Sequence!; + } + + /// A version made from with (positions on ), as a workspace sends it. + internal static ProjectVersion Draft(ProjectVersion parent, ProjectSettings settings, IEnumerable frames, string? note = null) + { + var portable = ProjectContent.ToPortable(settings, Photos(frames)); + return new ProjectVersion + { + Id = Guid.CreateVersion7(), + ProjectId = parent.ProjectId, + ParentId = parent.Id, + Number = parent.Number + 1, + Kind = ProjectVersionKind.Commit, + Author = "alberto", + ClientName = "test", + Note = note, + SettingsJson = portable, + ContentHash = ProjectContent.Hash(portable), + FramesVersion = parent.FramesVersion, + CreatedAt = DateTimeOffset.UtcNow, + }; + } + + [Test] + public async Task MainIsMadeFromTheSequenceSettingsWithItsPositionsParkedByPhoto() + { + var fx = await FixtureAsync(); + var (sequence, frames) = await UploadedAsync(fx, 5); + await WriteSettingsAsync(fx, sequence.Id, new ProjectSettings { ExcludedFrames = [1], HeroFrame = 3 }); + var projects = fx.Service.Projects; + + var main = await projects.MainAsync(sequence.Id, fx.Ct); + Assert.True(main.IsMain); + Assert.Equal((ProjectService.MainName, 1), (main.Name, main.HeadNumber)); + var head = await projects.HeadAsync(main.Id, fx.Ct); + Assert.Equal(ProjectVersionKind.Imported, head.Kind); + Assert.True(ProjectContent.Verifies(head.SettingsJson, head.ContentHash)); + var portable = ProjectSettings.FromJson(head.SettingsJson); + Assert.Empty(portable.ExcludedFrames); + Assert.Null(portable.HeroFrame); + Assert.Equal(2, portable.PendingFrames!.Count); + var placed = ProjectSettings.FromJson(ProjectContent.Place(head.SettingsJson, Photos(frames))); + Assert.SequenceEqual([1], placed.ExcludedFrames); + Assert.Equal(3, placed.HeroFrame); + + // The sidecars hold it: the record and an immutable version file. + var folder = fx.Paths.Sequence(sequence.Folder); + var record = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(main.Id)); + Assert.Equal((main.Id, sequence.Id, true, head.Id, 1), (record.Id, record.SequenceId, record.IsMain, record.Head!.Id, record.Head.Number)); + Assert.Equal(head.SettingsJson, ProjectSidecars.ReadVersion(folder.VersionPath(main.Id, 1, head.Id)).SettingsJson); + + // Asking again makes nothing new. + Assert.Equal(main.Id, (await projects.MainAsync(sequence.Id, fx.Ct)).Id); + Assert.Count(1, await projects.ListAsync(sequence.Id, fx.Ct)); + Assert.Count(1, await projects.HistoryAsync(main.Id, cancellationToken: fx.Ct)); + } + + [Test] + public async Task APushFastForwardsAndMirrorsMainIntoTheSequence() + { + var fx = await FixtureAsync(); + var (sequence, frames) = await UploadedAsync(fx, 5); + var projects = fx.Service.Projects; + var main = await projects.MainAsync(sequence.Id, fx.Ct); + var head = await projects.HeadAsync(main.Id, fx.Ct); + var before = (await fx.Db.Sequences.GetAsync(sequence.Id))!; + + var draft = Draft(head, new ProjectSettings { ExcludedFrames = [0, 4], HeroFrame = 2, Exposure = new ExposureSettings { ExposureOffsetEv = 0.7 } }, frames, + "first look"); + var result = await projects.PushAsync(main.Id, draft, fx.Ct); + + Assert.Equal(PushOutcome.Stored, result.Outcome); + var stored = Assert.NotNull(result.Version); + Assert.Equal((draft.Id, 2, "first look"), (stored.Id, stored.Number, stored.Note)); + Assert.NotNull(stored.StoredAt); + Assert.Equal(stored.Id, (await projects.HeadAsync(main.Id, fx.Ct)).Id); + Assert.SequenceEqual([2, 1], (await projects.HistoryAsync(main.Id, cancellationToken: fx.Ct)).Select(v => v.Number)); + Assert.FileExists(fx.Paths.Sequence(sequence.Folder).VersionPath(main.Id, 2, draft.Id)); + + // Main's head is the sequence's settings, positions on its frames, through the settings version check. + var after = (await fx.Db.Sequences.GetAsync(sequence.Id))!; + Assert.Greater(after.SettingsVersion, before.SettingsVersion); + var mirrored = ProjectSettings.FromJson(after.SettingsJson!); + Assert.SequenceEqual([0, 4], mirrored.ExcludedFrames); + Assert.Equal(0.7, mirrored.Exposure.ExposureOffsetEv); + Assert.Equal(frames[2].Id, after.HeroFrameId); + Assert.Equal(after.SettingsJson, File.ReadAllText(fx.Paths.Sequence(sequence.Folder).ProjectPath)); + var row = (await fx.Db.Projects.GetAsync(main.Id))!; + Assert.Equal((stored.Id, after.SettingsVersion), (row.MirroredVersionId, row.MirroredSettingsVersion)); + // The mirror is no outside change: no version is added for it. + Assert.Equal(2, (await projects.HeadAsync(main.Id, fx.Ct)).Number); + } + + [Test] + public async Task APushIsIdempotentDivergesFromAnOldParentAndChecksItsHash() + { + var fx = await FixtureAsync(); + var (sequence, frames) = await UploadedAsync(fx, 3); + var projects = fx.Service.Projects; + var main = await projects.MainAsync(sequence.Id, fx.Ct); + var first = await projects.HeadAsync(main.Id, fx.Ct); + var draft = Draft(first, new ProjectSettings { Stack = new StackSettings { Mode = StackSetting.Median } }, frames); + + Assert.Equal(PushOutcome.Stored, (await projects.PushAsync(main.Id, draft, fx.Ct)).Outcome); + var again = await projects.PushAsync(main.Id, draft, fx.Ct); + Assert.Equal((PushOutcome.AlreadyStored, 2), (again.Outcome, again.Version!.Number)); + Assert.Equal(2, (await projects.HistoryAsync(main.Id, cancellationToken: fx.Ct)).Count); + + var late = Draft(first, new ProjectSettings { Stack = new StackSettings { Mode = StackSetting.Mean } }, frames); + var diverged = await projects.PushAsync(main.Id, late, fx.Ct); + Assert.Equal(PushOutcome.Diverged, diverged.Outcome); + Assert.Equal(draft.Id, diverged.Head!.Id); + + var head = diverged.Head; + var damaged = Draft(head, new ProjectSettings(), frames) with { ContentHash = new string('f', 64) }; + Assert.Equal(PushOutcome.HashMismatch, (await projects.PushAsync(main.Id, damaged, fx.Ct)).Outcome); + var positional = new ProjectSettings { ExcludedFrames = [1] }.ToJson(); + var notPortable = Draft(head, new ProjectSettings(), frames) with { SettingsJson = positional, ContentHash = ProjectContent.Hash(positional) }; + Assert.Equal(PushOutcome.Invalid, (await projects.PushAsync(main.Id, notPortable, fx.Ct)).Outcome); + var newer = """{ "version": 9 }"""; + Assert.Equal(PushOutcome.NewerFormat, + (await projects.PushAsync(main.Id, Draft(head, new ProjectSettings(), frames) with { SettingsJson = newer, ContentHash = ProjectContent.Hash(newer) }, fx.Ct)).Outcome); + Assert.Equal(2, (await projects.HeadAsync(main.Id, fx.Ct)).Number); + } + + [Test] + public async Task SettingsAnotherPathWritesBecomeAnExternalVersionOfMain() + { + var fx = await FixtureAsync(); + var (sequence, _) = await UploadedAsync(fx, 3); + var projects = fx.Service.Projects; + var main = await projects.MainAsync(sequence.Id, fx.Ct); + + // The editor's old path (or an analysis, or the server's API) writes the settings. + await WriteSettingsAsync(fx, sequence.Id, new ProjectSettings { ExcludedFrames = [2], Exposure = new ExposureSettings { ExposureOffsetEv = -0.3 } }); + var history = await projects.HistoryAsync(main.Id, cancellationToken: fx.Ct); + Assert.Equal((2, ProjectVersionKind.External), (history[0].Number, history[0].Kind)); + Assert.Equal(history[1].Id, history[0].ParentId); + Assert.Equal(-0.3, ProjectSettings.FromJson(history[0].SettingsJson).Exposure.ExposureOffsetEv); + + // A title or tags edit moves the settings version too, but not the settings: nothing is added. + await fx.Db.Sequences.SetTagsAsync(sequence.Id, ["harbour"], fx.Ct); + Assert.Equal(2, (await projects.HistoryAsync(main.Id, cancellationToken: fx.Ct)).Count); + } + + [Test] + public async Task AMirrorACrashCutShortIsCompletedAndOneRacedByAnEditIsMerged() + { + var fx = await FixtureAsync(); + var (sequence, frames) = await UploadedAsync(fx, 4); + var projects = fx.Service.Projects; + var main = await projects.MainAsync(sequence.Id, fx.Ct); + var first = await projects.HeadAsync(main.Id, fx.Ct); + var pushed = (await projects.PushAsync(main.Id, Draft(first, new ProjectSettings { Stack = new StackSettings { Mode = StackSetting.Median } }, frames), + fx.Ct)).Version!; + + // As if the push had stopped before its mirror: the sequence still holds version 1, mirrored at its settings version. + var reverted = await WriteSettingsAsync(fx, sequence.Id, new ProjectSettings()); + await fx.Db.Projects.SetMirroredAsync(main.Id, first.Id, reverted.SettingsVersion, fx.Ct); + Assert.Equal(pushed.Id, (await projects.HeadAsync(main.Id, fx.Ct)).Id); + var completed = (await fx.Db.Sequences.GetAsync(sequence.Id))!; + Assert.Equal(StackSetting.Median, ProjectSettings.FromJson(completed.SettingsJson!).Stack.Mode); + + // Again, but someone wrote other settings meanwhile: both changes are kept in a merge version. + await WriteSettingsAsync(fx, sequence.Id, new ProjectSettings()); + await fx.Db.Projects.SetMirroredAsync(main.Id, first.Id, completed.SettingsVersion, fx.Ct); + await WriteSettingsAsync(fx, sequence.Id, new ProjectSettings { Exposure = new ExposureSettings { ExposureOffsetEv = 1.25 } }); + var merge = await projects.HeadAsync(main.Id, fx.Ct); + Assert.Equal((ProjectVersionKind.Merge, pushed.Id), (merge.Kind, merge.ParentId)); + var merged = ProjectSettings.FromJson((await fx.Db.Sequences.GetAsync(sequence.Id))!.SettingsJson!); + Assert.Equal((StackSetting.Median, 1.25), (merged.Stack.Mode, merged.Exposure.ExposureOffsetEv)); + } + + [Test] + public async Task ProjectsAreNamedOnceDeletedIntoTheirTrashAndMainStays() + { + var fx = await FixtureAsync(); + var (sequence, _) = await UploadedAsync(fx, 3); + var projects = fx.Service.Projects; + var main = await projects.MainAsync(sequence.Id, fx.Ct); + var mainHead = await projects.HeadAsync(main.Id, fx.Ct); + + var vertical = await projects.CreateAsync(sequence.Id, " Vertical social ", author: "alberto", cancellationToken: fx.Ct); + Assert.Equal(("Vertical social", false, 1), (vertical.Name, vertical.IsMain, vertical.HeadNumber)); + var copy = await projects.HeadAsync(vertical.Id, fx.Ct); + Assert.Equal((ProjectVersionKind.Imported, mainHead.ContentHash), (copy.Kind, copy.ContentHash)); + Assert.Equal(ProjectError.NameTaken, + (await Assert.ThrowsAsync(() => projects.CreateAsync(sequence.Id, "VERTICAL SOCIAL", cancellationToken: fx.Ct))).Reason); + Assert.Equal(ProjectError.InvalidName, (await Assert.ThrowsAsync(() => projects.CreateAsync(sequence.Id, " ", cancellationToken: fx.Ct))).Reason); + var landscape = await projects.CreateAsync(sequence.Id, "4K landscape", copy.Id, cancellationToken: fx.Ct); + Assert.Equal(ProjectError.NameTaken, (await Assert.ThrowsAsync(() => projects.RenameAsync(landscape.Id, "Main", fx.Ct))).Reason); + landscape = await projects.RenameAsync(landscape.Id, "4K wide", fx.Ct); + Assert.Equal("4K wide", ProjectSidecars.ReadRecord(fx.Paths.Sequence(sequence.Folder).ProjectRecordPath(landscape.Id)).Name); + Assert.SequenceEqual(["Main", "Vertical social", "4K wide"], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Name)); + + Assert.Equal(ProjectError.MainCannotBeDeleted, (await Assert.ThrowsAsync(() => projects.DeleteAsync(main.Id, fx.Ct))).Reason); + await projects.DeleteAsync(vertical.Id, fx.Ct); + var folder = fx.Paths.Sequence(sequence.Folder); + Assert.False(Directory.Exists(folder.ProjectDirectory(vertical.Id))); + var trashed = Directory.GetDirectories(folder.ProjectsTrashDirectory).Single(); + Assert.True(Path.GetFileName(trashed).StartsWith(vertical.Id.ToString("N"), StringComparison.Ordinal)); + Assert.NotNull(ProjectSidecars.ReadRecord(Path.Combine(trashed, "project.json")).Deleted); + Assert.SequenceEqual(["Main", "4K wide"], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Name)); + // Its name is free again. + await projects.CreateAsync(sequence.Id, "Vertical social", cancellationToken: fx.Ct); + } + + [Test] + public async Task AnIndexRebuiltFromTheSidecarsHasEveryProjectAndVersionBack() + { + var fx = await FixtureAsync(); + var (sequence, frames) = await UploadedAsync(fx, 4); + var projects = fx.Service.Projects; + var main = await projects.MainAsync(sequence.Id, fx.Ct); + var mainPush = (await projects.PushAsync(main.Id, Draft(await projects.HeadAsync(main.Id, fx.Ct), new ProjectSettings { ExcludedFrames = [3] }, frames), + fx.Ct)).Version!; + var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); + var verticalPush = (await projects.PushAsync(vertical.Id, + Draft(await projects.HeadAsync(vertical.Id, fx.Ct), new ProjectSettings { Camera = new CameraSettings { Aspect = new Auto().WithManual("9:16") } }, frames), + fx.Ct)).Version!; + var gone = await projects.CreateAsync(sequence.Id, "Old", cancellationToken: fx.Ct); + await projects.DeleteAsync(gone.Id, fx.Ct); + + await fx.ReopenWithEmptyIndexAsync(); + await fx.ScanAsync(ScanMode.Rebuild); + var rebuilt = fx.Service.Projects; + + var listed = await rebuilt.ListAsync(sequence.Id, fx.Ct); + Assert.SequenceEqual([main.Id, vertical.Id], listed.Select(p => p.Id)); + Assert.Equal((mainPush.Id, 2), (listed[0].HeadVersionId, listed[0].HeadNumber)); + Assert.Equal((verticalPush.Id, 2), (listed[1].HeadVersionId, listed[1].HeadNumber)); + Assert.Equal(verticalPush.SettingsJson, (await rebuilt.GetVersionAsync(verticalPush.Id, fx.Ct))!.SettingsJson); + Assert.NotNull((await fx.Db.Projects.GetAsync(gone.Id))!.DeletedAt); + // The sequence's settings are Main's head: the rebuilt index recognises the mirror and adds no version. + Assert.SequenceEqual([2, 1], (await rebuilt.HistoryAsync(main.Id, cancellationToken: fx.Ct)).Select(v => v.Number)); + Assert.Equal(mainPush.Id, (await fx.Db.Projects.GetAsync(main.Id))!.MirroredVersionId); + } + + [Test] + public async Task AVersionFileAPushLeftBehindItsRecordIsAdoptedAndAStrayOneSetAside() + { + var fx = await FixtureAsync(); + var (sequence, frames) = await UploadedAsync(fx, 3); + var projects = fx.Service.Projects; + var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct); + var head = await projects.HeadAsync(vertical.Id, fx.Ct); + var folder = fx.Paths.Sequence(sequence.Folder); + + // A push that wrote its version file and stopped before the record: the file follows the head. + var follower = Draft(head, new ProjectSettings { Stack = new StackSettings { Mode = StackSetting.StarTrails } }, frames) with { StoredAt = DateTimeOffset.UtcNow }; + ProjectSidecars.WriteVersion(folder.VersionPath(vertical.Id, 2, follower.Id), follower); + // And a file past the head that follows nothing. + var stray = Draft(head, new ProjectSettings(), frames) with { Number = 5, ParentId = Guid.CreateVersion7(), StoredAt = DateTimeOffset.UtcNow }; + ProjectSidecars.WriteVersion(folder.VersionPath(vertical.Id, 5, stray.Id), stray); + + Assert.Equal(follower.Id, (await projects.HeadAsync(vertical.Id, fx.Ct)).Id); + Assert.Equal(follower.Id, ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)).Head!.Id); + Assert.False(File.Exists(folder.VersionPath(vertical.Id, 5, stray.Id))); + Assert.Count(1, Directory.GetFiles(Path.Combine(folder.ProjectVersionsDirectory(vertical.Id), ".stray"))); + // A version file is immutable: another version at its name is refused. + Assert.Throws(() => ProjectSidecars.WriteVersion(folder.VersionPath(vertical.Id, 2, follower.Id), follower with { ContentHash = stray.ContentHash })); + } + + [Test] + public async Task AReadWhileAnotherWriterHoldsTheSequenceAnswersFromTheIndex() + { + var fx = await FixtureAsync(); + var (sequence, _) = await UploadedAsync(fx, 2); + var projects = fx.Service.Projects; + var main = await projects.MainAsync(sequence.Id, fx.Ct); + using (Assert.NotNull(LeaseFile.TryAcquire(fx.Paths.LocksDirectory, $"seq-{sequence.Id:N}", "another process"))) + { + var watch = System.Diagnostics.Stopwatch.StartNew(); + Assert.Equal(main.Id, (await projects.ListAsync(sequence.Id, fx.Ct)).Single().Id); + Assert.Equal(main.HeadVersionId, (await projects.HeadAsync(main.Id, fx.Ct)).Id); + // Each read waits a moment for the lease, never the writers' whole wait. + Assert.Less(watch.Elapsed.TotalSeconds, TestTiming.Budget(15)); + } + } + + [Test] + public async Task AReorganizedSequenceKeepsItsProjects() + { + var fx = await FixtureAsync(); + var (sequence, frames) = await UploadedAsync(fx, 3); + var projects = fx.Service.Projects; + var main = await projects.MainAsync(sequence.Id, fx.Ct); + var moved = "2026/Moved " + Guid.NewGuid().ToString("N")[..6]; + Directory.CreateDirectory(Path.GetDirectoryName(fx.Paths.Sequence(moved).Folder)!); + Directory.Move(fx.Paths.Sequence(sequence.Folder).Folder, fx.Paths.Sequence(moved).Folder); + await fx.Db.Sequences.RelocateAsync(sequence.Id, moved, sequence.Title, fx.Ct); + + var pushed = await projects.PushAsync(main.Id, Draft(await projects.HeadAsync(main.Id, fx.Ct), new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct); + Assert.Equal(PushOutcome.Stored, pushed.Outcome); + Assert.FileExists(fx.Paths.Sequence(moved).VersionPath(main.Id, 2, pushed.Version!.Id)); + } +} + +public sealed class SqliteProjectServiceTests(TestContext context) : ProjectServiceTests(context) +{ + protected override Backend Backend => Backend.SqliteFile; +} + +[RequiresEnvironment(PgTestServer.Variable)] +public sealed class PostgresProjectServiceTests(TestContext context) : ProjectServiceTests(context) +{ + protected override Backend Backend => Backend.Postgres; +} diff --git a/tests/Chronos.Data.Tests/Workspaces/WorkspaceFixture.cs b/tests/Chronos.Data.Tests/Workspaces/WorkspaceFixture.cs new file mode 100644 index 00000000..b21b271b --- /dev/null +++ b/tests/Chronos.Data.Tests/Workspaces/WorkspaceFixture.cs @@ -0,0 +1,108 @@ +using Chronos.Core.Model; +using Chronos.Data.Archiving; +using Chronos.Data.Jobs; +using Chronos.Data.Projects; +using Chronos.Data.Tests.Archiving; +using Chronos.Data.Tests.Projects; +using Chronos.Data.Workspaces; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Workspaces; + +/// An archive with an uploaded sequence as the projects' home, and workspaces that edit it (owner review 2, R8). +internal sealed class WorkspaceFixture : IAsyncDisposable +{ + private readonly TestContext _context; + private JobEngine? _engine; + + private WorkspaceFixture(TestContext context, ArchiveFixture archive, Sequence sequence, List frames) + { + _context = context; + Archive = archive; + Sequence = sequence; + Frames = frames; + Home = new ArchiveProjectHome(archive.Db, archive.Service.Projects, sequence.Id); + } + + public ArchiveFixture Archive { get; } + public Sequence Sequence { get; set; } + public List Frames { get; } + public ArchiveProjectHome Home { get; } + public ChronosDatabase Db => Archive.Db; + public CancellationToken Ct => Archive.Ct; + + public static async Task CreateAsync(TestContext context, ArchiveFixture archive, int frames = 5) + { + var (sequence, list) = await ProjectServiceTests.UploadedAsync(archive, frames); + return new WorkspaceFixture(context, archive, sequence, list); + } + + /// A workspace of its own (another PC, or another folder of this one) and its sync. + public (WorkspaceStore Store, WorkspaceSync Sync) Workspace(string name = "workspace") + { + var store = new WorkspaceStore(Path.Combine(_context.CreateTempDirectory(name), "workspace")); + return (store, new WorkspaceSync(store, "Chronos on " + name) { LockWait = TimeSpan.FromSeconds(TestTiming.Budget(10)) }); + } + + /// Main of the sequence checked out into 's workspace, its media copied. + public async Task CheckoutAsync(WorkspaceSync sync, Guid? projectId = null) + { + var id = projectId ?? (await Archive.Service.Projects.MainAsync(Sequence.Id, Ct)).Id; + var project = await sync.CheckoutAsync(Home, id, cancellationToken: Ct); + await CopyAsync(sync.Store, id); + return project; + } + + /// Runs a workspace copy job to its end. + public async Task CopyAsync(WorkspaceStore store, Guid projectId, JobState expected = JobState.Done, string? checkpoint = null, string media = "proxies") + { + var engine = await EngineAsync(); + var request = WorkspaceJobs.Copy(Sequence, store.Root, projectId, media); + var job = checkpoint is null ? await engine.EnqueueAsync(request, cancellationToken: Ct) : await Archive.EnqueueWithCheckpointAsync(engine, request, checkpoint); + return await Archive.WaitForAsync(job.Id, expected, (int)TestTiming.Budget(60)); + } + + private async Task EngineAsync() + { + if (_engine is not null) return _engine; + _engine = new JobEngine(Db, [.. Archive.Service.CreateJobHandlers(), .. WorkspaceJobs.CreateHandlers(Db, Archive.Service)], + new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) }); + await _engine.StartAsync(Ct); + return _engine; + } + + public async ValueTask DisposeAsync() + { + if (_engine is not null) await _engine.DisposeAsync(); + } +} + +/// A home that cannot be reached, standing for a server out of reach or an archive offline. +internal sealed class UnreachableHome(IProjectHome home) : IProjectHome +{ + public bool Reachable { get; set; } + + private T Reach(Func call) => Reachable ? call() : throw new HomeUnavailableException("The home cannot be reached."); + + public Task SequenceAsync(CancellationToken cancellationToken) => Reach(() => home.SequenceAsync(cancellationToken)); + public Task> ListAsync(CancellationToken cancellationToken) => Reach(() => home.ListAsync(cancellationToken)); + + public Task CreateAsync(string name, Guid? fromVersionId, string? author, CancellationToken cancellationToken) => + Reach(() => home.CreateAsync(name, fromVersionId, author, cancellationToken)); + + public Task HeadAsync(Guid projectId, CancellationToken cancellationToken) => Reach(() => home.HeadAsync(projectId, cancellationToken)); + + public Task GetVersionAsync(Guid projectId, Guid versionId, CancellationToken cancellationToken) => + Reach(() => home.GetVersionAsync(projectId, versionId, cancellationToken)); + + public Task> VersionsAfterAsync(Guid projectId, int afterNumber, int limit, CancellationToken cancellationToken) => + Reach(() => home.VersionsAfterAsync(projectId, afterNumber, limit, cancellationToken)); + + public Task> HistoryAsync(Guid projectId, int? beforeNumber, int limit, CancellationToken cancellationToken) => + Reach(() => home.HistoryAsync(projectId, beforeNumber, limit, cancellationToken)); + + public Task PushAsync(Guid projectId, ProjectVersion version, CancellationToken cancellationToken) => + Reach(() => home.PushAsync(projectId, version, cancellationToken)); + + public Task FramesAsync(CancellationToken cancellationToken) => Reach(() => home.FramesAsync(cancellationToken)); +} diff --git a/tests/Chronos.Data.Tests/Workspaces/WorkspaceTests.cs b/tests/Chronos.Data.Tests/Workspaces/WorkspaceTests.cs new file mode 100644 index 00000000..e01dce19 --- /dev/null +++ b/tests/Chronos.Data.Tests/Workspaces/WorkspaceTests.cs @@ -0,0 +1,389 @@ +using Chronos.Data.Tests.Projects; +using System.Security.Cryptography; +using Chronos.Core.Archiving; +using Chronos.Core.Localization; +using Chronos.Core.Model; +using Chronos.Core.Project; +using Chronos.Data.Projects; +using Chronos.Data.Tests.Archiving; +using Chronos.Data.Tests.Infrastructure; +using Chronos.Data.Workspaces; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Workspaces; + +/// The workspace (owner review 2, R8): its folder, the copy of a sequence's proxies, and sync with the archive as home. +[Category("workspace")] +public abstract class WorkspaceTests(TestContext context) : DatabaseTest(context) +{ + private async Task FixtureAsync(int frames = 5) + { + var archive = Own(await ArchiveFixture.CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true))); + return Own(await WorkspaceFixture.CreateAsync(Context, archive, frames)); + } + + private static string Sha(string path) => Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(path))); + + /// The working state edited as the editor does: read, changed, stored. + private static async Task EditAsync(WorkspaceSync sync, Guid projectId, Func edit, CancellationToken ct) + { + var session = await sync.OpenAsync(projectId, ct); + var edited = edit(session.Working); + await sync.SaveAsync(projectId, edited, ct); + return edited; + } + + [Test] + public void TheWorkspaceFolderHasItsManifestAndProjectFolders() + { + var root = Path.Combine(Context.CreateTempDirectory("ws"), "workspace"); + var store = new WorkspaceStore(root); + Assert.FileExists(Path.Combine(root, WorkspaceStore.ManifestFile)); + Assert.Equal((WorkspaceStore.FormatName, 1), (store.Manifest.Format, store.Manifest.FormatVersion)); + Assert.Equal(store.Id, new WorkspaceStore(root).Id); + Assert.Empty(store.List()); + + // Only folders named by a project id that hold a working state are projects. + Directory.CreateDirectory(Path.Combine(root, "notes")); + var empty = Guid.CreateVersion7(); + Directory.CreateDirectory(Path.Combine(root, empty.ToString("N"))); + Assert.Empty(store.List()); + Assert.Null(store.Find(empty)); + var project = store.Project(empty); + Assert.Equal(Path.Combine(root, empty.ToString("N"), "media", "proxies"), project.ProxiesDirectory); + Assert.Equal("proxies/" + empty.ToString("N") + ".jpg", WorkspaceProject.ProxyName(empty)); + + File.WriteAllText(Path.Combine(root, WorkspaceStore.ManifestFile), """{ "format": "chronos-workspace", "formatVersion": 7, "id": "0192a4c1-7b3e-7cc0-8a51-2f9d0e6b1a01" }"""); + Assert.Throws(() => new WorkspaceStore(root)); + } + + [Test] + public async Task AProjectIsHeldByOneWriterAtATime() + { + var store = new WorkspaceStore(Path.Combine(Context.CreateTempDirectory("ws"), "workspace")); + var project = store.Project(Guid.CreateVersion7()); + using (await project.LockAsync(TimeSpan.FromSeconds(1), Ct)) + await Assert.ThrowsAsync(() => project.LockAsync(TimeSpan.FromMilliseconds(200), Ct)); + using (await project.LockAsync(TimeSpan.FromSeconds(TestTiming.Budget(5)), Ct)) + { + } + } + + [Test] + public async Task TheCopyBringsVerifiedProxiesAndTheWorkingStateFollowsItsFrames() + { + var fx = await FixtureAsync(); + var (store, sync) = fx.Workspace(); + var main = await fx.Archive.Service.Projects.MainAsync(fx.Sequence.Id, fx.Ct); + var project = await sync.CheckoutAsync(fx.Home, main.Id, cancellationToken: fx.Ct); + Assert.Equal(WorkspaceState.MediaRemoved, (await sync.StatusAsync(main.Id, fx.Home, fx.Ct)).State); + Assert.Null((await sync.OpenAsync(main.Id, fx.Ct)).Sequence); + + var job = await fx.CopyAsync(store, main.Id); + Assert.Contains(Loc.L("Archive.Log.WorkspaceCopied", fx.Sequence.DisplayTitle, 5, 0, 0, 0), job.LogTail!); + var media = Assert.NotNull(project.ReadMedia()); + Assert.Equal((BundleMedia.Proxies, fx.Sequence.Id, fx.Frames.Count), (media.Kind, media.SequenceId, media.Frames.Count)); + foreach (var (frame, entry) in fx.Frames.Zip(media.Frames)) + { + Assert.Equal((frame.Id, frame.Sha256), (entry.Id, entry.Sha256)); + var file = project.MediaFile(entry.File!); + Assert.Equal(entry.FileSha256, Sha(file)); + // The archive's own proxy, copied byte for byte. + Assert.Equal(Sha(fx.Archive.Paths.Sequence(fx.Sequence.Folder).Proxy(frame.Index)), entry.FileSha256); + } + var session = await sync.OpenAsync(main.Id, fx.Ct); + var opened = Assert.NotNull(session.Sequence); + Assert.Equal(5, opened.Files.Count(f => f is not null)); + Assert.Equal((WorkspaceState.InSync, 0), ((await sync.StatusAsync(main.Id, fx.Home, fx.Ct)).State, (await sync.StatusAsync(main.Id, fx.Home, fx.Ct)).Ahead)); + + // Positions on the copied frames follow their photos when the home's frames change and the copy runs again. + await EditAsync(sync, main.Id, s => s with { ExcludedFrames = [2] }, fx.Ct); + var (path, size, sha) = fx.Archive.Stage(ArchiveFixture.Start.AddSeconds(-30), 999); + await fx.Archive.Service.Ingest.CommitAsync(new Chronos.Data.Archiving.IngestRequest(fx.Archive.Archive.Id, fx.Sequence.Id, null, null, path, "IMG_0000.JPG", + size, sha, null, null), fx.Ct); + fx.Sequence = (await fx.Db.Sequences.GetAsync(fx.Sequence.Id))!; + await fx.CopyAsync(store, main.Id); + var moved = await sync.OpenAsync(main.Id, fx.Ct); + Assert.Equal(6, moved.Media!.Frames.Count); + Assert.SequenceEqual([3], moved.Working.ExcludedFrames); + Assert.Equal(fx.Frames[2].Id, moved.Media.Frames[3].Id); + } + + [Test] + public async Task ACopyResumesFromItsCheckpointAndRewritesWhatDoesNotMatch() + { + var fx = await FixtureAsync(); + var (store, sync) = fx.Workspace(); + var main = await fx.Archive.Service.Projects.MainAsync(fx.Sequence.Id, fx.Ct); + var project = await sync.CheckoutAsync(fx.Home, main.Id, cancellationToken: fx.Ct); + await fx.CopyAsync(store, main.Id); + var copied = project.ReadMedia()!; + + // An earlier run copied the first two before it stopped; a later file was left damaged. + File.WriteAllText(project.MediaFile(copied.Frames[3].File!), "not a photo"); + File.Delete(project.MediaIndexPath); + var job = await fx.CopyAsync(store, main.Id, checkpoint: "copy:2"); + Assert.Contains(Loc.L("Archive.Log.WorkspaceCopied", fx.Sequence.DisplayTitle, 1, 0, 4, 0), job.LogTail!); + var media = Assert.NotNull(project.ReadMedia()); + Assert.SequenceEqual(copied.Frames.Select(f => f.FileSha256), media.Frames.Select(f => f.FileSha256)); + Assert.Equal(copied.Frames[3].FileSha256, Sha(project.MediaFile(media.Frames[3].File!))); + } + + [Test] + public async Task CopyingTheOriginalsIsRefusedForNowWithItsReason() + { + var fx = await FixtureAsync(2); + var (store, sync) = fx.Workspace(); + var main = await fx.Archive.Service.Projects.MainAsync(fx.Sequence.Id, fx.Ct); + await sync.CheckoutAsync(fx.Home, main.Id, BundleMedia.Originals, fx.Ct); + var job = await fx.CopyAsync(store, main.Id, JobState.Failed, media: BundleMedia.Originals); + Assert.Equal(Loc.L("Errors.Workspace.OriginalsLater"), job.Error); + } + + [Test] + public async Task AVersionSavedHereIsPushedAndBecomesTheSequenceSettings() + { + var fx = await FixtureAsync(); + var (_, sync) = fx.Workspace(); + var project = await fx.CheckoutAsync(sync); + var id = project.ProjectId; + + Assert.Null(await sync.CommitAsync(id, cancellationToken: fx.Ct)); + await EditAsync(sync, id, s => s with { ExcludedFrames = [1, 4], Exposure = s.Exposure with { ExposureOffsetEv = 0.4 } }, fx.Ct); + Assert.Equal(WorkspaceState.Modified, (await sync.StatusAsync(id, fx.Home, fx.Ct)).State); + var version = Assert.NotNull(await sync.CommitAsync(id, "warmer", "alberto", fx.Ct)); + Assert.Equal((ProjectVersionKind.Commit, 2, "warmer", (DateTimeOffset?)null), (version.Kind, version.Number, version.Note, version.StoredAt)); + var status = await sync.StatusAsync(id, fx.Home, fx.Ct); + Assert.Equal((WorkspaceState.Ahead, 1, false), (status.State, status.Ahead, status.Modified)); + + var pushed = await sync.PushAsync(id, fx.Home, fx.Ct); + Assert.Equal(version.Id, pushed.Pushed.Single().Id); + Assert.Equal(WorkspaceState.InSync, pushed.Status.State); + Assert.NotNull(project.Versions().Single(v => v.Id == version.Id).StoredAt); + var sequence = (await fx.Db.Sequences.GetAsync(fx.Sequence.Id))!; + var settings = ProjectSettings.FromJson(sequence.SettingsJson!); + Assert.SequenceEqual([1, 4], settings.ExcludedFrames); + Assert.Equal(0.4, settings.Exposure.ExposureOffsetEv); + + // Nothing else to push; the home's head is ours. + Assert.Empty((await sync.PushAsync(id, fx.Home, fx.Ct)).Pushed); + Assert.Equal(version.Id, (await fx.Home.HeadAsync(id, fx.Ct)).Id); + } + + [Test] + public async Task AnotherWorkspacesPushIsPulledIntoTheWorkingState() + { + var fx = await FixtureAsync(); + var (_, one) = fx.Workspace("one"); + var (_, two) = fx.Workspace("two"); + var id = (await fx.CheckoutAsync(one)).ProjectId; + await fx.CheckoutAsync(two, id); + + await EditAsync(one, id, s => s with { Stack = new StackSettings { Mode = StackSetting.Median, Length = 5 } }, fx.Ct); + await one.CommitAsync(id, cancellationToken: fx.Ct); + await one.PushAsync(id, fx.Home, fx.Ct); + + // Two has a change of its own not saved as a version yet: it stays, on top of what one pushed. + await EditAsync(two, id, s => s with { ExcludedFrames = [0] }, fx.Ct); + Assert.Equal((WorkspaceState.Behind, 1), ((await two.StatusAsync(id, fx.Home, fx.Ct)).State, (await two.StatusAsync(id, fx.Home, fx.Ct)).Behind)); + var pulled = await two.PullAsync(id, fx.Home, fx.Ct); + Assert.Equal(1, pulled.Pulled.Count); + Assert.Equal(WorkspaceState.Modified, pulled.Status.State); + var working = (await two.OpenAsync(id, fx.Ct)).Working; + Assert.Equal((StackSetting.Median, 5), (working.Stack.Mode, working.Stack.Length)); + Assert.SequenceEqual([0], working.ExcludedFrames); + } + + [Test] + public async Task SeveralLocalVersionsAreReplayedOnTopOfTheHomesWhenItMovedMeanwhile() + { + var fx = await FixtureAsync(); + var (_, one) = fx.Workspace("one"); + var (_, two) = fx.Workspace("two"); + var project = await fx.CheckoutAsync(one); + var id = project.ProjectId; + await fx.CheckoutAsync(two, id); + + await EditAsync(two, id, s => s with { Exposure = s.Exposure with { DeflickerStrength = new Auto().WithManual(0.6) } }, fx.Ct); + await two.CommitAsync(id, cancellationToken: fx.Ct); + await two.PushAsync(id, fx.Home, fx.Ct); + + // One saved two versions meanwhile, excluding another photo than two would have and changing other settings. + await EditAsync(one, id, s => s with { Exposure = s.Exposure with { ExposureOffsetEv = 0.3 } }, fx.Ct); + var first = (await one.CommitAsync(id, "offset", cancellationToken: fx.Ct))!; + await EditAsync(one, id, s => s with { ExcludedFrames = [3], Stack = new StackSettings { Mode = StackSetting.Mean } }, fx.Ct); + var second = (await one.CommitAsync(id, "stack", cancellationToken: fx.Ct))!; + Assert.Equal(2, (await one.StatusAsync(id, cancellationToken: fx.Ct)).Ahead); + + var pushed = await one.PushAsync(id, fx.Home, fx.Ct); + Assert.Equal(1, pushed.Pulled.Count); + Assert.SequenceEqual([3, 4], pushed.Pushed.Select(v => v.Number)); + Assert.SequenceEqual(["offset", "stack"], pushed.Pushed.Select(v => v.Note)); + Assert.False(pushed.Pushed.Any(v => v.Id == first.Id || v.Id == second.Id), "replayed versions are new versions"); + Assert.Equal(WorkspaceState.InSync, pushed.Status.State); + var head = await fx.Home.HeadAsync(id, fx.Ct); + var settings = ProjectSettings.FromJson(ProjectContent.Place(head.SettingsJson, ProjectServiceTests.Photos(fx.Frames))); + Assert.Equal((0.6, 0.3, StackSetting.Mean), (settings.Exposure.DeflickerStrength.Resolve(1), settings.Exposure.ExposureOffsetEv, settings.Stack.Mode)); + Assert.SequenceEqual([3], settings.ExcludedFrames); + // The first replayed version holds one's first change on top of two's, without the later ones. + var replayed = ProjectSettings.FromJson(pushed.Pushed[0].SettingsJson); + Assert.Equal((0.6, 0.3, StackSetting.None), (replayed.Exposure.DeflickerStrength.Resolve(1), replayed.Exposure.ExposureOffsetEv, replayed.Stack.Mode)); + // The versions as they were saved are kept. + Assert.Equal(2, Directory.GetFiles(Path.Combine(project.VersionsDirectory, WorkspaceProject.ReplacedFolder)).Length); + } + + [Test] + public async Task ExcludingDifferentPhotosOnEachSideMergesWithoutAConflict() + { + var fx = await FixtureAsync(); + var (_, one) = fx.Workspace("one"); + var (_, two) = fx.Workspace("two"); + var id = (await fx.CheckoutAsync(one)).ProjectId; + await fx.CheckoutAsync(two, id); + + await EditAsync(one, id, s => s with { ExcludedFrames = [1] }, fx.Ct); + await one.CommitAsync(id, cancellationToken: fx.Ct); + await one.PushAsync(id, fx.Home, fx.Ct); + await EditAsync(two, id, s => s with { ExcludedFrames = [3] }, fx.Ct); + await two.CommitAsync(id, cancellationToken: fx.Ct); + + var pushed = await two.PushAsync(id, fx.Home, fx.Ct); + Assert.Equal(WorkspaceState.InSync, pushed.Status.State); + Assert.SequenceEqual([1, 3], ProjectSettings.FromJson((await fx.Db.Sequences.GetAsync(fx.Sequence.Id))!.SettingsJson!).ExcludedFrames); + } + + [Test] + public async Task TheSameSettingChangedOnBothSidesIsAConflictToSettle() + { + var fx = await FixtureAsync(); + var (_, one) = fx.Workspace("one"); + var (_, two) = fx.Workspace("two"); + var id = (await fx.CheckoutAsync(one)).ProjectId; + await fx.CheckoutAsync(two, id); + await EditAsync(one, id, s => s with { Exposure = s.Exposure with { ExposureOffsetEv = 0.5 } }, fx.Ct); + await one.CommitAsync(id, cancellationToken: fx.Ct); + await one.PushAsync(id, fx.Home, fx.Ct); + + await EditAsync(two, id, s => s with { Exposure = s.Exposure with { ExposureOffsetEv = 1.0 }, Stack = new StackSettings { Mode = StackSetting.Median } }, fx.Ct); + await two.CommitAsync(id, cancellationToken: fx.Ct); + var stopped = await two.PushAsync(id, fx.Home, fx.Ct); + Assert.Empty(stopped.Pushed); + Assert.Equal(WorkspaceState.Conflict, stopped.Status.State); + var difference = stopped.Status.Conflict!.Differences.Single(); + Assert.Equal(("exposure.exposureOffsetEv", "1", "0.5"), (difference.Path, difference.Mine, difference.Theirs)); + await Assert.ThrowsAsync(() => two.CommitAsync(id, cancellationToken: fx.Ct)); + + // Take the server's value there: the other change of this workspace stays, and is pushed. + var settled = await two.ResolveAsync(id, fx.Home, ConflictChoice.TakeServer, cancellationToken: fx.Ct); + Assert.Equal(WorkspaceState.Ahead, settled.Status.State); + Assert.Equal(ProjectVersionKind.Merge, two.Unpushed(id).Single().Kind); + await two.PushAsync(id, fx.Home, fx.Ct); + var head = ProjectSettings.FromJson((await fx.Home.HeadAsync(id, fx.Ct)).SettingsJson); + Assert.Equal((0.5, StackSetting.Median), (head.Exposure.ExposureOffsetEv, head.Stack.Mode)); + } + + [Test] + public async Task KeepMineWinsWhereBothChangedAndSaveBothKeepsEachSide() + { + var fx = await FixtureAsync(); + var (_, one) = fx.Workspace("one"); + var (_, two) = fx.Workspace("two"); + var (_, three) = fx.Workspace("three"); + var id = (await fx.CheckoutAsync(one)).ProjectId; + await fx.CheckoutAsync(two, id); + await fx.CheckoutAsync(three, id); + await EditAsync(one, id, s => s with { Exposure = s.Exposure with { ExposureOffsetEv = 0.5 } }, fx.Ct); + await one.CommitAsync(id, cancellationToken: fx.Ct); + await one.PushAsync(id, fx.Home, fx.Ct); + + await EditAsync(two, id, s => s with { Exposure = s.Exposure with { ExposureOffsetEv = -1 } }, fx.Ct); + await two.CommitAsync(id, cancellationToken: fx.Ct); + Assert.Equal(WorkspaceState.Conflict, (await two.PullAsync(id, fx.Home, fx.Ct)).Status.State); + await two.ResolveAsync(id, fx.Home, ConflictChoice.KeepMine, cancellationToken: fx.Ct); + await two.PushAsync(id, fx.Home, fx.Ct); + Assert.Equal(-1, ProjectSettings.FromJson((await fx.Home.HeadAsync(id, fx.Ct)).SettingsJson).Exposure.ExposureOffsetEv); + + await EditAsync(three, id, s => s with { Exposure = s.Exposure with { ExposureOffsetEv = 2 } }, fx.Ct); + await three.CommitAsync(id, "bright", cancellationToken: fx.Ct); + await EditAsync(three, id, s => s with { Stack = new StackSettings { Mode = StackSetting.StarTrails } }, fx.Ct); + Assert.Equal(WorkspaceState.Conflict, (await three.PushAsync(id, fx.Home, fx.Ct)).Status.State); + var both = await three.ResolveAsync(id, fx.Home, ConflictChoice.SaveBoth, cancellationToken: fx.Ct); + + // Three's versions (its unsaved change saved first) live on in a new project; this project is the home's again. + var copy = Assert.NotNull(both.SavedAs); + Assert.Equal("Main (2)", copy.Name); + Assert.Equal(2, both.Pushed.Count); + var kept = ProjectSettings.FromJson((await fx.Home.HeadAsync(copy.Id, fx.Ct)).SettingsJson); + Assert.Equal((2, StackSetting.StarTrails), (kept.Exposure.ExposureOffsetEv, kept.Stack.Mode)); + Assert.Equal(WorkspaceState.InSync, both.Status.State); + Assert.Equal(-1, (await three.OpenAsync(id, fx.Ct)).Working.Exposure.ExposureOffsetEv); + Assert.Equal(2, (await fx.Home.ListAsync(fx.Ct)).Count); + } + + [Test] + public async Task ARestoredVersionIsTheWorkingStateAndTheNextVersionSaysSo() + { + var fx = await FixtureAsync(); + var (_, sync) = fx.Workspace(); + var id = (await fx.CheckoutAsync(sync)).ProjectId; + var original = await fx.Home.HeadAsync(id, fx.Ct); + await EditAsync(sync, id, s => s with { ExcludedFrames = [2] }, fx.Ct); + await sync.CommitAsync(id, cancellationToken: fx.Ct); + // An unsaved change before the restore is saved as a version first. + await EditAsync(sync, id, s => s with { Stack = new StackSettings { Mode = StackSetting.Mean } }, fx.Ct); + + var restored = await sync.RestoreAsync(id, original.Id, fx.Home, fx.Ct); + Assert.Empty(restored.ExcludedFrames); + Assert.Equal(2, sync.Unpushed(id).Count); + await EditAsync(sync, id, s => s with { Exposure = s.Exposure with { ExposureOffsetEv = 0.1 } }, fx.Ct); + var version = Assert.NotNull(await sync.CommitAsync(id, cancellationToken: fx.Ct)); + Assert.Equal((ProjectVersionKind.Restore, original.Id), (version.Kind, version.RestoredFrom)); + await EditAsync(sync, id, s => s with { Exposure = s.Exposure with { ExposureOffsetEv = 0.2 } }, fx.Ct); + var next = Assert.NotNull(await sync.CommitAsync(id, cancellationToken: fx.Ct)); + Assert.Equal((ProjectVersionKind.Commit, (Guid?)null), (next.Kind, next.RestoredFrom)); + } + + [Test] + public async Task AHomeOutOfReachIsOfflineAndTheWorkWaits() + { + var fx = await FixtureAsync(); + var (_, sync) = fx.Workspace(); + var id = (await fx.CheckoutAsync(sync)).ProjectId; + var away = new UnreachableHome(fx.Home); + await EditAsync(sync, id, s => s with { ExcludedFrames = [0] }, fx.Ct); + await sync.CommitAsync(id, cancellationToken: fx.Ct); + + var status = await sync.StatusAsync(id, away, fx.Ct); + Assert.Equal((WorkspaceState.Offline, 1, true), (status.State, status.Ahead, status.Offline)); + await Assert.ThrowsAsync(() => sync.PushAsync(id, away, fx.Ct)); + away.Reachable = true; + Assert.Equal(WorkspaceState.InSync, (await sync.PushAsync(id, away, fx.Ct)).Status.State); + } + + [Test] + public async Task SettingsOfANewerChronosOpenReadOnly() + { + var fx = await FixtureAsync(2); + var (_, sync) = fx.Workspace(); + var project = await fx.CheckoutAsync(sync); + var bundle = project.ReadBundle(); + File.WriteAllText(project.BundlePath, bundle.ToJson().Replace("\"version\": 1", "\"version\": 2", StringComparison.Ordinal)); + + var session = await sync.OpenAsync(project.ProjectId, fx.Ct); + Assert.True(session.IsReadOnly); + await Assert.ThrowsAsync(() => sync.SaveAsync(project.ProjectId, new ProjectSettings(), fx.Ct)); + await Assert.ThrowsAsync(() => sync.CommitAsync(project.ProjectId, cancellationToken: fx.Ct)); + Assert.True((await sync.StatusAsync(project.ProjectId, cancellationToken: fx.Ct)).ReadOnly); + } +} + +public sealed class SqliteWorkspaceTests(TestContext context) : WorkspaceTests(context) +{ + protected override Backend Backend => Backend.SqliteFile; +} + +[RequiresEnvironment(PgTestServer.Variable)] +public sealed class PostgresWorkspaceTests(TestContext context) : WorkspaceTests(context) +{ + protected override Backend Backend => Backend.Postgres; +}