feat(projects): workspace projects with linear versions, the local archive as their home

Owner review 2, phase 5a (R8): editing moves out of the library into a workspace on a fast disk, and a
sequence gets several projects, each with a linear version history pushed and pulled like git.

- Core: ProjectSettings keeps members a newer Chronos wrote (Extra) and reads a newer settings format
  read-only; ProjectContent is the portable form of a version (every frame position parked by photo,
  canonical text, SHA-256 content hash); ProjectBundle format 2 adds the workspace section, and a file
  without it is still written as format 1. JobKind.WorkspaceCopy = 20 (19 kept for Convert).
- Data: migration 0009 (projects, project_versions, the render's project columns); ProjectService on
  ArchiveService.Projects keeps projects as sidecars first (.chronos/projects/<id>/), then the index,
  under the sequence lease: create, rename, delete into the projects' trash, history, push with
  fast-forward, idempotent repeat, divergence and hash checks. Main is made from the sequence's settings
  and mirrored both ways, so every older code path keeps working; outside writes become "external"
  versions, a mirror a crash cut short is completed or merged. Full scans and rebuilds take the projects
  back from their sidecars.
- ProjectMerge moves from the App to Data/Projects, merges pending frames photo by photo and replays
  unpushed commits on a moved head.
- Workspaces: WorkspaceStore, WorkspaceSync (checkout, save, commit, push, pull with replay and merge,
  conflicts settled by keep mine / take server / save both, restore, status) and the WorkspaceCopy job
  (proxies copied hash-verified and resumable; originals refused until phase 5c).
- Renders of a project version place its positions on the frames and record the project and version.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit c4c1802884e403db8759ed0501f770a67a047f20)
This commit is contained in:
Alby96 committed 2026-10-27 22:51:32 +01:00
1 parent e3cbe30b29
commit 03d32f18d3
51 files changed
+4917 -108

No files matched your search

@@ -0,0 +1,340 @@
using System.Globalization;
using System.Security.Cryptography;
using System.Text.Json;
using Chronos.Core.Archiving;
using Chronos.Core.Diagnostics;
using Chronos.Core.IO;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Data.Archiving;
using Chronos.Data.Archiving.Jobs;
using Chronos.Data.Jobs;
namespace Chronos.Data.Workspaces;
/// <summary>Builds the jobs of the workspace (owner review 2, R8), and their handlers for the desktop's engine.</summary>
public static class WorkspaceJobs
{
/// <summary>
/// <see cref="JobKind.WorkspaceCopy"/>: the media of workspace project <paramref name="projectId"/> (checked out first,
/// <see cref="WorkspaceSync.CheckoutAsync"/>) copied from its home, the sequence of this PC's archive.
/// </summary>
public static ArchiveJobRequest Copy(Sequence sequence, string workspaceRoot, Guid projectId, string media = BundleMedia.Proxies)
{
ArgumentNullException.ThrowIfNull(sequence);
ArgumentException.ThrowIfNullOrWhiteSpace(workspaceRoot);
return new ArchiveJobRequest(JobKind.WorkspaceCopy, [sequence.Id], sequence.DisplayTitle,
JsonSerializer.Serialize(new WorkspaceCopyParameters(workspaceRoot, projectId, media), WorkspaceJson.Default.WorkspaceCopyParameters), JobPriority.User);
}
/// <summary>
/// The handlers of the workspace's jobs for a desktop's engine (never a server's: the job writes into a folder its
/// parameters name).
/// </summary>
public static IReadOnlyList<IJobHandler> CreateHandlers(ChronosDatabase db, ArchiveService archives) => [new WorkspaceCopyJobHandler(db, archives)];
}
/// <summary>
/// <see cref="JobKind.WorkspaceCopy"/> for a sequence of this PC's archive (owner review 2, R8): its proxies into the
/// workspace project's <c>media/proxies/&lt;frame id&gt;.jpg</c>, the archive's own when current, the missing ones made
/// first by the archive's proxy pass (made here from the original when the archive makes none), each written atomically
/// and read back to check its SHA-256; then the deep report (<c>media/analysis.json</c>) and, last, <c>media/media.json</c>.
/// Frames changed at the home during the copy make it start again (three times at most). The working state's positions
/// follow the new frames by photo. Checkpoint: <c>proxies:&lt;frame&gt;</c> during the archive's pass, then
/// <c>copy:&lt;frame&gt;</c>. Originals come in phase 5c: asked for, the job fails saying so.
/// </summary>
internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService archives) : IJobHandler
{
private const int PageSize = 1000;
private const int Passes = 3;
private static readonly TimeSpan LockWait = TimeSpan.FromMinutes(2);
private static readonly TimeSpan CheckpointEvery = TimeSpan.FromSeconds(5);
public JobKind Kind => JobKind.WorkspaceCopy;
public async Task RunAsync(JobContext context)
{
var ct = context.Cancellation;
var parameters = Parameters(context.Job);
if (parameters.Media == BundleMedia.Originals) throw new NotSupportedException(Loc.L("Errors.Workspace.OriginalsLater"));
if (parameters.Media != BundleMedia.Proxies) throw new InvalidDataException(Loc.L("Errors.Workspace.Parameters"));
if (context.Job.TargetIds is not [var sequenceId] || await db.Sequences.GetAsync(sequenceId, ct).ConfigureAwait(false) is not { } sequence
|| await db.Archives.GetAsync(sequence.ArchiveId, ct).ConfigureAwait(false) is not { } archive)
throw new InvalidOperationException(Loc.L("Archive.Log.SequenceMissing", context.Job.TargetIds.FirstOrDefault()));
var store = new WorkspaceStore(parameters.WorkspaceRoot);
var project = store.Find(parameters.ProjectId) ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.NotCheckedOut"));
if (project.ReadBundle().Workspace is not { Home: { IsArchive: true } home } || home.SequenceId != sequenceId)
throw new InvalidOperationException(Loc.L("Errors.Workspace.OtherHome"));
var paths = new ArchivePaths(archive.RootPath);
var (stage, next) = Parse(context.Checkpoint);
for (var pass = 1; ; pass++)
{
var (row, frames) = await FramesAsync(sequenceId, ct).ConfigureAwait(false);
var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy;
var makesProxies = policy.LongEdge != ProxyPolicy.Disabled && ArchiveVolume.Probe(paths) == ArchiveAvailability.Online;
var total = (long)frames.Count * (makesProxies ? 2 : 1);
long offset = 0;
if (makesProxies)
{
if (stage == Stage.Proxies && (row.Status & SequenceStatus.ProxiesBuilt) == 0)
{
var result = await archives.Images.BuildProxiesAsync(archive, row, force: false, next,
(frame, _) => context.CheckpointAsync(Checkpoint(Stage.Proxies, frame), frame, total), ct).ConfigureAwait(false);
context.Log(Loc.L("Archive.Log.ProxiesDone", row.DisplayTitle, result.Built, result.Reused, result.Undecodable, result.Waiting));
next = 0;
}
offset = frames.Count;
}
// Frames before a copy checkpoint were copied by an earlier run.
var copied = stage == Stage.Copy ? next : 0;
(stage, next) = (Stage.Copy, 0);
var folder = paths.Sequence(row.Folder);
var tally = new Tally();
var entries = new WorkspaceFrame[frames.Count];
Directory.CreateDirectory(project.ProxiesDirectory);
var clock = System.Diagnostics.Stopwatch.StartNew();
for (var i = 0; i < frames.Count; i++)
{
ct.ThrowIfCancellationRequested();
entries[i] = Copy(project, folder, paths, frames[i], policy, done: i < copied, tally, ct);
if (clock.Elapsed >= CheckpointEvery)
{
await context.CheckpointAsync(Checkpoint(Stage.Copy, i + 1), offset + i + 1, total).ConfigureAwait(false);
clock.Restart();
}
else
{
await context.ReportAsync(offset + i + 1, total).ConfigureAwait(false);
}
}
var analysis = CopyAnalysis(folder, project, frames.Count);
var after = await db.Sequences.GetAsync(sequenceId, ct).ConfigureAwait(false)
?? throw new InvalidOperationException(Loc.L("Archive.Log.SequenceMissing", sequenceId));
if (after.FramesVersion != row.FramesVersion && pass < Passes)
{
context.Log(Loc.L("Archive.Log.WorkspaceFramesChanged", row.DisplayTitle));
(stage, next) = (Stage.Proxies, 0);
continue;
}
var media = new WorkspaceMedia(WorkspaceMedia.FormatName, WorkspaceMedia.CurrentFormatVersion, BundleMedia.Proxies, sequenceId, row.FramesVersion, entries,
DateTimeOffset.UtcNow, analysis);
// The working state is the editor's meanwhile: it is held only while the media index and the positions move.
using (await project.LockAsync(LockWait, ct).ConfigureAwait(false))
{
Commit(project, project.ReadBundle(), media, row);
RemoveStale(project, media);
}
context.Log(Loc.L("Archive.Log.WorkspaceCopied", row.DisplayTitle, tally.Copied, tally.Made, tally.Kept, tally.Missing));
await context.CheckpointAsync(Checkpoint(Stage.Done, frames.Count), total, total).ConfigureAwait(false);
return;
}
}
/// <summary>
/// The frame's file in the workspace: kept when an earlier run copied it (before the checkpoint, or the same bytes as the
/// archive's proxy), else the archive's current proxy, else one made from the original; none for a frame without either.
/// </summary>
private WorkspaceFrame Copy(WorkspaceProject project, SequencePaths folder, ArchivePaths paths, Frame frame, ProxyPolicy policy, bool done, Tally tally,
CancellationToken cancellationToken)
{
var name = WorkspaceProject.ProxyName(frame.Id);
var target = project.MediaFile(name);
if (done && File.Exists(target))
{
tally.Kept++;
var kept = AtomicFile.ReadAllBytes(target);
return WorkspaceFrame.Of(frame, name, Sha(kept), kept.Length);
}
var source = folder.Proxy(frame.Index);
byte[]? bytes = null;
if (File.Exists(source) && ArchiveImages.IsProxyOf(source, frame.CapturedAt, frame.Width, frame.Height, null)) bytes = AtomicFile.ReadAllBytes(source);
var made = false;
if (bytes is null && (frame.Flags & FrameFlags.Missing) == 0)
{
var original = paths.FromRelative(frame.RelativePath);
try
{
var edge = policy.LongEdge == ProxyPolicy.Disabled ? new ProxyPolicy().LongEdge : policy.LongEdge;
bytes = File.Exists(original) ? ArchiveImages.MakeProxy(archives.Decoder, original, frame, null, edge, policy.Quality, cancellationToken) : null;
made = bytes is not null;
}
catch (IOException ex)
{
Log.Warning("workspace", $"No proxy of {original} could be made", ex);
}
}
if (bytes is null)
{
tally.Missing++;
return WorkspaceFrame.Of(frame, null, null, 0);
}
var sha = Sha(bytes);
if (File.Exists(target) && Sha(AtomicFile.ReadAllBytes(target)) == sha)
{
tally.Kept++;
return WorkspaceFrame.Of(frame, name, sha, bytes.Length);
}
WriteVerified(target, bytes, sha);
if (made) tally.Made++;
else tally.Copied++;
return WorkspaceFrame.Of(frame, name, sha, bytes.Length);
}
/// <summary>Writes atomically and reads back: a file that does not hash as its source is written again once, then refused.</summary>
private static void WriteVerified(string path, byte[] bytes, string sha)
{
for (var attempt = 1; ; attempt++)
{
AtomicFile.WriteAllBytes(path, bytes);
if (Sha(AtomicFile.ReadAllBytes(path)) == sha) return;
if (attempt == 2) throw new IOException(Loc.L("Errors.Workspace.CopyMismatch", path));
}
}
/// <summary>The home's deep report into the media folder when it describes these frames; its SHA-256, or null without one.</summary>
private static string? CopyAnalysis(SequencePaths folder, WorkspaceProject project, int frameCount)
{
if (!File.Exists(folder.AnalysisPath))
{
if (File.Exists(project.AnalysisPath)) File.Delete(project.AnalysisPath);
return null;
}
var bytes = AtomicFile.ReadAllBytes(folder.AnalysisPath);
try
{
if (Core.Analysis.Deep.DeepAnalysisReport.FromJson(System.Text.Encoding.UTF8.GetString(bytes)).FrameCount != frameCount) return null;
}
catch (Exception ex) when (ex is InvalidDataException or JsonException or IOException)
{
Log.Warning("workspace", $"{folder.AnalysisPath} cannot be read; the workspace takes no deep report", ex);
return null;
}
var sha = Sha(bytes);
WriteVerified(project.AnalysisPath, bytes, sha);
return sha;
}
/// <summary>
/// The media index and the working state moved onto it: first the working state with every position parked by photo
/// (valid whatever the frames), then the index, then the working state placed on the new frames. A crash in between
/// leaves positions that wait for their photos, which the next open places.
/// </summary>
private static void Commit(WorkspaceProject project, ProjectBundle bundle, WorkspaceMedia media, Sequence row)
{
var old = TryReadMedia(project);
var workspace = bundle.Workspace! with { Media = new BundleMedia(BundleMedia.Proxies, media.FramesVersion) };
var sequence = bundle.Sequence with { Folder = row.Folder, Title = row.DisplayTitle, Frames = media.Frames.Count };
var working = bundle.Project;
// Without the media index the positions were made against, they are taken as naming these frames.
if (!working.IsNewerFormat && old is not null)
{
var portable = ProjectSettings.FromJson(ProjectContent.ToPortable(working, old.Photos), allowNewer: true);
project.WriteBundle(bundle with { Project = portable, Workspace = workspace, Sequence = sequence });
project.WriteMedia(media);
working = ProjectSettings.FromJson(ProjectContent.Place(portable.ToJson(), media.Photos), allowNewer: true);
}
else
{
project.WriteMedia(media);
}
project.WriteBundle(bundle with { Project = working, Workspace = workspace, Sequence = sequence, Saved = DateTimeOffset.UtcNow });
}
/// <summary>Proxies of frames the sequence no longer has: regenerable copies, removed so the media folder holds the frames listed.</summary>
private static void RemoveStale(WorkspaceProject project, WorkspaceMedia media)
{
var listed = media.Frames.Where(f => f.File is not null).Select(f => Path.GetFileName(f.File!)).ToHashSet(StringComparer.OrdinalIgnoreCase);
foreach (var file in Directory.EnumerateFiles(project.ProxiesDirectory, "*.jpg"))
{
if (listed.Contains(Path.GetFileName(file))) continue;
try
{
File.Delete(file);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("workspace", $"{file} could not be removed", ex);
}
}
}
private static WorkspaceMedia? TryReadMedia(WorkspaceProject project)
{
try
{
return project.ReadMedia();
}
catch (InvalidDataException ex)
{
Log.Warning("workspace", $"The media index of {project.Folder} cannot be read; positions are taken as they are", ex);
return null;
}
}
/// <summary>The sequence row with its frames as one state (listed between two reads of the row at the same frames version).</summary>
private async Task<(Sequence Row, List<Frame> Frames)> FramesAsync(Guid sequenceId, CancellationToken cancellationToken)
{
for (var attempt = 0; ; attempt++)
{
var before = await db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
?? throw new InvalidOperationException(Loc.L("Archive.Log.SequenceMissing", sequenceId));
var frames = new List<Frame>();
for (var after = -1; ;)
{
var page = await db.Frames.ListBySequenceAsync(sequenceId, after, PageSize, cancellationToken).ConfigureAwait(false);
frames.AddRange(page);
if (page.Count < PageSize) break;
after = page[^1].Index;
}
var now = await db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? before;
if (now.FramesVersion == before.FramesVersion || attempt == 2) return (now, frames);
}
}
private static WorkspaceCopyParameters Parameters(Job job)
{
try
{
if (job.ParametersJson is { Length: > 0 } json
&& JsonSerializer.Deserialize(json, WorkspaceJson.Default.WorkspaceCopyParameters) is { WorkspaceRoot.Length: > 0 } parameters
&& parameters.ProjectId != Guid.Empty)
return parameters with { Media = parameters.Media ?? BundleMedia.Proxies };
}
catch (JsonException ex)
{
throw new InvalidDataException(Loc.L("Errors.Workspace.Parameters"), ex);
}
throw new InvalidDataException(Loc.L("Errors.Workspace.Parameters"));
}
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes));
private enum Stage
{
Proxies,
Copy,
Done,
}
private static string Checkpoint(Stage stage, int frame) =>
string.Create(CultureInfo.InvariantCulture, $"{stage.ToString().ToLowerInvariant()}:{frame}");
private static (Stage Stage, int Next) Parse(string? checkpoint)
{
var parts = checkpoint?.Split(':');
return parts is [var name, var number] && Enum.TryParse<Stage>(name, ignoreCase: true, out var stage)
&& int.TryParse(number, NumberStyles.None, CultureInfo.InvariantCulture, out var next)
? (stage == Stage.Done ? Stage.Copy : stage, stage == Stage.Done ? 0 : next)
: (Stage.Proxies, 0);
}
private sealed class Tally
{
public int Copied { get; set; }
public int Made { get; set; }
public int Kept { get; set; }
public int Missing { get; set; }
}
}