feat(projects): workspace projects with linear versions, the local archive as their home
Owner review 2, phase 5a (R8): editing moves out of the library into a workspace on a fast disk, and a sequence gets several projects, each with a linear version history pushed and pulled like git. - Core: ProjectSettings keeps members a newer Chronos wrote (Extra) and reads a newer settings format read-only; ProjectContent is the portable form of a version (every frame position parked by photo, canonical text, SHA-256 content hash); ProjectBundle format 2 adds the workspace section, and a file without it is still written as format 1. JobKind.WorkspaceCopy = 20 (19 kept for Convert). - Data: migration 0009 (projects, project_versions, the render's project columns); ProjectService on ArchiveService.Projects keeps projects as sidecars first (.chronos/projects/<id>/), then the index, under the sequence lease: create, rename, delete into the projects' trash, history, push with fast-forward, idempotent repeat, divergence and hash checks. Main is made from the sequence's settings and mirrored both ways, so every older code path keeps working; outside writes become "external" versions, a mirror a crash cut short is completed or merged. Full scans and rebuilds take the projects back from their sidecars. - ProjectMerge moves from the App to Data/Projects, merges pending frames photo by photo and replays unpushed commits on a moved head. - Workspaces: WorkspaceStore, WorkspaceSync (checkout, save, commit, push, pull with replay and merge, conflicts settled by keep mine / take server / save both, restore, status) and the WorkspaceCopy job (proxies copied hash-verified and resumable; originals refused until phase 5c). - Renders of a project version place its positions on the frames and record the project and version. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> (cherry picked from commit c4c1802884e403db8759ed0501f770a67a047f20)
This commit is contained in:
1 parent
e3cbe30b29
commit
03d32f18d3
51 files changed
+4917
-108
No files matched your search
@@ -0,0 +1,340 @@
|
||||
using System.Globalization;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text.Json;
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.Diagnostics;
|
||||
using Chronos.Core.IO;
|
||||
using Chronos.Core.Localization;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Core.Project;
|
||||
using Chronos.Data.Archiving;
|
||||
using Chronos.Data.Archiving.Jobs;
|
||||
using Chronos.Data.Jobs;
|
||||
|
||||
namespace Chronos.Data.Workspaces;
|
||||
|
||||
/// <summary>Builds the jobs of the workspace (owner review 2, R8), and their handlers for the desktop's engine.</summary>
|
||||
public static class WorkspaceJobs
|
||||
{
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.WorkspaceCopy"/>: the media of workspace project <paramref name="projectId"/> (checked out first,
|
||||
/// <see cref="WorkspaceSync.CheckoutAsync"/>) copied from its home, the sequence of this PC's archive.
|
||||
/// </summary>
|
||||
public static ArchiveJobRequest Copy(Sequence sequence, string workspaceRoot, Guid projectId, string media = BundleMedia.Proxies)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(sequence);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(workspaceRoot);
|
||||
return new ArchiveJobRequest(JobKind.WorkspaceCopy, [sequence.Id], sequence.DisplayTitle,
|
||||
JsonSerializer.Serialize(new WorkspaceCopyParameters(workspaceRoot, projectId, media), WorkspaceJson.Default.WorkspaceCopyParameters), JobPriority.User);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The handlers of the workspace's jobs for a desktop's engine (never a server's: the job writes into a folder its
|
||||
/// parameters name).
|
||||
/// </summary>
|
||||
public static IReadOnlyList<IJobHandler> CreateHandlers(ChronosDatabase db, ArchiveService archives) => [new WorkspaceCopyJobHandler(db, archives)];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.WorkspaceCopy"/> for a sequence of this PC's archive (owner review 2, R8): its proxies into the
|
||||
/// workspace project's <c>media/proxies/<frame id>.jpg</c>, the archive's own when current, the missing ones made
|
||||
/// first by the archive's proxy pass (made here from the original when the archive makes none), each written atomically
|
||||
/// and read back to check its SHA-256; then the deep report (<c>media/analysis.json</c>) and, last, <c>media/media.json</c>.
|
||||
/// Frames changed at the home during the copy make it start again (three times at most). The working state's positions
|
||||
/// follow the new frames by photo. Checkpoint: <c>proxies:<frame></c> during the archive's pass, then
|
||||
/// <c>copy:<frame></c>. Originals come in phase 5c: asked for, the job fails saying so.
|
||||
/// </summary>
|
||||
internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService archives) : IJobHandler
|
||||
{
|
||||
private const int PageSize = 1000;
|
||||
private const int Passes = 3;
|
||||
private static readonly TimeSpan LockWait = TimeSpan.FromMinutes(2);
|
||||
private static readonly TimeSpan CheckpointEvery = TimeSpan.FromSeconds(5);
|
||||
|
||||
public JobKind Kind => JobKind.WorkspaceCopy;
|
||||
|
||||
public async Task RunAsync(JobContext context)
|
||||
{
|
||||
var ct = context.Cancellation;
|
||||
var parameters = Parameters(context.Job);
|
||||
if (parameters.Media == BundleMedia.Originals) throw new NotSupportedException(Loc.L("Errors.Workspace.OriginalsLater"));
|
||||
if (parameters.Media != BundleMedia.Proxies) throw new InvalidDataException(Loc.L("Errors.Workspace.Parameters"));
|
||||
if (context.Job.TargetIds is not [var sequenceId] || await db.Sequences.GetAsync(sequenceId, ct).ConfigureAwait(false) is not { } sequence
|
||||
|| await db.Archives.GetAsync(sequence.ArchiveId, ct).ConfigureAwait(false) is not { } archive)
|
||||
throw new InvalidOperationException(Loc.L("Archive.Log.SequenceMissing", context.Job.TargetIds.FirstOrDefault()));
|
||||
var store = new WorkspaceStore(parameters.WorkspaceRoot);
|
||||
var project = store.Find(parameters.ProjectId) ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.NotCheckedOut"));
|
||||
if (project.ReadBundle().Workspace is not { Home: { IsArchive: true } home } || home.SequenceId != sequenceId)
|
||||
throw new InvalidOperationException(Loc.L("Errors.Workspace.OtherHome"));
|
||||
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
var (stage, next) = Parse(context.Checkpoint);
|
||||
for (var pass = 1; ; pass++)
|
||||
{
|
||||
var (row, frames) = await FramesAsync(sequenceId, ct).ConfigureAwait(false);
|
||||
var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy;
|
||||
var makesProxies = policy.LongEdge != ProxyPolicy.Disabled && ArchiveVolume.Probe(paths) == ArchiveAvailability.Online;
|
||||
var total = (long)frames.Count * (makesProxies ? 2 : 1);
|
||||
long offset = 0;
|
||||
if (makesProxies)
|
||||
{
|
||||
if (stage == Stage.Proxies && (row.Status & SequenceStatus.ProxiesBuilt) == 0)
|
||||
{
|
||||
var result = await archives.Images.BuildProxiesAsync(archive, row, force: false, next,
|
||||
(frame, _) => context.CheckpointAsync(Checkpoint(Stage.Proxies, frame), frame, total), ct).ConfigureAwait(false);
|
||||
context.Log(Loc.L("Archive.Log.ProxiesDone", row.DisplayTitle, result.Built, result.Reused, result.Undecodable, result.Waiting));
|
||||
next = 0;
|
||||
}
|
||||
offset = frames.Count;
|
||||
}
|
||||
// Frames before a copy checkpoint were copied by an earlier run.
|
||||
var copied = stage == Stage.Copy ? next : 0;
|
||||
(stage, next) = (Stage.Copy, 0);
|
||||
var folder = paths.Sequence(row.Folder);
|
||||
var tally = new Tally();
|
||||
var entries = new WorkspaceFrame[frames.Count];
|
||||
Directory.CreateDirectory(project.ProxiesDirectory);
|
||||
var clock = System.Diagnostics.Stopwatch.StartNew();
|
||||
for (var i = 0; i < frames.Count; i++)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
entries[i] = Copy(project, folder, paths, frames[i], policy, done: i < copied, tally, ct);
|
||||
if (clock.Elapsed >= CheckpointEvery)
|
||||
{
|
||||
await context.CheckpointAsync(Checkpoint(Stage.Copy, i + 1), offset + i + 1, total).ConfigureAwait(false);
|
||||
clock.Restart();
|
||||
}
|
||||
else
|
||||
{
|
||||
await context.ReportAsync(offset + i + 1, total).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
var analysis = CopyAnalysis(folder, project, frames.Count);
|
||||
var after = await db.Sequences.GetAsync(sequenceId, ct).ConfigureAwait(false)
|
||||
?? throw new InvalidOperationException(Loc.L("Archive.Log.SequenceMissing", sequenceId));
|
||||
if (after.FramesVersion != row.FramesVersion && pass < Passes)
|
||||
{
|
||||
context.Log(Loc.L("Archive.Log.WorkspaceFramesChanged", row.DisplayTitle));
|
||||
(stage, next) = (Stage.Proxies, 0);
|
||||
continue;
|
||||
}
|
||||
var media = new WorkspaceMedia(WorkspaceMedia.FormatName, WorkspaceMedia.CurrentFormatVersion, BundleMedia.Proxies, sequenceId, row.FramesVersion, entries,
|
||||
DateTimeOffset.UtcNow, analysis);
|
||||
// The working state is the editor's meanwhile: it is held only while the media index and the positions move.
|
||||
using (await project.LockAsync(LockWait, ct).ConfigureAwait(false))
|
||||
{
|
||||
Commit(project, project.ReadBundle(), media, row);
|
||||
RemoveStale(project, media);
|
||||
}
|
||||
context.Log(Loc.L("Archive.Log.WorkspaceCopied", row.DisplayTitle, tally.Copied, tally.Made, tally.Kept, tally.Missing));
|
||||
await context.CheckpointAsync(Checkpoint(Stage.Done, frames.Count), total, total).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The frame's file in the workspace: kept when an earlier run copied it (before the checkpoint, or the same bytes as the
|
||||
/// archive's proxy), else the archive's current proxy, else one made from the original; none for a frame without either.
|
||||
/// </summary>
|
||||
private WorkspaceFrame Copy(WorkspaceProject project, SequencePaths folder, ArchivePaths paths, Frame frame, ProxyPolicy policy, bool done, Tally tally,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var name = WorkspaceProject.ProxyName(frame.Id);
|
||||
var target = project.MediaFile(name);
|
||||
if (done && File.Exists(target))
|
||||
{
|
||||
tally.Kept++;
|
||||
var kept = AtomicFile.ReadAllBytes(target);
|
||||
return WorkspaceFrame.Of(frame, name, Sha(kept), kept.Length);
|
||||
}
|
||||
var source = folder.Proxy(frame.Index);
|
||||
byte[]? bytes = null;
|
||||
if (File.Exists(source) && ArchiveImages.IsProxyOf(source, frame.CapturedAt, frame.Width, frame.Height, null)) bytes = AtomicFile.ReadAllBytes(source);
|
||||
var made = false;
|
||||
if (bytes is null && (frame.Flags & FrameFlags.Missing) == 0)
|
||||
{
|
||||
var original = paths.FromRelative(frame.RelativePath);
|
||||
try
|
||||
{
|
||||
var edge = policy.LongEdge == ProxyPolicy.Disabled ? new ProxyPolicy().LongEdge : policy.LongEdge;
|
||||
bytes = File.Exists(original) ? ArchiveImages.MakeProxy(archives.Decoder, original, frame, null, edge, policy.Quality, cancellationToken) : null;
|
||||
made = bytes is not null;
|
||||
}
|
||||
catch (IOException ex)
|
||||
{
|
||||
Log.Warning("workspace", $"No proxy of {original} could be made", ex);
|
||||
}
|
||||
}
|
||||
if (bytes is null)
|
||||
{
|
||||
tally.Missing++;
|
||||
return WorkspaceFrame.Of(frame, null, null, 0);
|
||||
}
|
||||
var sha = Sha(bytes);
|
||||
if (File.Exists(target) && Sha(AtomicFile.ReadAllBytes(target)) == sha)
|
||||
{
|
||||
tally.Kept++;
|
||||
return WorkspaceFrame.Of(frame, name, sha, bytes.Length);
|
||||
}
|
||||
WriteVerified(target, bytes, sha);
|
||||
if (made) tally.Made++;
|
||||
else tally.Copied++;
|
||||
return WorkspaceFrame.Of(frame, name, sha, bytes.Length);
|
||||
}
|
||||
|
||||
/// <summary>Writes atomically and reads back: a file that does not hash as its source is written again once, then refused.</summary>
|
||||
private static void WriteVerified(string path, byte[] bytes, string sha)
|
||||
{
|
||||
for (var attempt = 1; ; attempt++)
|
||||
{
|
||||
AtomicFile.WriteAllBytes(path, bytes);
|
||||
if (Sha(AtomicFile.ReadAllBytes(path)) == sha) return;
|
||||
if (attempt == 2) throw new IOException(Loc.L("Errors.Workspace.CopyMismatch", path));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The home's deep report into the media folder when it describes these frames; its SHA-256, or null without one.</summary>
|
||||
private static string? CopyAnalysis(SequencePaths folder, WorkspaceProject project, int frameCount)
|
||||
{
|
||||
if (!File.Exists(folder.AnalysisPath))
|
||||
{
|
||||
if (File.Exists(project.AnalysisPath)) File.Delete(project.AnalysisPath);
|
||||
return null;
|
||||
}
|
||||
var bytes = AtomicFile.ReadAllBytes(folder.AnalysisPath);
|
||||
try
|
||||
{
|
||||
if (Core.Analysis.Deep.DeepAnalysisReport.FromJson(System.Text.Encoding.UTF8.GetString(bytes)).FrameCount != frameCount) return null;
|
||||
}
|
||||
catch (Exception ex) when (ex is InvalidDataException or JsonException or IOException)
|
||||
{
|
||||
Log.Warning("workspace", $"{folder.AnalysisPath} cannot be read; the workspace takes no deep report", ex);
|
||||
return null;
|
||||
}
|
||||
var sha = Sha(bytes);
|
||||
WriteVerified(project.AnalysisPath, bytes, sha);
|
||||
return sha;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The media index and the working state moved onto it: first the working state with every position parked by photo
|
||||
/// (valid whatever the frames), then the index, then the working state placed on the new frames. A crash in between
|
||||
/// leaves positions that wait for their photos, which the next open places.
|
||||
/// </summary>
|
||||
private static void Commit(WorkspaceProject project, ProjectBundle bundle, WorkspaceMedia media, Sequence row)
|
||||
{
|
||||
var old = TryReadMedia(project);
|
||||
var workspace = bundle.Workspace! with { Media = new BundleMedia(BundleMedia.Proxies, media.FramesVersion) };
|
||||
var sequence = bundle.Sequence with { Folder = row.Folder, Title = row.DisplayTitle, Frames = media.Frames.Count };
|
||||
var working = bundle.Project;
|
||||
// Without the media index the positions were made against, they are taken as naming these frames.
|
||||
if (!working.IsNewerFormat && old is not null)
|
||||
{
|
||||
var portable = ProjectSettings.FromJson(ProjectContent.ToPortable(working, old.Photos), allowNewer: true);
|
||||
project.WriteBundle(bundle with { Project = portable, Workspace = workspace, Sequence = sequence });
|
||||
project.WriteMedia(media);
|
||||
working = ProjectSettings.FromJson(ProjectContent.Place(portable.ToJson(), media.Photos), allowNewer: true);
|
||||
}
|
||||
else
|
||||
{
|
||||
project.WriteMedia(media);
|
||||
}
|
||||
project.WriteBundle(bundle with { Project = working, Workspace = workspace, Sequence = sequence, Saved = DateTimeOffset.UtcNow });
|
||||
}
|
||||
|
||||
/// <summary>Proxies of frames the sequence no longer has: regenerable copies, removed so the media folder holds the frames listed.</summary>
|
||||
private static void RemoveStale(WorkspaceProject project, WorkspaceMedia media)
|
||||
{
|
||||
var listed = media.Frames.Where(f => f.File is not null).Select(f => Path.GetFileName(f.File!)).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var file in Directory.EnumerateFiles(project.ProxiesDirectory, "*.jpg"))
|
||||
{
|
||||
if (listed.Contains(Path.GetFileName(file))) continue;
|
||||
try
|
||||
{
|
||||
File.Delete(file);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("workspace", $"{file} could not be removed", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static WorkspaceMedia? TryReadMedia(WorkspaceProject project)
|
||||
{
|
||||
try
|
||||
{
|
||||
return project.ReadMedia();
|
||||
}
|
||||
catch (InvalidDataException ex)
|
||||
{
|
||||
Log.Warning("workspace", $"The media index of {project.Folder} cannot be read; positions are taken as they are", ex);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The sequence row with its frames as one state (listed between two reads of the row at the same frames version).</summary>
|
||||
private async Task<(Sequence Row, List<Frame> Frames)> FramesAsync(Guid sequenceId, CancellationToken cancellationToken)
|
||||
{
|
||||
for (var attempt = 0; ; attempt++)
|
||||
{
|
||||
var before = await db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
|
||||
?? throw new InvalidOperationException(Loc.L("Archive.Log.SequenceMissing", sequenceId));
|
||||
var frames = new List<Frame>();
|
||||
for (var after = -1; ;)
|
||||
{
|
||||
var page = await db.Frames.ListBySequenceAsync(sequenceId, after, PageSize, cancellationToken).ConfigureAwait(false);
|
||||
frames.AddRange(page);
|
||||
if (page.Count < PageSize) break;
|
||||
after = page[^1].Index;
|
||||
}
|
||||
var now = await db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? before;
|
||||
if (now.FramesVersion == before.FramesVersion || attempt == 2) return (now, frames);
|
||||
}
|
||||
}
|
||||
|
||||
private static WorkspaceCopyParameters Parameters(Job job)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (job.ParametersJson is { Length: > 0 } json
|
||||
&& JsonSerializer.Deserialize(json, WorkspaceJson.Default.WorkspaceCopyParameters) is { WorkspaceRoot.Length: > 0 } parameters
|
||||
&& parameters.ProjectId != Guid.Empty)
|
||||
return parameters with { Media = parameters.Media ?? BundleMedia.Proxies };
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
throw new InvalidDataException(Loc.L("Errors.Workspace.Parameters"), ex);
|
||||
}
|
||||
throw new InvalidDataException(Loc.L("Errors.Workspace.Parameters"));
|
||||
}
|
||||
|
||||
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes));
|
||||
|
||||
private enum Stage
|
||||
{
|
||||
Proxies,
|
||||
Copy,
|
||||
Done,
|
||||
}
|
||||
|
||||
private static string Checkpoint(Stage stage, int frame) =>
|
||||
string.Create(CultureInfo.InvariantCulture, $"{stage.ToString().ToLowerInvariant()}:{frame}");
|
||||
|
||||
private static (Stage Stage, int Next) Parse(string? checkpoint)
|
||||
{
|
||||
var parts = checkpoint?.Split(':');
|
||||
return parts is [var name, var number] && Enum.TryParse<Stage>(name, ignoreCase: true, out var stage)
|
||||
&& int.TryParse(number, NumberStyles.None, CultureInfo.InvariantCulture, out var next)
|
||||
? (stage == Stage.Done ? Stage.Copy : stage, stage == Stage.Done ? 0 : next)
|
||||
: (Stage.Proxies, 0);
|
||||
}
|
||||
|
||||
private sealed class Tally
|
||||
{
|
||||
public int Copied { get; set; }
|
||||
public int Made { get; set; }
|
||||
public int Kept { get; set; }
|
||||
public int Missing { get; set; }
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user