feat: owner review 2 phase 5b: the server as a project home (#17)
CI / Build and test (push) Successful in 9m58s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 6s
Release / Build, test and publish (push) Successful in 14m8s
CI / PostgreSQL 16 integration (push) Successful in 8m55s
CI / PostgreSQL 17 integration (push) Successful in 8m50s
CI / Docker image (push) Successful in 57s
Release / Server image (push) Successful in 1m22s
CI / Build and test (push) Successful in 9m58s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 6s
Release / Build, test and publish (push) Successful in 14m8s
CI / PostgreSQL 16 integration (push) Successful in 8m55s
CI / PostgreSQL 17 integration (push) Successful in 8m50s
CI / Docker image (push) Successful in 57s
Release / Server image (push) Successful in 1m22s
A Chronos Server keeps its sequences' projects, as a PC's archive does: Main and named projects with versions under /api/v1/sequences/{id}/projects, the projects capability and the projectChanged event. The desktop opens a server's sequences in the workspace (proxies downloaded, hash-checked, resumable) with Save version, Sync, conflicts, History, Restore and the project actions against the server. Renders can be of a project version. The admin panel lists a sequence's projects and versions. Strings in all 38 languages.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
This commit was merged in pull request #17.
This commit is contained in:
1 parent
255e149087
commit
81c303f484
325 files changed
+11477
-516
No files matched your search
@@ -1,4 +1,5 @@
|
||||
using System.Globalization;
|
||||
using System.Net;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text.Json;
|
||||
using Chronos.Core.Archiving;
|
||||
@@ -12,6 +13,9 @@ using Chronos.Data.Archiving.Jobs;
|
||||
using Chronos.Data.Jobs;
|
||||
using Chronos.Data.Projects;
|
||||
using Chronos.Data.Sync;
|
||||
using Chronos.Protocol;
|
||||
using Chronos.Protocol.Client;
|
||||
using SequenceManifest = Chronos.Protocol.SequenceManifest;
|
||||
|
||||
namespace Chronos.Data.Workspaces;
|
||||
|
||||
@@ -31,10 +35,28 @@ public static class WorkspaceJobs
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The handlers of the workspace's jobs for a desktop's engine (never a server's: the job writes into a folder its
|
||||
/// parameters name).
|
||||
/// <see cref="JobKind.WorkspaceCopy"/> of a project whose home is a server (phase 5b): the media of workspace project
|
||||
/// <paramref name="projectId"/> (checked out first) downloaded from sequence <paramref name="sequenceId"/> of
|
||||
/// <paramref name="server"/> (<see cref="SyncService.ServerKey"/>), into the workspace alone. The job's target is the
|
||||
/// server's sequence, as for a sync download.
|
||||
/// </summary>
|
||||
public static IReadOnlyList<IJobHandler> CreateHandlers(ChronosDatabase db, ArchiveService archives) => [new WorkspaceCopyJobHandler(db, archives)];
|
||||
public static ArchiveJobRequest CopyFromServer(Guid sequenceId, string title, string server, string workspaceRoot, Guid projectId, string media = BundleMedia.Proxies)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(server);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(workspaceRoot);
|
||||
return new ArchiveJobRequest(JobKind.WorkspaceCopy, [sequenceId], title,
|
||||
JsonSerializer.Serialize(new WorkspaceCopyParameters(workspaceRoot, projectId, media, server), WorkspaceJson.Default.WorkspaceCopyParameters), JobPriority.User);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The handlers of the workspace's jobs for a desktop's engine (never a server's: the job writes into a folder its
|
||||
/// parameters name); copies from a server fail without a connection to it.
|
||||
/// </summary>
|
||||
public static IReadOnlyList<IJobHandler> CreateHandlers(ChronosDatabase db, ArchiveService archives) => CreateHandlers(db, archives, null);
|
||||
|
||||
/// <param name="servers">The connection a copy from a server's sequence downloads through (the desktop's signed-in session).</param>
|
||||
public static IReadOnlyList<IJobHandler> CreateHandlers(ChronosDatabase db, ArchiveService archives, ISyncConnector? servers) =>
|
||||
[new WorkspaceCopyJobHandler(db, archives, servers)];
|
||||
|
||||
/// <summary>
|
||||
/// What a first copy of a sequence's proxies writes into a workspace, as the copy reckons it before it starts (the
|
||||
@@ -70,14 +92,32 @@ public static class WorkspaceJobs
|
||||
/// project goes on with them (<see cref="RemovePartialMediaAsync"/>); one cancelled while paused or failed, which no handler
|
||||
/// runs, has them removed the same way (<see cref="CancelledAsync"/>).
|
||||
/// Originals come in phase 5c: asked for, the job fails saying so.
|
||||
/// A project whose home is a server (phase 5b, <see cref="WorkspaceJobs.CopyFromServer"/>) has its media downloaded from
|
||||
/// the server through the desktop's session (<see cref="ISyncConnector"/>), never through this PC's archive: the frames as
|
||||
/// of one frames version with the aliases of the server's conversions (<c>GET …/projects/frames</c>), each frame's proxy as
|
||||
/// the proxy cache fetches it (<see cref="SyncTransfers.DownloadFirstAsync"/>: the proxy, else the thumbnail standing in
|
||||
/// while the server has none, recorded as such and replaced by a later copy once the server has the proxy), hashed as it
|
||||
/// arrives and checked against the file written, four at a time; then the server's deep report. A file the editor's media
|
||||
/// or this run hold of the same photo is hashed again and kept; the editor's file is never written over, as above. The
|
||||
/// space, the passes, the checkpoint and a cancelled first copy go as for a copy from this PC's archive (the server's
|
||||
/// folder of the sequence in the checkpoint). Like the sync's downloads it is not held on a metered connection.
|
||||
/// </summary>
|
||||
internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService archives) : ICancelledJobHandler
|
||||
internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService archives, ISyncConnector? servers = null) : ICancelledJobHandler
|
||||
{
|
||||
private const int PageSize = 1000;
|
||||
private const int Passes = 3;
|
||||
|
||||
/// <summary>Frames downloaded at once from a server (as the sync's downloads).</summary>
|
||||
private const int Downloads = 4;
|
||||
|
||||
/// <summary>Frames downloaded between two looks at the checkpoint clock.</summary>
|
||||
private const int DownloadBatch = 32;
|
||||
private static readonly TimeSpan LockWait = TimeSpan.FromMinutes(2);
|
||||
private static readonly TimeSpan CheckpointEvery = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>Beside a server's thumbnail downloaded for a frame until the copy commits: what it is, should the run stop first.</summary>
|
||||
internal const string StandInSuffix = ".thumb";
|
||||
|
||||
/// <summary>Bytes left free on the workspace's drive besides the copy, as a download keeps (<see cref="SyncTransfers"/>).</summary>
|
||||
internal const long SpaceReserve = 512L << 20;
|
||||
|
||||
@@ -97,6 +137,11 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
var parameters = Parameters(context.Job);
|
||||
if (parameters.Media == BundleMedia.Originals) throw new NotSupportedException(Loc.L("Errors.Workspace.OriginalsLater"));
|
||||
if (parameters.Media != BundleMedia.Proxies) throw new InvalidDataException(Loc.L("Errors.Workspace.Parameters"));
|
||||
if (parameters.Server is { Length: > 0 } server)
|
||||
{
|
||||
await RunFromServerAsync(context, parameters, server).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
if (context.Job.TargetIds is not [var sequenceId] || await db.Sequences.GetAsync(sequenceId, ct).ConfigureAwait(false) is not { } sequence
|
||||
|| await db.Archives.GetAsync(sequence.ArchiveId, ct).ConfigureAwait(false) is not { } archive)
|
||||
throw new InvalidOperationException(Loc.L("Archive.Log.SequenceMissing", context.Job.TargetIds.FirstOrDefault()));
|
||||
@@ -336,7 +381,7 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
// The working state is the editor's meanwhile: it is held only while the media index and the positions move.
|
||||
using (await project.LockAsync(LockWait, ct).ConfigureAwait(false))
|
||||
{
|
||||
Commit(project, project.ReadBundle(), media, row);
|
||||
Commit(project, project.ReadBundle(), media, row.Folder, row.DisplayTitle);
|
||||
RemoveStale(project, media);
|
||||
}
|
||||
context.Log(Loc.L("Archive.Log.WorkspaceCopied", row.DisplayTitle, tally.Copied, tally.Made, tally.Kept, tally.Missing));
|
||||
@@ -345,6 +390,293 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The copy of a project whose home is a server (see the class remarks): its frames and their files downloaded into the
|
||||
/// workspace, then the media index and the working state moved onto them as for a copy from this PC's archive.
|
||||
/// </summary>
|
||||
private async Task RunFromServerAsync(JobContext context, WorkspaceCopyParameters parameters, string server)
|
||||
{
|
||||
var ct = context.Cancellation;
|
||||
if (context.Job.TargetIds is not [var sequenceId]) throw new InvalidDataException(Loc.L("Errors.Workspace.Parameters"));
|
||||
// Asked before the store is opened, which makes its folder (as for a copy from this PC's archive).
|
||||
if (await InsideALibraryAsync(parameters.WorkspaceRoot, ct).ConfigureAwait(false)) throw new InvalidOperationException(Loc.L("Errors.Workspace.InsideLibrary"));
|
||||
var store = new WorkspaceStore(parameters.WorkspaceRoot);
|
||||
var project = store.Find(parameters.ProjectId) ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.NotCheckedOut"));
|
||||
if (project.ReadBundle().Workspace is not { Home: { Kind: BundleHome.ServerKind } home } || home.SequenceId != sequenceId
|
||||
|| !string.Equals(home.Server, server, StringComparison.OrdinalIgnoreCase))
|
||||
throw new InvalidOperationException(Loc.L("Errors.Workspace.OtherHome"));
|
||||
try
|
||||
{
|
||||
using (WritingMedia(project))
|
||||
{
|
||||
using var remote = (servers is null ? null : await servers.ConnectAsync(server, ct).ConfigureAwait(false))
|
||||
?? throw new InvalidOperationException(Loc.L("Errors.Sync.NotConnected"));
|
||||
await DownloadAsync(context, project, remote, sequenceId, parameters, ct).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
// Stopped writing: what a first copy the user cancelled wrote goes (a pause or a shutdown resumes it).
|
||||
await RemovePartialMediaAsync(context.Job, project).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task DownloadAsync(JobContext context, WorkspaceProject project, ISyncRemote remote, Guid sequenceId, WorkspaceCopyParameters parameters,
|
||||
CancellationToken ct)
|
||||
{
|
||||
var (copied, checkpointVersion, checkpointFolder) = Parse(context.Checkpoint);
|
||||
// What this run downloaded, pass after pass: kept where a frame still shows the same photo.
|
||||
var fetched = new Dictionary<Guid, WorkspaceFrame>();
|
||||
for (var pass = 1; ; pass++)
|
||||
{
|
||||
var (sequence, frames, assets, aliases, framesVersion, spec) = await ServerFramesAsync(remote, sequenceId, ct).ConfigureAwait(false);
|
||||
var folder = sequence.Folder ?? string.Empty;
|
||||
if ((checkpointVersion is { } version && version != framesVersion) || (checkpointFolder is { } read && read != folder)) copied = 0;
|
||||
var total = (long)frames.Count;
|
||||
var tally = new Tally();
|
||||
var entries = new WorkspaceFrame[frames.Count];
|
||||
// The media the editor reads until this copy commits: their files stay until then (RemoveStale).
|
||||
var current = TryReadMedia(project)?.Frames.Where(f => f.File is not null).ToDictionary(f => f.Id) ?? [];
|
||||
Directory.CreateDirectory(project.ProxiesDirectory);
|
||||
// The workspace is on this PC's own drive: the copy never fills it (the server's files estimated at its proxy size).
|
||||
var remaining = RemainingDownloads(project, frames, assets, spec);
|
||||
EnsureSpace(parameters.WorkspaceRoot, remaining[0]);
|
||||
var clock = System.Diagnostics.Stopwatch.StartNew();
|
||||
var options = new ParallelOptions { MaxDegreeOfParallelism = Downloads, CancellationToken = ct };
|
||||
for (var start = 0; start < frames.Count; start += DownloadBatch)
|
||||
{
|
||||
var end = Math.Min(frames.Count, start + DownloadBatch);
|
||||
var done = copied;
|
||||
await Parallel.ForEachAsync(Enumerable.Range(start, end - start), options, async (i, token) =>
|
||||
entries[i] = await DownloadAsync(remote, project, frames[i], assets.GetValueOrDefault(frames[i].Id), i < done, current.GetValueOrDefault(frames[i].Id),
|
||||
fetched.GetValueOrDefault(frames[i].Id), tally, token).ConfigureAwait(false)).ConfigureAwait(false);
|
||||
if (clock.Elapsed >= CheckpointEvery || end == frames.Count)
|
||||
{
|
||||
await context.CheckpointAsync(Checkpoint(Stage.Copy, end, framesVersion, folder), end, total).ConfigureAwait(false);
|
||||
// Another program may take the space meanwhile.
|
||||
EnsureSpace(parameters.WorkspaceRoot, remaining[end]);
|
||||
clock.Restart();
|
||||
}
|
||||
else
|
||||
{
|
||||
await context.ReportAsync(end, total).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
foreach (var entry in entries.Where(e => e.File is not null)) fetched[entry.Id] = entry;
|
||||
var analysis = await CopyServerAnalysisAsync(remote, sequenceId, project, frames.Count, ct).ConfigureAwait(false);
|
||||
var after = await remote.GetSequenceAsync(sequenceId, ct).ConfigureAwait(false);
|
||||
if (after.FramesVersion != framesVersion && pass < Passes)
|
||||
{
|
||||
// The files read may be of other frames now: the next pass reads the frames again, keeping what still shows them,
|
||||
// and so does a run resumed after a crash in it (the checkpoint stored says so).
|
||||
context.Log(Loc.L("Archive.Log.WorkspaceFramesChanged", sequence.DisplayTitle));
|
||||
await context.CheckpointAsync(Checkpoint(Stage.Copy, 0, after.FramesVersion, after.Folder ?? string.Empty), 0, total).ConfigureAwait(false);
|
||||
(copied, checkpointVersion, checkpointFolder) = (0, null, null);
|
||||
continue;
|
||||
}
|
||||
// Frames still arriving are copied as they were read (Update media takes the rest).
|
||||
var media = new WorkspaceMedia(WorkspaceMedia.FormatName, WorkspaceMedia.CurrentFormatVersion, BundleMedia.Proxies, sequenceId, framesVersion, entries,
|
||||
DateTimeOffset.UtcNow, analysis, aliases.Count > 0 ? aliases : null);
|
||||
// The working state is the editor's meanwhile: it is held only while the media index and the positions move.
|
||||
using (await project.LockAsync(LockWait, ct).ConfigureAwait(false))
|
||||
{
|
||||
Commit(project, project.ReadBundle(), media, folder, sequence.DisplayTitle);
|
||||
RemoveStale(project, media);
|
||||
}
|
||||
context.Log(Loc.L("Archive.Log.WorkspaceDownloaded", sequence.DisplayTitle, tally.Copied, tally.Made, tally.Kept, tally.Missing));
|
||||
await context.CheckpointAsync(Checkpoint(Stage.Done, frames.Count, framesVersion, folder), total, total).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A server sequence's frames in index order as of one frames version, the assets the server holds of each, the links
|
||||
/// of its conversions, that frames version and the server's proxy size: the frames read twice around the manifest until
|
||||
/// both are of the same one.
|
||||
/// </summary>
|
||||
private static async Task<(SequenceDto Sequence, List<Frame> Frames, Dictionary<Guid, ManifestFrame> Assets, IReadOnlyList<PhotoAlias> Aliases, int FramesVersion,
|
||||
ProxySpec? Spec)> ServerFramesAsync(ISyncRemote remote, Guid sequenceId, CancellationToken cancellationToken)
|
||||
{
|
||||
for (var attempt = 1; ; attempt++)
|
||||
{
|
||||
ProjectFramesDto frames;
|
||||
SequenceManifest manifest;
|
||||
SequenceDto sequence;
|
||||
try
|
||||
{
|
||||
sequence = await remote.GetSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false);
|
||||
frames = await remote.GetProjectFramesAsync(sequenceId, cancellationToken).ConfigureAwait(false);
|
||||
manifest = await remote.GetManifestAsync(sequenceId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.NotFound)
|
||||
{
|
||||
// The sequence is gone, or the server keeps no projects (its API knows no such call).
|
||||
throw new InvalidOperationException(ex.Error.Details == "sequence" ? Loc.L("Archive.Log.SequenceMissing", sequenceId) : Loc.L("Errors.Workspace.NoProjects"), ex);
|
||||
}
|
||||
if (manifest.FramesVersion == frames.FramesVersion || attempt == Passes)
|
||||
{
|
||||
var home = ProjectWire.ToHomeFrames(frames);
|
||||
var aliases = (frames.Aliases ?? []).Where(a => !string.IsNullOrEmpty(a.Sha256) && !string.IsNullOrEmpty(a.Replacement))
|
||||
.Select(a => new PhotoAlias(a.Sha256, a.Replacement!)).ToList();
|
||||
return (sequence, [.. home.Frames], manifest.Frames.ToDictionary(f => f.Id), aliases, frames.FramesVersion, manifest.Proxies);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A frame's file from the server: kept when this run downloaded it before it stopped or in an earlier pass, or the
|
||||
/// editor's media hold one of the same photo that is no thumbnail standing in for a proxy the server has now (hashed
|
||||
/// again: a file that no longer matches is fetched anew); else its proxy, or the thumbnail while the server has none,
|
||||
/// hashed as it arrives and checked against the file written (downloaded once more when they differ, then refused);
|
||||
/// none for a frame without either, or missing or damaged on the server. The editor's file (<paramref name="current"/>)
|
||||
/// is never written over: other bytes for it go under a name of their own, as for a copy from this PC's archive.
|
||||
/// </summary>
|
||||
private static async Task<WorkspaceFrame> DownloadAsync(ISyncRemote remote, WorkspaceProject project, Frame frame, ManifestFrame? asset, bool done,
|
||||
WorkspaceFrame? current, WorkspaceFrame? fetched, Tally tally, CancellationToken cancellationToken)
|
||||
{
|
||||
var name = WorkspaceProject.ProxyName(frame.Id);
|
||||
var target = project.MediaFile(name);
|
||||
var listed = current is not null && string.Equals(current.File, name, StringComparison.OrdinalIgnoreCase);
|
||||
IReadOnlyList<string> offered = asset is null || (frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0 ? [] : SyncTransfers.Kinds(asset);
|
||||
if (done && File.Exists(target))
|
||||
{
|
||||
var keptSha = await ContentHash.Sha256Async(target, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
// The editor's file of another photo of the frame (a conversion since) was left by an earlier copy, not this run.
|
||||
// A run stopped before it recorded the file's kind left a marker beside a thumbnail it took for a missing proxy.
|
||||
var keptKind = KindOf(fetched, keptSha) ?? KindOf(current, keptSha)
|
||||
?? (File.Exists(StandInMarker(target)) ? ProxyCache.Thumb : offered.FirstOrDefault() ?? ProxyCache.Proxy);
|
||||
if ((!listed || keptSha != current!.FileSha256 || SamePhoto(current, frame)) && !(keptKind == ProxyCache.Thumb && offered.Contains(ProxyCache.Proxy)))
|
||||
{
|
||||
tally.Keep();
|
||||
return WorkspaceFrame.Of(frame, name, keptSha, new FileInfo(target).Length, keptKind);
|
||||
}
|
||||
}
|
||||
// What this run downloaded in an earlier pass, then the editor's file of the same photo, unless a thumbnail stands in
|
||||
// there for a proxy the server has now.
|
||||
foreach (var held in new[] { fetched, current })
|
||||
{
|
||||
if (held is not { File: { } file, FileSha256: { } heldSha } || !SamePhoto(held, frame) || (held.IsStandIn && offered.Contains(ProxyCache.Proxy))) continue;
|
||||
var path = project.MediaFile(file);
|
||||
if (!File.Exists(path)
|
||||
|| !string.Equals(await ContentHash.Sha256Async(path, cancellationToken: cancellationToken).ConfigureAwait(false), heldSha, StringComparison.OrdinalIgnoreCase))
|
||||
continue;
|
||||
tally.Keep();
|
||||
return WorkspaceFrame.Of(frame, file, heldSha.ToLowerInvariant(), new FileInfo(path).Length, held.FileKind ?? ProxyCache.Proxy);
|
||||
}
|
||||
if (offered.Count == 0)
|
||||
{
|
||||
// A file an earlier copy left for the frame is never kept as its photo: a regenerable copy, it goes, unless the
|
||||
// editor's media name it (it goes once this copy commits).
|
||||
if (!listed && File.Exists(target)) TryDelete(target);
|
||||
tally.Miss();
|
||||
return WorkspaceFrame.Of(frame, null, null, 0);
|
||||
}
|
||||
var part = target + ".part";
|
||||
for (var attempt = 1; ; attempt++)
|
||||
{
|
||||
HashingStream? hashing = null;
|
||||
string? kind;
|
||||
string received;
|
||||
long bytes;
|
||||
try
|
||||
{
|
||||
kind = await SyncTransfers.DownloadFirstAsync(remote, frame.Id, offered, part, file =>
|
||||
{
|
||||
hashing?.Dispose();
|
||||
return hashing = new HashingStream(file);
|
||||
}, cancellationToken, durable: true).ConfigureAwait(false);
|
||||
(received, bytes) = hashing is null ? (string.Empty, 0L) : (hashing.Sha256(), hashing.Written);
|
||||
}
|
||||
finally
|
||||
{
|
||||
hashing?.Dispose();
|
||||
}
|
||||
if (kind is null || bytes == 0)
|
||||
{
|
||||
if (File.Exists(part)) File.Delete(part);
|
||||
if (!listed && File.Exists(target)) TryDelete(target);
|
||||
tally.Miss();
|
||||
return WorkspaceFrame.Of(frame, null, null, 0);
|
||||
}
|
||||
// Other bytes for the file the editor's media name: under a name of their own, never over the editor's file.
|
||||
var (file, path) = (name, target);
|
||||
if (listed && current!.FileSha256 is { } editors && !string.Equals(editors, received, StringComparison.OrdinalIgnoreCase)
|
||||
&& Holds(target, editors.ToLowerInvariant()))
|
||||
{
|
||||
file = WorkspaceProject.ProxyName(frame.Id, received);
|
||||
path = project.MediaFile(file);
|
||||
}
|
||||
// The kind is known on disk before the file is: a run stopped after the move resumes with it.
|
||||
if (kind == ProxyCache.Thumb) File.WriteAllBytes(StandInMarker(path), []);
|
||||
else if (File.Exists(StandInMarker(path))) File.Delete(StandInMarker(path));
|
||||
File.Move(part, path, overwrite: true);
|
||||
if (await ContentHash.Sha256Async(path, cancellationToken: cancellationToken).ConfigureAwait(false) == received)
|
||||
{
|
||||
if (kind == ProxyCache.Proxy) tally.Copy();
|
||||
else tally.Make();
|
||||
return WorkspaceFrame.Of(frame, file, received, bytes, kind);
|
||||
}
|
||||
if (attempt == 2) throw new IOException(Loc.L("Errors.Workspace.CopyMismatch", path));
|
||||
}
|
||||
|
||||
static string StandInMarker(string file) => file + StandInSuffix;
|
||||
|
||||
static string? KindOf(WorkspaceFrame? entry, string sha) =>
|
||||
entry is { FileSha256: { } held } && string.Equals(held, sha, StringComparison.OrdinalIgnoreCase) ? entry.FileKind ?? ProxyCache.Proxy : null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The bytes a copy from a server still writes from each frame on (index <c>i</c>: frames <c>i</c> to the last; one more
|
||||
/// entry, 0): nothing for a frame the workspace holds a file of already or the server has neither proxy nor thumbnail of,
|
||||
/// else an estimate at the server's proxy size (its manifest gives no file's size).
|
||||
/// </summary>
|
||||
private static long[] RemainingDownloads(WorkspaceProject project, List<Frame> frames, Dictionary<Guid, ManifestFrame> assets, ProxySpec? spec)
|
||||
{
|
||||
var held = Directory.Exists(project.ProxiesDirectory) ? HeldFrames(Directory.EnumerateFiles(project.ProxiesDirectory, "*.jpg")) : [];
|
||||
var policy = spec is null ? new ProxyPolicy() : new ProxyPolicy { LongEdge = spec.LongEdge, Quality = spec.Quality };
|
||||
var remaining = new long[frames.Count + 1];
|
||||
for (var i = frames.Count - 1; i >= 0; i--)
|
||||
{
|
||||
var frame = frames[i];
|
||||
var bytes = held.Contains(frame.Id) || assets.GetValueOrDefault(frame.Id) is not { } asset || SyncTransfers.Kinds(asset).Length == 0 ? 0 : Estimate(frame, policy);
|
||||
remaining[i] = remaining[i + 1] + bytes;
|
||||
}
|
||||
return remaining;
|
||||
}
|
||||
|
||||
/// <summary>The server's deep report into the media folder when it describes these frames; its SHA-256, or null without one.</summary>
|
||||
private static async Task<string?> CopyServerAnalysisAsync(ISyncRemote remote, Guid sequenceId, WorkspaceProject project, int frameCount,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
string? json;
|
||||
try
|
||||
{
|
||||
json = await remote.GetAnalysisReportAsync(sequenceId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.NotFound)
|
||||
{
|
||||
json = null;
|
||||
}
|
||||
try
|
||||
{
|
||||
if (json is null || Core.Analysis.Deep.DeepAnalysisReport.FromJson(json).FrameCount != frameCount)
|
||||
{
|
||||
if (File.Exists(project.AnalysisPath)) File.Delete(project.AnalysisPath);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is InvalidDataException or JsonException)
|
||||
{
|
||||
Log.Warning("workspace", $"The deep report of server sequence {sequenceId} cannot be read; the workspace takes none", ex);
|
||||
return null;
|
||||
}
|
||||
var bytes = System.Text.Encoding.UTF8.GetBytes(json);
|
||||
var sha = Sha(bytes);
|
||||
WriteVerified(project.AnalysisPath, bytes, sha);
|
||||
return sha;
|
||||
}
|
||||
|
||||
/// <summary>Whether <paramref name="folder"/> is an archive's root of this index or lies inside one.</summary>
|
||||
private async Task<bool> InsideALibraryAsync(string folder, CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -397,7 +729,7 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
// The editor's file of another photo of the frame (a conversion since) was left by an earlier copy, not this run.
|
||||
if (!listed || keptSha != current!.FileSha256 || SamePhoto(current, frame))
|
||||
{
|
||||
tally.Kept++;
|
||||
tally.Keep();
|
||||
return WorkspaceFrame.Of(frame, name, keptSha, kept.Length);
|
||||
}
|
||||
}
|
||||
@@ -424,7 +756,7 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
// A file an earlier copy left for the frame (of other frames, or another folder's) is never kept as its photo by a
|
||||
// resumed run: a regenerable copy, it goes, unless the editor's media name it (it goes once this copy commits).
|
||||
if (!listed && File.Exists(target)) TryDelete(target);
|
||||
tally.Missing++;
|
||||
tally.Miss();
|
||||
return WorkspaceFrame.Of(frame, null, null, 0);
|
||||
}
|
||||
var sha = Sha(bytes);
|
||||
@@ -432,12 +764,12 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
if (current?.File is { } held && !string.Equals(held, name, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(current.FileSha256, sha, StringComparison.OrdinalIgnoreCase) && Holds(project.MediaFile(held), sha))
|
||||
{
|
||||
tally.Kept++;
|
||||
tally.Keep();
|
||||
return WorkspaceFrame.Of(frame, held, sha, bytes.Length);
|
||||
}
|
||||
if (Holds(target, sha))
|
||||
{
|
||||
tally.Kept++;
|
||||
tally.Keep();
|
||||
return WorkspaceFrame.Of(frame, name, sha, bytes.Length);
|
||||
}
|
||||
// Other bytes for the file the editor's media name (a conversion since): written under a name of their own, never over
|
||||
@@ -450,13 +782,13 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
target = project.MediaFile(name);
|
||||
if (Holds(target, sha))
|
||||
{
|
||||
tally.Kept++;
|
||||
tally.Keep();
|
||||
return WorkspaceFrame.Of(frame, name, sha, bytes.Length);
|
||||
}
|
||||
}
|
||||
WriteVerified(target, bytes, sha);
|
||||
if (made) tally.Made++;
|
||||
else tally.Copied++;
|
||||
if (made) tally.Make();
|
||||
else tally.Copy();
|
||||
return WorkspaceFrame.Of(frame, name, sha, bytes.Length);
|
||||
}
|
||||
|
||||
@@ -504,11 +836,11 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
/// as the photo they hold for it, so one whose file a conversion replaced (the same frame, another SHA-256) keeps its
|
||||
/// positions, as the archive's own settings do (<see cref="ProjectPositions"/>).
|
||||
/// </summary>
|
||||
private static void Commit(WorkspaceProject project, ProjectBundle bundle, WorkspaceMedia media, Sequence row)
|
||||
private static void Commit(WorkspaceProject project, ProjectBundle bundle, WorkspaceMedia media, string folder, string title)
|
||||
{
|
||||
var old = TryReadMedia(project);
|
||||
var workspace = bundle.Workspace! with { Media = new BundleMedia(BundleMedia.Proxies, media.FramesVersion) };
|
||||
var sequence = bundle.Sequence with { Folder = row.Folder, Title = row.DisplayTitle, Frames = media.Frames.Count };
|
||||
var sequence = bundle.Sequence with { Folder = folder, Title = title, Frames = media.Frames.Count };
|
||||
var working = bundle.Project;
|
||||
// Without the media index the positions were made against, they are taken as naming these frames.
|
||||
if (!working.IsNewerFormat && old is not null)
|
||||
@@ -534,14 +866,19 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
held.Size == frame.Size && held.CapturedAt == frame.CapturedAt
|
||||
&& (string.IsNullOrEmpty(held.Sha256) || string.IsNullOrEmpty(frame.Sha256) || string.Equals(held.Sha256, frame.Sha256, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
/// <summary>Proxies of frames the sequence no longer has: regenerable copies, removed so the media folder holds the frames listed.</summary>
|
||||
/// <summary>
|
||||
/// Proxies of frames the sequence no longer has, and downloads a stopped copy from a server left half done: regenerable
|
||||
/// copies, removed so the media folder holds the frames listed.
|
||||
/// </summary>
|
||||
private static void RemoveStale(WorkspaceProject project, WorkspaceMedia media)
|
||||
{
|
||||
var listed = media.Frames.Where(f => f.File is not null).Select(f => Path.GetFileName(f.File!)).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var file in Directory.EnumerateFiles(project.ProxiesDirectory, "*.jpg"))
|
||||
foreach (var file in Directory.EnumerateFiles(project.ProxiesDirectory, "*.jpg").Concat(Directory.EnumerateFiles(project.ProxiesDirectory, "*.part")))
|
||||
{
|
||||
if (!listed.Contains(Path.GetFileName(file))) TryDelete(file);
|
||||
}
|
||||
// The media committed record each file's kind: the markers of a run are done with.
|
||||
foreach (var marker in Directory.EnumerateFiles(project.ProxiesDirectory, "*" + StandInSuffix)) TryDelete(marker);
|
||||
}
|
||||
|
||||
private static void TryDelete(string file)
|
||||
@@ -718,11 +1055,30 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
return (next, version, parts.Length == 4 ? parts[3] : null);
|
||||
}
|
||||
|
||||
/// <summary>What a copy did with each frame, counted from the workers that download them at once.</summary>
|
||||
private sealed class Tally
|
||||
{
|
||||
public int Copied { get; set; }
|
||||
public int Made { get; set; }
|
||||
public int Kept { get; set; }
|
||||
public int Missing { get; set; }
|
||||
private int _copied;
|
||||
private int _made;
|
||||
private int _kept;
|
||||
private int _missing;
|
||||
|
||||
/// <summary>Proxies copied (from the archive) or downloaded (from a server).</summary>
|
||||
public int Copied => Volatile.Read(ref _copied);
|
||||
|
||||
/// <summary>Proxies made from the originals (from the archive), or thumbnails standing in for them (from a server).</summary>
|
||||
public int Made => Volatile.Read(ref _made);
|
||||
|
||||
public int Kept => Volatile.Read(ref _kept);
|
||||
|
||||
public int Missing => Volatile.Read(ref _missing);
|
||||
|
||||
public void Copy() => Interlocked.Increment(ref _copied);
|
||||
|
||||
public void Make() => Interlocked.Increment(ref _made);
|
||||
|
||||
public void Keep() => Interlocked.Increment(ref _kept);
|
||||
|
||||
public void Miss() => Interlocked.Increment(ref _missing);
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,10 @@ namespace Chronos.Data.Workspaces;
|
||||
/// editor read) and the file the workspace holds for it, with its SHA-256 and size as copied.
|
||||
/// </summary>
|
||||
/// <param name="File">The file relative to the media folder ("proxies/<frame id>.jpg"); null when the workspace holds none.</param>
|
||||
/// <param name="FileKind">
|
||||
/// What the file is when a server sent it (phase 5b): <c>proxy</c>, or <c>thumb</c>, a thumbnail standing in while the server
|
||||
/// had no proxy of the frame, which a later copy replaces once it has one; null for a proxy (copies from this PC's archive).
|
||||
/// </param>
|
||||
public sealed record WorkspaceFrame(
|
||||
Guid Id,
|
||||
int Index,
|
||||
@@ -41,18 +45,23 @@ public sealed record WorkspaceFrame(
|
||||
FrameFlags Flags,
|
||||
string? File,
|
||||
string? FileSha256,
|
||||
long FileBytes)
|
||||
long FileBytes,
|
||||
string? FileKind = null)
|
||||
{
|
||||
/// <summary>The record of a home's frame, with the file the workspace holds for it.</summary>
|
||||
public static WorkspaceFrame Of(Frame frame, string? file, string? fileSha256, long fileBytes)
|
||||
/// <summary>The record of a home's frame, with the file the workspace holds for it (<paramref name="fileKind"/>: what a server sent).</summary>
|
||||
public static WorkspaceFrame Of(Frame frame, string? file, string? fileSha256, long fileBytes, string? fileKind = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(frame);
|
||||
return new WorkspaceFrame(frame.Id, frame.Index, frame.FileName, frame.OriginalFileName, frame.Size, frame.Sha256, frame.CapturedAt, frame.UtcOffsetMinutes,
|
||||
frame.TimestampSource, frame.ExposureTime, frame.FNumber, frame.Iso, frame.FocalLength, frame.ExposureBias, frame.WhiteBalance, frame.Orientation,
|
||||
frame.Width, frame.Height, frame.Format, frame.PreviewWidth, frame.PreviewHeight, frame.MeanLogLuminance, frame.ClipLow, frame.ClipHigh, frame.Flags,
|
||||
file, fileSha256, fileBytes);
|
||||
file, fileSha256, fileBytes, fileKind);
|
||||
}
|
||||
|
||||
/// <summary>A thumbnail stands in for the frame's proxy (a server had none when it was copied).</summary>
|
||||
[JsonIgnore]
|
||||
public bool IsStandIn => FileKind == Sync.ProxyCache.Thumb;
|
||||
|
||||
/// <summary>The frame as the processing engine reads it; flagged missing while the workspace holds no file for it.</summary>
|
||||
public Frame ToFrame(Guid sequenceId) => new()
|
||||
{
|
||||
|
||||
@@ -93,7 +93,8 @@ public sealed record WorkspaceConflict(Guid TheirsVersionId, int TheirsNumber, G
|
||||
/// Parameters of a <see cref="JobKind.WorkspaceCopy"/> job: target = the home's sequence; the workspace folder, the
|
||||
/// project whose media to copy and what to copy (<see cref="BundleMedia.Proxies"/>; originals in phase 5c).
|
||||
/// </summary>
|
||||
public sealed record WorkspaceCopyParameters(string WorkspaceRoot, Guid ProjectId, string Media);
|
||||
/// <param name="Server">The server the home's sequence is on (<see cref="SyncService.ServerKey"/>, phase 5b); null for a sequence of this PC's archive.</param>
|
||||
public sealed record WorkspaceCopyParameters(string WorkspaceRoot, Guid ProjectId, string Media, string? Server = null);
|
||||
|
||||
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
UseStringEnumConverter = true, WriteIndented = true)]
|
||||
|
||||
@@ -50,10 +50,11 @@ public enum WorkspaceState
|
||||
|
||||
/// <summary>
|
||||
/// A workspace project's state (owner review 2, R8): the leading <see cref="State"/> and every fact behind it. Without a
|
||||
/// home to ask, <see cref="Behind"/> is 0 and <see cref="Offline"/> false.
|
||||
/// home to ask, <see cref="Behind"/> is 0 and <see cref="Offline"/> and <see cref="Gone"/> false.
|
||||
/// </summary>
|
||||
/// <param name="Base">The home's version the project was last in step with.</param>
|
||||
/// <param name="HomeHead">The home's newest version, when it was asked.</param>
|
||||
/// <param name="Gone">The home was asked and no longer has the project (<see cref="WorkspaceState.Gone"/>).</param>
|
||||
public sealed record WorkspaceStatus(WorkspaceState State, bool Modified, int Ahead, int Behind, bool Offline, bool MediaRemoved, bool ReadOnly,
|
||||
WorkspaceConflict? Conflict, BundleBase? Base, ProjectVersion? HomeHead)
|
||||
{
|
||||
@@ -369,10 +370,50 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName)
|
||||
return new WorkspaceSyncResult(await StatusLockedAsync(project, home, cancellationToken).ConfigureAwait(false), [], pulled);
|
||||
}
|
||||
if (project.ReadSync().Conflict is null) PruneReplaced(project);
|
||||
await FollowNameLockedAsync(project, home, cancellationToken).ConfigureAwait(false);
|
||||
return new WorkspaceSyncResult(await StatusLockedAsync(project, home, cancellationToken).ConfigureAwait(false), [], pulled);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The project's name (and whether it is Main) as its home has them now, written to the copy here: a rename made at the
|
||||
/// home reaches the workspace. True when the copy's name changed; false when it already matched, or the home could not
|
||||
/// say (unreachable, or no longer listing the project): the next pull or status tells those.
|
||||
/// </summary>
|
||||
public async Task<bool> FollowNameAsync(Guid projectId, IProjectHome home, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(home);
|
||||
var project = Existing(projectId);
|
||||
using (await project.LockAsync(LockWait, cancellationToken).ConfigureAwait(false))
|
||||
return await FollowNameLockedAsync(project, home, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="FollowNameAsync"/> under the project's lock. It runs after a pull that succeeded, so it never fails: whatever
|
||||
/// the home or the copy answers (unreachable, a refusal such as a 403, a file in use) is logged and the name is followed
|
||||
/// next time; only a cancellation is passed on.
|
||||
/// </summary>
|
||||
private static async Task<bool> FollowNameLockedAsync(WorkspaceProject project, IProjectHome home, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
var listed = (await home.ListAsync(cancellationToken).ConfigureAwait(false)).FirstOrDefault(p => p.Id == project.ProjectId);
|
||||
var bundle = project.ReadBundle();
|
||||
if (listed is null || bundle.Workspace is not { } workspace || (workspace.Name == listed.Name && workspace.IsMain == listed.IsMain)) return false;
|
||||
project.WriteBundle(bundle with { Workspace = workspace with { Name = listed.Name, IsMain = listed.IsMain } });
|
||||
return true;
|
||||
}
|
||||
catch (Exception ex) when (ex is HomeUnavailableException or ProjectException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException || !cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
Log.Warning("workspace", $"The name of project {project.ProjectId} could not be followed from its home; tried again at the next pull", ex);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Settles the conflict a pull or push stopped at, as the user was shown it (<paramref name="shown"/>, else the one
|
||||
/// recorded). <see cref="ConflictChoice.KeepMine"/> keeps this workspace's values (but for the members in
|
||||
|
||||
Reference in new issue
Block a user