feat(workspace): send local renders home with their project and version
Phase 5c P3. A render made in a workspace must reach its home with the project and version it was rendered from, and transfers with a server must respect a metered connection, downloads into the workspace included. - WorkspaceSendJobHandler: per unsent ledger entry, hash the file again (missing or changed: marked failed, file left in place), push the named version first when the home lacks it (failure fails the job, entry pending), then file it in this PC's archive (staged copy read back against its SHA-256, project and version checked against the sequence) or upload it to the server with ProjectId/ProjectVersionId; mark it sent. Idempotent by render id. A send to a server waits while HoldTransfers says so. - SyncTransfers: extract FileRenderAsync, the lease-held filing of a staged render into a sequence's exports, shared by GetExportsAsync and the send. - ISyncConnector.ProjectHome (default null; SyncHost already implements it) so the send can push a version to a server home. - Server: RenderUploads checks the project fields at start and completion (422 project_mismatch) and records them; RenderDto carries them. - UploadHold also pauses WorkspaceCopy and WorkspaceSend jobs naming a server; Server screen checkbox reads "Pause metered transfers". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
2e1c67f531
commit
00b8dcd249
12 files changed
+894
-53
No files matched your search
@@ -1,25 +1,303 @@
|
||||
using System.Net;
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.Diagnostics;
|
||||
using Chronos.Core.Localization;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Core.Project;
|
||||
using Chronos.Data.Archiving;
|
||||
using Chronos.Data.Jobs;
|
||||
using Chronos.Data.Projects;
|
||||
using Chronos.Data.Sync;
|
||||
using Chronos.Protocol;
|
||||
using Chronos.Protocol.Client;
|
||||
|
||||
namespace Chronos.Data.Workspaces;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.WorkspaceSend"/> (phase 5c, <see cref="WorkspaceJobs.Send"/>): the local renders of a workspace project
|
||||
/// sent home with their project and version, per unsent ledger entry: the file hashed again (another is refused and stays),
|
||||
/// the version it names pushed first when the home lacks it, then filed in this PC's archive (as the sync files a server's
|
||||
/// exports) or uploaded to its server (<c>RenderUploadRequest</c> with <c>ProjectId</c> and <c>ProjectVersionId</c>), and the
|
||||
/// entry marked sent. Idempotent by render id: a send repeated after a cut finds the home holding it already.
|
||||
/// sent home with their project and version, per unsent ledger entry (<see cref="WorkspaceRenders"/>; the ids the job names,
|
||||
/// or every unsent one): the file hashed again (one missing or changed since it was rendered is refused, marked failed in the
|
||||
/// ledger and left where it is), the version it names pushed first when the home lacks it (a push that fails fails the job,
|
||||
/// the entry left as it was), then filed in this PC's archive (copied into the archive's staging folder, read back against
|
||||
/// its SHA-256, and filed under the sequence's lease as the sync files a server's exports,
|
||||
/// <see cref="SyncTransfers.FileRenderAsync"/>, after checking the project and version are the sequence's) or uploaded to its
|
||||
/// server (<see cref="RenderUploadRequest"/> with <c>ProjectId</c> and <c>ProjectVersionId</c>, which the server checks the
|
||||
/// same way), and the entry marked sent. The file stays in <c>exports/</c> until a clean-up removes it.
|
||||
/// Idempotent by render id: a send repeated after a cut finds the home holding the render already and only marks it sent; a
|
||||
/// render id the home files under another sequence is refused. The ledger is read and written under the project's lease,
|
||||
/// each entry read again before it is written, so a render finished meanwhile is kept.
|
||||
/// A send to a server is held while <see cref="WorkspaceJobOptions.HoldTransfers"/> says so (a metered connection): it waits
|
||||
/// before each upload, and the desktop pauses it meanwhile. A send to this PC's archive is never held.
|
||||
/// </summary>
|
||||
internal sealed class WorkspaceSendJobHandler(ChronosDatabase db, ArchiveService archives, ISyncConnector? servers, WorkspaceJobOptions options) : IJobHandler
|
||||
{
|
||||
private static readonly TimeSpan LockWait = TimeSpan.FromMinutes(2);
|
||||
|
||||
/// <summary>How often a send held on a metered connection looks again.</summary>
|
||||
internal static TimeSpan HoldPoll { get; set; } = TimeSpan.FromSeconds(2);
|
||||
|
||||
public JobKind Kind => JobKind.WorkspaceSend;
|
||||
|
||||
public Task RunAsync(JobContext context)
|
||||
public async Task RunAsync(JobContext context)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
_ = (db, archives, servers, options);
|
||||
throw new NotImplementedException("Sending a workspace project's renders home comes with phase 5c (P3).");
|
||||
var ct = context.Cancellation;
|
||||
var parameters = WorkspaceJobs.SendParameters(context.Job) ?? throw new InvalidDataException(Loc.L("Errors.Workspace.JobParameters"));
|
||||
if (context.Job.TargetIds is not [var sequenceId] || string.IsNullOrWhiteSpace(parameters.WorkspaceRoot))
|
||||
throw new InvalidDataException(Loc.L("Errors.Workspace.JobParameters"));
|
||||
var server = parameters.Server is { Length: > 0 } named ? named : null;
|
||||
var project = new WorkspaceStore(parameters.WorkspaceRoot).Find(parameters.ProjectId)
|
||||
?? throw new InvalidOperationException(Loc.L("Errors.Workspace.NotCheckedOut"));
|
||||
var workspace = project.ReadBundle().Workspace ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.NotCheckedOut"));
|
||||
var home = workspace.Home;
|
||||
if (home.SequenceId != sequenceId || (server is null
|
||||
? !home.IsArchive
|
||||
: home.Kind != BundleHome.ServerKind || !string.Equals(home.Server, server, StringComparison.OrdinalIgnoreCase)))
|
||||
throw new InvalidOperationException(Loc.L("Errors.Workspace.OtherHome"));
|
||||
|
||||
var entries = await ToSendAsync(project, parameters.RenderIds, ct).ConfigureAwait(false);
|
||||
await context.ReportAsync(0, entries.Count).ConfigureAwait(false);
|
||||
if (entries.Count == 0)
|
||||
{
|
||||
context.Log(Loc.L("Archive.Log.WorkspaceSent", workspace.Name, 0, 0));
|
||||
return;
|
||||
}
|
||||
if (server is null)
|
||||
{
|
||||
var sequence = await db.Sequences.GetAsync(sequenceId, ct).ConfigureAwait(false)
|
||||
?? throw new InvalidOperationException(Loc.L("Archive.Log.SequenceMissing", sequenceId));
|
||||
var archive = await db.Archives.GetAsync(sequence.ArchiveId, ct).ConfigureAwait(false)
|
||||
?? throw new InvalidOperationException(Loc.L("Errors.Sync.NoArchive"));
|
||||
var projectHome = new ArchiveProjectHome(db, archives.Projects, sequenceId);
|
||||
await SendAsync(context, project, workspace, entries, () => projectHome, held: false,
|
||||
(entry, file, token) => FileAsync(sequence, archive, entry, file, token)).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
using var remote = (servers is null ? null : await servers.ConnectAsync(server, ct).ConfigureAwait(false))
|
||||
?? throw new InvalidOperationException(Loc.L("Errors.Sync.NotConnected"));
|
||||
await SendAsync(context, project, workspace, entries,
|
||||
() => servers!.ProjectHome(home) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.NotConnected")), held: true,
|
||||
(entry, file, token) => UploadAsync(remote, sequenceId, entry, file, token)).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The entries to send, oldest first: those of <paramref name="renderIds"/> the ledger holds unsent (an id it no longer
|
||||
/// holds was discarded meanwhile, one sent already is left), every unsent one when none is named.
|
||||
/// </summary>
|
||||
private static async Task<IReadOnlyList<WorkspaceRenderEntry>> ToSendAsync(WorkspaceProject project, IReadOnlyList<Guid>? renderIds, CancellationToken ct)
|
||||
{
|
||||
WorkspaceRenders ledger;
|
||||
using (await project.LockAsync(LockWait, ct).ConfigureAwait(false)) ledger = project.ReadRenders();
|
||||
if (renderIds is not { Count: > 0 }) return ledger.Unsent;
|
||||
var named = renderIds.ToHashSet();
|
||||
return [.. ledger.Entries.Where(e => named.Contains(e.RenderId) && e.IsUnsent)];
|
||||
}
|
||||
|
||||
/// <summary>Delivers one checked render home: true when the home took it now, false when it held it already.</summary>
|
||||
private delegate Task<bool> Deliver(WorkspaceRenderEntry entry, string file, CancellationToken cancellationToken);
|
||||
|
||||
private async Task SendAsync(JobContext context, WorkspaceProject project, BundleWorkspace workspace, IReadOnlyList<WorkspaceRenderEntry> entries,
|
||||
Func<IProjectHome> home, bool held, Deliver deliver)
|
||||
{
|
||||
var ct = context.Cancellation;
|
||||
var (sent, there, refused) = (0, 0, 0);
|
||||
// The versions this run found at the home, or sent there first.
|
||||
var versions = new HashSet<Guid>();
|
||||
for (var i = 0; i < entries.Count; i++)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
if (held) await WhileHeldAsync(context).ConfigureAwait(false);
|
||||
var entry = entries[i];
|
||||
var file = project.ExportFile(entry.File);
|
||||
var refusal = await CheckAsync(project, entry, file, ct).ConfigureAwait(false);
|
||||
if (refusal is null)
|
||||
{
|
||||
if (entry.ProjectVersionId is { } versionId && !versions.Contains(versionId))
|
||||
{
|
||||
await EnsureVersionAsync(context, project, workspace, home(), entry.ProjectId, versionId, ct).ConfigureAwait(false);
|
||||
versions.Add(versionId);
|
||||
}
|
||||
try
|
||||
{
|
||||
if (await deliver(entry, file, ct).ConfigureAwait(false)) sent++;
|
||||
else there++;
|
||||
// The home holds it: recorded whatever stops the job now.
|
||||
await UpdateAsync(project, entry.RenderId, e => e.AsSent(DateTimeOffset.UtcNow)).ConfigureAwait(false);
|
||||
}
|
||||
catch (SendRefusedException ex)
|
||||
{
|
||||
refusal = ex.Message;
|
||||
}
|
||||
}
|
||||
if (refusal is not null)
|
||||
{
|
||||
refused++;
|
||||
context.Log(refusal);
|
||||
await UpdateAsync(project, entry.RenderId, e => e.AsFailed(refusal)).ConfigureAwait(false);
|
||||
}
|
||||
await context.ReportAsync(i + 1, entries.Count).ConfigureAwait(false);
|
||||
}
|
||||
context.Log(Loc.L("Archive.Log.WorkspaceSent", workspace.Name, sent, there));
|
||||
if (refused > 0) throw new InvalidOperationException(Loc.L("Errors.Workspace.SendFailed", refused, entries.Count));
|
||||
}
|
||||
|
||||
/// <summary>Why the render cannot be sent as it is (gone from the exports, or another file than the one rendered); null when it can.</summary>
|
||||
private static async Task<string?> CheckAsync(WorkspaceProject project, WorkspaceRenderEntry entry, string file, CancellationToken ct)
|
||||
{
|
||||
var name = Path.GetFileName(entry.File);
|
||||
var exports = Path.TrimEndingDirectorySeparator(Path.GetFullPath(project.ExportsDirectory)) + Path.DirectorySeparatorChar;
|
||||
// A ledger names files inside the exports only; one pointing elsewhere is never read.
|
||||
if (!Path.GetFullPath(file).StartsWith(exports, StringComparison.OrdinalIgnoreCase) || !File.Exists(file)) return Loc.L("Errors.Workspace.SendMissing", name);
|
||||
if (new FileInfo(file).Length != entry.Size
|
||||
|| !string.Equals(await ChronosClient.HashFileAsync(file, ct).ConfigureAwait(false), entry.Sha256, StringComparison.OrdinalIgnoreCase))
|
||||
return Loc.L("Errors.Workspace.SendHashMismatch", name);
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Version <paramref name="versionId"/> of the project at the home: pushed there first (every local version, in order, as a
|
||||
/// sync pushes them) when the home lacks it. A push that does not bring it there fails the job.
|
||||
/// </summary>
|
||||
private static async Task EnsureVersionAsync(JobContext context, WorkspaceProject project, BundleWorkspace workspace, IProjectHome home, Guid projectId,
|
||||
Guid versionId, CancellationToken ct)
|
||||
{
|
||||
if (await home.GetVersionAsync(projectId, versionId, ct).ConfigureAwait(false) is not null) return;
|
||||
try
|
||||
{
|
||||
await new WorkspaceSync(project.Store, Environment.MachineName) { LockWait = LockWait }.PushAsync(projectId, home, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is HomeUnavailableException or WorkspaceSyncException or ProjectException or LeaseUnavailableException
|
||||
or InvalidOperationException)
|
||||
{
|
||||
context.Log(ex.Message);
|
||||
throw new InvalidOperationException(Loc.L("Errors.Workspace.SendVersion"), ex);
|
||||
}
|
||||
var pushed = await home.GetVersionAsync(projectId, versionId, ct).ConfigureAwait(false)
|
||||
?? throw new InvalidOperationException(Loc.L("Errors.Workspace.SendVersion"));
|
||||
context.Log(Loc.L("Archive.Log.WorkspaceVersionPushed", workspace.Name, pushed.Number));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The render filed in this PC's archive, the home's sequence: copied into the archive's staging folder, read back against
|
||||
/// the ledger's SHA-256, then filed under the sequence's lease with the project and version (once both are checked to be
|
||||
/// the sequence's). A render id the index holds already for this sequence is the same render, sent before.
|
||||
/// </summary>
|
||||
private async Task<bool> FileAsync(Sequence sequence, Archive archive, WorkspaceRenderEntry entry, string file, CancellationToken ct)
|
||||
{
|
||||
if (await db.Renders.GetAsync(entry.RenderId, ct).ConfigureAwait(false) is { } known)
|
||||
return known.SequenceId == sequence.Id ? false : throw new SendRefusedException(Loc.L("Server.Error.RenderExists"));
|
||||
if (!await ProjectOfAsync(sequence.Id, entry, ct).ConfigureAwait(false)) throw new SendRefusedException(Loc.L("Server.Error.ProjectMismatch"));
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
SyncTransfers.EnsureSpace(paths.Root, entry.Size);
|
||||
Directory.CreateDirectory(paths.StagingDirectory);
|
||||
var staged = Path.Combine(paths.StagingDirectory, "workspace-render-" + entry.RenderId.ToString("N") + ".part");
|
||||
try
|
||||
{
|
||||
await using (var source = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 20, FileOptions.Asynchronous | FileOptions.SequentialScan))
|
||||
await using (var target = new FileStream(staged, FileMode.Create, FileAccess.Write, FileShare.None, 1 << 20, FileOptions.Asynchronous))
|
||||
{
|
||||
await source.CopyToAsync(target, ct).ConfigureAwait(false);
|
||||
target.Flush(flushToDisk: true);
|
||||
}
|
||||
// Read back from the archive's volume: what is filed is the render as it was written.
|
||||
if (!string.Equals(await ChronosClient.HashFileAsync(staged, ct).ConfigureAwait(false), entry.Sha256, StringComparison.OrdinalIgnoreCase))
|
||||
throw new IOException(Loc.L("Errors.Workspace.CopyMismatch", staged));
|
||||
var name = SyncTransfers.SafeFileName(Path.GetFileName(entry.File)) ?? entry.RenderId.ToString("N");
|
||||
var filed = await SyncTransfers.FileRenderAsync(archives, db, sequence, paths, staged, name, new Render
|
||||
{
|
||||
Id = entry.RenderId,
|
||||
SequenceId = sequence.Id,
|
||||
PresetName = entry.PresetName,
|
||||
OutputPath = string.Empty,
|
||||
Codec = entry.Codec,
|
||||
Width = entry.Width,
|
||||
Height = entry.Height,
|
||||
Fps = entry.Fps,
|
||||
DurationSeconds = entry.DurationSeconds,
|
||||
Created = entry.Rendered,
|
||||
ProducedBy = options.Processing.ProducedBy,
|
||||
ProjectId = entry.ProjectId,
|
||||
ProjectVersionId = entry.ProjectVersionId,
|
||||
}, ct).ConfigureAwait(false);
|
||||
return filed is not null ? true : throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
|
||||
}
|
||||
finally
|
||||
{
|
||||
TryDelete(staged);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The project the entry names is one of the sequence's, and its version (when it names one) one of that project's.</summary>
|
||||
private async Task<bool> ProjectOfAsync(Guid sequenceId, WorkspaceRenderEntry entry, CancellationToken ct)
|
||||
{
|
||||
var project = await db.Projects.GetAsync(entry.ProjectId, ct).ConfigureAwait(false);
|
||||
var version = entry.ProjectVersionId is { } versionId ? await db.Projects.GetVersionAsync(versionId, ct).ConfigureAwait(false) : null;
|
||||
if (project is null || (entry.ProjectVersionId is not null && version is null))
|
||||
{
|
||||
// One the index does not know yet may be in the sequence's sidecars: listing reads them first.
|
||||
await archives.Projects.ListAsync(sequenceId, ct).ConfigureAwait(false);
|
||||
project = await db.Projects.GetAsync(entry.ProjectId, ct).ConfigureAwait(false);
|
||||
if (entry.ProjectVersionId is { } id) version = await db.Projects.GetVersionAsync(id, ct).ConfigureAwait(false);
|
||||
}
|
||||
return project is not null && project.SequenceId == sequenceId && (entry.ProjectVersionId is null || version?.ProjectId == project.Id);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The render uploaded to its server's sequence, with its project and version, resumable and checked against its SHA-256
|
||||
/// there; the server answers that it holds the render already when it does. Refused for good (the server takes the
|
||||
/// project for another sequence's, the id for another sequence's render, the file for too large or not the one hashed):
|
||||
/// marked failed; anything else (the server out of reach, busy, the sign-in expired) fails the job, the entry left as it was.
|
||||
/// </summary>
|
||||
private async Task<bool> UploadAsync(ISyncRemote remote, Guid sequenceId, WorkspaceRenderEntry entry, string file, CancellationToken ct)
|
||||
{
|
||||
var name = SyncTransfers.SafeFileName(Path.GetFileName(entry.File)) ?? entry.RenderId.ToString("N");
|
||||
var request = new RenderUploadRequest(entry.RenderId, name, entry.Size, entry.Sha256.ToLowerInvariant(), entry.PresetName, entry.Codec, entry.Width, entry.Height,
|
||||
entry.Fps, entry.DurationSeconds, options.Processing.ProducedBy, entry.ProjectId, entry.ProjectVersionId);
|
||||
try
|
||||
{
|
||||
return await remote.UploadRenderAsync(sequenceId, file, request, ct).ConfigureAwait(false) is not null;
|
||||
}
|
||||
catch (ChronosApiException ex) when (ex.Status is HttpStatusCode.BadRequest or HttpStatusCode.Conflict or HttpStatusCode.RequestEntityTooLarge
|
||||
or HttpStatusCode.UnprocessableEntity)
|
||||
{
|
||||
throw new SendRefusedException(string.IsNullOrWhiteSpace(ex.Error.Message) ? ex.Message : ex.Error.Message);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Waits while transfers with a server are held (a metered connection), saying so once.</summary>
|
||||
private async Task WhileHeldAsync(JobContext context)
|
||||
{
|
||||
if (!options.HoldTransfers()) return;
|
||||
context.Log(Loc.L("Archive.Log.WorkspaceSendHeld"));
|
||||
while (options.HoldTransfers()) await Task.Delay(HoldPoll, context.Cancellation).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The ledger entry of <paramref name="renderId"/> changed by <paramref name="change"/>, under the project's lease, read again
|
||||
/// first (a render finished meanwhile stays); an entry discarded meanwhile stays discarded. Written whatever stops the job.
|
||||
/// </summary>
|
||||
private static async Task UpdateAsync(WorkspaceProject project, Guid renderId, Func<WorkspaceRenderEntry, WorkspaceRenderEntry> change)
|
||||
{
|
||||
using (await project.LockAsync(LockWait, CancellationToken.None).ConfigureAwait(false))
|
||||
{
|
||||
var ledger = project.ReadRenders();
|
||||
if (ledger.Find(renderId) is { } entry) project.WriteRenders(ledger.With(change(entry)));
|
||||
}
|
||||
}
|
||||
|
||||
private static void TryDelete(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (File.Exists(path)) File.Delete(path);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Debug("workspace", $"{path} could not be removed: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The home will not take this render as it is: it is marked failed with this message, and the others go on.</summary>
|
||||
private sealed class SendRefusedException(string message) : Exception(message);
|
||||
}
|
||||
Reference in new issue
Block a user