diff --git a/src/Chronos.App/Infrastructure/AppSettings.cs b/src/Chronos.App/Infrastructure/AppSettings.cs index a2e2e6ac..3780f173 100644 --- a/src/Chronos.App/Infrastructure/AppSettings.cs +++ b/src/Chronos.App/Infrastructure/AppSettings.cs @@ -236,7 +236,7 @@ public sealed class AppSettings /// Send and take project settings changed on one side without asking (Server.html). public bool SyncSettingsAutomatically { get; set; } = true; - /// Pause upload jobs while Windows reports the connection as metered (Server.html). + /// Pause transfers with a server (uploads, workspace copies and renders sent) while Windows reports the connection as metered (Server.html). public bool PauseUploadsOnMetered { get; set; } // ---- Jobs ---- diff --git a/src/Chronos.App/Sections/ServerView.xaml b/src/Chronos.App/Sections/ServerView.xaml index 8fc9283e..3e039e1c 100644 --- a/src/Chronos.App/Sections/ServerView.xaml +++ b/src/Chronos.App/Sections/ServerView.xaml @@ -160,7 +160,7 @@ - + diff --git a/src/Chronos.App/Server/UploadHold.cs b/src/Chronos.App/Server/UploadHold.cs index 5cee850c..8bfd77ed 100644 --- a/src/Chronos.App/Server/UploadHold.cs +++ b/src/Chronos.App/Server/UploadHold.cs @@ -4,14 +4,17 @@ using Chronos.App.Infrastructure; using Chronos.Core.Diagnostics; using Chronos.Core.Model; using Chronos.Data.Jobs; +using Chronos.Data.Workspaces; namespace Chronos.App.Server; /// -/// "Pause uploads on metered networks" (Server.html): while is on and -/// Windows reports the connection as metered, upload jobs are paused (a running one stops after its current chunk and -/// keeps its checkpoint); they are resumed when the connection is no longer metered or the option goes off. Only the -/// uploads it paused are resumed. Checked when the network or an upload job changes, and every minute. +/// "Pause metered transfers" (Server.html): while is on and Windows reports +/// the connection as metered, the transfers with a server are paused (a running one stops after its current chunk or file +/// and keeps its checkpoint): upload jobs, and since phase 5c the workspace's copies from a server and its renders sent to +/// one (); a copy from or a send to this PC's archive is never held. They are resumed when the connection +/// is no longer metered or the option goes off. Only the jobs it paused are resumed. Checked when the network or one of +/// those jobs changes, and every minute. /// public sealed class UploadHold : IDisposable { @@ -58,7 +61,26 @@ public sealed class UploadHold : IDisposable private void OnJobChanged(Job job) { - if (job is { Kind: JobKind.UploadCommit, State: JobState.Queued or JobState.Running }) Request(); + if (job is { State: JobState.Queued or JobState.Running } && Holds(job)) Request(); + } + + /// The kinds of job a metered connection may hold ( says which of them). + internal static IReadOnlyList Kinds { get; } = [JobKind.UploadCommit, JobKind.WorkspaceCopy, JobKind.WorkspaceSend]; + + /// + /// Whether a metered connection holds : an upload, or a workspace copy or send whose home is a server + /// (its parameters name one). One whose parameters cannot be read is not held: it fails with its reason. + /// + internal static bool Holds(Job job) + { + ArgumentNullException.ThrowIfNull(job); + return job.Kind switch + { + JobKind.UploadCommit => true, + JobKind.WorkspaceCopy => WorkspaceJobs.CopyParameters(job)?.Server is { Length: > 0 }, + JobKind.WorkspaceSend => WorkspaceJobs.SendParameters(job)?.Server is { Length: > 0 }, + _ => false, + }; } private async Task ApplyAsync() @@ -71,10 +93,13 @@ public sealed class UploadHold : IDisposable { foreach (var state in new[] { JobState.Queued, JobState.Running }) { - foreach (var job in (await db.Jobs.ListAsync(state, JobKind.UploadCommit, 500)).Items) + foreach (var kind in Kinds) { - await engine.PauseAsync(job.Id); - _held.Add(job.Id); + foreach (var job in (await db.Jobs.ListAsync(state, kind, 500)).Items.Where(Holds)) + { + await engine.PauseAsync(job.Id); + _held.Add(job.Id); + } } } return; @@ -87,7 +112,7 @@ public sealed class UploadHold : IDisposable } catch (Data.DatabaseException ex) { - Log.Warning("sync", "Uploads could not be paused or resumed for the metered connection", ex); + Log.Warning("sync", "Transfers could not be paused or resumed for the metered connection", ex); } finally { diff --git a/src/Chronos.Data/Sync/SyncRemote.cs b/src/Chronos.Data/Sync/SyncRemote.cs index ef1c01a2..3ef8b7cb 100644 --- a/src/Chronos.Data/Sync/SyncRemote.cs +++ b/src/Chronos.Data/Sync/SyncRemote.cs @@ -92,6 +92,13 @@ public interface ISyncConnector { /// A signed-in remote of owned by the caller; null when this PC is not connected to that server. ValueTask ConnectAsync(string server, CancellationToken cancellationToken); + + /// + /// The project home of a workspace project whose home is (a server's sequence) over the same + /// signed-in session: where a send of its renders pushes the version they name first (phase 5c). Null when this PC is not + /// connected to that server, or the connection keeps no projects. + /// + Projects.IProjectHome? ProjectHome(Core.Project.BundleHome home) => null; } /// over a signed-in , which it disposes when it owns it. diff --git a/src/Chronos.Data/Sync/SyncTransfers.cs b/src/Chronos.Data/Sync/SyncTransfers.cs index a8a816f6..117ad8ce 100644 --- a/src/Chronos.Data/Sync/SyncTransfers.cs +++ b/src/Chronos.Data/Sync/SyncTransfers.cs @@ -500,44 +500,58 @@ internal sealed class SyncTransfers(SyncService sync) foreach (var render in wanted) { var (part, name) = await FetchRenderAsync(remote, render, staging, ct).ConfigureAwait(false); - // Filed under the sequence's lease into its folder as the index has it then, as a server files a render: a - // Reorganize may have moved it since the sequence was read, and moves the row's path along when it moves it later. - // One another index moved is found by this index's next scan; until then nothing is filed (the download stays - // staged for the next attempt), so the folder it left is not created again, nor filed into when it holds another - // sequence now. - var filed = await sync.Archives.UnderLeaseAsync(local, async row => + var filed = await FileRenderAsync(sync.Archives, db, local, paths, part, name, new Render { - if (row is null) throw new InvalidOperationException(Loc.L("Errors.Sync.LocalMissing")); - var folder = paths.Sequence(row.Folder); - if (!SequenceFolders.Holds(folder, row.Id)) return false; - Directory.CreateDirectory(folder.Exports); - var target = Unique(folder.Exports, name); - File.Move(part, target); - // Recorded even if the job is being cancelled: a filed export is never left without its row. - await db.Renders.InsertAsync(new Render - { - Id = render.Id, - SequenceId = row.Id, - PresetName = render.PresetName, - OutputPath = paths.ToRelative(target), - Codec = render.Codec, - Width = render.Width, - Height = render.Height, - Fps = render.Fps, - DurationSeconds = render.DurationSeconds, - Bytes = new FileInfo(target).Length, - Created = render.Created, - ProducedBy = render.ProducedBy, - }, CancellationToken.None).ConfigureAwait(false); - return true; + Id = render.Id, + SequenceId = local.Id, + PresetName = render.PresetName, + OutputPath = string.Empty, + Codec = render.Codec, + Width = render.Width, + Height = render.Height, + Fps = render.Fps, + DurationSeconds = render.DurationSeconds, + Created = render.Created, + ProducedBy = render.ProducedBy, }, ct).ConfigureAwait(false); - if (!filed) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved")); + if (filed is null) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved")); await progress.StepAsync().ConfigureAwait(false); } TryDeleteDirectory(staging); await sync.RecompareAsync(remote, remoteId, ct).ConfigureAwait(false); } + /// + /// Files the render staged at (on the archive's volume) into the exports of + /// under its lease, into its folder as the index has it then, as a server files a render: a + /// Reorganize may have moved it since the sequence was read, and moves the row's path along when it moves it later. The + /// file takes , or "name (2)" and so on (an export is never replaced), and the index records + /// for it, with the sequence, the path and the size it has there (recorded even if the job is + /// being cancelled: a filed export is never left without its row). Used by the sync's downloads of a server's exports and + /// by a workspace's renders sent home (). Returns the archive-relative path + /// it was filed at; null when the folder does not hold the sequence now (another index moved it; this index's next scan + /// finds it): nothing is filed then, the staged file stays for the next attempt, and the folder it left is not created + /// again, nor filed into when it holds another sequence now. + /// + /// The sequence left the index. + /// The archive is offline or read-only. + /// Another writer held the sequence for longer than the lease wait. + internal static Task FileRenderAsync(ArchiveService archives, ChronosDatabase db, Sequence local, ArchivePaths paths, string part, string name, + Render render, CancellationToken cancellationToken) => + archives.UnderLeaseAsync(local, async row => + { + if (row is null) throw new InvalidOperationException(Loc.L("Errors.Sync.LocalMissing")); + var folder = paths.Sequence(row.Folder); + if (!SequenceFolders.Holds(folder, row.Id)) return null; + Directory.CreateDirectory(folder.Exports); + var target = Unique(folder.Exports, name); + File.Move(part, target); + var relative = paths.ToRelative(target); + await db.Renders.InsertAsync(render with { SequenceId = row.Id, OutputPath = relative, Bytes = new FileInfo(target).Length }, CancellationToken.None) + .ConfigureAwait(false); + return (string?)relative; + }, cancellationToken); + /// A render's file in staging (resumed with Range; started over once if the partial copy no longer fits) and the name the server gives it. private static async Task<(string Part, string Name)> FetchRenderAsync(ISyncRemote remote, RenderDto render, string staging, CancellationToken cancellationToken) { @@ -833,7 +847,7 @@ internal sealed class SyncTransfers(SyncService sync) private static Task EnqueueFollowUpAsync(JobContext context, ArchiveJobRequest request, CancellationToken cancellationToken) => context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, cancellationToken); - private static void EnsureSpace(string root, long bytes) + internal static void EnsureSpace(string root, long bytes) { long? free; try diff --git a/src/Chronos.Data/Workspaces/WorkspaceSendJobHandler.cs b/src/Chronos.Data/Workspaces/WorkspaceSendJobHandler.cs index 2434dca6..9b99f99a 100644 --- a/src/Chronos.Data/Workspaces/WorkspaceSendJobHandler.cs +++ b/src/Chronos.Data/Workspaces/WorkspaceSendJobHandler.cs @@ -1,25 +1,303 @@ +using System.Net; +using Chronos.Core.Archiving; +using Chronos.Core.Diagnostics; +using Chronos.Core.Localization; using Chronos.Core.Model; +using Chronos.Core.Project; using Chronos.Data.Archiving; using Chronos.Data.Jobs; +using Chronos.Data.Projects; using Chronos.Data.Sync; +using Chronos.Protocol; +using Chronos.Protocol.Client; namespace Chronos.Data.Workspaces; /// /// (phase 5c, ): the local renders of a workspace project -/// sent home with their project and version, per unsent ledger entry: the file hashed again (another is refused and stays), -/// the version it names pushed first when the home lacks it, then filed in this PC's archive (as the sync files a server's -/// exports) or uploaded to its server (RenderUploadRequest with ProjectId and ProjectVersionId), and the -/// entry marked sent. Idempotent by render id: a send repeated after a cut finds the home holding it already. +/// sent home with their project and version, per unsent ledger entry (; the ids the job names, +/// or every unsent one): the file hashed again (one missing or changed since it was rendered is refused, marked failed in the +/// ledger and left where it is), the version it names pushed first when the home lacks it (a push that fails fails the job, +/// the entry left as it was), then filed in this PC's archive (copied into the archive's staging folder, read back against +/// its SHA-256, and filed under the sequence's lease as the sync files a server's exports, +/// , after checking the project and version are the sequence's) or uploaded to its +/// server ( with ProjectId and ProjectVersionId, which the server checks the +/// same way), and the entry marked sent. The file stays in exports/ until a clean-up removes it. +/// Idempotent by render id: a send repeated after a cut finds the home holding the render already and only marks it sent; a +/// render id the home files under another sequence is refused. The ledger is read and written under the project's lease, +/// each entry read again before it is written, so a render finished meanwhile is kept. +/// A send to a server is held while says so (a metered connection): it waits +/// before each upload, and the desktop pauses it meanwhile. A send to this PC's archive is never held. /// internal sealed class WorkspaceSendJobHandler(ChronosDatabase db, ArchiveService archives, ISyncConnector? servers, WorkspaceJobOptions options) : IJobHandler { + private static readonly TimeSpan LockWait = TimeSpan.FromMinutes(2); + + /// How often a send held on a metered connection looks again. + internal static TimeSpan HoldPoll { get; set; } = TimeSpan.FromSeconds(2); + public JobKind Kind => JobKind.WorkspaceSend; - public Task RunAsync(JobContext context) + public async Task RunAsync(JobContext context) { ArgumentNullException.ThrowIfNull(context); - _ = (db, archives, servers, options); - throw new NotImplementedException("Sending a workspace project's renders home comes with phase 5c (P3)."); + var ct = context.Cancellation; + var parameters = WorkspaceJobs.SendParameters(context.Job) ?? throw new InvalidDataException(Loc.L("Errors.Workspace.JobParameters")); + if (context.Job.TargetIds is not [var sequenceId] || string.IsNullOrWhiteSpace(parameters.WorkspaceRoot)) + throw new InvalidDataException(Loc.L("Errors.Workspace.JobParameters")); + var server = parameters.Server is { Length: > 0 } named ? named : null; + var project = new WorkspaceStore(parameters.WorkspaceRoot).Find(parameters.ProjectId) + ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.NotCheckedOut")); + var workspace = project.ReadBundle().Workspace ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.NotCheckedOut")); + var home = workspace.Home; + if (home.SequenceId != sequenceId || (server is null + ? !home.IsArchive + : home.Kind != BundleHome.ServerKind || !string.Equals(home.Server, server, StringComparison.OrdinalIgnoreCase))) + throw new InvalidOperationException(Loc.L("Errors.Workspace.OtherHome")); + + var entries = await ToSendAsync(project, parameters.RenderIds, ct).ConfigureAwait(false); + await context.ReportAsync(0, entries.Count).ConfigureAwait(false); + if (entries.Count == 0) + { + context.Log(Loc.L("Archive.Log.WorkspaceSent", workspace.Name, 0, 0)); + return; + } + if (server is null) + { + var sequence = await db.Sequences.GetAsync(sequenceId, ct).ConfigureAwait(false) + ?? throw new InvalidOperationException(Loc.L("Archive.Log.SequenceMissing", sequenceId)); + var archive = await db.Archives.GetAsync(sequence.ArchiveId, ct).ConfigureAwait(false) + ?? throw new InvalidOperationException(Loc.L("Errors.Sync.NoArchive")); + var projectHome = new ArchiveProjectHome(db, archives.Projects, sequenceId); + await SendAsync(context, project, workspace, entries, () => projectHome, held: false, + (entry, file, token) => FileAsync(sequence, archive, entry, file, token)).ConfigureAwait(false); + return; + } + using var remote = (servers is null ? null : await servers.ConnectAsync(server, ct).ConfigureAwait(false)) + ?? throw new InvalidOperationException(Loc.L("Errors.Sync.NotConnected")); + await SendAsync(context, project, workspace, entries, + () => servers!.ProjectHome(home) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.NotConnected")), held: true, + (entry, file, token) => UploadAsync(remote, sequenceId, entry, file, token)).ConfigureAwait(false); } + + /// + /// The entries to send, oldest first: those of the ledger holds unsent (an id it no longer + /// holds was discarded meanwhile, one sent already is left), every unsent one when none is named. + /// + private static async Task> ToSendAsync(WorkspaceProject project, IReadOnlyList? renderIds, CancellationToken ct) + { + WorkspaceRenders ledger; + using (await project.LockAsync(LockWait, ct).ConfigureAwait(false)) ledger = project.ReadRenders(); + if (renderIds is not { Count: > 0 }) return ledger.Unsent; + var named = renderIds.ToHashSet(); + return [.. ledger.Entries.Where(e => named.Contains(e.RenderId) && e.IsUnsent)]; + } + + /// Delivers one checked render home: true when the home took it now, false when it held it already. + private delegate Task Deliver(WorkspaceRenderEntry entry, string file, CancellationToken cancellationToken); + + private async Task SendAsync(JobContext context, WorkspaceProject project, BundleWorkspace workspace, IReadOnlyList entries, + Func home, bool held, Deliver deliver) + { + var ct = context.Cancellation; + var (sent, there, refused) = (0, 0, 0); + // The versions this run found at the home, or sent there first. + var versions = new HashSet(); + for (var i = 0; i < entries.Count; i++) + { + ct.ThrowIfCancellationRequested(); + if (held) await WhileHeldAsync(context).ConfigureAwait(false); + var entry = entries[i]; + var file = project.ExportFile(entry.File); + var refusal = await CheckAsync(project, entry, file, ct).ConfigureAwait(false); + if (refusal is null) + { + if (entry.ProjectVersionId is { } versionId && !versions.Contains(versionId)) + { + await EnsureVersionAsync(context, project, workspace, home(), entry.ProjectId, versionId, ct).ConfigureAwait(false); + versions.Add(versionId); + } + try + { + if (await deliver(entry, file, ct).ConfigureAwait(false)) sent++; + else there++; + // The home holds it: recorded whatever stops the job now. + await UpdateAsync(project, entry.RenderId, e => e.AsSent(DateTimeOffset.UtcNow)).ConfigureAwait(false); + } + catch (SendRefusedException ex) + { + refusal = ex.Message; + } + } + if (refusal is not null) + { + refused++; + context.Log(refusal); + await UpdateAsync(project, entry.RenderId, e => e.AsFailed(refusal)).ConfigureAwait(false); + } + await context.ReportAsync(i + 1, entries.Count).ConfigureAwait(false); + } + context.Log(Loc.L("Archive.Log.WorkspaceSent", workspace.Name, sent, there)); + if (refused > 0) throw new InvalidOperationException(Loc.L("Errors.Workspace.SendFailed", refused, entries.Count)); + } + + /// Why the render cannot be sent as it is (gone from the exports, or another file than the one rendered); null when it can. + private static async Task CheckAsync(WorkspaceProject project, WorkspaceRenderEntry entry, string file, CancellationToken ct) + { + var name = Path.GetFileName(entry.File); + var exports = Path.TrimEndingDirectorySeparator(Path.GetFullPath(project.ExportsDirectory)) + Path.DirectorySeparatorChar; + // A ledger names files inside the exports only; one pointing elsewhere is never read. + if (!Path.GetFullPath(file).StartsWith(exports, StringComparison.OrdinalIgnoreCase) || !File.Exists(file)) return Loc.L("Errors.Workspace.SendMissing", name); + if (new FileInfo(file).Length != entry.Size + || !string.Equals(await ChronosClient.HashFileAsync(file, ct).ConfigureAwait(false), entry.Sha256, StringComparison.OrdinalIgnoreCase)) + return Loc.L("Errors.Workspace.SendHashMismatch", name); + return null; + } + + /// + /// Version of the project at the home: pushed there first (every local version, in order, as a + /// sync pushes them) when the home lacks it. A push that does not bring it there fails the job. + /// + private static async Task EnsureVersionAsync(JobContext context, WorkspaceProject project, BundleWorkspace workspace, IProjectHome home, Guid projectId, + Guid versionId, CancellationToken ct) + { + if (await home.GetVersionAsync(projectId, versionId, ct).ConfigureAwait(false) is not null) return; + try + { + await new WorkspaceSync(project.Store, Environment.MachineName) { LockWait = LockWait }.PushAsync(projectId, home, ct).ConfigureAwait(false); + } + catch (Exception ex) when (ex is HomeUnavailableException or WorkspaceSyncException or ProjectException or LeaseUnavailableException + or InvalidOperationException) + { + context.Log(ex.Message); + throw new InvalidOperationException(Loc.L("Errors.Workspace.SendVersion"), ex); + } + var pushed = await home.GetVersionAsync(projectId, versionId, ct).ConfigureAwait(false) + ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.SendVersion")); + context.Log(Loc.L("Archive.Log.WorkspaceVersionPushed", workspace.Name, pushed.Number)); + } + + /// + /// The render filed in this PC's archive, the home's sequence: copied into the archive's staging folder, read back against + /// the ledger's SHA-256, then filed under the sequence's lease with the project and version (once both are checked to be + /// the sequence's). A render id the index holds already for this sequence is the same render, sent before. + /// + private async Task FileAsync(Sequence sequence, Archive archive, WorkspaceRenderEntry entry, string file, CancellationToken ct) + { + if (await db.Renders.GetAsync(entry.RenderId, ct).ConfigureAwait(false) is { } known) + return known.SequenceId == sequence.Id ? false : throw new SendRefusedException(Loc.L("Server.Error.RenderExists")); + if (!await ProjectOfAsync(sequence.Id, entry, ct).ConfigureAwait(false)) throw new SendRefusedException(Loc.L("Server.Error.ProjectMismatch")); + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + SyncTransfers.EnsureSpace(paths.Root, entry.Size); + Directory.CreateDirectory(paths.StagingDirectory); + var staged = Path.Combine(paths.StagingDirectory, "workspace-render-" + entry.RenderId.ToString("N") + ".part"); + try + { + await using (var source = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 20, FileOptions.Asynchronous | FileOptions.SequentialScan)) + await using (var target = new FileStream(staged, FileMode.Create, FileAccess.Write, FileShare.None, 1 << 20, FileOptions.Asynchronous)) + { + await source.CopyToAsync(target, ct).ConfigureAwait(false); + target.Flush(flushToDisk: true); + } + // Read back from the archive's volume: what is filed is the render as it was written. + if (!string.Equals(await ChronosClient.HashFileAsync(staged, ct).ConfigureAwait(false), entry.Sha256, StringComparison.OrdinalIgnoreCase)) + throw new IOException(Loc.L("Errors.Workspace.CopyMismatch", staged)); + var name = SyncTransfers.SafeFileName(Path.GetFileName(entry.File)) ?? entry.RenderId.ToString("N"); + var filed = await SyncTransfers.FileRenderAsync(archives, db, sequence, paths, staged, name, new Render + { + Id = entry.RenderId, + SequenceId = sequence.Id, + PresetName = entry.PresetName, + OutputPath = string.Empty, + Codec = entry.Codec, + Width = entry.Width, + Height = entry.Height, + Fps = entry.Fps, + DurationSeconds = entry.DurationSeconds, + Created = entry.Rendered, + ProducedBy = options.Processing.ProducedBy, + ProjectId = entry.ProjectId, + ProjectVersionId = entry.ProjectVersionId, + }, ct).ConfigureAwait(false); + return filed is not null ? true : throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved")); + } + finally + { + TryDelete(staged); + } + } + + /// The project the entry names is one of the sequence's, and its version (when it names one) one of that project's. + private async Task ProjectOfAsync(Guid sequenceId, WorkspaceRenderEntry entry, CancellationToken ct) + { + var project = await db.Projects.GetAsync(entry.ProjectId, ct).ConfigureAwait(false); + var version = entry.ProjectVersionId is { } versionId ? await db.Projects.GetVersionAsync(versionId, ct).ConfigureAwait(false) : null; + if (project is null || (entry.ProjectVersionId is not null && version is null)) + { + // One the index does not know yet may be in the sequence's sidecars: listing reads them first. + await archives.Projects.ListAsync(sequenceId, ct).ConfigureAwait(false); + project = await db.Projects.GetAsync(entry.ProjectId, ct).ConfigureAwait(false); + if (entry.ProjectVersionId is { } id) version = await db.Projects.GetVersionAsync(id, ct).ConfigureAwait(false); + } + return project is not null && project.SequenceId == sequenceId && (entry.ProjectVersionId is null || version?.ProjectId == project.Id); + } + + /// + /// The render uploaded to its server's sequence, with its project and version, resumable and checked against its SHA-256 + /// there; the server answers that it holds the render already when it does. Refused for good (the server takes the + /// project for another sequence's, the id for another sequence's render, the file for too large or not the one hashed): + /// marked failed; anything else (the server out of reach, busy, the sign-in expired) fails the job, the entry left as it was. + /// + private async Task UploadAsync(ISyncRemote remote, Guid sequenceId, WorkspaceRenderEntry entry, string file, CancellationToken ct) + { + var name = SyncTransfers.SafeFileName(Path.GetFileName(entry.File)) ?? entry.RenderId.ToString("N"); + var request = new RenderUploadRequest(entry.RenderId, name, entry.Size, entry.Sha256.ToLowerInvariant(), entry.PresetName, entry.Codec, entry.Width, entry.Height, + entry.Fps, entry.DurationSeconds, options.Processing.ProducedBy, entry.ProjectId, entry.ProjectVersionId); + try + { + return await remote.UploadRenderAsync(sequenceId, file, request, ct).ConfigureAwait(false) is not null; + } + catch (ChronosApiException ex) when (ex.Status is HttpStatusCode.BadRequest or HttpStatusCode.Conflict or HttpStatusCode.RequestEntityTooLarge + or HttpStatusCode.UnprocessableEntity) + { + throw new SendRefusedException(string.IsNullOrWhiteSpace(ex.Error.Message) ? ex.Message : ex.Error.Message); + } + } + + /// Waits while transfers with a server are held (a metered connection), saying so once. + private async Task WhileHeldAsync(JobContext context) + { + if (!options.HoldTransfers()) return; + context.Log(Loc.L("Archive.Log.WorkspaceSendHeld")); + while (options.HoldTransfers()) await Task.Delay(HoldPoll, context.Cancellation).ConfigureAwait(false); + } + + /// + /// The ledger entry of changed by , under the project's lease, read again + /// first (a render finished meanwhile stays); an entry discarded meanwhile stays discarded. Written whatever stops the job. + /// + private static async Task UpdateAsync(WorkspaceProject project, Guid renderId, Func change) + { + using (await project.LockAsync(LockWait, CancellationToken.None).ConfigureAwait(false)) + { + var ledger = project.ReadRenders(); + if (ledger.Find(renderId) is { } entry) project.WriteRenders(ledger.With(change(entry))); + } + } + + private static void TryDelete(string path) + { + try + { + if (File.Exists(path)) File.Delete(path); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Debug("workspace", $"{path} could not be removed: {ex.Message}"); + } + } + + /// The home will not take this render as it is: it is marked failed with this message, and the others go on. + private sealed class SendRefusedException(string message) : Exception(message); } diff --git a/src/Chronos.Protocol/Sequences.cs b/src/Chronos.Protocol/Sequences.cs index 1dcfe9be..617c9169 100644 --- a/src/Chronos.Protocol/Sequences.cs +++ b/src/Chronos.Protocol/Sequences.cs @@ -109,6 +109,13 @@ public sealed record ManifestFrame(Guid Id, int Index, string FileName, long Siz /// How the server's proxies are made (BRIEF §8.5): long edge in pixels (0 = full size) and JPEG quality. public sealed record ProxySpec(int LongEdge, int Quality); +/// A render of a sequence, filed under its exports. +/// +/// The project whose version was rendered (owner review 2, R8): a render node's output of a job that names one, or a render a +/// workspace made on a client and sent home (phase 5c); null for a render of the sequence's own settings, and from servers +/// before phase 5c. +/// +/// The project version rendered; null when none is named (see ). public sealed record RenderDto( Guid Id, string PresetName, @@ -119,7 +126,9 @@ public sealed record RenderDto( double DurationSeconds, long Bytes, DateTimeOffset Created, - string? ProducedBy); + string? ProducedBy, + Guid? ProjectId = null, + Guid? ProjectVersionId = null); /// Aggregates for the archive dashboard and the admin panel. /// Free bytes of the space volume: the server's space path when one is set, else the archive's volume. diff --git a/src/Chronos.Server/Api/Dto.cs b/src/Chronos.Server/Api/Dto.cs index 0a5b6720..32a2a16c 100644 --- a/src/Chronos.Server/Api/Dto.cs +++ b/src/Chronos.Server/Api/Dto.cs @@ -23,7 +23,8 @@ public static class Dto f.Id, f.SequenceId, f.Index, f.FileName, f.Size, f.Sha256, f.CapturedAt, f.UtcOffsetMinutes, f.ExposureTime, f.FNumber, f.Iso, f.FocalLength, f.Orientation, f.Width, f.Height, f.Format.ToString().ToLowerInvariant(), (long)f.Flags, f.PreviewWidth, f.PreviewHeight); - public static RenderDto Render(Render r) => new(r.Id, r.PresetName, r.Codec, r.Width, r.Height, r.Fps, r.DurationSeconds, r.Bytes, r.Created, r.ProducedBy); + public static RenderDto Render(Render r) => new(r.Id, r.PresetName, r.Codec, r.Width, r.Height, r.Fps, r.DurationSeconds, r.Bytes, r.Created, r.ProducedBy, + r.ProjectId, r.ProjectVersionId); public static JobDto Job(Job j) => new( j.Id, JobWire.Kind(j.Kind), j.TargetIds, j.TargetLabel, JobWire.State(j.State), JobWire.Priority(j.Priority), j.ProgressDone, diff --git a/src/Chronos.Server/Storage/RenderUploads.cs b/src/Chronos.Server/Storage/RenderUploads.cs index 80e55955..08747830 100644 --- a/src/Chronos.Server/Storage/RenderUploads.cs +++ b/src/Chronos.Server/Storage/RenderUploads.cs @@ -3,6 +3,7 @@ using System.Text.Json; using Chronos.Core.Archiving; using Chronos.Core.Model; using Chronos.Data; +using Chronos.Data.Archiving; using Chronos.Data.Repositories; using Chronos.Protocol; using Chronos.Server.Api; @@ -17,7 +18,12 @@ namespace Chronos.Server.Storage; /// Renders a client sends on request (BRIEF §14.5): chunked like frame uploads (same store, chunk endpoint and /// retention), verified against their SHA-256 at completion and filed by under the /// sequence's exports with the client's render id, so both sides recognise the same render. +/// A render a workspace made (phase 5c) names its project and version: both are checked against the sequence when the upload +/// starts and again when it completes (422 project_mismatch: the project is not one of the sequence's, or the version +/// not one of the project's) and recorded with the render; a version alone names its project. Without them a render is filed +/// as before, recording neither. /// +/// The archive engine whose projects are read when the index does not know one yet; one of its own when none is registered. public sealed class RenderUploads( ChronosDatabase db, ServerOptions options, @@ -26,7 +32,8 @@ public sealed class RenderUploads( RenderStore renders, EventHub events, VolumeSpace volumes, - TimeProvider time) + TimeProvider time, + ArchiveService? archives = null) { public async Task> StartAsync(Caller caller, Guid sequenceId, RenderUploadRequest request, CancellationToken cancellationToken) { @@ -45,6 +52,8 @@ public sealed class RenderUploads( ? new StartUploadResponse(null, options.ChunkSizeBytes, [], AlreadyExists: true, null) : Outcome.Fail(409, "render_exists"); } + // Checked before a byte is sent, and again at completion. + if (await ProjectOfAsync(sequenceId, request, cancellationToken) is null) return Outcome.Fail(422, "project_mismatch"); var sha = request.Sha256.ToLowerInvariant(); if (await db.Uploads.FindReceivingAsync(caller.Id, sha, cancellationToken) is { } open && open.SequenceId == sequenceId && open.Size == request.Size && Read(open)?.RenderId == request.RenderId && store.Exists(open.Id)) @@ -81,6 +90,8 @@ public sealed class RenderUploads( if (upload.State != UploadState.Receiving) return Outcome.Fail(409, "upload_state"); if (!upload.IsComplete) return Outcome.Fail(409, "incomplete", (upload.ChunkCount - upload.ReceivedCount).ToString(CultureInfo.InvariantCulture)); + // Asked again: the upload may have been started before the project was deleted from the index, or by an earlier server. + if (await ProjectOfAsync(sequenceId, output, cancellationToken) is not { } provenance) return Outcome.Fail(422, "project_mismatch"); if (!uploads.ArchiveHasRoom(upload)) return Outcome.Fail(507, "insufficient_storage"); if (!await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Receiving, UploadState.Committing, cancellationToken: cancellationToken)) return Outcome.Fail(409, "upload_state"); @@ -93,7 +104,7 @@ public sealed class RenderUploads( try { render = sequence is null ? null : await renders.CommitAsync(sequence, staged, result, output.ProducedBy ?? caller.ClientName ?? caller.User.Username, - output.RenderId, cancellationToken); + output.RenderId, provenance.ProjectId, provenance.ProjectVersionId, cancellationToken); } catch (Exception ex) when (ex is LeaseUnavailableException or ArchiveOfflineException or FolderMovedException) { @@ -113,6 +124,33 @@ public sealed class RenderUploads( return Dto.Render(render); } + /// + /// The project and version names, checked against sequence : the + /// project one of its own (deleted ones included: their versions are kept), the version one of that project's (a version + /// alone names its project). One the index does not know yet may be in the sequence's sidecars: listing reads them first. + /// Both null when the request names neither; null when they do not hold. + /// + private async Task<(Guid? ProjectId, Guid? ProjectVersionId)?> ProjectOfAsync(Guid sequenceId, RenderUploadRequest request, CancellationToken cancellationToken) + { + if (request.ProjectId is null && request.ProjectVersionId is null) return (null, null); + var (project, version) = await FindAsync(); + if (project is null || (request.ProjectVersionId is not null && version is null)) + { + await LibraryEndpoints.Engine(archives, db).Projects.ListAsync(sequenceId, cancellationToken); + (project, version) = await FindAsync(); + } + if (project is null || project.SequenceId != sequenceId || (version is not null && version.ProjectId != project.Id)) return null; + return (project.Id, version?.Id); + + async Task<(SequenceProject? Project, ProjectVersion? Version)> FindAsync() + { + var known = request.ProjectVersionId is { } versionId ? await db.Projects.GetVersionAsync(versionId, cancellationToken) : null; + if (request.ProjectVersionId is not null && known is null) return (null, null); + var projectId = request.ProjectId ?? known!.ProjectId; + return (await db.Projects.GetAsync(projectId, cancellationToken), known); + } + } + private static RenderUploadRequest? Read(Upload upload) { if (upload.RenderJson is not { } json) return null; diff --git a/tests/Chronos.App.Tests/Server/UploadHoldTests.cs b/tests/Chronos.App.Tests/Server/UploadHoldTests.cs new file mode 100644 index 00000000..e2d9844e --- /dev/null +++ b/tests/Chronos.App.Tests/Server/UploadHoldTests.cs @@ -0,0 +1,62 @@ +using Chronos.App.Server; +using Chronos.Core.Model; +using Chronos.Data.Archiving.Jobs; +using Chronos.Data.Workspaces; +using Chronos.Testing; + +namespace Chronos.App.Tests.Server; + +/// +/// Which jobs "Pause metered transfers" holds (Server.html, phase 5c): uploads, and the workspace's copies from and renders +/// sent to a server; never a copy from or a send to this PC's archive. +/// +public sealed class UploadHoldTests +{ + private const string Server = "https://nas.local:8443"; + private static readonly Guid SequenceId = Guid.CreateVersion7(); + private static readonly Guid ProjectId = Guid.CreateVersion7(); + private const string Root = @"C:\Users\me\Chronos\workspace"; + + private static Job Of(ArchiveJobRequest request) => new() + { + Id = Guid.CreateVersion7(), + Kind = request.Kind, + TargetIds = request.Targets, + TargetLabel = request.Label, + ParametersJson = request.ParametersJson, + State = JobState.Queued, + Created = DateTimeOffset.UtcNow, + }; + + private static Job Of(JobKind kind, string? parameters = null) => new() + { + Id = Guid.CreateVersion7(), + Kind = kind, + TargetIds = [SequenceId], + ParametersJson = parameters, + State = JobState.Queued, + Created = DateTimeOffset.UtcNow, + }; + + [Test] + public void TransfersWithAServerAreHeld() + { + Assert.True(UploadHold.Holds(Of(JobKind.UploadCommit))); + Assert.True(UploadHold.Holds(Of(WorkspaceJobs.CopyFromServer(SequenceId, "Harbour", Server, Root, ProjectId)))); + Assert.True(UploadHold.Holds(Of(WorkspaceJobs.Send(SequenceId, "Harbour", Root, ProjectId, [], Server)))); + Assert.SequenceEqual([JobKind.UploadCommit, JobKind.WorkspaceCopy, JobKind.WorkspaceSend], UploadHold.Kinds); + } + + [Test] + public void TransfersWithThisPcsLibraryAndOtherJobsAreNot() + { + var sequence = new Sequence { Id = SequenceId, ArchiveId = Guid.CreateVersion7(), Title = "Harbour", Folder = "2026/harbour" }; + Assert.False(UploadHold.Holds(Of(WorkspaceJobs.Copy(sequence, Root, ProjectId)))); + Assert.False(UploadHold.Holds(Of(WorkspaceJobs.Send(SequenceId, "Harbour", Root, ProjectId, [])))); + Assert.False(UploadHold.Holds(Of(JobKind.Download))); + Assert.False(UploadHold.Holds(Of(JobKind.Render))); + // Parameters that cannot be read are not held: the job fails with its reason. + Assert.False(UploadHold.Holds(Of(JobKind.WorkspaceSend, "{not json"))); + Assert.False(UploadHold.Holds(Of(JobKind.WorkspaceCopy))); + } +} diff --git a/tests/Chronos.Data.Tests/Workspaces/WorkspaceSendTests.cs b/tests/Chronos.Data.Tests/Workspaces/WorkspaceSendTests.cs new file mode 100644 index 00000000..6559dcc2 --- /dev/null +++ b/tests/Chronos.Data.Tests/Workspaces/WorkspaceSendTests.cs @@ -0,0 +1,221 @@ +using System.Security.Cryptography; +using Chronos.Core.Localization; +using Chronos.Core.Model; +using Chronos.Core.Project; +using Chronos.Data.Archiving; +using Chronos.Data.Jobs; +using Chronos.Data.Tests.Archiving; +using Chronos.Data.Tests.Infrastructure; +using Chronos.Data.Workspaces; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Workspaces; + +/// +/// A workspace's local renders sent home (phase 5c, ) to this PC's archive: filed with +/// their project and version, the version pushed first when the home lacks it, a changed or missing file refused and left in +/// the workspace, a send repeated after a cut harmless, and never held on a metered connection. +/// +[Category("workspace")] +public abstract class WorkspaceSendTests(TestContext context) : DatabaseTest(context) +{ + private static readonly DateTimeOffset Rendered = new(2026, 11, 4, 9, 0, 0, TimeSpan.Zero); + + private async Task FixtureAsync() + { + var archive = Own(await ArchiveFixture.CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true))); + return Own(await WorkspaceFixture.CreateAsync(Context, archive, 3)); + } + + /// An engine with the workspace's handlers; stands for a metered connection. + private async Task EngineAsync(WorkspaceFixture fx, Func? hold = null) + { + var options = WorkspaceJobOptions.For(fx.Archive.Service) with { HoldTransfers = hold ?? (() => false) }; + var engine = new JobEngine(fx.Db, WorkspaceJobs.CreateHandlers(fx.Db, fx.Archive.Service, null, options), + new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) }); + await engine.StartAsync(Ct); + Own(engine); + return engine; + } + + /// Main checked out into a workspace of its own (its media are not needed to send). + private static async Task<(WorkspaceStore Store, WorkspaceSync Sync, WorkspaceProject Project)> CheckoutAsync(WorkspaceFixture fx) + { + var (store, sync) = fx.Workspace(); + var main = await fx.Archive.Service.Projects.MainAsync(fx.Sequence.Id, fx.Ct); + return (store, sync, await sync.CheckoutAsync(fx.Home, main.Id, cancellationToken: fx.Ct)); + } + + /// An output in the project's exports with its ledger entry, as a local render leaves them. + private static WorkspaceRenderEntry Render(WorkspaceProject project, Guid? versionId, string name, int seed) + { + Directory.CreateDirectory(project.ExportsDirectory); + var bytes = new byte[150_000 + seed]; + new Random(seed).NextBytes(bytes); + File.WriteAllBytes(Path.Combine(project.ExportsDirectory, name), bytes); + var entry = new WorkspaceRenderEntry(Guid.CreateVersion7(), Guid.CreateVersion7(), name, Convert.ToHexStringLower(SHA256.HashData(bytes)), bytes.Length, + "1080p H.264", "h264", 1920, 1080, 24, 2.5, project.ProjectId, versionId, null, Rendered); + project.WriteRenders(project.ReadRenders().With(entry)); + return entry; + } + + private static string Sha(string path) => Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(path))); + + private static async Task SendAsync(WorkspaceFixture fx, JobEngine engine, WorkspaceStore store, Guid projectId, JobState expected, + params Guid[] renderIds) + { + var job = await engine.EnqueueAsync(WorkspaceJobs.Send(fx.Sequence.Id, fx.Sequence.DisplayTitle, store.Root, projectId, renderIds), cancellationToken: fx.Ct); + return await fx.Archive.WaitForAsync(job.Id, expected, (int)TestTiming.Budget(60)); + } + + [Test] + public async Task TheSendFilesTheRenderInTheLibraryWithItsProjectAndVersion() + { + var fx = await FixtureAsync(); + var engine = await EngineAsync(fx); + var (store, _, project) = await CheckoutAsync(fx); + var head = project.ReadBundle().Workspace!.Base!.VersionId; + var entry = Render(project, head, "Main 1080p.mp4", 1); + + await SendAsync(fx, engine, store, project.ProjectId, JobState.Done); + + var row = Assert.NotNull(await fx.Db.Renders.GetAsync(entry.RenderId, fx.Ct)); + Assert.Equal((fx.Sequence.Id, (Guid?)project.ProjectId, (Guid?)head, "1080p H.264", entry.Size, 1920), + (row.SequenceId, row.ProjectId, row.ProjectVersionId, row.PresetName, row.Bytes, row.Width)); + var filed = Path.Combine(fx.Archive.Archive.RootPath, row.OutputPath); + Assert.Equal(entry.Sha256, Sha(filed)); + Assert.Equal("Main 1080p.mp4", Path.GetFileName(filed)); + // The workspace keeps its file until a clean-up; the ledger says it went home. + Assert.Equal(entry.Sha256, Sha(project.ExportFile(entry.File))); + var sent = Assert.NotNull(project.ReadRenders().Find(entry.RenderId)); + Assert.True(sent.IsSent); + Assert.NotNull(sent.Sent); + Assert.Empty(Directory.GetFiles(fx.Archive.Paths.StagingDirectory, "workspace-render-*")); + } + + [Test] + public async Task ASendRepeatedAfterACutFilesNothingTwice() + { + var fx = await FixtureAsync(); + var engine = await EngineAsync(fx); + var (store, _, project) = await CheckoutAsync(fx); + var entry = Render(project, project.ReadBundle().Workspace!.Base!.VersionId, "Main.mp4", 2); + await SendAsync(fx, engine, store, project.ProjectId, JobState.Done, entry.RenderId); + + // Cut after the home filed it, before the ledger said so: the next send finds it there and only marks it sent. + project.WriteRenders(project.ReadRenders().With(entry)); + Assert.True(project.ReadRenders().Find(entry.RenderId)!.IsUnsent); + var again = await SendAsync(fx, engine, store, project.ProjectId, JobState.Done); + Assert.Contains(Loc.L("Archive.Log.WorkspaceSent", project.ReadBundle().Workspace!.Name, 0, 1), again.LogTail); + Assert.True(project.ReadRenders().Find(entry.RenderId)!.IsSent); + Assert.Equal(1, (await fx.Db.Renders.ListBySequenceAsync(fx.Sequence.Id, fx.Ct)).Count); + var exports = Path.GetDirectoryName(Path.Combine(fx.Archive.Archive.RootPath, (await fx.Db.Renders.GetAsync(entry.RenderId, fx.Ct))!.OutputPath))!; + Assert.Equal(1, Directory.GetFiles(exports).Length); + + // A send with nothing unsent is done at once. + var nothing = await SendAsync(fx, engine, store, project.ProjectId, JobState.Done); + Assert.Contains(Loc.L("Archive.Log.WorkspaceSent", project.ReadBundle().Workspace!.Name, 0, 0), nothing.LogTail); + } + + [Test] + public async Task AVersionTheHomeLacksIsSentFirst() + { + var fx = await FixtureAsync(); + var engine = await EngineAsync(fx); + // With its media: a version is saved only of frames the workspace holds. + var (store, sync) = fx.Workspace(); + var project = await fx.CheckoutAsync(sync); + var session = await sync.OpenAsync(project.ProjectId, fx.Ct); + await sync.SaveAsync(project.ProjectId, session.Working with { HeroFrame = 2 }, fx.Ct); + var local = Assert.NotNull(await sync.CommitAsync(project.ProjectId, "Render", cancellationToken: fx.Ct)); + Assert.Null(await fx.Home.GetVersionAsync(project.ProjectId, local.Id, fx.Ct)); + var entry = Render(project, local.Id, "Hero.mp4", 3); + + var job = await SendAsync(fx, engine, store, project.ProjectId, JobState.Done); + + var pushed = Assert.NotNull(await fx.Home.GetVersionAsync(project.ProjectId, local.Id, fx.Ct)); + Assert.Contains(Loc.L("Archive.Log.WorkspaceVersionPushed", project.ReadBundle().Workspace!.Name, pushed.Number), job.LogTail); + Assert.Equal((Guid?)local.Id, (await fx.Db.Renders.GetAsync(entry.RenderId, fx.Ct))!.ProjectVersionId); + Assert.Equal(WorkspaceState.InSync, (await sync.StatusAsync(project.ProjectId, fx.Home, fx.Ct)).State); + } + + [Test] + public async Task AChangedOrMissingRenderIsRefusedAndTheOthersGoOn() + { + var fx = await FixtureAsync(); + var engine = await EngineAsync(fx); + var (store, _, project) = await CheckoutAsync(fx); + var head = project.ReadBundle().Workspace!.Base!.VersionId; + var changed = Render(project, head, "Changed.mp4", 4); + var gone = Render(project, head, "Gone.mp4", 5); + var good = Render(project, head, "Good.mp4", 6); + var original = File.ReadAllBytes(project.ExportFile(changed.File)); + File.AppendAllText(project.ExportFile(changed.File), "edited"); + File.Delete(project.ExportFile(gone.File)); + + var failed = await SendAsync(fx, engine, store, project.ProjectId, JobState.Failed); + + Assert.Equal(Loc.L("Errors.Workspace.SendFailed", 2, 3), failed.Error); + var ledger = project.ReadRenders(); + Assert.True(ledger.Find(good.RenderId)!.IsSent); + Assert.Equal((WorkspaceRenderEntry.Failed, Loc.L("Errors.Workspace.SendHashMismatch", "Changed.mp4")), + (ledger.Find(changed.RenderId)!.State, ledger.Find(changed.RenderId)!.Error)); + Assert.Equal((WorkspaceRenderEntry.Failed, Loc.L("Errors.Workspace.SendMissing", "Gone.mp4")), + (ledger.Find(gone.RenderId)!.State, ledger.Find(gone.RenderId)!.Error)); + Assert.Null(await fx.Db.Renders.GetAsync(changed.RenderId, fx.Ct)); + Assert.Null(await fx.Db.Renders.GetAsync(gone.RenderId, fx.Ct)); + // The changed file stays in the workspace as it is. + Assert.FileExists(project.ExportFile(changed.File)); + + // Put back as rendered (and the missing one discarded), the next send takes it. + File.WriteAllBytes(project.ExportFile(changed.File), original); + project.WriteRenders(project.ReadRenders().Without(gone.RenderId)); + await SendAsync(fx, engine, store, project.ProjectId, JobState.Done); + Assert.True(project.ReadRenders().Find(changed.RenderId)!.IsSent); + Assert.Equal(2, (await fx.Db.Renders.ListBySequenceAsync(fx.Sequence.Id, fx.Ct)).Count); + } + + [Test] + public async Task AVersionOfAnotherProjectIsNeverFiledWithTheRender() + { + var fx = await FixtureAsync(); + var engine = await EngineAsync(fx); + var (store, _, project) = await CheckoutAsync(fx); + var other = await fx.Home.CreateAsync("Vertical", null, null, fx.Ct); + var foreign = (await fx.Home.HeadAsync(other.Id, fx.Ct)).Id; + var entry = Render(project, foreign, "Foreign.mp4", 7); + + var failed = await SendAsync(fx, engine, store, project.ProjectId, JobState.Failed); + + Assert.Equal(Loc.L("Errors.Workspace.SendVersion"), failed.Error); + Assert.Null(await fx.Db.Renders.GetAsync(entry.RenderId, fx.Ct)); + // Not the render's fault: it waits, unsent, for the next send. + Assert.Equal(WorkspaceRenderEntry.Pending, project.ReadRenders().Find(entry.RenderId)!.State); + } + + [Test] + public async Task ASendToThisPcsLibraryIsNeverHeldOnAMeteredConnection() + { + var fx = await FixtureAsync(); + var engine = await EngineAsync(fx, hold: () => true); + var (store, _, project) = await CheckoutAsync(fx); + var entry = Render(project, null, "Working.mp4", 8); + + var job = await SendAsync(fx, engine, store, project.ProjectId, JobState.Done); + + Assert.False(job.LogTail?.Contains(Loc.L("Archive.Log.WorkspaceSendHeld"), StringComparison.Ordinal) ?? false); + var row = Assert.NotNull(await fx.Db.Renders.GetAsync(entry.RenderId, fx.Ct)); + Assert.Equal(((Guid?)project.ProjectId, (Guid?)null), (row.ProjectId, row.ProjectVersionId)); + } +} + +public sealed class SqliteWorkspaceSendTests(TestContext context) : WorkspaceSendTests(context) +{ + protected override Backend Backend => Backend.SqliteFile; +} + +[RequiresEnvironment(PgTestServer.Variable)] +public sealed class PostgresWorkspaceSendTests(TestContext context) : WorkspaceSendTests(context) +{ + protected override Backend Backend => Backend.Postgres; +} diff --git a/tests/Chronos.Server.Tests/Workspaces/WorkspaceSendServerTests.cs b/tests/Chronos.Server.Tests/Workspaces/WorkspaceSendServerTests.cs new file mode 100644 index 00000000..adaa1692 --- /dev/null +++ b/tests/Chronos.Server.Tests/Workspaces/WorkspaceSendServerTests.cs @@ -0,0 +1,186 @@ +using System.Net; +using System.Security.Cryptography; +using Chronos.Core.Codecs.Jpeg; +using Chronos.Core.Localization; +using Chronos.Core.Model; +using Chronos.Core.Project; +using Chronos.Data; +using Chronos.Data.Archiving; +using Chronos.Data.Jobs; +using Chronos.Data.Projects; +using Chronos.Data.Sync; +using Chronos.Data.Workspaces; +using Chronos.Protocol; +using Chronos.Protocol.Client; +using Chronos.Server.Storage; +using Chronos.Testing; + +namespace Chronos.Server.Tests.Workspaces; + +/// +/// A render made in a workspace and sent to its server (phase 5c): the server checks the project and version it names +/// against the sequence (422 project_mismatch) and records them with the render; one naming neither is filed as +/// before. The desktop's uploads it with them, and waits while a metered connection holds +/// transfers with the server. +/// +[Category("server")] +[Category("workspace")] +public sealed class WorkspaceSendServerTests(TestContext context) +{ + private const string Title = "Rooftop"; + private static readonly TimeSpan Wait = TimeSpan.FromSeconds(TestTiming.Budget(60)); + + [Test] + public async Task TheServerFilesAWorkspaceRenderWithItsProjectAndVersion() + { + await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); + var (http, _, token) = await server.UserAsync("alberto", "editor"); + var sid = (await Uploads.UploadAsync(http, Photo(1), "IMG_0001.JPG", sessionKey: "roof")).SequenceId; + using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "PC-A") { Token = token }; + var home = new ServerProjectHome(client, sid, "Test server"); + var main = (await home.ListAsync(context.Cancellation)).Single(); + var head = await home.HeadAsync(main.Id, context.Cancellation); + + var (path, request) = Output(1); + var dto = Assert.NotNull(await client.UploadRenderAsync(sid, path, request with { ProjectId = main.Id, ProjectVersionId = head.Id }, ct: context.Cancellation)); + Assert.Equal(((Guid?)main.Id, (Guid?)head.Id), (dto.ProjectId, dto.ProjectVersionId)); + var row = Assert.NotNull(await server.Database.Renders.GetAsync(request.RenderId)); + Assert.Equal((sid, (Guid?)main.Id, (Guid?)head.Id), (row.SequenceId, row.ProjectId, row.ProjectVersionId)); + Assert.Contains((await client.GetManifestAsync(sid, context.Cancellation)).Renders, r => r.Id == request.RenderId && r.ProjectVersionId == head.Id); + + // A version alone names its project. + var (alone, versionOnly) = Output(2); + var named = Assert.NotNull(await client.UploadRenderAsync(sid, alone, versionOnly with { ProjectVersionId = head.Id }, ct: context.Cancellation)); + Assert.Equal(((Guid?)main.Id, (Guid?)head.Id), (named.ProjectId, named.ProjectVersionId)); + + // Sent again: the server holds it already, and nothing is filed twice. + Assert.Null(await client.UploadRenderAsync(sid, path, request with { ProjectId = main.Id, ProjectVersionId = head.Id }, ct: context.Cancellation)); + Assert.Equal(2, (await server.Database.Renders.ListBySequenceAsync(sid)).Count); + + // Without them, as from an earlier client: filed naming neither. + var (plain, earlier) = Output(3); + var before = Assert.NotNull(await client.UploadRenderAsync(sid, plain, earlier, ct: context.Cancellation)); + Assert.Equal(((Guid?)null, (Guid?)null), (before.ProjectId, before.ProjectVersionId)); + Assert.Null((await server.Database.Renders.GetAsync(earlier.RenderId))!.ProjectVersionId); + } + + [Test] + public async Task AProjectOrVersionOfAnotherSequenceIsRefused() + { + await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); + var (http, _, token) = await server.UserAsync("alberto", "editor"); + var sid = (await Uploads.UploadAsync(http, Photo(1), "IMG_0001.JPG", sessionKey: "roof")).SequenceId; + var other = (await Uploads.UploadAsync(http, Photo(2), "IMG_0002.JPG", sessionKey: "lake", capturedAt: new DateTime(2026, 10, 3, 9, 0, 0))).SequenceId; + Assert.NotEqual(sid, other); + using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "PC-A") { Token = token }; + var main = (await new ServerProjectHome(client, sid).ListAsync(context.Cancellation)).Single(); + var foreignHome = new ServerProjectHome(client, other); + var foreign = (await foreignHome.ListAsync(context.Cancellation)).Single(); + var foreignHead = await foreignHome.HeadAsync(foreign.Id, context.Cancellation); + + var (path, request) = Output(4); + foreach (var named in new[] + { + request with { ProjectId = main.Id, ProjectVersionId = foreignHead.Id }, + request with { ProjectId = foreign.Id }, + request with { ProjectVersionId = foreignHead.Id }, + request with { ProjectId = Guid.CreateVersion7() }, + }) + { + var refused = await Assert.ThrowsAsync(() => client.UploadRenderAsync(sid, path, named, ct: context.Cancellation)); + Assert.Equal((HttpStatusCode.UnprocessableEntity, "project_mismatch"), (refused.Status, refused.Error.Code)); + } + Assert.Null(await server.Database.Renders.GetAsync(request.RenderId)); + } + + [Test] + public async Task TheSendUploadsTheRenderWithItsVersionOnceTheConnectionIsNoLongerMetered() + { + await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); + var (http, _, token) = await server.UserAsync("alberto", "editor"); + var sid = (await Uploads.UploadAsync(http, Photo(1), "IMG_0001.JPG", sessionKey: "roof")).SequenceId; + using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "PC-A") { Token = token }; + var serverKey = SyncService.ServerKey(client.BaseAddress); + await using var db = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("pc-db"), "chronos.db")), context.Cancellation); + var archives = new ArchiveService(db, new ArchiveServiceOptions { Owner = "PC-A" }); + var metered = true; + var options = WorkspaceJobOptions.For(archives) with { HoldTransfers = () => Volatile.Read(ref metered) }; + await using var engine = new JobEngine(db, WorkspaceJobs.CreateHandlers(db, archives, new Connector(client), options), + new JobEngineOptions { Workers = 1, WorkerId = "PC-A", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) }); + await engine.StartAsync(context.Cancellation); + var store = new WorkspaceStore(Path.Combine(context.CreateTempDirectory("pc-ws"), "workspace")); + var sync = new WorkspaceSync(store, "Chronos on PC-A") { LockWait = TimeSpan.FromSeconds(TestTiming.Budget(10)) }; + var home = new ServerProjectHome(client, sid, "Test server"); + var main = (await home.ListAsync(context.Cancellation)).Single(); + var project = await sync.CheckoutAsync(home, main.Id, cancellationToken: context.Cancellation); + var head = project.ReadBundle().Workspace!.Base!.VersionId; + + Directory.CreateDirectory(project.ExportsDirectory); + var bytes = RandomNumberGenerator.GetBytes(120_000); + File.WriteAllBytes(Path.Combine(project.ExportsDirectory, "Roof 1080p.mp4"), bytes); + var entry = new WorkspaceRenderEntry(Guid.CreateVersion7(), Guid.CreateVersion7(), "Roof 1080p.mp4", Convert.ToHexStringLower(SHA256.HashData(bytes)), + bytes.Length, "1080p H.264", "h264", 1920, 1080, 24, 2.5, main.Id, head, null, DateTimeOffset.UtcNow); + project.WriteRenders(WorkspaceRenders.Empty.With(entry)); + + // Held while the connection is metered: nothing goes out. + var job = await engine.EnqueueAsync(WorkspaceJobs.Send(sid, Title, store.Root, main.Id, [entry.RenderId], serverKey), cancellationToken: context.Cancellation); + await Task.Delay(TimeSpan.FromSeconds(1), context.Cancellation); + Assert.Equal(JobState.Running, (await db.Jobs.GetAsync(job.Id, context.Cancellation))!.State); + Assert.Null(await server.Database.Renders.GetAsync(entry.RenderId)); + Assert.True(project.ReadRenders().Find(entry.RenderId)!.IsUnsent); + + Volatile.Write(ref metered, false); + var done = await WaitAsync(db, job.Id, JobState.Done); + Assert.Contains(Loc.L("Archive.Log.WorkspaceSendHeld"), done.LogTail!); + Assert.Contains(Loc.L("Archive.Log.WorkspaceSent", main.Name, 1, 0), done.LogTail!); + var row = Assert.NotNull(await server.Database.Renders.GetAsync(entry.RenderId)); + Assert.Equal((sid, (Guid?)main.Id, (Guid?)head, entry.Size), (row.SequenceId, row.ProjectId, row.ProjectVersionId, row.Bytes)); + Assert.True(project.ReadRenders().Find(entry.RenderId)!.IsSent); + Assert.FileExists(project.ExportFile(entry.File)); + // This PC's library is never written: the server is the home. + Assert.Empty(await db.Renders.ListBySequenceAsync(sid, context.Cancellation)); + } + + /// A render's file on this PC and the offer of it, naming no project. + private (string Path, RenderUploadRequest Request) Output(int seed) + { + var data = Uploads.Photo(4000 + seed, seed); + var path = System.IO.Path.Combine(context.CreateTempDirectory("render"), $"Roof {seed}.mp4"); + File.WriteAllBytes(path, data); + return (path, new RenderUploadRequest(Guid.CreateVersion7(), System.IO.Path.GetFileName(path), data.Length, Uploads.Sha(data), "1080p H.264", "h264", 1920, 1080, + 24, 2.5, "PC-A")); + } + + private static async Task WaitAsync(ChronosDatabase db, Guid jobId, JobState state) + { + var until = DateTime.UtcNow + Wait; + while (true) + { + var job = await db.Jobs.GetAsync(jobId); + if (job?.State == state) return job; + if (job?.State is JobState.Failed or JobState.Done or JobState.Cancelled) Assert.Fail($"Job ended {job.State}: {job.Error}\n{job.LogTail}"); + if (DateTime.UtcNow > until) Assert.Fail($"Job {jobId} did not reach {state} (now {job?.State})"); + await Task.Delay(25); + } + } + + private static byte[] Photo(int seed) + { + const int width = 96, height = 64; + var pixels = new byte[width * height * 3]; + for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)((i * 7 + seed * 31) % 251); + return JpegEncoder.Encode(pixels, width, height, 3, new JpegEncodeOptions { Quality = 90 }); + } + + /// The desktop's session with the test server: a remote and the project home over the signed-in client. + private sealed class Connector(ChronosClient client) : ISyncConnector + { + public ValueTask ConnectAsync(string server, CancellationToken cancellationToken) + { + ISyncRemote remote = new ChronosSyncRemote(client, ownsClient: false); + return ValueTask.FromResult(remote.Server == server ? remote : null); + } + + public IProjectHome? ProjectHome(BundleHome home) => new ServerProjectHome(client, home.SequenceId, "Test server"); + } +}