using System.Net; using Chronos.Core.Archiving; using Chronos.Core.Diagnostics; using Chronos.Core.Localization; using Chronos.Core.Model; using Chronos.Core.Project; using Chronos.Data.Archiving; using Chronos.Data.Jobs; using Chronos.Data.Projects; using Chronos.Data.Sync; using Chronos.Protocol; using Chronos.Protocol.Client; namespace Chronos.Data.Workspaces; /// /// (phase 5c, ): the local renders of a workspace project /// sent home with their project and version, per unsent ledger entry (; the ids the job names, /// or every unsent one): the file hashed again (one missing or changed since it was rendered is refused, marked failed in the /// ledger and left where it is), the version it names pushed first when the home lacks it (a push that fails fails the job, /// the entry left as it was; a version the push replayed under a new id, after the home moved on, is found again by the /// content hash of what rendered, else the render goes with its project alone, and the entry says so from then on), then /// filed in this PC's archive (copied into the archive's staging folder, read back against /// its SHA-256, and filed under the sequence's lease as the sync files a server's exports, /// , after checking the project and version are the sequence's) or uploaded to its /// server ( with ProjectId and ProjectVersionId, which the server checks the /// same way), and the entry marked sent. The file stays in exports/ until a clean-up removes it. /// Idempotent by render id: a send repeated after a cut finds the home holding the render already and only marks it sent; a /// render id the home files under another sequence is refused. The ledger is read and written under the project's lease, /// each entry read again before it is written, so a render finished meanwhile is kept. /// A send to a server is held while says so (a metered connection): it waits /// before each upload, and the desktop pauses it meanwhile. A send to this PC's archive is never held. A server from before /// phase 5c files the render without its project and version: the job's log says so. A send queued before the workspace /// moved works where the workspace is now, and none runs while it is being moved (). /// internal sealed class WorkspaceSendJobHandler(ChronosDatabase db, ArchiveService archives, ISyncConnector? servers, WorkspaceJobOptions options) : IJobHandler { private static readonly TimeSpan LockWait = TimeSpan.FromMinutes(2); /// How often a send held on a metered connection looks again. internal static TimeSpan HoldPoll { get; set; } = TimeSpan.FromSeconds(2); public JobKind Kind => JobKind.WorkspaceSend; public async Task RunAsync(JobContext context) { ArgumentNullException.ThrowIfNull(context); var ct = context.Cancellation; var parameters = WorkspaceJobs.SendParameters(context.Job) ?? throw new InvalidDataException(Loc.L("Errors.Workspace.JobParameters")); if (context.Job.TargetIds is not [var sequenceId] || string.IsNullOrWhiteSpace(parameters.WorkspaceRoot)) throw new InvalidDataException(Loc.L("Errors.Workspace.JobParameters")); var server = parameters.Server is { Length: > 0 } named ? named : null; // Where the project is now (a workspace moved since the job was queued), never while it is being moved; nothing is // made where it is not. var root = WorkspaceJobRoots.Resolve(parameters.WorkspaceRoot, parameters.ProjectId, options.CurrentRoot, "Errors.Workspace.NotCheckedOut"); var project = new WorkspaceStore(root).Find(parameters.ProjectId) ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.NotCheckedOut")); var workspace = project.ReadBundle().Workspace ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.NotCheckedOut")); var home = workspace.Home; if (home.SequenceId != sequenceId || (server is null ? !home.IsArchive : home.Kind != BundleHome.ServerKind || !string.Equals(home.Server, server, StringComparison.OrdinalIgnoreCase))) throw new InvalidOperationException(Loc.L("Errors.Workspace.OtherHome")); var entries = await ToSendAsync(project, parameters.RenderIds, ct).ConfigureAwait(false); await context.ReportAsync(0, entries.Count).ConfigureAwait(false); if (entries.Count == 0) { context.Log(Loc.L("Archive.Log.WorkspaceSent", workspace.Name, 0, 0)); return; } if (server is null) { var sequence = await db.Sequences.GetAsync(sequenceId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Archive.Log.SequenceMissing", sequenceId)); var archive = await db.Archives.GetAsync(sequence.ArchiveId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.NoArchive")); var projectHome = new ArchiveProjectHome(db, archives.Projects, sequenceId); await SendAsync(context, project, workspace, entries, () => projectHome, held: false, (entry, file, token) => FileAsync(sequence, archive, entry, file, token)).ConfigureAwait(false); return; } using var remote = (servers is null ? null : await servers.ConnectAsync(server, ct).ConfigureAwait(false)) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.NotConnected")); await SendAsync(context, project, workspace, entries, () => servers!.ProjectHome(home) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.NotConnected")), held: true, (entry, file, token) => UploadAsync(context, remote, sequenceId, entry, file, token)).ConfigureAwait(false); } /// /// The entries to send, oldest first: those of the ledger holds unsent (an id it no longer /// holds was discarded meanwhile, one sent already is left), every unsent one when none is named. /// private static async Task> ToSendAsync(WorkspaceProject project, IReadOnlyList? renderIds, CancellationToken ct) { WorkspaceRenders ledger; using (await project.LockAsync(LockWait, ct).ConfigureAwait(false)) ledger = project.ReadRenders(); if (renderIds is not { Count: > 0 }) return ledger.Unsent; var named = renderIds.ToHashSet(); return [.. ledger.Entries.Where(e => named.Contains(e.RenderId) && e.IsUnsent)]; } /// Delivers one checked render home: true when the home took it now, false when it held it already. private delegate Task Deliver(WorkspaceRenderEntry entry, string file, CancellationToken cancellationToken); private async Task SendAsync(JobContext context, WorkspaceProject project, BundleWorkspace workspace, IReadOnlyList entries, Func home, bool held, Deliver deliver) { var ct = context.Cancellation; var (sent, there, refused) = (0, 0, 0); // The versions this run found at the home, or sent there first, with the id the home holds each by (a replay gives another). var versions = new Dictionary(); for (var i = 0; i < entries.Count; i++) { ct.ThrowIfCancellationRequested(); if (held) await WhileHeldAsync(context).ConfigureAwait(false); var entry = entries[i]; var file = project.ContainedExportFile(entry.File); var refusal = file is null ? Loc.L("Errors.Workspace.SendMissing", Path.GetFileName(entry.File ?? string.Empty)) : await CheckAsync(entry, file, folders: !held, ct).ConfigureAwait(false); if (refusal is null) { if (entry.ProjectVersionId is { } versionId) { if (!versions.TryGetValue(versionId, out var atHome)) { atHome = await EnsureVersionAsync(context, project, workspace, home(), entry, versionId, ct).ConfigureAwait(false); versions[versionId] = atHome; } if (atHome != versionId) { // The ledger says from now on what the home holds: later sends agree with this one. entry = entry with { ProjectVersionId = atHome }; var mapped = atHome; await UpdateAsync(project, entry.RenderId, e => e with { ProjectVersionId = mapped }).ConfigureAwait(false); } } try { if (await deliver(entry, file!, ct).ConfigureAwait(false)) sent++; else there++; // The home holds it: recorded whatever stops the job now. await UpdateAsync(project, entry.RenderId, e => e.AsSent(DateTimeOffset.UtcNow)).ConfigureAwait(false); } catch (SendRefusedException ex) { refusal = ex.Message; } } if (refusal is not null) { refused++; context.Log(refusal); await UpdateAsync(project, entry.RenderId, e => e.AsFailed(refusal)).ConfigureAwait(false); } await context.ReportAsync(i + 1, entries.Count).ConfigureAwait(false); } context.Log(Loc.L("Archive.Log.WorkspaceSent", workspace.Name, sent, there)); if (refused > 0) throw new InvalidOperationException(Loc.L("Errors.Workspace.SendFailed", refused, entries.Count)); } /// /// Why the render cannot be sent as it is (gone from the exports, another file than the one rendered, or an image /// sequence's folder for a home that takes one file per render); null when it can. A folder (an image sequence, only /// when : this PC's archive files folders as its own renders do) is checked by its total size /// and the hash of its listing, as the render recorded it (). /// /// A ledger names files inside the exports only (): one pointing elsewhere is never read. private static async Task CheckAsync(WorkspaceRenderEntry entry, string file, bool folders, CancellationToken ct) { var name = Path.GetFileName(entry.File); if (Directory.Exists(file)) { if (!folders) return Loc.L("Errors.Workspace.SendSequence", name); return WorkspaceCleaner.Measure(file).Bytes != entry.Size || !string.Equals(await WorkspaceRenderJobHandler.HashAsync(file).ConfigureAwait(false), entry.Sha256, StringComparison.OrdinalIgnoreCase) ? Loc.L("Errors.Workspace.SendHashMismatch", name) : null; } if (!File.Exists(file)) return Loc.L("Errors.Workspace.SendMissing", name); if (new FileInfo(file).Length != entry.Size || !string.Equals(await ChronosClient.HashFileAsync(file, ct).ConfigureAwait(false), entry.Sha256, StringComparison.OrdinalIgnoreCase)) return Loc.L("Errors.Workspace.SendHashMismatch", name); return null; } /// How many of the home's newest versions are read for one of the same content, when a replay gave a rendered version another id. private const int ReplayedLookBack = 100; /// /// Version of the project at the home, as the id the home holds it by: pushed there first /// (every local version, in order, as a sync pushes them) when the home lacks it. A push that replays it on a home that /// moved on gives it another id (and one that changes nothing is not sent at all): the home's version of the content /// hash that rendered () stands for it, else none (null: the render goes /// with its project alone, as a render of a version the home never had). A push that fails fails the job. /// private static async Task EnsureVersionAsync(JobContext context, WorkspaceProject project, BundleWorkspace workspace, IProjectHome home, WorkspaceRenderEntry entry, Guid versionId, CancellationToken ct) { var projectId = entry.ProjectId; if (await home.GetVersionAsync(projectId, versionId, ct).ConfigureAwait(false) is not null) return versionId; try { await new WorkspaceSync(project.Store, Environment.MachineName) { LockWait = LockWait }.PushAsync(projectId, home, ct).ConfigureAwait(false); } catch (Exception ex) when (ex is HomeUnavailableException or WorkspaceSyncException or ProjectException or LeaseUnavailableException or InvalidOperationException) { context.Log(ex.Message); throw new InvalidOperationException(Loc.L("Errors.Workspace.SendVersion"), ex); } if (await home.GetVersionAsync(projectId, versionId, ct).ConfigureAwait(false) is { } pushed) { context.Log(Loc.L("Archive.Log.WorkspaceVersionPushed", workspace.Name, pushed.Number)); return versionId; } ProjectVersion? same = null; if (entry.SettingsHash is { Length: > 0 } hash) { var history = await home.HistoryAsync(projectId, null, ReplayedLookBack, ct).ConfigureAwait(false); // Of the same content whatever its line ends: a version stored by a Windows build before every build wrote LF // holds the CRLF text and its hash. same = history.FirstOrDefault(v => string.Equals(v.ContentHash, hash, StringComparison.OrdinalIgnoreCase) || (v.SettingsJson is { Length: > 0 } text && ProjectContent.MatchesHash(text, hash))); } if (same is not null) { Log.Info("workspace", $"Version {versionId} of project {projectId} reached the home as version {same.Id} (a replay): render {entry.RenderId} names that one"); context.Log(Loc.L("Archive.Log.WorkspaceVersionReplayed", workspace.Name, same.Number)); return same.Id; } Log.Info("workspace", $"Version {versionId} of project {projectId} is not at the home after the push, nor one of the same content: render {entry.RenderId} goes with its project alone"); context.Log(Loc.L("Archive.Log.WorkspaceVersionNotHome", workspace.Name)); return null; } /// /// The render filed in this PC's archive, the home's sequence: copied into the archive's staging folder, read back against /// the ledger's SHA-256, then filed under the sequence's lease with the project and version (once both are checked to be /// the sequence's). A render id the index holds already for this sequence is the same render, sent before. /// private async Task FileAsync(Sequence sequence, Archive archive, WorkspaceRenderEntry entry, string file, CancellationToken ct) { if (await db.Renders.GetAsync(entry.RenderId, ct).ConfigureAwait(false) is { } known) return known.SequenceId == sequence.Id ? false : throw new SendRefusedException(Loc.L("Server.Error.RenderExists")); if (!await ProjectOfAsync(sequence.Id, entry, ct).ConfigureAwait(false)) throw new SendRefusedException(Loc.L("Server.Error.ProjectMismatch")); var paths = new ArchivePaths(archive.RootPath); ArchiveVolume.EnsureWritable(paths); SyncTransfers.EnsureSpace(paths.Root, entry.Size); Directory.CreateDirectory(paths.StagingDirectory); var staged = Path.Combine(paths.StagingDirectory, "workspace-render-" + entry.RenderId.ToString("N") + ".part"); var folder = Directory.Exists(file); try { TryDelete(staged); if (folder) { // An image sequence: every file of its folder, the folder filed whole as the archive's own renders are. foreach (var source in Directory.EnumerateFiles(file, "*", SearchOption.AllDirectories)) { var target = Path.Combine(staged, Path.GetRelativePath(file, source)); Directory.CreateDirectory(Path.GetDirectoryName(target)!); await CopyAsync(source, target, ct).ConfigureAwait(false); } Directory.CreateDirectory(staged); } else await CopyAsync(file, staged, ct).ConfigureAwait(false); // Read back from the archive's volume: what is filed is the render as it was written. var readBack = folder ? await WorkspaceRenderJobHandler.HashAsync(staged).ConfigureAwait(false) : await ChronosClient.HashFileAsync(staged, ct).ConfigureAwait(false); if (!string.Equals(readBack, entry.Sha256, StringComparison.OrdinalIgnoreCase)) throw new IOException(Loc.L("Errors.Workspace.CopyMismatch", staged)); var name = SyncTransfers.SafeFileName(Path.GetFileName(entry.File)) ?? entry.RenderId.ToString("N"); var filed = await SyncTransfers.FileRenderAsync(archives, db, sequence, paths, staged, name, new Render { Id = entry.RenderId, SequenceId = sequence.Id, PresetName = entry.PresetName, OutputPath = string.Empty, Codec = entry.Codec, Width = entry.Width, Height = entry.Height, Fps = entry.Fps, DurationSeconds = entry.DurationSeconds, Created = entry.Rendered, ProducedBy = options.Processing.ProducedBy, ProjectId = entry.ProjectId, ProjectVersionId = entry.ProjectVersionId, }, ct).ConfigureAwait(false); return filed is not null ? true : throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved")); } finally { TryDelete(staged); } } /// Copies to , flushed to the disk. private static async Task CopyAsync(string source, string target, CancellationToken ct) { await using var from = new FileStream(source, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 20, FileOptions.Asynchronous | FileOptions.SequentialScan); await using var to = new FileStream(target, FileMode.Create, FileAccess.Write, FileShare.None, 1 << 20, FileOptions.Asynchronous); await from.CopyToAsync(to, ct).ConfigureAwait(false); to.Flush(flushToDisk: true); } /// The project the entry names is one of the sequence's, and its version (when it names one) one of that project's. private async Task ProjectOfAsync(Guid sequenceId, WorkspaceRenderEntry entry, CancellationToken ct) { var project = await db.Projects.GetAsync(entry.ProjectId, ct).ConfigureAwait(false); var version = entry.ProjectVersionId is { } versionId ? await db.Projects.GetVersionAsync(versionId, ct).ConfigureAwait(false) : null; if (project is null || (entry.ProjectVersionId is not null && version is null)) { // One the index does not know yet may be in the sequence's sidecars: listing reads them first. await archives.Projects.ListAsync(sequenceId, ct).ConfigureAwait(false); project = await db.Projects.GetAsync(entry.ProjectId, ct).ConfigureAwait(false); if (entry.ProjectVersionId is { } id) version = await db.Projects.GetVersionAsync(id, ct).ConfigureAwait(false); } return project is not null && project.SequenceId == sequenceId && (entry.ProjectVersionId is null || version?.ProjectId == project.Id); } /// /// The render uploaded to its server's sequence, with its project and version, resumable and checked against its SHA-256 /// there; the server answers that it holds the render already when it does. Refused for good (the server takes the /// project for another sequence's, the id for another sequence's render, the file for too large or not the one hashed): /// marked failed; anything else (the server out of reach, busy, the sign-in expired) fails the job, the entry left as it was. /// private async Task UploadAsync(JobContext context, ISyncRemote remote, Guid sequenceId, WorkspaceRenderEntry entry, string file, CancellationToken ct) { var name = SyncTransfers.SafeFileName(Path.GetFileName(entry.File)) ?? entry.RenderId.ToString("N"); var request = new RenderUploadRequest(entry.RenderId, name, entry.Size, entry.Sha256.ToLowerInvariant(), entry.PresetName, entry.Codec, entry.Width, entry.Height, entry.Fps, entry.DurationSeconds, options.Processing.ProducedBy, entry.ProjectId, entry.ProjectVersionId); try { if (await remote.UploadRenderAsync(sequenceId, file, request, ct).ConfigureAwait(false) is not { } filed) return false; // A server from before phase 5c ignores the project and version: the render is there, without them. if (filed.ProjectId != entry.ProjectId || filed.ProjectVersionId != entry.ProjectVersionId) { Log.Warning("workspace", $"The server filed render {entry.RenderId} without its project {entry.ProjectId} and version {entry.ProjectVersionId} (a server before phase 5c)"); context.Log(Loc.L("Archive.Log.WorkspaceSendNoProvenance", name)); } return true; } catch (ChronosApiException ex) when (ex.Status is HttpStatusCode.BadRequest or HttpStatusCode.Conflict or HttpStatusCode.RequestEntityTooLarge or HttpStatusCode.UnprocessableEntity) { throw new SendRefusedException(string.IsNullOrWhiteSpace(ex.Error.Message) ? ex.Message : ex.Error.Message); } } /// Waits while transfers with a server are held (a metered connection), saying so once. private async Task WhileHeldAsync(JobContext context) { if (!options.HoldTransfers()) return; context.Log(Loc.L("Archive.Log.WorkspaceSendHeld")); while (options.HoldTransfers()) await Task.Delay(HoldPoll, context.Cancellation).ConfigureAwait(false); } /// /// The ledger entry of changed by , under the project's lease, read again /// first (a render finished meanwhile stays); an entry discarded meanwhile stays discarded. Written whatever stops the job. /// private static async Task UpdateAsync(WorkspaceProject project, Guid renderId, Func change) { using (await project.LockAsync(LockWait, CancellationToken.None).ConfigureAwait(false)) { var ledger = project.ReadRenders(); if (ledger.Find(renderId) is { } entry) project.WriteRenders(ledger.With(change(entry))); } } private static void TryDelete(string path) { try { if (Directory.Exists(path)) Directory.Delete(path, recursive: true); else if (File.Exists(path)) File.Delete(path); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { Log.Debug("workspace", $"{path} could not be removed: {ex.Message}"); } } /// The home will not take this render as it is: it is marked failed with this message, and the others go on. private sealed class SendRefusedException(string message) : Exception(message); }