using System.Net;
using Chronos.Core.Archiving;
using Chronos.Core.Diagnostics;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Data.Archiving;
using Chronos.Data.Jobs;
using Chronos.Data.Projects;
using Chronos.Data.Sync;
using Chronos.Protocol;
using Chronos.Protocol.Client;
namespace Chronos.Data.Workspaces;
///
/// (phase 5c, ): the local renders of a workspace project
/// sent home with their project and version, per unsent ledger entry (; the ids the job names,
/// or every unsent one): the file hashed again (one missing or changed since it was rendered is refused, marked failed in the
/// ledger and left where it is), the version it names pushed first when the home lacks it (a push that fails fails the job,
/// the entry left as it was; a version the push replayed under a new id, after the home moved on, is found again by the
/// content hash of what rendered, else the render goes with its project alone, and the entry says so from then on), then
/// filed in this PC's archive (copied into the archive's staging folder, read back against
/// its SHA-256, and filed under the sequence's lease as the sync files a server's exports,
/// , after checking the project and version are the sequence's) or uploaded to its
/// server ( with ProjectId and ProjectVersionId, which the server checks the
/// same way), and the entry marked sent. The file stays in exports/ until a clean-up removes it.
/// Idempotent by render id: a send repeated after a cut finds the home holding the render already and only marks it sent; a
/// render id the home files under another sequence is refused. The ledger is read and written under the project's lease,
/// each entry read again before it is written, so a render finished meanwhile is kept.
/// A send to a server is held while says so (a metered connection): it waits
/// before each upload, and the desktop pauses it meanwhile. A send to this PC's archive is never held. A server from before
/// phase 5c files the render without its project and version: the job's log says so. A send queued before the workspace
/// moved works where the workspace is now, and none runs while it is being moved ().
///
internal sealed class WorkspaceSendJobHandler(ChronosDatabase db, ArchiveService archives, ISyncConnector? servers, WorkspaceJobOptions options) : IJobHandler
{
private static readonly TimeSpan LockWait = TimeSpan.FromMinutes(2);
/// How often a send held on a metered connection looks again.
internal static TimeSpan HoldPoll { get; set; } = TimeSpan.FromSeconds(2);
public JobKind Kind => JobKind.WorkspaceSend;
public async Task RunAsync(JobContext context)
{
ArgumentNullException.ThrowIfNull(context);
var ct = context.Cancellation;
var parameters = WorkspaceJobs.SendParameters(context.Job) ?? throw new InvalidDataException(Loc.L("Errors.Workspace.JobParameters"));
if (context.Job.TargetIds is not [var sequenceId] || string.IsNullOrWhiteSpace(parameters.WorkspaceRoot))
throw new InvalidDataException(Loc.L("Errors.Workspace.JobParameters"));
var server = parameters.Server is { Length: > 0 } named ? named : null;
// Where the project is now (a workspace moved since the job was queued), never while it is being moved; nothing is
// made where it is not.
var root = WorkspaceJobRoots.Resolve(parameters.WorkspaceRoot, parameters.ProjectId, options.CurrentRoot, "Errors.Workspace.NotCheckedOut");
var project = WorkspaceStore.ForJob(root).Find(parameters.ProjectId)
?? throw new InvalidOperationException(Loc.L("Errors.Workspace.NotCheckedOut"));
var workspace = project.ReadBundle().Workspace ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.NotCheckedOut"));
var home = workspace.Home;
if (home.SequenceId != sequenceId || (server is null
? !home.IsArchive
: home.Kind != BundleHome.ServerKind || !options.IsSameServer(home.Server, server)))
throw new InvalidOperationException(Loc.L("Errors.Workspace.OtherHome"));
var entries = await ToSendAsync(project, parameters.RenderIds, ct).ConfigureAwait(false);
await context.ReportAsync(0, entries.Count).ConfigureAwait(false);
if (entries.Count == 0)
{
context.Log(Loc.L("Archive.Log.WorkspaceSent", workspace.Name, 0, 0));
return;
}
if (server is null)
{
var sequence = await db.Sequences.GetAsync(sequenceId, ct).ConfigureAwait(false)
?? throw new InvalidOperationException(Loc.L("Archive.Log.SequenceMissing", sequenceId));
var archive = await db.Archives.GetAsync(sequence.ArchiveId, ct).ConfigureAwait(false)
?? throw new InvalidOperationException(Loc.L("Errors.Sync.NoArchive"));
var projectHome = new ArchiveProjectHome(db, archives.Projects, sequenceId);
await SendAsync(context, project, workspace, entries, () => projectHome, held: false,
(entry, file, token) => FileAsync(sequence, archive, entry, file, token)).ConfigureAwait(false);
return;
}
using var remote = (servers is null ? null : await servers.ConnectAsync(server, ct).ConfigureAwait(false))
?? throw new InvalidOperationException(Loc.L("Errors.Sync.NotConnected"));
await SendAsync(context, project, workspace, entries,
() => servers!.ProjectHome(home) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.NotConnected")), held: true,
(entry, file, token) => UploadAsync(context, remote, sequenceId, entry, file, token)).ConfigureAwait(false);
}
///
/// The entries to send, oldest first: those of the ledger holds unsent (an id it no longer
/// holds was discarded meanwhile, one sent already is left), every unsent one when none is named.
///
private static async Task> ToSendAsync(WorkspaceProject project, IReadOnlyList? renderIds, CancellationToken ct)
{
WorkspaceRenders ledger;
using (await project.LockAsync(LockWait, ct).ConfigureAwait(false)) ledger = project.ReadRenders();
if (renderIds is not { Count: > 0 }) return ledger.Unsent;
var named = renderIds.ToHashSet();
return [.. ledger.Entries.Where(e => named.Contains(e.RenderId) && e.IsUnsent)];
}
/// Delivers one checked render home: true when the home took it now, false when it held it already.
private delegate Task Deliver(WorkspaceRenderEntry entry, string file, CancellationToken cancellationToken);
private async Task SendAsync(JobContext context, WorkspaceProject project, BundleWorkspace workspace, IReadOnlyList entries,
Func home, bool held, Deliver deliver)
{
var ct = context.Cancellation;
var (sent, there, refused) = (0, 0, 0);
// The versions this run found at the home, or sent there first, with the id the home holds each by (a replay gives another).
var versions = new Dictionary();
for (var i = 0; i < entries.Count; i++)
{
ct.ThrowIfCancellationRequested();
if (held) await WhileHeldAsync(context).ConfigureAwait(false);
var entry = entries[i];
await context.ReportAsync(i, entries.Count, new JobDetail(JobStep.Upload, Path.GetFileName(entry.File), 0, 0, i + 1, entries.Count, JobPartUnit.Render))
.ConfigureAwait(false);
var file = project.ContainedExportFile(entry.File);
var refusal = file is null ? Loc.L("Errors.Workspace.SendMissing", Path.GetFileName(entry.File ?? string.Empty)) : await CheckAsync(entry, file, folders: !held, ct).ConfigureAwait(false);
if (refusal is null)
{
if (entry.ProjectVersionId is { } versionId)
{
if (!versions.TryGetValue(versionId, out var atHome))
{
atHome = await EnsureVersionAsync(context, project, workspace, home(), entry, versionId, ct).ConfigureAwait(false);
versions[versionId] = atHome;
}
if (atHome != versionId)
{
// The ledger says from now on what the home holds: later sends agree with this one.
entry = entry with { ProjectVersionId = atHome };
var mapped = atHome;
await UpdateAsync(project, entry.RenderId, e => e with { ProjectVersionId = mapped }).ConfigureAwait(false);
}
}
try
{
if (await deliver(entry, file!, ct).ConfigureAwait(false)) sent++;
else there++;
// The home holds it: recorded whatever stops the job now.
await UpdateAsync(project, entry.RenderId, e => e.AsSent(DateTimeOffset.UtcNow)).ConfigureAwait(false);
}
catch (SendRefusedException ex)
{
refusal = ex.Message;
}
}
if (refusal is not null)
{
refused++;
context.Log(refusal);
await UpdateAsync(project, entry.RenderId, e => e.AsFailed(refusal)).ConfigureAwait(false);
}
await context.ReportAsync(i + 1, entries.Count).ConfigureAwait(false);
}
context.Log(Loc.L("Archive.Log.WorkspaceSent", workspace.Name, sent, there));
if (refused > 0) throw new InvalidOperationException(Loc.L("Errors.Workspace.SendFailed", refused, entries.Count));
}
///
/// Why the render cannot be sent as it is (gone from the exports, another file than the one rendered, or an image
/// sequence's folder for a home that takes one file per render); null when it can. A folder (an image sequence, only
/// when : this PC's archive files folders as its own renders do) is checked by its total size
/// and the hash of its listing, as the render recorded it ().
///
/// A ledger names files inside the exports only (): one pointing elsewhere is never read.
private static async Task CheckAsync(WorkspaceRenderEntry entry, string file, bool folders, CancellationToken ct)
{
var name = Path.GetFileName(entry.File);
if (Directory.Exists(file))
{
if (!folders) return Loc.L("Errors.Workspace.SendSequence", name);
return WorkspaceCleaner.Measure(file).Bytes != entry.Size
|| !string.Equals(await WorkspaceRenderJobHandler.HashAsync(file).ConfigureAwait(false), entry.Sha256, StringComparison.OrdinalIgnoreCase)
? Loc.L("Errors.Workspace.SendHashMismatch", name)
: null;
}
if (!File.Exists(file)) return Loc.L("Errors.Workspace.SendMissing", name);
if (new FileInfo(file).Length != entry.Size
|| !string.Equals(await ChronosClient.HashFileAsync(file, ct).ConfigureAwait(false), entry.Sha256, StringComparison.OrdinalIgnoreCase))
return Loc.L("Errors.Workspace.SendHashMismatch", name);
return null;
}
/// How many of the home's newest versions are read for one of the same content, when a replay gave a rendered version another id.
private const int ReplayedLookBack = 100;
///
/// Version of the project at the home, as the id the home holds it by: pushed there first
/// (every local version, in order, as a sync pushes them) when the home lacks it. A push that replays it on a home that
/// moved on gives it another id (and one that changes nothing is not sent at all): the home's version of the content
/// hash that rendered () stands for it, else none (null: the render goes
/// with its project alone, as a render of a version the home never had). A push that fails fails the job.
///
private static async Task EnsureVersionAsync(JobContext context, WorkspaceProject project, BundleWorkspace workspace, IProjectHome home,
WorkspaceRenderEntry entry, Guid versionId, CancellationToken ct)
{
var projectId = entry.ProjectId;
if (await home.GetVersionAsync(projectId, versionId, ct).ConfigureAwait(false) is not null) return versionId;
try
{
await new WorkspaceSync(project.Store, Environment.MachineName) { LockWait = LockWait }.PushAsync(projectId, home, ct).ConfigureAwait(false);
}
catch (Exception ex) when (ex is HomeUnavailableException or WorkspaceSyncException or ProjectException or LeaseUnavailableException
or InvalidOperationException)
{
context.Log(ex.Message);
throw new InvalidOperationException(Loc.L("Errors.Workspace.SendVersion"), ex);
}
if (await home.GetVersionAsync(projectId, versionId, ct).ConfigureAwait(false) is { } pushed)
{
context.Log(Loc.L("Archive.Log.WorkspaceVersionPushed", workspace.Name, pushed.Number));
return versionId;
}
ProjectVersion? same = null;
if (entry.SettingsHash is { Length: > 0 } hash)
{
var history = await home.HistoryAsync(projectId, null, ReplayedLookBack, ct).ConfigureAwait(false);
// Of the same content whatever its line ends: a version stored by a Windows build before every build wrote LF
// holds the CRLF text and its hash.
same = history.FirstOrDefault(v => string.Equals(v.ContentHash, hash, StringComparison.OrdinalIgnoreCase)
|| (v.SettingsJson is { Length: > 0 } text && ProjectContent.MatchesHash(text, hash)));
}
if (same is not null)
{
Log.Info("workspace", $"Version {versionId} of project {projectId} reached the home as version {same.Id} (a replay): render {entry.RenderId} names that one");
context.Log(Loc.L("Archive.Log.WorkspaceVersionReplayed", workspace.Name, same.Number));
return same.Id;
}
Log.Info("workspace", $"Version {versionId} of project {projectId} is not at the home after the push, nor one of the same content: render {entry.RenderId} goes with its project alone");
context.Log(Loc.L("Archive.Log.WorkspaceVersionNotHome", workspace.Name));
return null;
}
///
/// The render filed in this PC's archive, the home's sequence: copied into the archive's staging folder, read back against
/// the ledger's SHA-256, then filed under the sequence's lease with the project and version (once both are checked to be
/// the sequence's). A render id the index holds already for this sequence is the same render, sent before.
///
private async Task FileAsync(Sequence sequence, Archive archive, WorkspaceRenderEntry entry, string file, CancellationToken ct)
{
if (await db.Renders.GetAsync(entry.RenderId, ct).ConfigureAwait(false) is { } known)
return known.SequenceId == sequence.Id ? false : throw new SendRefusedException(Loc.L("Server.Error.RenderExists"));
if (!await ProjectOfAsync(sequence.Id, entry, ct).ConfigureAwait(false)) throw new SendRefusedException(Loc.L("Server.Error.ProjectMismatch"));
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
SyncTransfers.EnsureSpace(paths.Root, entry.Size);
Directory.CreateDirectory(paths.StagingDirectory);
var staged = Path.Combine(paths.StagingDirectory, "workspace-render-" + entry.RenderId.ToString("N") + ".part");
var folder = Directory.Exists(file);
try
{
TryDelete(staged);
if (folder)
{
// An image sequence: every file of its folder, the folder filed whole as the archive's own renders are.
foreach (var source in Directory.EnumerateFiles(file, "*", SearchOption.AllDirectories))
{
var target = Path.Combine(staged, Path.GetRelativePath(file, source));
Directory.CreateDirectory(Path.GetDirectoryName(target)!);
await CopyAsync(source, target, ct).ConfigureAwait(false);
}
Directory.CreateDirectory(staged);
}
else await CopyAsync(file, staged, ct).ConfigureAwait(false);
// Read back from the archive's volume: what is filed is the render as it was written.
var readBack = folder
? await WorkspaceRenderJobHandler.HashAsync(staged).ConfigureAwait(false)
: await ChronosClient.HashFileAsync(staged, ct).ConfigureAwait(false);
if (!string.Equals(readBack, entry.Sha256, StringComparison.OrdinalIgnoreCase))
throw new IOException(Loc.L("Errors.Workspace.CopyMismatch", staged));
var name = SyncTransfers.SafeFileName(Path.GetFileName(entry.File)) ?? entry.RenderId.ToString("N");
var filed = await SyncTransfers.FileRenderAsync(archives, db, sequence, paths, staged, name, new Render
{
Id = entry.RenderId,
SequenceId = sequence.Id,
PresetName = entry.PresetName,
OutputPath = string.Empty,
Codec = entry.Codec,
Width = entry.Width,
Height = entry.Height,
Fps = entry.Fps,
DurationSeconds = entry.DurationSeconds,
Created = entry.Rendered,
ProducedBy = options.Processing.ProducedBy,
ProjectId = entry.ProjectId,
ProjectVersionId = entry.ProjectVersionId,
// A file's hash is the file's own (an image sequence's ledger hash is of its listing, not a render's).
Sha256 = folder ? null : entry.Sha256.ToLowerInvariant(),
Kind = entry.Kind,
}, ct).ConfigureAwait(false);
return filed is not null ? true : throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
}
finally
{
TryDelete(staged);
}
}
/// Copies to , flushed to the disk.
private static async Task CopyAsync(string source, string target, CancellationToken ct)
{
await using var from = new FileStream(source, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 20, FileOptions.Asynchronous | FileOptions.SequentialScan);
await using var to = new FileStream(target, FileMode.Create, FileAccess.Write, FileShare.None, 1 << 20, FileOptions.Asynchronous);
await from.CopyToAsync(to, ct).ConfigureAwait(false);
to.Flush(flushToDisk: true);
}
/// The project the entry names is one of the sequence's, and its version (when it names one) one of that project's.
private async Task ProjectOfAsync(Guid sequenceId, WorkspaceRenderEntry entry, CancellationToken ct)
{
var project = await db.Projects.GetAsync(entry.ProjectId, ct).ConfigureAwait(false);
var version = entry.ProjectVersionId is { } versionId ? await db.Projects.GetVersionAsync(versionId, ct).ConfigureAwait(false) : null;
if (project is null || (entry.ProjectVersionId is not null && version is null))
{
// One the index does not know yet may be in the sequence's sidecars: listing reads them first.
await archives.Projects.ListAsync(sequenceId, ct).ConfigureAwait(false);
project = await db.Projects.GetAsync(entry.ProjectId, ct).ConfigureAwait(false);
if (entry.ProjectVersionId is { } id) version = await db.Projects.GetVersionAsync(id, ct).ConfigureAwait(false);
}
return project is not null && project.SequenceId == sequenceId && (entry.ProjectVersionId is null || version?.ProjectId == project.Id);
}
///
/// The render uploaded to its server's sequence, with its project and version, resumable and checked against its SHA-256
/// there; the server answers that it holds the render already when it does. Refused for good (the server takes the
/// project for another sequence's, the id for another sequence's render, the file for too large or not the one hashed):
/// marked failed; anything else (the server out of reach, busy, the sign-in expired) fails the job, the entry left as it was.
///
private async Task UploadAsync(JobContext context, ISyncRemote remote, Guid sequenceId, WorkspaceRenderEntry entry, string file, CancellationToken ct)
{
var name = SyncTransfers.SafeFileName(Path.GetFileName(entry.File)) ?? entry.RenderId.ToString("N");
var request = new RenderUploadRequest(entry.RenderId, name, entry.Size, entry.Sha256.ToLowerInvariant(), entry.PresetName, entry.Codec, entry.Width, entry.Height,
entry.Fps, entry.DurationSeconds, options.Processing.ProducedBy, entry.ProjectId, entry.ProjectVersionId, entry.Kind);
try
{
if (await remote.UploadRenderAsync(sequenceId, file, request, ct).ConfigureAwait(false) is not { } filed) return false;
// A server from before phase 5c ignores the project and version: the render is there, without them.
if (filed.ProjectId != entry.ProjectId || filed.ProjectVersionId != entry.ProjectVersionId)
{
Log.Warning("workspace", $"The server filed render {entry.RenderId} without its project {entry.ProjectId} and version {entry.ProjectVersionId} (a server before phase 5c)");
context.Log(Loc.L("Archive.Log.WorkspaceSendNoProvenance", name));
}
return true;
}
catch (ChronosApiException ex) when (ex.Status is HttpStatusCode.BadRequest or HttpStatusCode.Conflict or HttpStatusCode.RequestEntityTooLarge
or HttpStatusCode.UnprocessableEntity)
{
throw new SendRefusedException(string.IsNullOrWhiteSpace(ex.Error.Message) ? ex.Message : ex.Error.Message);
}
}
/// Waits while transfers with a server are held (a metered connection), saying so once.
private async Task WhileHeldAsync(JobContext context)
{
if (!options.HoldTransfers()) return;
context.Log(Loc.L("Archive.Log.WorkspaceSendHeld"));
while (options.HoldTransfers()) await Task.Delay(HoldPoll, context.Cancellation).ConfigureAwait(false);
}
///
/// The ledger entry of changed by , under the project's lease, read again
/// first (a render finished meanwhile stays); an entry discarded meanwhile stays discarded. Written whatever stops the job.
///
private static async Task UpdateAsync(WorkspaceProject project, Guid renderId, Func change)
{
using (await project.LockAsync(LockWait, CancellationToken.None).ConfigureAwait(false))
{
var ledger = project.ReadRenders();
if (ledger.Find(renderId) is { } entry) project.WriteRenders(ledger.With(change(entry)));
}
}
private static void TryDelete(string path)
{
try
{
if (Directory.Exists(path)) Directory.Delete(path, recursive: true);
else if (File.Exists(path)) File.Delete(path);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Debug("workspace", $"{path} could not be removed: {ex.Message}");
}
}
/// The home will not take this render as it is: it is marked failed with this message, and the others go on.
private sealed class SendRefusedException(string message) : Exception(message);
}