CI / Build and test (push) Successful in 8m14s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 7s
Release / Build, test and publish (push) Successful in 13m19s
CI / PostgreSQL 16 integration (push) Successful in 4m49s
CI / PostgreSQL 17 integration (push) Successful in 5m28s
CI / Docker image (push) Successful in 53s
Release / Server image (push) Failing after 10s
The owner renamed the product and moved the repository to the self-hosted Gitea (https://gitea.encke-hake.ts.net/BalboSoft/Chronos) for more control, and asked for Actions and a first automatic release with the server package and the Windows exe (docs/requests/2026-10-23-chronos-gitea.md). - Chronos everywhere: a byte-exact rename of contents and paths, then a per-language review of the forms the old name's ending had dictated. - Gitea instead of GitHub: the update check, licence, issue and support links read ChronosReleasesUrl; the update check takes the highest stable version, offers a pre-release build its final release and reports a repository it cannot see as a failed check. EULA, security policy and Gitea issue forms live at the repository root. - Gitea Actions for the Linux act_runner (.gitea/workflows): CI, release, nightly image and performance gates in the .NET SDK container with git checkout and the system SQLite. The release cross-compiles the Windows programs, checks their icon, version and DPI manifest, packs installers and zips, publishes the Gitea release and pushes the server image (REGISTRY_TOKEN). Windows-only tests run in an opt-in job on a Windows runner (WINDOWS_RUNNER). - New Chronos.Tools commands: zip, gitea-release, postgres-ci (replaces the docker-exec script, which Gitea's service containers do not allow; the throw-away server runs without durability) and pe-check. - Timed tests take their budget from TestTiming (CHRONOS_TEST_TIME_FACTOR, 3 on the shared runner; the weekly performance gates keep theirs). BREAKING CHANGE: programs, settings, data folders, file types, environment variables and the Docker image use the new name; nothing is migrated from Titano, which never had a release. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
265 lines
16 KiB
C#
265 lines
16 KiB
C#
using System.Globalization;
|
|
using System.Text.Json;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Data;
|
|
using Chronos.Data.Jobs;
|
|
using Chronos.Data.Repositories;
|
|
using Chronos.Protocol;
|
|
using Chronos.Server.Api;
|
|
using Chronos.Server.Configuration;
|
|
using Chronos.Server.Events;
|
|
using Chronos.Server.Security;
|
|
using Chronos.Server.Storage;
|
|
using UploadState = Chronos.Data.Repositories.UploadState;
|
|
|
|
namespace Chronos.Server.Jobs;
|
|
|
|
/// <summary>
|
|
/// Render nodes (BRIEF §14.7): clients with the render-node right register their encoders, heartbeat, claim queued
|
|
/// render jobs they can encode, report progress, and upload the output in resumable chunks that become a render of
|
|
/// the sequence. A claimed job lists every frame with its flags (a node renders none missing, corrupt or excluded) and
|
|
/// the proxies the server asks for (<see cref="ServerProxies"/>), which the node sends from the originals it fetched.
|
|
/// A node that stops heart-beating loses its jobs to the queue (<see cref="RenderNodeMonitor"/>).
|
|
/// </summary>
|
|
public sealed class RenderNodeService(
|
|
ChronosDatabase db,
|
|
ServerOptions options,
|
|
ServerArchive archive,
|
|
UploadService uploads,
|
|
UploadStore store,
|
|
RenderStore renders,
|
|
JobEngineHost jobs,
|
|
EventHub events,
|
|
TimeProvider time)
|
|
{
|
|
public const string Idle = "idle";
|
|
public const string Busy = "busy";
|
|
public const string Offline = "offline";
|
|
private const int LogTailLines = 40;
|
|
|
|
public async Task<Outcome<RenderNodeDto>> RegisterAsync(Caller caller, RegisterNodeRequest request, CancellationToken cancellationToken)
|
|
{
|
|
var name = request.ClientName?.Trim();
|
|
if (string.IsNullOrEmpty(name) || name.Length > 100) return Outcome<RenderNodeDto>.Fail(400, "bad_request", "clientName");
|
|
if (request.Encoders is null || request.Encoders.Count > 64 || request.Encoders.Any(e => string.IsNullOrWhiteSpace(e?.Codec)))
|
|
return Outcome<RenderNodeDto>.Fail(400, "bad_request", "encoders");
|
|
|
|
var existing = (await db.RenderNodes.ListAsync(cancellationToken)).FirstOrDefault(n => n.UserId == caller.Id && n.ClientName == name);
|
|
// A node registers when it starts, holding no job: what the server still lists as running on it was abandoned
|
|
// by its previous run (stopped, restarted or crashed within the heartbeat timeout) and goes back to the queue.
|
|
if (existing is not null) await ReleaseJobsAsync(db, jobs, existing, $"Render node {name} restarted.", cancellationToken);
|
|
var node = new RenderNode
|
|
{
|
|
Id = existing?.Id ?? Guid.CreateVersion7(),
|
|
UserId = caller.Id,
|
|
ClientName = name,
|
|
CapabilitiesJson = JsonSerializer.Serialize(request with { ClientName = name }, ProtocolJson.Default.RegisterNodeRequest),
|
|
LastHeartbeat = time.GetUtcNow(),
|
|
State = Idle,
|
|
};
|
|
await db.RenderNodes.RegisterAsync(node, cancellationToken);
|
|
events.Publish(new NodeStateEvent(node.Id, node.ClientName, node.State));
|
|
return Dto.Node(node, caller.User.Username);
|
|
}
|
|
|
|
public async Task<Outcome<bool>> HeartbeatAsync(Caller caller, Guid nodeId, CancellationToken cancellationToken)
|
|
{
|
|
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome<bool>.Fail(404, "not_found", "node");
|
|
await TouchAsync(node, cancellationToken);
|
|
return true;
|
|
}
|
|
|
|
/// <summary>The next queued render job this node can encode (highest priority, then oldest), or null when none fits.</summary>
|
|
public async Task<Outcome<ClaimedJob?>> ClaimAsync(Caller caller, Guid nodeId, CancellationToken cancellationToken)
|
|
{
|
|
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome<ClaimedJob?>.Fail(404, "not_found", "node");
|
|
var capabilities = Dto.Capabilities(node);
|
|
foreach (var queued in await db.Jobs.ListQueuedAsync(JobKind.Render, 50, cancellationToken))
|
|
{
|
|
// A render the server's own engine started (it left a checkpoint) resumes there: only it knows that attempt's
|
|
// leftovers and the batch entries it recorded.
|
|
if (queued.Checkpoint is not null) continue;
|
|
if (JobEndpoints.ParseRender(queued.ParametersJson) is not { } parameters || capabilities is null || !CanEncode(capabilities, parameters)) continue;
|
|
if (await db.Jobs.TryClaimAsync(queued.Id, NodeKey(node), cancellationToken) is not { } job) continue;
|
|
jobs.Publish(job);
|
|
if (job.TargetIds.Count != 1 || await db.Sequences.GetAsync(job.TargetIds[0], cancellationToken) is not { } sequence)
|
|
{
|
|
await FinishAsync(node, job, JobState.Failed, "The sequence no longer exists.", cancellationToken);
|
|
continue;
|
|
}
|
|
await TouchAsync(node, cancellationToken);
|
|
var spec = ServerProxies.Spec(archive.Current);
|
|
return new ClaimedJob(Dto.Job(job), sequence.Id, parameters.SettingsJson ?? sequence.SettingsJson ?? "{}", parameters.PresetJson,
|
|
parameters.UseProxies, await ServerProxies.FramesAsync(db, archive.Current, sequence, spec, cancellationToken), spec);
|
|
}
|
|
await TouchAsync(node, cancellationToken);
|
|
return (ClaimedJob?)null;
|
|
}
|
|
|
|
public async Task<Outcome<bool>> ProgressAsync(Caller caller, Guid nodeId, Guid jobId, NodeProgress progress, CancellationToken cancellationToken)
|
|
{
|
|
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome<bool>.Fail(404, "not_found", "node");
|
|
if (await AssignedJobAsync(node, jobId, cancellationToken) is not { } job) return Outcome<bool>.Fail(409, "job_not_assigned");
|
|
// A node reports any rate (a tiny one when it stalls): only a usable time left is stored and broadcast.
|
|
var rate = progress.ItemsPerSecond is { } r && double.IsFinite(r) && r > 0 ? r : (double?)null;
|
|
var eta = JobEta.Of(progress.Total - progress.Done, rate);
|
|
var tail = AppendLog(job.LogTail, progress.LogLine);
|
|
if (!await db.Jobs.UpdateNodeProgressAsync(jobId, NodeKey(node), progress.Done, progress.Total, rate, eta, tail, cancellationToken))
|
|
return Outcome<bool>.Fail(409, "job_not_assigned");
|
|
await TouchAsync(node, cancellationToken);
|
|
events.Publish(new JobProgressEvent(jobId, progress.Done, progress.Total, rate, eta?.TotalSeconds));
|
|
return true;
|
|
}
|
|
|
|
public async Task<Outcome<bool>> FailAsync(Caller caller, Guid nodeId, Guid jobId, NodeJobFailure failure, CancellationToken cancellationToken)
|
|
{
|
|
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome<bool>.Fail(404, "not_found", "node");
|
|
if (await AssignedJobAsync(node, jobId, cancellationToken) is not { } job) return Outcome<bool>.Fail(409, "job_not_assigned");
|
|
var error = string.IsNullOrWhiteSpace(failure.Error) ? "Render failed on the node." : failure.Error.Trim()[..Math.Min(failure.Error.Trim().Length, 500)];
|
|
return await FinishAsync(node, job, JobState.Failed, error, cancellationToken) ? true : Outcome<bool>.Fail(409, "job_not_assigned");
|
|
}
|
|
|
|
/// <summary>Starts (or resumes) the chunked upload of a job's output.</summary>
|
|
public async Task<Outcome<StartUploadResponse>> StartResultAsync(Caller caller, Guid nodeId, Guid jobId, RenderResultRequest request, CancellationToken cancellationToken)
|
|
{
|
|
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome<StartUploadResponse>.Fail(404, "not_found", "node");
|
|
if (await AssignedJobAsync(node, jobId, cancellationToken) is not { } job) return Outcome<StartUploadResponse>.Fail(409, "job_not_assigned");
|
|
if (UploadRules.SafeFileName(request.FileName) is not { } fileName) return Outcome<StartUploadResponse>.Fail(400, "invalid_file_name");
|
|
if (!UploadRules.IsSha256(request.Sha256)) return Outcome<StartUploadResponse>.Fail(400, "invalid_hash");
|
|
if (request.Size <= 0) return Outcome<StartUploadResponse>.Fail(400, "bad_request", "size");
|
|
if (request.Size > options.MaxUploadBytes)
|
|
return Outcome<StartUploadResponse>.Fail(413, "too_large", options.MaxUploadBytes.ToString(CultureInfo.InvariantCulture));
|
|
if (string.IsNullOrWhiteSpace(request.PresetName)) return Outcome<StartUploadResponse>.Fail(400, "bad_request", "presetName");
|
|
|
|
var sha = request.Sha256.ToLowerInvariant();
|
|
if (await db.Uploads.FindByJobAsync(jobId, cancellationToken) is { State: UploadState.Receiving } open && open.Sha256 == sha && open.Size == request.Size
|
|
&& store.Exists(open.Id))
|
|
return Started(open);
|
|
if (UploadRules.FreeBytes(options.UploadsPath) is { } free && free < request.Size) return Outcome<StartUploadResponse>.Fail(507, "insufficient_storage");
|
|
|
|
var now = time.GetUtcNow();
|
|
var upload = await db.Uploads.CreateAsync(new Upload
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
UserId = caller.Id,
|
|
SequenceId = job.TargetIds.Count == 1 ? job.TargetIds[0] : null,
|
|
FileName = fileName,
|
|
Size = request.Size,
|
|
Sha256 = sha,
|
|
ChunkSize = options.ChunkSizeBytes,
|
|
State = UploadState.Receiving,
|
|
Created = now,
|
|
UpdatedAt = now,
|
|
JobId = jobId,
|
|
RenderJson = JsonSerializer.Serialize(request with { FileName = fileName, Sha256 = sha }, ProtocolJson.Default.RenderResultRequest),
|
|
}, cancellationToken);
|
|
store.Create(upload.Id, upload.Size);
|
|
return Started(upload);
|
|
}
|
|
|
|
/// <summary>Verifies the uploaded output, files it as a render of the sequence and marks the job done.</summary>
|
|
public async Task<Outcome<RenderDto>> CompleteResultAsync(Caller caller, Guid nodeId, Guid jobId, CancellationToken cancellationToken)
|
|
{
|
|
if (await OwnedNodeAsync(caller, nodeId, cancellationToken) is not { } node) return Outcome<RenderDto>.Fail(404, "not_found", "node");
|
|
if (await AssignedJobAsync(node, jobId, cancellationToken) is not { } job) return Outcome<RenderDto>.Fail(409, "job_not_assigned");
|
|
var upload = await db.Uploads.FindByJobAsync(jobId, cancellationToken);
|
|
if (upload is not { State: UploadState.Receiving } || upload.RenderJson is null) return Outcome<RenderDto>.Fail(409, "upload_state");
|
|
if (!upload.IsComplete)
|
|
return Outcome<RenderDto>.Fail(409, "incomplete", (upload.ChunkCount - upload.ReceivedCount).ToString(CultureInfo.InvariantCulture));
|
|
if (!await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Receiving, UploadState.Committing, cancellationToken: cancellationToken))
|
|
return Outcome<RenderDto>.Fail(409, "upload_state");
|
|
|
|
if (await uploads.StageVerifiedAsync(upload, cancellationToken) is not { } staged) return Outcome<RenderDto>.Fail(422, "hash_mismatch");
|
|
var output = JsonSerializer.Deserialize(upload.RenderJson, ProtocolJson.Default.RenderResultRequest)!;
|
|
var sequence = await db.Sequences.GetAsync(job.TargetIds[0], cancellationToken);
|
|
var render = sequence is null ? null : await renders.CommitAsync(sequence, staged, output, node.ClientName, cancellationToken);
|
|
if (render is null)
|
|
{
|
|
UploadStore.TryDelete(staged);
|
|
await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Failed, cancellationToken: CancellationToken.None);
|
|
store.Delete(upload.Id);
|
|
await FinishAsync(node, job, JobState.Failed, "The sequence has no folder in the archive.", CancellationToken.None);
|
|
return Outcome<RenderDto>.Fail(409, "upload_state", "sequence");
|
|
}
|
|
await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Committed, cancellationToken: CancellationToken.None);
|
|
store.Delete(upload.Id);
|
|
await FinishAsync(node, job, JobState.Done, null, CancellationToken.None);
|
|
if (await db.Sequences.GetAsync(sequence!.Id, CancellationToken.None) is { } updated)
|
|
events.Publish(new SequenceChangedEvent(updated.Id, "renders", updated.FramesVersion, updated.SettingsVersion));
|
|
return Dto.Render(render);
|
|
}
|
|
|
|
/// <summary>Running-job owner key stored in <c>jobs.node_id</c>.</summary>
|
|
public static string NodeKey(RenderNode node) => node.Id.ToString();
|
|
|
|
/// <summary>
|
|
/// Gives the jobs running on <paramref name="node"/> back to the queue with <paramref name="reason"/> in their log
|
|
/// (checkpoint and progress kept); a job that already lost its node <see cref="RenderNodeMonitor.MaxAttempts"/> times
|
|
/// fails instead. Returns how many went back.
|
|
/// </summary>
|
|
internal static async Task<int> ReleaseJobsAsync(ChronosDatabase db, JobEngineHost jobs, RenderNode node, string reason, CancellationToken cancellationToken)
|
|
{
|
|
var key = NodeKey(node);
|
|
var lost = await db.Jobs.ListRunningByNodeAsync(key, cancellationToken);
|
|
foreach (var job in lost)
|
|
{
|
|
var line = AppendLog(job.LogTail, reason);
|
|
if (job.Attempts >= RenderNodeMonitor.MaxAttempts)
|
|
await db.Jobs.FinishNodeJobAsync(job.Id, key, JobState.Failed, $"Lost its render node {job.Attempts} times.", line, cancellationToken);
|
|
else
|
|
await db.Jobs.UpdateNodeProgressAsync(job.Id, key, job.ProgressDone, job.ProgressTotal, null, null, line, cancellationToken);
|
|
}
|
|
var requeued = await db.Jobs.RequeueRunningAsync(key, cancellationToken);
|
|
foreach (var job in lost)
|
|
{
|
|
if (await db.Jobs.GetAsync(job.Id, cancellationToken) is { } current) jobs.Publish(current);
|
|
}
|
|
return requeued;
|
|
}
|
|
|
|
private async Task<RenderNode?> OwnedNodeAsync(Caller caller, Guid nodeId, CancellationToken cancellationToken) =>
|
|
await db.RenderNodes.GetAsync(nodeId, cancellationToken) is { } node && node.UserId == caller.Id ? node : null;
|
|
|
|
private async Task<Job?> AssignedJobAsync(RenderNode node, Guid jobId, CancellationToken cancellationToken) =>
|
|
await db.Jobs.GetAsync(jobId, cancellationToken) is { State: JobState.Running } job && job.NodeId == NodeKey(node) ? job : null;
|
|
|
|
private async Task<bool> FinishAsync(RenderNode node, Job job, JobState state, string? error, CancellationToken cancellationToken)
|
|
{
|
|
var tail = error is null ? null : AppendLog(job.LogTail, error);
|
|
if (!await db.Jobs.FinishNodeJobAsync(job.Id, NodeKey(node), state, error, tail, cancellationToken)) return false;
|
|
if (await db.Jobs.GetAsync(job.Id, cancellationToken) is { } finished) jobs.Publish(finished);
|
|
await TouchAsync(node, cancellationToken);
|
|
return true;
|
|
}
|
|
|
|
/// <summary>Records a sign of life and publishes the node's state when it changed.</summary>
|
|
private async Task TouchAsync(RenderNode node, CancellationToken cancellationToken)
|
|
{
|
|
var state = (await db.Jobs.ListRunningByNodeAsync(NodeKey(node), cancellationToken)).Count > 0 ? Busy : Idle;
|
|
await db.RenderNodes.HeartbeatAsync(node.Id, state, time.GetUtcNow(), cancellationToken);
|
|
if (state != node.State) events.Publish(new NodeStateEvent(node.Id, node.ClientName, state));
|
|
}
|
|
|
|
/// <summary>
|
|
/// The node has an encoder for the job's codec that handles its resolution: the encoder's own limit, else the node's
|
|
/// (0 = no announced limit).
|
|
/// </summary>
|
|
internal static bool CanEncode(RegisterNodeRequest node, RenderJobParameters job) =>
|
|
node.Encoders.Any(e => string.Equals(e.Codec, job.Codec, StringComparison.OrdinalIgnoreCase)
|
|
&& Fits(job.Width, e.MaxWidth > 0 ? e.MaxWidth : node.MaxWidth)
|
|
&& Fits(job.Height, e.MaxHeight > 0 ? e.MaxHeight : node.MaxHeight));
|
|
|
|
private static bool Fits(int size, int limit) => limit <= 0 || size <= limit;
|
|
|
|
internal static string? AppendLog(string? tail, string? line)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(line)) return tail;
|
|
var lines = (tail ?? string.Empty).Split('\n', StringSplitOptions.RemoveEmptyEntries).ToList();
|
|
lines.Add(string.Create(CultureInfo.InvariantCulture, $"{DateTime.UtcNow:HH:mm:ss} {line.Trim()[..Math.Min(line.Trim().Length, 300)]}"));
|
|
return string.Join('\n', lines.TakeLast(LogTailLines));
|
|
}
|
|
|
|
private static StartUploadResponse Started(Upload upload) =>
|
|
new(upload.Id, upload.ChunkSize, UploadRules.ReceivedChunks(upload), AlreadyExists: false, FrameId: null);
|
|
}
|