Owner review 1, phase 2 (docs/requests/2026-10-15-owner-review-1.md, R2). Jobs now run one at a time on the desktop and on the server. Parallelism (Settings > Rendering, TITANO_PARALLELISM) is how many photos the running job processes at once: thumbnail and proxy passes draw that many photos together and still write the pack and checkpoints in frame order, decode tiers are probed that many at a time, the deep analysis measures that many frames and a render decodes that many ahead. Wherever originals are developed whole (TIFF, PNG, RAW without a usable preview), fewer run at once within the memory budget, which the running job now has whole; the server's frame window is sized to the host instead of 1 GiB. A new job waits its turn and pausing the running job starts the next one. An optional priority queue gives each job kind a level (renders, imports and downloads high, verifies and proxies low by default): a queued job of a higher level pauses the running one without asking, and that job goes back to the queue with its checkpoint and resumes by itself afterwards. Jobs of the same level, and a job's own follow-ups, never interrupt it. The Jobs screen and the admin panel show the parallelism and the queue; Settings > Rendering and the admin panel's Settings set them, at once. TITANO_WORKERS and the Workers key of server.json still set the parallelism (1 to 64, with a warning at start for the variable). Titano.Tools gains i18n prune, which removed the renamed keys from the 36 machine-translated languages; the new strings are translated in all 38. Found while testing: uploads and imports failed when a pass read the sidecar while they wrote it (atomic writes now retry the rename while a reader holds the file, and never for a read-only one), and taking a lease just released while another process read it failed on Windows (the create now waits for the reader). Five adversarial review rounds checked the phase before this commit. Also updates BRIEF (12.8, 14.6, 14.8, 14.9, 24), DECISIONS, CHANGELOG, ARCHITECTURE, STATUS, the request file and the compose/Unraid templates. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
264 lines
15 KiB
C#
264 lines
15 KiB
C#
using System.Net;
|
|
using System.Net.Http.Json;
|
|
using System.Text.Json;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Titano.Core.Archiving;
|
|
using Titano.Core.Codecs.Jpeg;
|
|
using Titano.Core.Diagnostics.Synthetic;
|
|
using Titano.Core.Metadata;
|
|
using Titano.Core.Model;
|
|
using Titano.Core.Localization;
|
|
using Titano.Core.Project;
|
|
using Titano.Data.Processing;
|
|
using Titano.Protocol;
|
|
using Titano.Server.Events;
|
|
using Titano.Server.Jobs;
|
|
using Titano.Server.Storage;
|
|
using Titano.Server.Tests.Jobs;
|
|
using Titano.Testing;
|
|
|
|
namespace Titano.Server.Tests.Storage;
|
|
|
|
/// <summary>
|
|
/// The production composition: frames committed by the archive engine are light-analysed by the processing handlers,
|
|
/// and the server renders the managed formats itself.
|
|
/// </summary>
|
|
[Category("processing")]
|
|
public sealed class ProcessingModuleTests(TestContext context)
|
|
{
|
|
private const int Frames = 10;
|
|
|
|
/// <summary>The registrations of Program.cs.</summary>
|
|
private static void Production(IServiceCollection services)
|
|
{
|
|
ArchiveModule.Register(services);
|
|
ProcessingModule.Register(services);
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheRunningJobHasTheParallelismAndTheHostsMemoryBudget()
|
|
{
|
|
// Owner review 1: one job at a time, the parallelism working inside it, its frame window sized to the host.
|
|
await using var server = await TestServer.StartAsync(context, o => o.Parallelism = 3, Production);
|
|
var processing = server.Services.GetRequiredService<ProcessingService>();
|
|
var tuned = processing.Options.Tune!(processing.Options);
|
|
Assert.Equal((3, 3, 3), (tuned.DeepAnalysis.MaxDegreeOfParallelism, tuned.FrameWindow.MaxConcurrentLoads, processing.Options.Archive.Parallelism()));
|
|
Assert.Equal(processing.Options.Archive.MemoryBudget(), tuned.FrameWindow.MemoryBudget);
|
|
Assert.True(tuned.FrameWindow.MemoryBudget >= 1L << 30, "at least the window's own default");
|
|
// A change in the admin panel applies to the next job.
|
|
server.Options.Parallelism = 5;
|
|
Assert.Equal(5, processing.Options.Tune!(processing.Options).FrameWindow.MaxConcurrentLoads);
|
|
}
|
|
|
|
[Test]
|
|
public async Task UploadedFramesAreLightAnalysedAndAdvised()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, services: Production);
|
|
using var events = server.Services.GetRequiredService<EventHub>().Subscribe();
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
|
var sequenceId = await UploadSceneAsync(editor);
|
|
|
|
// The commits queue a light analysis; its handler comes from the processing module.
|
|
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
|
Sequence sequence;
|
|
while (true)
|
|
{
|
|
sequence = (await server.Database.Sequences.GetAsync(sequenceId))!;
|
|
if (sequence.Status.HasFlag(SequenceStatus.AnalyzedLight) && sequence.FrameCount == Frames
|
|
&& !await server.Database.Jobs.IsQueuedAsync(JobKind.LightAnalyse, sequenceId)) break;
|
|
Assert.True(DateTime.UtcNow < until, $"not analysed: {sequence.Status}, {sequence.FrameCount} frames");
|
|
await Task.Delay(100);
|
|
}
|
|
var project = ProjectSettings.FromJson(Assert.NotNull(sequence.SettingsJson));
|
|
Assert.True(project.Timing.Mode.HasAdvice, "timing mode advised");
|
|
var folder = new ArchivePaths(server.Options.ArchivePath).Sequence(sequence.Folder);
|
|
Assert.FileExists(folder.ProjectPath);
|
|
Assert.True(ManifestStore.ReadSequence(folder.ManifestPath).Status.HasFlag(SequenceStatus.AnalyzedLight), "sidecar status");
|
|
// Connected clients learn that the Auto values changed (BRIEF §14.3).
|
|
Assert.Contains(Drain(events), e => e is SequenceChangedEvent { Change: "settings" } changed && changed.SequenceId == sequenceId);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AProxyThatArrivesHasTheSequenceAnalysedAgain()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, services: Production);
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
|
// A HEIF original the server cannot decode (BRIEF §2.3): what it renders depends on the proxy a client sends.
|
|
var upload = await Uploads.UploadAsync(editor, [0, 0, 0, 24, .. "ftypheic"u8, 0, 0, 0, 0, .. "mif1heic"u8, .. Uploads.Photo(2000, 1)], "IMG_0001.HEIC",
|
|
sessionKey: "roof");
|
|
await LightAnalysesAsync(server, upload.SequenceId, 1);
|
|
var frame = Assert.NotNull(await server.Database.Frames.GetAsync(upload.FrameId));
|
|
|
|
using var image = new Core.Imaging.ImageBuffer(320, 240);
|
|
var exif = ExifWriter.BuildJpegApp1(new FrameMetadata { CapturedAt = frame.CapturedAt, Software = "Titano" });
|
|
using var proxy = new ByteArrayContent(JpegEncoder.Encode(image, new JpegEncodeOptions { App1Segments = [exif[4..]] }));
|
|
proxy.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("image/jpeg");
|
|
using (var stored = await editor.PutAsync($"/api/v1/frames/{frame.Id}/proxy", proxy))
|
|
await stored.ExpectAsync(HttpStatusCode.NoContent);
|
|
|
|
// A frame read from its proxy is no longer preview-only, so the advisor's Auto values are worked out again.
|
|
await LightAnalysesAsync(server, upload.SequenceId, 2);
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheServerRendersManagedFormatsAndLeavesMp4ToRenderNodes()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, services: Production);
|
|
using var events = server.Services.GetRequiredService<EventHub>().Subscribe();
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
|
var sequence = (await server.Database.Sequences.GetAsync(await UploadSceneAsync(editor)))!;
|
|
// A client's export folder means nothing on the server: the output still goes to the sequence's exports.
|
|
var avi = new ExportSettings
|
|
{
|
|
Preset = new Auto<string>().WithManual("mjpeg-avi"),
|
|
Overrides = new ExportOverrides { Width = 96, Height = 64 },
|
|
NameTemplate = "clip",
|
|
Folder = context.CreateTempDirectory("elsewhere"),
|
|
};
|
|
var hevc = new ExportSettings { Preset = new Auto<string>().WithManual("4k-hevc") };
|
|
|
|
var mp4 = await JobApiTests.CreateAsync(editor, "render", sequence.Id, ProcessingJobs.Render(sequence, hevc, 96, 64).ParametersJson);
|
|
var managed = await JobApiTests.CreateAsync(editor, "render", sequence.Id, ProcessingJobs.Render(sequence, avi, 96, 64).ParametersJson);
|
|
var done = await JobApiTests.WaitForAsync(editor, managed.Id, "done");
|
|
|
|
Assert.Equal(JobEngineHost.WorkerId, done.NodeId?.Split('#')[0]);
|
|
var render = (await server.Database.Renders.ListBySequenceAsync(sequence.Id)).Single();
|
|
Assert.Equal(($"{sequence.Folder}/exports/clip.avi", server.Options.ServerName, "Mjpeg"), (render.OutputPath, render.ProducedBy, render.Codec));
|
|
Assert.FileExists(Path.Combine(server.Options.ArchivePath, render.OutputPath));
|
|
Assert.Empty(Directory.EnumerateFileSystemEntries(avi.Folder));
|
|
var waiting = await JobApiTests.GetAsync(editor, mp4.Id);
|
|
Assert.Equal(("queued", (string?)null), (waiting.State, waiting.NodeId));
|
|
Assert.Contains(Drain(events), e => e is SequenceChangedEvent { Change: "renders" } changed && changed.SequenceId == sequence.Id);
|
|
}
|
|
|
|
[Test]
|
|
public async Task Mp4JobsNoRenderNodeCanTakeAreRenderedAsMjpegAndTheLogSaysSo()
|
|
{
|
|
var clock = new ManualTimeProvider();
|
|
await using var server = await TestServer.StartAsync(context, o => o.NodeTimeoutSeconds = 60, s =>
|
|
{
|
|
Production(s);
|
|
s.AddSingleton<TimeProvider>(clock);
|
|
});
|
|
var engine = server.Services.GetRequiredService<JobEngineHost>().Engine;
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor", renderNode: true);
|
|
var sequence = (await server.Database.Sequences.GetAsync(await UploadSceneAsync(editor)))!;
|
|
var hevc = new ExportSettings
|
|
{
|
|
Preset = new Auto<string>().WithManual("4k-hevc"),
|
|
Overrides = new ExportOverrides { Width = 96, Height = 64 },
|
|
NameTemplate = "clip-{codec}",
|
|
};
|
|
var job = await JobApiTests.CreateAsync(editor, "render", sequence.Id, ProcessingJobs.Render(sequence, hevc, 96, 64).ParametersJson);
|
|
|
|
// A node that encodes HEVC is online: the job waits for it, however long it has been queued.
|
|
using (var registered = await editor.PostAsJsonAsync("/api/v1/nodes/register",
|
|
new RegisterNodeRequest("studio-pc", [new NodeEncoder("hevc", "NVENC HEVC", true, true, 8192, 8192)], 8192, 8192), ProtocolJson.Default.RegisterNodeRequest))
|
|
await registered.ExpectAsync(HttpStatusCode.OK);
|
|
clock.Advance(TimeSpan.FromSeconds(61));
|
|
var node = (await server.Database.RenderNodes.ListAsync()).Single();
|
|
Assert.True(await server.Database.RenderNodes.HeartbeatAsync(node.Id, "idle", clock.GetUtcNow()));
|
|
engine.Wake();
|
|
await Task.Delay(1500);
|
|
Assert.Equal("queued", (await JobApiTests.GetAsync(editor, job.Id)).State);
|
|
|
|
// The node goes silent: the server renders the job itself in a managed format.
|
|
clock.Advance(TimeSpan.FromSeconds(61));
|
|
engine.Wake();
|
|
var done = await JobApiTests.WaitForAsync(editor, job.Id, "done");
|
|
Assert.Equal(JobEngineHost.WorkerId, done.NodeId?.Split('#')[0]);
|
|
var render = (await server.Database.Renders.ListBySequenceAsync(sequence.Id)).Single();
|
|
Assert.Equal(($"{sequence.Folder}/exports/clip-MJPEG.avi", "Mjpeg", 96, 64), (render.OutputPath, render.Codec, render.Width, render.Height));
|
|
Assert.Contains(Loc.L("Export.Log.NoRenderNode", Loc.L("Export.Preset.Custom"), "HEVC", "MJPEG"), done.LogTail);
|
|
}
|
|
|
|
[Test]
|
|
public async Task APostedBatchBecomesOneJobPerPreset()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, services: Production);
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
|
var sequence = (await server.Database.Sequences.GetAsync(await UploadSceneAsync(editor)))!;
|
|
var batch = new ExportSettings
|
|
{
|
|
Preset = new Auto<string>().WithManual("mjpeg-avi"),
|
|
Overrides = new ExportOverrides { Width = 96, Height = 64 },
|
|
NameTemplate = "{title} - {codec}",
|
|
Batch = ["gif-preview", "1080p-h264"],
|
|
};
|
|
|
|
var main = await JobApiTests.CreateAsync(editor, "render", sequence.Id, ProcessingJobs.Render(sequence, batch, 96, 64).ParametersJson);
|
|
var jobs = (await server.Database.Jobs.ListAsync(null, JobKind.Render)).Items;
|
|
Assert.Count(3, jobs);
|
|
var mp4 = jobs.Single(j => j.TargetLabel!.EndsWith(Loc.L("Export.Preset.1080pShare"), StringComparison.Ordinal));
|
|
var gif = jobs.Single(j => j.TargetLabel!.EndsWith(Loc.L("Export.Preset.Gif"), StringComparison.Ordinal));
|
|
await JobApiTests.WaitForAsync(editor, main.Id, "done");
|
|
await JobApiTests.WaitForAsync(editor, gif.Id, "done");
|
|
|
|
// The server renders the managed presets; the MP4 one waits for a render node, at the preset's size.
|
|
Assert.SequenceEqual(["Gif", "Mjpeg"], (await server.Database.Renders.ListBySequenceAsync(sequence.Id)).Select(r => r.Codec!).Order(StringComparer.Ordinal));
|
|
Assert.Equal("queued", (await JobApiTests.GetAsync(editor, mp4.Id)).State);
|
|
var parameters = JsonSerializer.Deserialize(mp4.ParametersJson!, ProtocolJson.Default.RenderJobParameters)!;
|
|
Assert.Equal(("h264", 1920, 1080), (parameters.Codec, parameters.Width, parameters.Height));
|
|
}
|
|
|
|
[Test]
|
|
public async Task RenderJobsThatCannotBeReadNeverStopTheEngine()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, services: Production);
|
|
var (editor, user, _) = await server.UserAsync("alberto", "editor");
|
|
var sequence = (await server.Database.Sequences.GetAsync(await UploadSceneAsync(editor)))!;
|
|
const string mjpeg = """{"preset":{"isAuto":false,"manual":"mjpeg-avi"},"overrides":{"width":96,"height":64}""";
|
|
|
|
// The API refuses a preset that gives a member twice; one already queued fails with its reason.
|
|
using (var refused = await editor.PostAsJsonAsync("/api/v1/jobs",
|
|
new CreateJobRequest("render", [sequence.Id], Parameters(mjpeg + ""","Preset":{}}""")), ProtocolJson.Default.CreateJobRequest))
|
|
await refused.ExpectErrorAsync(HttpStatusCode.BadRequest, "invalid_parameters");
|
|
var engine = server.Services.GetRequiredService<JobEngineHost>().Engine;
|
|
var broken = await engine.EnqueueAsync(JobKind.Render, [sequence.Id], "broken", Parameters(mjpeg + ""","Preset":{}}"""), JobPriority.User, user.Id);
|
|
var failed = await JobApiTests.WaitForAsync(editor, broken.Id, "failed");
|
|
Assert.Equal(Loc.L("Errors.Render.Parameters"), failed.Error);
|
|
|
|
// The workers carry on: a null batch is no batch.
|
|
var fine = await JobApiTests.CreateAsync(editor, "render", sequence.Id, Parameters(mjpeg + ""","batch":null}"""));
|
|
await JobApiTests.WaitForAsync(editor, fine.Id, "done");
|
|
}
|
|
|
|
private static string Parameters(string presetJson) =>
|
|
JsonSerializer.Serialize(new RenderJobParameters("MJPEG", presetJson, "mjpeg", 96, 64, false, null), ProtocolJson.Default.RenderJobParameters);
|
|
|
|
private static List<ServerEvent> Drain(EventHub.Subscription events)
|
|
{
|
|
var drained = new List<ServerEvent>();
|
|
while (events.Reader.TryRead(out var e)) drained.Add(e);
|
|
return drained;
|
|
}
|
|
|
|
/// <summary>Waits until <paramref name="count"/> light analyses of the sequence have finished.</summary>
|
|
private static async Task LightAnalysesAsync(TestServer server, Guid sequenceId, int count)
|
|
{
|
|
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
|
while (true)
|
|
{
|
|
var done = (await server.Database.Jobs.ListAsync(JobState.Done, JobKind.LightAnalyse, 200)).Items.Count(j => j.TargetIds.Contains(sequenceId));
|
|
if (done >= count) return;
|
|
Assert.True(DateTime.UtcNow < until, $"light analyses finished: {done} of {count}");
|
|
await Task.Delay(100);
|
|
}
|
|
}
|
|
|
|
/// <summary>Uploads a synthetic scene with Exif into one new sequence; returns its id.</summary>
|
|
private static async Task<Guid> UploadSceneAsync(HttpClient editor)
|
|
{
|
|
var scene = SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = Frames, StartEv = 0.5, EndEv = -0.5 });
|
|
Guid sequenceId = default;
|
|
for (var i = 0; i < scene.Count; i++)
|
|
{
|
|
using var image = scene.Render(i);
|
|
var exif = ExifWriter.BuildJpegApp1(scene.Metadata[i]);
|
|
var photo = JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 90, App1Segments = [exif[4..]] });
|
|
sequenceId = (await Uploads.UploadAsync(editor, photo, $"IMG_{i + 1:0000}.JPG", sessionKey: "roof")).SequenceId;
|
|
}
|
|
return sequenceId;
|
|
}
|
|
}
|