Files
Chronos/tests/Chronos.Server.Tests/Storage/VolumeSpaceTests.cs
T
Alby96 67a6d51cbe fix(archive): convert job applies exactly the confirmed plan, resumes safely and watches its space
The convert job planned every sequence again when it ran and checked only the frames
version, so a verify, a repaired upload or a tier change between the dry run and the job
let it replace photos the confirmed plan had listed as staying unchanged.

- Each sequence of a convert job now carries the digest of the photos it converts, each
  with its SHA-256 (ConvertSequencePlan.Digest, also in the plan's fingerprint). Under the
  sequence's lease, its sidecar settled, the job plans it again and goes on only while the
  digest matches; it then pins those photos (convert-<job>.pins), so photos that come in
  or are verified later stay as they are, a retry included. Stage B checks under the lease
  that every pinned photo still holds the content converted instead of holding the frames
  version, so a sequence still receiving uploads converts what was confirmed.
- A resumed job settles the journals a crash left in a sequence (a half-done replacement
  among them) before it plans, and reports a sequence its replacement completed as
  Converted, never Nothing or a partial swap.
- The passes a converted or new sequence needs are queued as soon as its outcome is
  journaled; the journal records the statuses they follow from and whether they were
  queued, so a pause, yield, stop or crash no longer loses them.
- Staged conversions of a blocked or failed sequence are kept for the retry, which
  re-verifies and takes them; maintenance keeps a convert job's staging while the job is
  queued, running or paused, and removes its journal and staging once the job is done or
  gone from the history.
- A dry run counts the sequences of a queued conversion as busy, so the same conversion
  cannot be confirmed twice.
- New sequence copies are owned by the source's owner, else by the requesting user's id
  (never the user name); a staged copy that vanished fails the sequence instead of leaving
  its photo out; the job log states how many photos stayed in the source alone.
- Free space is measured where the conversion writes: the server takes the tighter of the
  space path's volume and the archive's own (Unraid cache), the desktop asks Windows
  (GetDiskFreeSpaceEx), which also measures UNC archives; the job checks again before each
  sequence and each photo, scaling estimates by the files staged so far, and stops before
  the volume fills with what it staged kept.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit 2d7d461c33b9f257bf57793cc7cf331d0425c14e)
2026-10-27 22:06:55 +01:00

354 lines
19 KiB
C#

using System.IO.Compression;
using System.Net;
using System.Net.Http.Json;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Chronos.Protocol;
using Chronos.Server.Configuration;
using Chronos.Server.Storage;
using Chronos.Server.Tests.Jobs;
using Chronos.Testing;
namespace Chronos.Server.Tests.Storage;
/// <summary>Owner review 2 (R2): the free space of the Unraid array instead of the cache, and of the archive before an upload is filed.</summary>
[Category("server")]
public sealed class VolumeSpaceTests(TestContext context)
{
private const long GB = 1_000_000_000;
/// <summary>
/// /proc/self/mountinfo of a Chronos container on Unraid: the overlay root, the share bind-mounted at /archive (FUSE
/// shfs, whose root is the share's folder), its array view at /space, appdata at /data, a disk mounted inside the
/// archive under a name with a space, and two mounts stacked on /tmp.
/// </summary>
private const string Unraid = """
574 520 0:52 / / rw,relatime master:255 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/ABC:/var/lib/docker/overlay2/l/DEF,upperdir=/var/lib/docker/overlay2/1f/diff
575 574 0:55 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw
590 574 0:45 /Timelapse /archive rw,noatime - fuse.shfs shfs rw,user_id=0,group_id=0,default_permissions,allow_other
591 574 0:46 /Timelapse /space ro,noatime - fuse.shfs shfs rw,user_id=0,group_id=0,default_permissions,allow_other
592 574 0:31 /appdata/chronos /data rw,noatime shared:5 master:3 - btrfs /dev/nvme0n1p1 rw,ssd,space_cache=v2,subvolid=5,subvol=/
593 590 9:1 / /archive/Big\040Trip rw,noatime - xfs /dev/md1p1 rw,attr2,inode64
594 574 0:60 / /tmp rw - tmpfs tmpfs rw
595 574 0:61 / /tmp rw - tmpfs scratch rw,size=1g
this line is not a mount
""";
[Theory]
[Case("/archive", "/archive", "fuse.shfs")]
[Case("/archive/2026/Roof/IMG_0001.CR3", "/archive", "fuse.shfs")]
[Case("/archive/", "/archive", "fuse.shfs")]
[Case("/space", "/space", "fuse.shfs")]
[Case("/data/uploads", "/data", "btrfs")]
[Case("/archive/Big Trip/IMG_0002.CR3", "/archive/Big Trip", "xfs")]
[Case("/archive/Big", "/archive", "fuse.shfs")]
[Case("/archived", "/", "overlay")]
[Case("/", "/", "overlay")]
[Case("/app/Chronos.Server.dll", "/", "overlay")]
public void TheLongestMountPointHoldsAPath(string path, string mountPoint, string fileSystem)
{
var mount = Assert.NotNull(VolumeSpace.FindMount(Unraid, path));
Assert.Equal((mountPoint, fileSystem), (mount.MountPoint, mount.FileSystem));
}
[Test]
public void MountInfoLinesAreReadField()
{
var share = Assert.NotNull(VolumeSpace.FindMount(Unraid, "/archive"));
Assert.Equal("shfs", share.Source);
// Of mounts stacked on one point, the last listed is on top.
Assert.Equal("scratch", Assert.NotNull(VolumeSpace.FindMount(Unraid, "/tmp/x")).Source);
// Optional fields, a Windows line end and escapes in any field.
var data = Assert.NotNull(VolumeSpace.ParseMountLine("592 574 0:31 / /mnt/My\\040Disk rw shared:5 master:3 propagate_from:2 - ext4 /dev/sd\\134b rw\r"));
Assert.Equal(("/mnt/My Disk", "ext4", "/dev/sd\\b"), (data.MountPoint, data.FileSystem, data.Source));
Assert.Null(VolumeSpace.ParseMountLine("592 574 0:31 / /data rw shared:5 btrfs /dev/sdb1 rw"));
Assert.Null(VolumeSpace.ParseMountLine(""));
Assert.Null(VolumeSpace.FindMount("", "/archive"));
Assert.Null(VolumeSpace.FindMount("590 574 0:45 /Timelapse /archive rw - fuse.shfs shfs rw", "/data"));
}
[Test]
public void TheFileSystemOfAFolderIsKnown()
{
var folder = context.CreateTempDirectory("fs");
var fileSystem = Assert.NotNull(VolumeSpace.FileSystemOf(folder));
Assert.NotEmpty(fileSystem);
if (OperatingSystem.IsLinux()) Assert.Equal(fileSystem, Assert.NotNull(VolumeSpace.MountOf(folder)).FileSystem);
else Assert.Null(VolumeSpace.MountOf(folder));
Assert.Null(VolumeSpace.FileSystemOf(Path.Combine(folder, "missing")));
}
[Test]
public void TheSpacePathIsShownWithTheWriteVolumeApartWhenTheyDiffer()
{
var root = context.CreateTempDirectory("report");
var archive = Directory.CreateDirectory(Path.Combine(root, "archive")).FullName;
var space = Directory.CreateDirectory(Path.Combine(root, "space")).FullName;
var options = new ServerOptions { ArchivePath = archive };
var logger = new CountingLogger();
var volumes = new FakeVolumeSpace(options, logger);
volumes.Set(archive, new VolumeFigures(120 * GB, 500 * GB));
// No space path: the archive's own volume.
var plain = volumes.Report(archive);
Assert.Equal((120 * GB, false, (VolumeFigures?)null), (plain.Space!.FreeBytes, plain.FromSpacePath, plain.Write));
// The cache and the array: the array is shown, the cache apart.
options.SpacePath = space;
volumes.Set(space, new VolumeFigures(9_000 * GB, 16_000 * GB));
var separate = volumes.Report(archive);
Assert.Equal((9_000 * GB, true), (separate.Space!.FreeBytes, separate.FromSpacePath));
Assert.Equal(new VolumeFigures(120 * GB, 500 * GB), separate.Write);
Assert.Equal(VolumeSpace.FileSystemOf(space), separate.FileSystem);
// One volume read twice a moment apart: no second line.
volumes.Set(space, new VolumeFigures((120 * GB) - (8 << 20), 500 * GB));
Assert.Null(volumes.Report(archive).Write);
Assert.Equal(0, logger.Warnings);
}
[Test]
public void AConversionCountsOnTheTighterOfTheSpaceAndTheWriteVolume()
{
var root = context.CreateTempDirectory("writable");
var archive = Directory.CreateDirectory(Path.Combine(root, "archive")).FullName;
var space = Directory.CreateDirectory(Path.Combine(root, "space")).FullName;
var options = new ServerOptions { ArchivePath = archive };
var volumes = new FakeVolumeSpace(options, new CountingLogger());
volumes.Set(archive, new VolumeFigures(120 * GB, 500 * GB));
Assert.Equal(120 * GB, volumes.WritableFree(archive));
// Unraid: the array has room, the cache the share writes to until the mover runs has less, and it is what fills.
options.SpacePath = space;
volumes.Set(space, new VolumeFigures(9_000 * GB, 16_000 * GB));
Assert.Equal(120 * GB, volumes.WritableFree(archive));
// An array fuller than the cache is the limit then.
volumes.Set(space, new VolumeFigures(40 * GB, 16_000 * GB));
Assert.Equal(40 * GB, volumes.WritableFree(archive));
Assert.Null(new FakeVolumeSpace(new ServerOptions { ArchivePath = archive }, new CountingLogger()).WritableFree(Path.Combine(root, "missing")));
}
[Test]
public void ASpacePathThatCannotBeMeasuredFallsBackWithOneWarning()
{
var root = context.CreateTempDirectory("fallback");
var archive = Directory.CreateDirectory(Path.Combine(root, "archive")).FullName;
var space = Path.Combine(root, "not-mapped");
var logger = new CountingLogger();
var volumes = new VolumeSpace(new ServerOptions { ArchivePath = archive, SpacePath = space }, logger);
for (var i = 0; i < 3; i++)
{
var report = volumes.Report(archive);
Assert.False(report.FromSpacePath);
Assert.True(Assert.NotNull(report.Space).TotalBytes > 0);
Assert.Null(report.Write);
}
Assert.Equal(1, logger.Warnings);
// Mapped later: measured again, and a later outage is warned about once more.
Directory.CreateDirectory(space);
Assert.True(volumes.Report(archive).FromSpacePath);
Directory.Delete(space);
volumes.Report(archive);
volumes.Report(archive);
Assert.Equal(2, logger.Warnings);
}
[Test]
public async Task TheStatsReportTheSpacePathAndTheWriteVolume()
{
var space = context.CreateTempDirectory("space");
await using var server = await TestServer.StartAsync(context, o => o.SpacePath = space, s =>
{
s.AddSingleton<FakeVolumeSpace>();
s.AddSingleton<VolumeSpace>(sp => sp.GetRequiredService<FakeVolumeSpace>());
});
var admin = await server.AdminAsync();
var stats = await admin.GetFromJsonAsync("/api/v1/archive/stats", ProtocolJson.Default.ArchiveStatsDto);
Assert.True(Assert.NotNull(stats).UsesSpacePath);
Assert.True(stats.CapacityBytes > 0 && stats.FreeBytes <= stats.CapacityBytes);
Assert.Equal(VolumeSpace.FileSystemOf(space), stats.FileSystem);
// Temporary folders share one volume: no write volume apart.
Assert.Null(stats.WriteFreeBytes);
// The archive's own volume (an Unraid share's cache) differs: the panel gets it apart.
server.Services.GetRequiredService<FakeVolumeSpace>().Set(server.Options.ArchivePath, new VolumeFigures(40 * GB, 250 * GB));
using var dashboard = await admin.GetAsync("/api/v1/server/dashboard");
var archive = (await dashboard.ReadAsync(ProtocolJson.Default.DashboardDto)).Archive;
Assert.Equal((40 * GB, 250 * GB, true), (archive.WriteFreeBytes, archive.WriteCapacityBytes, archive.UsesSpacePath));
Assert.NotEqual(40 * GB, archive.FreeBytes);
}
[Test]
public async Task AMissingSpacePathShowsTheArchiveVolume()
{
var root = context.CreateTempDirectory("unmapped");
await using var server = await TestServer.StartAsync(context, o => o.SpacePath = Path.Combine(root, "space"));
var admin = await server.AdminAsync();
for (var i = 0; i < 2; i++)
{
var stats = Assert.NotNull(await admin.GetFromJsonAsync("/api/v1/archive/stats", ProtocolJson.Default.ArchiveStatsDto));
Assert.False(stats.UsesSpacePath);
Assert.True(stats.CapacityBytes > 0);
Assert.Null(stats.WriteFreeBytes);
Assert.Equal(VolumeSpace.FileSystemOf(server.Options.ArchivePath), stats.FileSystem);
}
// Warned at start, not at every request.
var warnings = Directory.EnumerateFiles(server.Options.LogPath).Sum(log =>
{
using var reader = new StreamReader(new FileStream(log, FileMode.Open, FileAccess.Read, FileShare.ReadWrite));
return reader.ReadToEnd().Split('\n').Count(line => line.Contains("CHRONOS_SPACE_PATH", StringComparison.Ordinal));
});
Assert.Equal(1, warnings);
}
[Test]
public async Task TheDiagnosticsReportBothVolumes()
{
var space = context.CreateTempDirectory("space");
await using var server = await TestServer.StartAsync(context, o => o.SpacePath = space);
var admin = await server.AdminAsync();
using var response = await admin.GetAsync("/api/v1/server/diagnostics");
await response.ExpectAsync(HttpStatusCode.OK);
await using var body = await response.Content.ReadAsStreamAsync();
using var zip = new ZipArchive(body, ZipArchiveMode.Read);
using var reader = new StreamReader(Assert.NotNull(zip.GetEntry("environment.txt")).Open());
var report = await reader.ReadToEndAsync();
var fileSystem = Assert.NotNull(VolumeSpace.FileSystemOf(space));
Assert.Contains($"Archive {server.Options.ArchivePath} (", report);
Assert.Contains($"Space {space} (", report);
Assert.Contains($" GB, {fileSystem}", report);
}
[Test]
public async Task AFullArchiveVolumeRefusesTheCompletionUntilThereIsRoom()
{
const int chunk = 64 * 1024;
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = chunk, s =>
{
s.AddSingleton<FakeVolumeSpace>();
s.AddSingleton<VolumeSpace>(sp => sp.GetRequiredService<FakeVolumeSpace>());
});
var volumes = server.Services.GetRequiredService<FakeVolumeSpace>();
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var data = Uploads.Photo(chunk + 500, 71);
var start = await Uploads.StartAsync(editor, data, sessionKey: "roof");
var id = Assert.NotNull(start.UploadId);
for (var i = 0; i < 2; i++)
{
using var put = await Uploads.PutChunkAsync(editor, id, i, Uploads.Chunk(data, chunk, i));
await put.ExpectAsync(HttpStatusCode.OK);
}
// The upload volume (/data) has room, the archive's does not: nothing is staged, the upload keeps its chunks.
volumes.Set(server.Options.ArchivePath, new VolumeFigures(32L << 20, 500 * GB));
using (var full = await editor.PostAsync($"/api/v1/uploads/{id}/complete", null))
await full.ExpectErrorAsync(HttpStatusCode.InsufficientStorage, "insufficient_storage");
using (var state = await editor.GetAsync($"/api/v1/uploads/{id}"))
Assert.Equal("receiving", (await state.ReadAsync(ProtocolJson.Default.UploadState)).State);
Assert.True(server.Services.GetRequiredService<UploadStore>().Exists(id));
Assert.False(Directory.Exists(server.Options.StagingPath) && Directory.EnumerateFiles(server.Options.StagingPath).Any());
volumes.Set(server.Options.ArchivePath, new VolumeFigures(GB, 500 * GB));
Guid sequence;
using (var room = await editor.PostAsync($"/api/v1/uploads/{id}/complete", null))
{
await room.ExpectAsync(HttpStatusCode.OK);
sequence = (await room.ReadAsync(ProtocolJson.Default.CompleteUploadResponse)).SequenceId;
}
// A render sent into the archive is checked the same way.
var output = Uploads.Photo(3000, 72);
var request = new RenderUploadRequest(Guid.CreateVersion7(), "Roof.mp4", output.Length, Uploads.Sha(output), "4K HEVC", "hevc", 3840, 2160, 30, 10, null);
using var started = await editor.PostAsJsonAsync($"/api/v1/sequences/{sequence}/renders", request, ProtocolJson.Default.RenderUploadRequest);
await started.ExpectAsync(HttpStatusCode.Created);
var render = Assert.NotNull((await started.ReadAsync(ProtocolJson.Default.StartUploadResponse)).UploadId);
using (var put = await Uploads.PutChunkAsync(editor, render, 0, output)) await put.ExpectAsync(HttpStatusCode.OK);
volumes.Set(server.Options.ArchivePath, new VolumeFigures(0, 500 * GB));
using (var full = await editor.PostAsync($"/api/v1/sequences/{sequence}/renders/{render}/complete", null))
await full.ExpectErrorAsync(HttpStatusCode.InsufficientStorage, "insufficient_storage");
volumes.Set(server.Options.ArchivePath, null);
using (var room = await editor.PostAsync($"/api/v1/sequences/{sequence}/renders/{render}/complete", null))
await room.ExpectAsync(HttpStatusCode.OK);
}
[Test]
public async Task AFullArchiveVolumeRefusesARenderNodeResultUntilThereIsRoom()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = 64 * 1024, s =>
{
s.AddSingleton<FakeVolumeSpace>();
s.AddSingleton<VolumeSpace>(sp => sp.GetRequiredService<FakeVolumeSpace>());
});
var volumes = server.Services.GetRequiredService<FakeVolumeSpace>();
var (editor, _, _) = await server.UserAsync("alberto", "editor", renderNode: true);
var frame = await Uploads.UploadAsync(editor, Uploads.Photo(5000, 73), "IMG_0001.CR3");
var job = await JobApiTests.CreateAsync(editor, "render", frame.SequenceId, RenderNodeTests.HevcJob);
var node = await RenderNodeTests.RegisterAsync(editor, "studio-pc", "hevc", 8192);
using (var claim = await editor.PostAsync($"/api/v1/nodes/{node.Id}/claim", null)) await claim.ExpectAsync(HttpStatusCode.OK);
var output = Uploads.Photo(3000, 74);
var request = new RenderResultRequest("Roof - 4K HEVC.mp4", output.Length, Uploads.Sha(output), "4K HEVC", "hevc", 3840, 2160, 25, 120);
using var started = await editor.PostAsJsonAsync($"/api/v1/nodes/{node.Id}/jobs/{job.Id}/result", request, ProtocolJson.Default.RenderResultRequest);
await started.ExpectAsync(HttpStatusCode.Created);
var upload = Assert.NotNull((await started.ReadAsync(ProtocolJson.Default.StartUploadResponse)).UploadId);
using (var put = await Uploads.PutChunkAsync(editor, upload, 0, output)) await put.ExpectAsync(HttpStatusCode.OK);
// Refused before the commit starts: the upload keeps receiving with its chunks and the node keeps the job.
var complete = $"/api/v1/nodes/{node.Id}/jobs/{job.Id}/result/complete";
volumes.Set(server.Options.ArchivePath, new VolumeFigures(0, 500 * GB));
using (var full = await editor.PostAsync(complete, null))
await full.ExpectErrorAsync(HttpStatusCode.InsufficientStorage, "insufficient_storage");
Assert.Equal(Chronos.Data.Repositories.UploadState.Receiving, Assert.NotNull(await server.Database.Uploads.FindByJobAsync(job.Id)).State);
Assert.True(server.Services.GetRequiredService<UploadStore>().Exists(upload));
var waiting = await JobApiTests.GetAsync(editor, job.Id);
Assert.Equal(("running", node.Id.ToString()), (waiting.State, waiting.NodeId));
volumes.Set(server.Options.ArchivePath, null);
using (var room = await editor.PostAsync(complete, null))
{
await room.ExpectAsync(HttpStatusCode.OK);
Assert.Equal("studio-pc", (await room.ReadAsync(ProtocolJson.Default.RenderDto)).ProducedBy);
}
Assert.Equal("done", (await JobApiTests.GetAsync(editor, job.Id)).State);
Assert.Equal(Chronos.Data.Repositories.UploadState.Committed, Assert.NotNull(await server.Database.Uploads.FindByJobAsync(job.Id)).State);
}
[Test]
public void TheSpacePathComesFromTheFileOrTheEnvironment()
{
var root = context.CreateTempDirectory("space-option");
File.WriteAllText(Path.Combine(root, "server.json"), """{ "SpacePath": "/space" }""");
var file = ServerOptions.Load(new Dictionary<string, string?> { ["CHRONOS_DATA_PATH"] = root, ["CHRONOS_DB"] = "postgres" });
Assert.Equal("/space", file.SpacePath);
Assert.DoesNotContain("SpacePath", file.Locked);
var environment = ServerOptions.Load(new Dictionary<string, string?>
{
["CHRONOS_DATA_PATH"] = root,
["CHRONOS_DB"] = "postgres",
["CHRONOS_SPACE_PATH"] = "/mnt/array",
});
Assert.Equal("/mnt/array", environment.SpacePath);
Assert.Contains("SpacePath", environment.Locked);
Assert.Null(new ServerOptions().SpacePath);
}
/// <summary>Counts the warnings a <see cref="VolumeSpace"/> logs.</summary>
private sealed class CountingLogger : ILogger<VolumeSpace>
{
private int _warnings;
public int Warnings => Volatile.Read(ref _warnings);
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
public bool IsEnabled(LogLevel logLevel) => true;
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
{
if (logLevel == LogLevel.Warning) Interlocked.Increment(ref _warnings);
}
}
}