The Unraid share tooltip told users only to map the share's array view to /space. With CHRONOS_SPACE_PATH unset (the shipped docker-compose file, or a template whose Space path was cleared) the server never looks at /space, so following the tip changed nothing. The tip (en, it) and BRIEF 14.8 now also say to set CHRONOS_SPACE_PATH=/space; the English text was shortened to stay within the 200-character tooltip budget. A render node completing its result on a full archive volume answers 507 before the upload moves to Committing, but no test covered that path. The new VolumeSpaceTests case checks that the upload keeps receiving with its chunks and the node keeps the job, then that the same call succeeds once there is room. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
333 lines
18 KiB
C#
333 lines
18 KiB
C#
using System.IO.Compression;
|
|
using System.Net;
|
|
using System.Net.Http.Json;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Logging;
|
|
using Chronos.Protocol;
|
|
using Chronos.Server.Configuration;
|
|
using Chronos.Server.Storage;
|
|
using Chronos.Server.Tests.Jobs;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Server.Tests.Storage;
|
|
|
|
/// <summary>Owner review 2 (R2): the free space of the Unraid array instead of the cache, and of the archive before an upload is filed.</summary>
|
|
[Category("server")]
|
|
public sealed class VolumeSpaceTests(TestContext context)
|
|
{
|
|
private const long GB = 1_000_000_000;
|
|
|
|
/// <summary>
|
|
/// /proc/self/mountinfo of a Chronos container on Unraid: the overlay root, the share bind-mounted at /archive (FUSE
|
|
/// shfs, whose root is the share's folder), its array view at /space, appdata at /data, a disk mounted inside the
|
|
/// archive under a name with a space, and two mounts stacked on /tmp.
|
|
/// </summary>
|
|
private const string Unraid = """
|
|
574 520 0:52 / / rw,relatime master:255 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/ABC:/var/lib/docker/overlay2/l/DEF,upperdir=/var/lib/docker/overlay2/1f/diff
|
|
575 574 0:55 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw
|
|
590 574 0:45 /Timelapse /archive rw,noatime - fuse.shfs shfs rw,user_id=0,group_id=0,default_permissions,allow_other
|
|
591 574 0:46 /Timelapse /space ro,noatime - fuse.shfs shfs rw,user_id=0,group_id=0,default_permissions,allow_other
|
|
592 574 0:31 /appdata/chronos /data rw,noatime shared:5 master:3 - btrfs /dev/nvme0n1p1 rw,ssd,space_cache=v2,subvolid=5,subvol=/
|
|
593 590 9:1 / /archive/Big\040Trip rw,noatime - xfs /dev/md1p1 rw,attr2,inode64
|
|
594 574 0:60 / /tmp rw - tmpfs tmpfs rw
|
|
595 574 0:61 / /tmp rw - tmpfs scratch rw,size=1g
|
|
this line is not a mount
|
|
""";
|
|
|
|
[Theory]
|
|
[Case("/archive", "/archive", "fuse.shfs")]
|
|
[Case("/archive/2026/Roof/IMG_0001.CR3", "/archive", "fuse.shfs")]
|
|
[Case("/archive/", "/archive", "fuse.shfs")]
|
|
[Case("/space", "/space", "fuse.shfs")]
|
|
[Case("/data/uploads", "/data", "btrfs")]
|
|
[Case("/archive/Big Trip/IMG_0002.CR3", "/archive/Big Trip", "xfs")]
|
|
[Case("/archive/Big", "/archive", "fuse.shfs")]
|
|
[Case("/archived", "/", "overlay")]
|
|
[Case("/", "/", "overlay")]
|
|
[Case("/app/Chronos.Server.dll", "/", "overlay")]
|
|
public void TheLongestMountPointHoldsAPath(string path, string mountPoint, string fileSystem)
|
|
{
|
|
var mount = Assert.NotNull(VolumeSpace.FindMount(Unraid, path));
|
|
Assert.Equal((mountPoint, fileSystem), (mount.MountPoint, mount.FileSystem));
|
|
}
|
|
|
|
[Test]
|
|
public void MountInfoLinesAreReadField()
|
|
{
|
|
var share = Assert.NotNull(VolumeSpace.FindMount(Unraid, "/archive"));
|
|
Assert.Equal("shfs", share.Source);
|
|
// Of mounts stacked on one point, the last listed is on top.
|
|
Assert.Equal("scratch", Assert.NotNull(VolumeSpace.FindMount(Unraid, "/tmp/x")).Source);
|
|
// Optional fields, a Windows line end and escapes in any field.
|
|
var data = Assert.NotNull(VolumeSpace.ParseMountLine("592 574 0:31 / /mnt/My\\040Disk rw shared:5 master:3 propagate_from:2 - ext4 /dev/sd\\134b rw\r"));
|
|
Assert.Equal(("/mnt/My Disk", "ext4", "/dev/sd\\b"), (data.MountPoint, data.FileSystem, data.Source));
|
|
Assert.Null(VolumeSpace.ParseMountLine("592 574 0:31 / /data rw shared:5 btrfs /dev/sdb1 rw"));
|
|
Assert.Null(VolumeSpace.ParseMountLine(""));
|
|
Assert.Null(VolumeSpace.FindMount("", "/archive"));
|
|
Assert.Null(VolumeSpace.FindMount("590 574 0:45 /Timelapse /archive rw - fuse.shfs shfs rw", "/data"));
|
|
}
|
|
|
|
[Test]
|
|
public void TheFileSystemOfAFolderIsKnown()
|
|
{
|
|
var folder = context.CreateTempDirectory("fs");
|
|
var fileSystem = Assert.NotNull(VolumeSpace.FileSystemOf(folder));
|
|
Assert.NotEmpty(fileSystem);
|
|
if (OperatingSystem.IsLinux()) Assert.Equal(fileSystem, Assert.NotNull(VolumeSpace.MountOf(folder)).FileSystem);
|
|
else Assert.Null(VolumeSpace.MountOf(folder));
|
|
Assert.Null(VolumeSpace.FileSystemOf(Path.Combine(folder, "missing")));
|
|
}
|
|
|
|
[Test]
|
|
public void TheSpacePathIsShownWithTheWriteVolumeApartWhenTheyDiffer()
|
|
{
|
|
var root = context.CreateTempDirectory("report");
|
|
var archive = Directory.CreateDirectory(Path.Combine(root, "archive")).FullName;
|
|
var space = Directory.CreateDirectory(Path.Combine(root, "space")).FullName;
|
|
var options = new ServerOptions { ArchivePath = archive };
|
|
var logger = new CountingLogger();
|
|
var volumes = new FakeVolumeSpace(options, logger);
|
|
volumes.Set(archive, new VolumeFigures(120 * GB, 500 * GB));
|
|
|
|
// No space path: the archive's own volume.
|
|
var plain = volumes.Report(archive);
|
|
Assert.Equal((120 * GB, false, (VolumeFigures?)null), (plain.Space!.FreeBytes, plain.FromSpacePath, plain.Write));
|
|
|
|
// The cache and the array: the array is shown, the cache apart.
|
|
options.SpacePath = space;
|
|
volumes.Set(space, new VolumeFigures(9_000 * GB, 16_000 * GB));
|
|
var separate = volumes.Report(archive);
|
|
Assert.Equal((9_000 * GB, true), (separate.Space!.FreeBytes, separate.FromSpacePath));
|
|
Assert.Equal(new VolumeFigures(120 * GB, 500 * GB), separate.Write);
|
|
Assert.Equal(VolumeSpace.FileSystemOf(space), separate.FileSystem);
|
|
|
|
// One volume read twice a moment apart: no second line.
|
|
volumes.Set(space, new VolumeFigures((120 * GB) - (8 << 20), 500 * GB));
|
|
Assert.Null(volumes.Report(archive).Write);
|
|
Assert.Equal(0, logger.Warnings);
|
|
}
|
|
|
|
[Test]
|
|
public void ASpacePathThatCannotBeMeasuredFallsBackWithOneWarning()
|
|
{
|
|
var root = context.CreateTempDirectory("fallback");
|
|
var archive = Directory.CreateDirectory(Path.Combine(root, "archive")).FullName;
|
|
var space = Path.Combine(root, "not-mapped");
|
|
var logger = new CountingLogger();
|
|
var volumes = new VolumeSpace(new ServerOptions { ArchivePath = archive, SpacePath = space }, logger);
|
|
|
|
for (var i = 0; i < 3; i++)
|
|
{
|
|
var report = volumes.Report(archive);
|
|
Assert.False(report.FromSpacePath);
|
|
Assert.True(Assert.NotNull(report.Space).TotalBytes > 0);
|
|
Assert.Null(report.Write);
|
|
}
|
|
Assert.Equal(1, logger.Warnings);
|
|
|
|
// Mapped later: measured again, and a later outage is warned about once more.
|
|
Directory.CreateDirectory(space);
|
|
Assert.True(volumes.Report(archive).FromSpacePath);
|
|
Directory.Delete(space);
|
|
volumes.Report(archive);
|
|
volumes.Report(archive);
|
|
Assert.Equal(2, logger.Warnings);
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheStatsReportTheSpacePathAndTheWriteVolume()
|
|
{
|
|
var space = context.CreateTempDirectory("space");
|
|
await using var server = await TestServer.StartAsync(context, o => o.SpacePath = space, s =>
|
|
{
|
|
s.AddSingleton<FakeVolumeSpace>();
|
|
s.AddSingleton<VolumeSpace>(sp => sp.GetRequiredService<FakeVolumeSpace>());
|
|
});
|
|
var admin = await server.AdminAsync();
|
|
var stats = await admin.GetFromJsonAsync("/api/v1/archive/stats", ProtocolJson.Default.ArchiveStatsDto);
|
|
Assert.True(Assert.NotNull(stats).UsesSpacePath);
|
|
Assert.True(stats.CapacityBytes > 0 && stats.FreeBytes <= stats.CapacityBytes);
|
|
Assert.Equal(VolumeSpace.FileSystemOf(space), stats.FileSystem);
|
|
// Temporary folders share one volume: no write volume apart.
|
|
Assert.Null(stats.WriteFreeBytes);
|
|
|
|
// The archive's own volume (an Unraid share's cache) differs: the panel gets it apart.
|
|
server.Services.GetRequiredService<FakeVolumeSpace>().Set(server.Options.ArchivePath, new VolumeFigures(40 * GB, 250 * GB));
|
|
using var dashboard = await admin.GetAsync("/api/v1/server/dashboard");
|
|
var archive = (await dashboard.ReadAsync(ProtocolJson.Default.DashboardDto)).Archive;
|
|
Assert.Equal((40 * GB, 250 * GB, true), (archive.WriteFreeBytes, archive.WriteCapacityBytes, archive.UsesSpacePath));
|
|
Assert.NotEqual(40 * GB, archive.FreeBytes);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AMissingSpacePathShowsTheArchiveVolume()
|
|
{
|
|
var root = context.CreateTempDirectory("unmapped");
|
|
await using var server = await TestServer.StartAsync(context, o => o.SpacePath = Path.Combine(root, "space"));
|
|
var admin = await server.AdminAsync();
|
|
for (var i = 0; i < 2; i++)
|
|
{
|
|
var stats = Assert.NotNull(await admin.GetFromJsonAsync("/api/v1/archive/stats", ProtocolJson.Default.ArchiveStatsDto));
|
|
Assert.False(stats.UsesSpacePath);
|
|
Assert.True(stats.CapacityBytes > 0);
|
|
Assert.Null(stats.WriteFreeBytes);
|
|
Assert.Equal(VolumeSpace.FileSystemOf(server.Options.ArchivePath), stats.FileSystem);
|
|
}
|
|
// Warned at start, not at every request.
|
|
var warnings = Directory.EnumerateFiles(server.Options.LogPath).Sum(log =>
|
|
{
|
|
using var reader = new StreamReader(new FileStream(log, FileMode.Open, FileAccess.Read, FileShare.ReadWrite));
|
|
return reader.ReadToEnd().Split('\n').Count(line => line.Contains("CHRONOS_SPACE_PATH", StringComparison.Ordinal));
|
|
});
|
|
Assert.Equal(1, warnings);
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheDiagnosticsReportBothVolumes()
|
|
{
|
|
var space = context.CreateTempDirectory("space");
|
|
await using var server = await TestServer.StartAsync(context, o => o.SpacePath = space);
|
|
var admin = await server.AdminAsync();
|
|
using var response = await admin.GetAsync("/api/v1/server/diagnostics");
|
|
await response.ExpectAsync(HttpStatusCode.OK);
|
|
await using var body = await response.Content.ReadAsStreamAsync();
|
|
using var zip = new ZipArchive(body, ZipArchiveMode.Read);
|
|
using var reader = new StreamReader(Assert.NotNull(zip.GetEntry("environment.txt")).Open());
|
|
var report = await reader.ReadToEndAsync();
|
|
var fileSystem = Assert.NotNull(VolumeSpace.FileSystemOf(space));
|
|
Assert.Contains($"Archive {server.Options.ArchivePath} (", report);
|
|
Assert.Contains($"Space {space} (", report);
|
|
Assert.Contains($" GB, {fileSystem}", report);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AFullArchiveVolumeRefusesTheCompletionUntilThereIsRoom()
|
|
{
|
|
const int chunk = 64 * 1024;
|
|
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = chunk, s =>
|
|
{
|
|
s.AddSingleton<FakeVolumeSpace>();
|
|
s.AddSingleton<VolumeSpace>(sp => sp.GetRequiredService<FakeVolumeSpace>());
|
|
});
|
|
var volumes = server.Services.GetRequiredService<FakeVolumeSpace>();
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
|
var data = Uploads.Photo(chunk + 500, 71);
|
|
var start = await Uploads.StartAsync(editor, data, sessionKey: "roof");
|
|
var id = Assert.NotNull(start.UploadId);
|
|
for (var i = 0; i < 2; i++)
|
|
{
|
|
using var put = await Uploads.PutChunkAsync(editor, id, i, Uploads.Chunk(data, chunk, i));
|
|
await put.ExpectAsync(HttpStatusCode.OK);
|
|
}
|
|
|
|
// The upload volume (/data) has room, the archive's does not: nothing is staged, the upload keeps its chunks.
|
|
volumes.Set(server.Options.ArchivePath, new VolumeFigures(32L << 20, 500 * GB));
|
|
using (var full = await editor.PostAsync($"/api/v1/uploads/{id}/complete", null))
|
|
await full.ExpectErrorAsync(HttpStatusCode.InsufficientStorage, "insufficient_storage");
|
|
using (var state = await editor.GetAsync($"/api/v1/uploads/{id}"))
|
|
Assert.Equal("receiving", (await state.ReadAsync(ProtocolJson.Default.UploadState)).State);
|
|
Assert.True(server.Services.GetRequiredService<UploadStore>().Exists(id));
|
|
Assert.False(Directory.Exists(server.Options.StagingPath) && Directory.EnumerateFiles(server.Options.StagingPath).Any());
|
|
|
|
volumes.Set(server.Options.ArchivePath, new VolumeFigures(GB, 500 * GB));
|
|
Guid sequence;
|
|
using (var room = await editor.PostAsync($"/api/v1/uploads/{id}/complete", null))
|
|
{
|
|
await room.ExpectAsync(HttpStatusCode.OK);
|
|
sequence = (await room.ReadAsync(ProtocolJson.Default.CompleteUploadResponse)).SequenceId;
|
|
}
|
|
|
|
// A render sent into the archive is checked the same way.
|
|
var output = Uploads.Photo(3000, 72);
|
|
var request = new RenderUploadRequest(Guid.CreateVersion7(), "Roof.mp4", output.Length, Uploads.Sha(output), "4K HEVC", "hevc", 3840, 2160, 30, 10, null);
|
|
using var started = await editor.PostAsJsonAsync($"/api/v1/sequences/{sequence}/renders", request, ProtocolJson.Default.RenderUploadRequest);
|
|
await started.ExpectAsync(HttpStatusCode.Created);
|
|
var render = Assert.NotNull((await started.ReadAsync(ProtocolJson.Default.StartUploadResponse)).UploadId);
|
|
using (var put = await Uploads.PutChunkAsync(editor, render, 0, output)) await put.ExpectAsync(HttpStatusCode.OK);
|
|
volumes.Set(server.Options.ArchivePath, new VolumeFigures(0, 500 * GB));
|
|
using (var full = await editor.PostAsync($"/api/v1/sequences/{sequence}/renders/{render}/complete", null))
|
|
await full.ExpectErrorAsync(HttpStatusCode.InsufficientStorage, "insufficient_storage");
|
|
volumes.Set(server.Options.ArchivePath, null);
|
|
using (var room = await editor.PostAsync($"/api/v1/sequences/{sequence}/renders/{render}/complete", null))
|
|
await room.ExpectAsync(HttpStatusCode.OK);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AFullArchiveVolumeRefusesARenderNodeResultUntilThereIsRoom()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = 64 * 1024, s =>
|
|
{
|
|
s.AddSingleton<FakeVolumeSpace>();
|
|
s.AddSingleton<VolumeSpace>(sp => sp.GetRequiredService<FakeVolumeSpace>());
|
|
});
|
|
var volumes = server.Services.GetRequiredService<FakeVolumeSpace>();
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor", renderNode: true);
|
|
var frame = await Uploads.UploadAsync(editor, Uploads.Photo(5000, 73), "IMG_0001.CR3");
|
|
var job = await JobApiTests.CreateAsync(editor, "render", frame.SequenceId, RenderNodeTests.HevcJob);
|
|
var node = await RenderNodeTests.RegisterAsync(editor, "studio-pc", "hevc", 8192);
|
|
using (var claim = await editor.PostAsync($"/api/v1/nodes/{node.Id}/claim", null)) await claim.ExpectAsync(HttpStatusCode.OK);
|
|
|
|
var output = Uploads.Photo(3000, 74);
|
|
var request = new RenderResultRequest("Roof - 4K HEVC.mp4", output.Length, Uploads.Sha(output), "4K HEVC", "hevc", 3840, 2160, 25, 120);
|
|
using var started = await editor.PostAsJsonAsync($"/api/v1/nodes/{node.Id}/jobs/{job.Id}/result", request, ProtocolJson.Default.RenderResultRequest);
|
|
await started.ExpectAsync(HttpStatusCode.Created);
|
|
var upload = Assert.NotNull((await started.ReadAsync(ProtocolJson.Default.StartUploadResponse)).UploadId);
|
|
using (var put = await Uploads.PutChunkAsync(editor, upload, 0, output)) await put.ExpectAsync(HttpStatusCode.OK);
|
|
|
|
// Refused before the commit starts: the upload keeps receiving with its chunks and the node keeps the job.
|
|
var complete = $"/api/v1/nodes/{node.Id}/jobs/{job.Id}/result/complete";
|
|
volumes.Set(server.Options.ArchivePath, new VolumeFigures(0, 500 * GB));
|
|
using (var full = await editor.PostAsync(complete, null))
|
|
await full.ExpectErrorAsync(HttpStatusCode.InsufficientStorage, "insufficient_storage");
|
|
Assert.Equal(Chronos.Data.Repositories.UploadState.Receiving, Assert.NotNull(await server.Database.Uploads.FindByJobAsync(job.Id)).State);
|
|
Assert.True(server.Services.GetRequiredService<UploadStore>().Exists(upload));
|
|
var waiting = await JobApiTests.GetAsync(editor, job.Id);
|
|
Assert.Equal(("running", node.Id.ToString()), (waiting.State, waiting.NodeId));
|
|
|
|
volumes.Set(server.Options.ArchivePath, null);
|
|
using (var room = await editor.PostAsync(complete, null))
|
|
{
|
|
await room.ExpectAsync(HttpStatusCode.OK);
|
|
Assert.Equal("studio-pc", (await room.ReadAsync(ProtocolJson.Default.RenderDto)).ProducedBy);
|
|
}
|
|
Assert.Equal("done", (await JobApiTests.GetAsync(editor, job.Id)).State);
|
|
Assert.Equal(Chronos.Data.Repositories.UploadState.Committed, Assert.NotNull(await server.Database.Uploads.FindByJobAsync(job.Id)).State);
|
|
}
|
|
|
|
[Test]
|
|
public void TheSpacePathComesFromTheFileOrTheEnvironment()
|
|
{
|
|
var root = context.CreateTempDirectory("space-option");
|
|
File.WriteAllText(Path.Combine(root, "server.json"), """{ "SpacePath": "/space" }""");
|
|
var file = ServerOptions.Load(new Dictionary<string, string?> { ["CHRONOS_DATA_PATH"] = root, ["CHRONOS_DB"] = "postgres" });
|
|
Assert.Equal("/space", file.SpacePath);
|
|
Assert.DoesNotContain("SpacePath", file.Locked);
|
|
var environment = ServerOptions.Load(new Dictionary<string, string?>
|
|
{
|
|
["CHRONOS_DATA_PATH"] = root,
|
|
["CHRONOS_DB"] = "postgres",
|
|
["CHRONOS_SPACE_PATH"] = "/mnt/array",
|
|
});
|
|
Assert.Equal("/mnt/array", environment.SpacePath);
|
|
Assert.Contains("SpacePath", environment.Locked);
|
|
Assert.Null(new ServerOptions().SpacePath);
|
|
}
|
|
|
|
/// <summary>Counts the warnings a <see cref="VolumeSpace"/> logs.</summary>
|
|
private sealed class CountingLogger : ILogger<VolumeSpace>
|
|
{
|
|
private int _warnings;
|
|
|
|
public int Warnings => Volatile.Read(ref _warnings);
|
|
|
|
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
|
|
|
|
public bool IsEnabled(LogLevel logLevel) => true;
|
|
|
|
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
|
|
{
|
|
if (logLevel == LogLevel.Warning) Interlocked.Increment(ref _warnings);
|
|
}
|
|
}
|
|
}
|