using System.IO.Compression; using System.Net; using System.Net.Http.Json; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Chronos.Protocol; using Chronos.Server.Configuration; using Chronos.Server.Storage; using Chronos.Server.Tests.Jobs; using Chronos.Testing; namespace Chronos.Server.Tests.Storage; /// Owner review 2 (R2): the free space of the Unraid array instead of the cache, and of the archive before an upload is filed. [Category("server")] public sealed class VolumeSpaceTests(TestContext context) { private const long GB = 1_000_000_000; /// /// /proc/self/mountinfo of a Chronos container on Unraid: the overlay root, the share bind-mounted at /archive (FUSE /// shfs, whose root is the share's folder), its array view at /space, appdata at /data, a disk mounted inside the /// archive under a name with a space, and two mounts stacked on /tmp. /// private const string Unraid = """ 574 520 0:52 / / rw,relatime master:255 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/ABC:/var/lib/docker/overlay2/l/DEF,upperdir=/var/lib/docker/overlay2/1f/diff 575 574 0:55 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw 590 574 0:45 /Timelapse /archive rw,noatime - fuse.shfs shfs rw,user_id=0,group_id=0,default_permissions,allow_other 591 574 0:46 /Timelapse /space ro,noatime - fuse.shfs shfs rw,user_id=0,group_id=0,default_permissions,allow_other 592 574 0:31 /appdata/chronos /data rw,noatime shared:5 master:3 - btrfs /dev/nvme0n1p1 rw,ssd,space_cache=v2,subvolid=5,subvol=/ 593 590 9:1 / /archive/Big\040Trip rw,noatime - xfs /dev/md1p1 rw,attr2,inode64 594 574 0:60 / /tmp rw - tmpfs tmpfs rw 595 574 0:61 / /tmp rw - tmpfs scratch rw,size=1g this line is not a mount """; [Theory] [Case("/archive", "/archive", "fuse.shfs")] [Case("/archive/2026/Roof/IMG_0001.CR3", "/archive", "fuse.shfs")] [Case("/archive/", "/archive", "fuse.shfs")] [Case("/space", "/space", "fuse.shfs")] [Case("/data/uploads", "/data", "btrfs")] [Case("/archive/Big Trip/IMG_0002.CR3", "/archive/Big Trip", "xfs")] [Case("/archive/Big", "/archive", "fuse.shfs")] [Case("/archived", "/", "overlay")] [Case("/", "/", "overlay")] [Case("/app/Chronos.Server.dll", "/", "overlay")] public void TheLongestMountPointHoldsAPath(string path, string mountPoint, string fileSystem) { var mount = Assert.NotNull(VolumeSpace.FindMount(Unraid, path)); Assert.Equal((mountPoint, fileSystem), (mount.MountPoint, mount.FileSystem)); } [Test] public void MountInfoLinesAreReadField() { var share = Assert.NotNull(VolumeSpace.FindMount(Unraid, "/archive")); Assert.Equal("shfs", share.Source); // Of mounts stacked on one point, the last listed is on top. Assert.Equal("scratch", Assert.NotNull(VolumeSpace.FindMount(Unraid, "/tmp/x")).Source); // Optional fields, a Windows line end and escapes in any field. var data = Assert.NotNull(VolumeSpace.ParseMountLine("592 574 0:31 / /mnt/My\\040Disk rw shared:5 master:3 propagate_from:2 - ext4 /dev/sd\\134b rw\r")); Assert.Equal(("/mnt/My Disk", "ext4", "/dev/sd\\b"), (data.MountPoint, data.FileSystem, data.Source)); Assert.Null(VolumeSpace.ParseMountLine("592 574 0:31 / /data rw shared:5 btrfs /dev/sdb1 rw")); Assert.Null(VolumeSpace.ParseMountLine("")); Assert.Null(VolumeSpace.FindMount("", "/archive")); Assert.Null(VolumeSpace.FindMount("590 574 0:45 /Timelapse /archive rw - fuse.shfs shfs rw", "/data")); } [Test] public void TheFileSystemOfAFolderIsKnown() { var folder = context.CreateTempDirectory("fs"); var fileSystem = Assert.NotNull(VolumeSpace.FileSystemOf(folder)); Assert.NotEmpty(fileSystem); if (OperatingSystem.IsLinux()) Assert.Equal(fileSystem, Assert.NotNull(VolumeSpace.MountOf(folder)).FileSystem); else Assert.Null(VolumeSpace.MountOf(folder)); Assert.Null(VolumeSpace.FileSystemOf(Path.Combine(folder, "missing"))); } [Test] public void TheSpacePathIsShownWithTheWriteVolumeApartWhenTheyDiffer() { var root = context.CreateTempDirectory("report"); var archive = Directory.CreateDirectory(Path.Combine(root, "archive")).FullName; var space = Directory.CreateDirectory(Path.Combine(root, "space")).FullName; var options = new ServerOptions { ArchivePath = archive }; var logger = new CountingLogger(); var volumes = new FakeVolumeSpace(options, logger); volumes.Set(archive, new VolumeFigures(120 * GB, 500 * GB)); // No space path: the archive's own volume. var plain = volumes.Report(archive); Assert.Equal((120 * GB, false, (VolumeFigures?)null), (plain.Space!.FreeBytes, plain.FromSpacePath, plain.Write)); // The cache and the array: the array is shown, the cache apart. options.SpacePath = space; volumes.Set(space, new VolumeFigures(9_000 * GB, 16_000 * GB)); var separate = volumes.Report(archive); Assert.Equal((9_000 * GB, true), (separate.Space!.FreeBytes, separate.FromSpacePath)); Assert.Equal(new VolumeFigures(120 * GB, 500 * GB), separate.Write); Assert.Equal(VolumeSpace.FileSystemOf(space), separate.FileSystem); // One volume read twice a moment apart: no second line. volumes.Set(space, new VolumeFigures((120 * GB) - (8 << 20), 500 * GB)); Assert.Null(volumes.Report(archive).Write); Assert.Equal(0, logger.Warnings); } [Test] public void AConversionCountsOnTheTighterOfTheSpaceAndTheWriteVolume() { var root = context.CreateTempDirectory("writable"); var archive = Directory.CreateDirectory(Path.Combine(root, "archive")).FullName; var space = Directory.CreateDirectory(Path.Combine(root, "space")).FullName; var options = new ServerOptions { ArchivePath = archive }; var volumes = new FakeVolumeSpace(options, new CountingLogger()); volumes.Set(archive, new VolumeFigures(120 * GB, 500 * GB)); Assert.Equal(120 * GB, volumes.WritableFree(archive)); // Unraid: the array has room, the cache the share writes to until the mover runs has less, and it is what fills. options.SpacePath = space; volumes.Set(space, new VolumeFigures(9_000 * GB, 16_000 * GB)); Assert.Equal(120 * GB, volumes.WritableFree(archive)); // An array fuller than the cache is the limit then. volumes.Set(space, new VolumeFigures(40 * GB, 16_000 * GB)); Assert.Equal(40 * GB, volumes.WritableFree(archive)); Assert.Null(new FakeVolumeSpace(new ServerOptions { ArchivePath = archive }, new CountingLogger()).WritableFree(Path.Combine(root, "missing"))); } [Test] public void ASpacePathThatCannotBeMeasuredFallsBackWithOneWarning() { var root = context.CreateTempDirectory("fallback"); var archive = Directory.CreateDirectory(Path.Combine(root, "archive")).FullName; var space = Path.Combine(root, "not-mapped"); var logger = new CountingLogger(); var volumes = new VolumeSpace(new ServerOptions { ArchivePath = archive, SpacePath = space }, logger); for (var i = 0; i < 3; i++) { var report = volumes.Report(archive); Assert.False(report.FromSpacePath); Assert.True(Assert.NotNull(report.Space).TotalBytes > 0); Assert.Null(report.Write); } Assert.Equal(1, logger.Warnings); // Mapped later: measured again, and a later outage is warned about once more. Directory.CreateDirectory(space); Assert.True(volumes.Report(archive).FromSpacePath); Directory.Delete(space); volumes.Report(archive); volumes.Report(archive); Assert.Equal(2, logger.Warnings); } [Test] public async Task TheStatsReportTheSpacePathAndTheWriteVolume() { var space = context.CreateTempDirectory("space"); await using var server = await TestServer.StartAsync(context, o => o.SpacePath = space, s => { s.AddSingleton(); s.AddSingleton(sp => sp.GetRequiredService()); }); var admin = await server.AdminAsync(); var stats = await admin.GetFromJsonAsync("/api/v1/archive/stats", ProtocolJson.Default.ArchiveStatsDto); Assert.True(Assert.NotNull(stats).UsesSpacePath); Assert.True(stats.CapacityBytes > 0 && stats.FreeBytes <= stats.CapacityBytes); Assert.Equal(VolumeSpace.FileSystemOf(space), stats.FileSystem); // Temporary folders share one volume: no write volume apart. Assert.Null(stats.WriteFreeBytes); // The archive's own volume (an Unraid share's cache) differs: the panel gets it apart. server.Services.GetRequiredService().Set(server.Options.ArchivePath, new VolumeFigures(40 * GB, 250 * GB)); using var dashboard = await admin.GetAsync("/api/v1/server/dashboard"); var archive = (await dashboard.ReadAsync(ProtocolJson.Default.DashboardDto)).Archive; Assert.Equal((40 * GB, 250 * GB, true), (archive.WriteFreeBytes, archive.WriteCapacityBytes, archive.UsesSpacePath)); Assert.NotEqual(40 * GB, archive.FreeBytes); } [Test] public async Task AMissingSpacePathShowsTheArchiveVolume() { var root = context.CreateTempDirectory("unmapped"); await using var server = await TestServer.StartAsync(context, o => o.SpacePath = Path.Combine(root, "space")); var admin = await server.AdminAsync(); for (var i = 0; i < 2; i++) { var stats = Assert.NotNull(await admin.GetFromJsonAsync("/api/v1/archive/stats", ProtocolJson.Default.ArchiveStatsDto)); Assert.False(stats.UsesSpacePath); Assert.True(stats.CapacityBytes > 0); Assert.Null(stats.WriteFreeBytes); Assert.Equal(VolumeSpace.FileSystemOf(server.Options.ArchivePath), stats.FileSystem); } // Warned at start, not at every request. var warnings = Directory.EnumerateFiles(server.Options.LogPath).Sum(log => { using var reader = new StreamReader(new FileStream(log, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)); return reader.ReadToEnd().Split('\n').Count(line => line.Contains("CHRONOS_SPACE_PATH", StringComparison.Ordinal)); }); Assert.Equal(1, warnings); } [Test] public async Task TheDiagnosticsReportBothVolumes() { var space = context.CreateTempDirectory("space"); await using var server = await TestServer.StartAsync(context, o => o.SpacePath = space); var admin = await server.AdminAsync(); using var response = await admin.GetAsync("/api/v1/server/diagnostics"); await response.ExpectAsync(HttpStatusCode.OK); await using var body = await response.Content.ReadAsStreamAsync(); using var zip = new ZipArchive(body, ZipArchiveMode.Read); using var reader = new StreamReader(Assert.NotNull(zip.GetEntry("environment.txt")).Open()); var report = await reader.ReadToEndAsync(); var fileSystem = Assert.NotNull(VolumeSpace.FileSystemOf(space)); Assert.Contains($"Archive {server.Options.ArchivePath} (", report); Assert.Contains($"Space {space} (", report); Assert.Contains($" GB, {fileSystem}", report); } [Test] public async Task AFullArchiveVolumeRefusesTheCompletionUntilThereIsRoom() { const int chunk = 64 * 1024; await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = chunk, s => { s.AddSingleton(); s.AddSingleton(sp => sp.GetRequiredService()); }); var volumes = server.Services.GetRequiredService(); var (editor, _, _) = await server.UserAsync("alberto", "editor"); var data = Uploads.Photo(chunk + 500, 71); var start = await Uploads.StartAsync(editor, data, sessionKey: "roof"); var id = Assert.NotNull(start.UploadId); for (var i = 0; i < 2; i++) { using var put = await Uploads.PutChunkAsync(editor, id, i, Uploads.Chunk(data, chunk, i)); await put.ExpectAsync(HttpStatusCode.OK); } // The upload volume (/data) has room, the archive's does not: nothing is staged, the upload keeps its chunks. volumes.Set(server.Options.ArchivePath, new VolumeFigures(32L << 20, 500 * GB)); using (var full = await editor.PostAsync($"/api/v1/uploads/{id}/complete", null)) await full.ExpectErrorAsync(HttpStatusCode.InsufficientStorage, "insufficient_storage"); using (var state = await editor.GetAsync($"/api/v1/uploads/{id}")) Assert.Equal("receiving", (await state.ReadAsync(ProtocolJson.Default.UploadState)).State); Assert.True(server.Services.GetRequiredService().Exists(id)); Assert.False(Directory.Exists(server.Options.StagingPath) && Directory.EnumerateFiles(server.Options.StagingPath).Any()); volumes.Set(server.Options.ArchivePath, new VolumeFigures(GB, 500 * GB)); Guid sequence; using (var room = await editor.PostAsync($"/api/v1/uploads/{id}/complete", null)) { await room.ExpectAsync(HttpStatusCode.OK); sequence = (await room.ReadAsync(ProtocolJson.Default.CompleteUploadResponse)).SequenceId; } // A render sent into the archive is checked the same way. var output = Uploads.Photo(3000, 72); var request = new RenderUploadRequest(Guid.CreateVersion7(), "Roof.mp4", output.Length, Uploads.Sha(output), "4K HEVC", "hevc", 3840, 2160, 30, 10, null); using var started = await editor.PostAsJsonAsync($"/api/v1/sequences/{sequence}/renders", request, ProtocolJson.Default.RenderUploadRequest); await started.ExpectAsync(HttpStatusCode.Created); var render = Assert.NotNull((await started.ReadAsync(ProtocolJson.Default.StartUploadResponse)).UploadId); using (var put = await Uploads.PutChunkAsync(editor, render, 0, output)) await put.ExpectAsync(HttpStatusCode.OK); volumes.Set(server.Options.ArchivePath, new VolumeFigures(0, 500 * GB)); using (var full = await editor.PostAsync($"/api/v1/sequences/{sequence}/renders/{render}/complete", null)) await full.ExpectErrorAsync(HttpStatusCode.InsufficientStorage, "insufficient_storage"); volumes.Set(server.Options.ArchivePath, null); using (var room = await editor.PostAsync($"/api/v1/sequences/{sequence}/renders/{render}/complete", null)) await room.ExpectAsync(HttpStatusCode.OK); } [Test] public async Task AFullArchiveVolumeRefusesARenderNodeResultUntilThereIsRoom() { await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = 64 * 1024, s => { s.AddSingleton(); s.AddSingleton(sp => sp.GetRequiredService()); }); var volumes = server.Services.GetRequiredService(); var (editor, _, _) = await server.UserAsync("alberto", "editor", renderNode: true); var frame = await Uploads.UploadAsync(editor, Uploads.Photo(5000, 73), "IMG_0001.CR3"); var job = await JobApiTests.CreateAsync(editor, "render", frame.SequenceId, RenderNodeTests.HevcJob); var node = await RenderNodeTests.RegisterAsync(editor, "studio-pc", "hevc", 8192); using (var claim = await editor.PostAsync($"/api/v1/nodes/{node.Id}/claim", null)) await claim.ExpectAsync(HttpStatusCode.OK); var output = Uploads.Photo(3000, 74); var request = new RenderResultRequest("Roof - 4K HEVC.mp4", output.Length, Uploads.Sha(output), "4K HEVC", "hevc", 3840, 2160, 25, 120); using var started = await editor.PostAsJsonAsync($"/api/v1/nodes/{node.Id}/jobs/{job.Id}/result", request, ProtocolJson.Default.RenderResultRequest); await started.ExpectAsync(HttpStatusCode.Created); var upload = Assert.NotNull((await started.ReadAsync(ProtocolJson.Default.StartUploadResponse)).UploadId); using (var put = await Uploads.PutChunkAsync(editor, upload, 0, output)) await put.ExpectAsync(HttpStatusCode.OK); // Refused before the commit starts: the upload keeps receiving with its chunks and the node keeps the job. var complete = $"/api/v1/nodes/{node.Id}/jobs/{job.Id}/result/complete"; volumes.Set(server.Options.ArchivePath, new VolumeFigures(0, 500 * GB)); using (var full = await editor.PostAsync(complete, null)) await full.ExpectErrorAsync(HttpStatusCode.InsufficientStorage, "insufficient_storage"); Assert.Equal(Chronos.Data.Repositories.UploadState.Receiving, Assert.NotNull(await server.Database.Uploads.FindByJobAsync(job.Id)).State); Assert.True(server.Services.GetRequiredService().Exists(upload)); var waiting = await JobApiTests.GetAsync(editor, job.Id); Assert.Equal(("running", node.Id.ToString()), (waiting.State, waiting.NodeId)); volumes.Set(server.Options.ArchivePath, null); using (var room = await editor.PostAsync(complete, null)) { await room.ExpectAsync(HttpStatusCode.OK); Assert.Equal("studio-pc", (await room.ReadAsync(ProtocolJson.Default.RenderDto)).ProducedBy); } Assert.Equal("done", (await JobApiTests.GetAsync(editor, job.Id)).State); Assert.Equal(Chronos.Data.Repositories.UploadState.Committed, Assert.NotNull(await server.Database.Uploads.FindByJobAsync(job.Id)).State); } [Test] public void TheSpacePathComesFromTheFileOrTheEnvironment() { var root = context.CreateTempDirectory("space-option"); File.WriteAllText(Path.Combine(root, "server.json"), """{ "SpacePath": "/space" }"""); var file = ServerOptions.Load(new Dictionary { ["CHRONOS_DATA_PATH"] = root, ["CHRONOS_DB"] = "postgres" }); Assert.Equal("/space", file.SpacePath); Assert.DoesNotContain("SpacePath", file.Locked); var environment = ServerOptions.Load(new Dictionary { ["CHRONOS_DATA_PATH"] = root, ["CHRONOS_DB"] = "postgres", ["CHRONOS_SPACE_PATH"] = "/mnt/array", }); Assert.Equal("/mnt/array", environment.SpacePath); Assert.Contains("SpacePath", environment.Locked); Assert.Null(new ServerOptions().SpacePath); } /// Counts the warnings a logs. private sealed class CountingLogger : ILogger { private int _warnings; public int Warnings => Volatile.Read(ref _warnings); public IDisposable? BeginScope(TState state) where TState : notnull => null; public bool IsEnabled(LogLevel logLevel) => true; public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) { if (logLevel == LogLevel.Warning) Interlocked.Increment(ref _warnings); } } }