Files
Chronos/tests/Chronos.Server.Tests/Storage/VolumeSpaceTests.cs
T
Alby96andClaude Opus 5.5 d560b41b81 feat(server): show the Unraid array's free space and check the archive volume before filing uploads
On Unraid the archive is a user share (FUSE shfs), which reports the share's
primary storage, often the cache, while the mover sends the photos to the
array: the admin panel showed the cache's free space (owner review 2, R2).

- New option SpacePath (server.json) / CHRONOS_SPACE_PATH: the admin panel
  shows that folder's volume; empty, or a folder that cannot be measured, falls
  back to the archive's volume (one warning, at start, not per request).
- VolumeSpace measures folders (DriveInfo; the path root on Windows) and reads
  the filesystem type and mount from /proc/self/mountinfo on Linux (longest
  mount point on segment boundaries, octal escapes, optional fields, the top of
  stacked mounts), since DriveFormat calls an shfs share only "fuse". It is an
  injectable service, so tests simulate a full volume; the /data checks use it.
- ArchiveStatsDto keeps FreeBytes/CapacityBytes for the shown volume and gains
  optional trailing members: the write volume's figures (when a space path is
  in use and they differ), the filesystem type and UsesSpacePath.
- Admin panel (dashboard and archive): a tooltip on the storage figures names
  the volume and its filesystem and, on an shfs share without a space path,
  says they are the primary storage and to map the array view to /space; a
  "Write volume" line shows the archive's own volume when it differs.
- Completing an upload, a render upload or a render node's result checks the
  archive volume for its size plus the 64 MiB reserve before the staging copy
  and answers 507 insufficient_storage, the upload still receiving.
- Diagnostics list the archive and space volumes with filesystem and mount.
- Unraid template: optional read-only "Array view" /space (default
  /mnt/user0/Timelapse) and "Space path" CHRONOS_SPACE_PATH=/space; compose and
  the releases README describe the same as an option.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-26 21:58:27 +01:00

290 lines
15 KiB
C#

using System.IO.Compression;
using System.Net;
using System.Net.Http.Json;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Chronos.Protocol;
using Chronos.Server.Configuration;
using Chronos.Server.Storage;
using Chronos.Testing;
namespace Chronos.Server.Tests.Storage;
/// <summary>Owner review 2 (R2): the free space of the Unraid array instead of the cache, and of the archive before an upload is filed.</summary>
[Category("server")]
public sealed class VolumeSpaceTests(TestContext context)
{
private const long GB = 1_000_000_000;
/// <summary>
/// /proc/self/mountinfo of a Chronos container on Unraid: the overlay root, the share bind-mounted at /archive (FUSE
/// shfs, whose root is the share's folder), its array view at /space, appdata at /data, a disk mounted inside the
/// archive under a name with a space, and two mounts stacked on /tmp.
/// </summary>
private const string Unraid = """
574 520 0:52 / / rw,relatime master:255 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/ABC:/var/lib/docker/overlay2/l/DEF,upperdir=/var/lib/docker/overlay2/1f/diff
575 574 0:55 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw
590 574 0:45 /Timelapse /archive rw,noatime - fuse.shfs shfs rw,user_id=0,group_id=0,default_permissions,allow_other
591 574 0:46 /Timelapse /space ro,noatime - fuse.shfs shfs rw,user_id=0,group_id=0,default_permissions,allow_other
592 574 0:31 /appdata/chronos /data rw,noatime shared:5 master:3 - btrfs /dev/nvme0n1p1 rw,ssd,space_cache=v2,subvolid=5,subvol=/
593 590 9:1 / /archive/Big\040Trip rw,noatime - xfs /dev/md1p1 rw,attr2,inode64
594 574 0:60 / /tmp rw - tmpfs tmpfs rw
595 574 0:61 / /tmp rw - tmpfs scratch rw,size=1g
this line is not a mount
""";
[Theory]
[Case("/archive", "/archive", "fuse.shfs")]
[Case("/archive/2026/Roof/IMG_0001.CR3", "/archive", "fuse.shfs")]
[Case("/archive/", "/archive", "fuse.shfs")]
[Case("/space", "/space", "fuse.shfs")]
[Case("/data/uploads", "/data", "btrfs")]
[Case("/archive/Big Trip/IMG_0002.CR3", "/archive/Big Trip", "xfs")]
[Case("/archive/Big", "/archive", "fuse.shfs")]
[Case("/archived", "/", "overlay")]
[Case("/", "/", "overlay")]
[Case("/app/Chronos.Server.dll", "/", "overlay")]
public void TheLongestMountPointHoldsAPath(string path, string mountPoint, string fileSystem)
{
var mount = Assert.NotNull(VolumeSpace.FindMount(Unraid, path));
Assert.Equal((mountPoint, fileSystem), (mount.MountPoint, mount.FileSystem));
}
[Test]
public void MountInfoLinesAreReadField()
{
var share = Assert.NotNull(VolumeSpace.FindMount(Unraid, "/archive"));
Assert.Equal("shfs", share.Source);
// Of mounts stacked on one point, the last listed is on top.
Assert.Equal("scratch", Assert.NotNull(VolumeSpace.FindMount(Unraid, "/tmp/x")).Source);
// Optional fields, a Windows line end and escapes in any field.
var data = Assert.NotNull(VolumeSpace.ParseMountLine("592 574 0:31 / /mnt/My\\040Disk rw shared:5 master:3 propagate_from:2 - ext4 /dev/sd\\134b rw\r"));
Assert.Equal(("/mnt/My Disk", "ext4", "/dev/sd\\b"), (data.MountPoint, data.FileSystem, data.Source));
Assert.Null(VolumeSpace.ParseMountLine("592 574 0:31 / /data rw shared:5 btrfs /dev/sdb1 rw"));
Assert.Null(VolumeSpace.ParseMountLine(""));
Assert.Null(VolumeSpace.FindMount("", "/archive"));
Assert.Null(VolumeSpace.FindMount("590 574 0:45 /Timelapse /archive rw - fuse.shfs shfs rw", "/data"));
}
[Test]
public void TheFileSystemOfAFolderIsKnown()
{
var folder = context.CreateTempDirectory("fs");
var fileSystem = Assert.NotNull(VolumeSpace.FileSystemOf(folder));
Assert.NotEmpty(fileSystem);
if (OperatingSystem.IsLinux()) Assert.Equal(fileSystem, Assert.NotNull(VolumeSpace.MountOf(folder)).FileSystem);
else Assert.Null(VolumeSpace.MountOf(folder));
Assert.Null(VolumeSpace.FileSystemOf(Path.Combine(folder, "missing")));
}
[Test]
public void TheSpacePathIsShownWithTheWriteVolumeApartWhenTheyDiffer()
{
var root = context.CreateTempDirectory("report");
var archive = Directory.CreateDirectory(Path.Combine(root, "archive")).FullName;
var space = Directory.CreateDirectory(Path.Combine(root, "space")).FullName;
var options = new ServerOptions { ArchivePath = archive };
var logger = new CountingLogger();
var volumes = new FakeVolumeSpace(options, logger);
volumes.Set(archive, new VolumeFigures(120 * GB, 500 * GB));
// No space path: the archive's own volume.
var plain = volumes.Report(archive);
Assert.Equal((120 * GB, false, (VolumeFigures?)null), (plain.Space!.FreeBytes, plain.FromSpacePath, plain.Write));
// The cache and the array: the array is shown, the cache apart.
options.SpacePath = space;
volumes.Set(space, new VolumeFigures(9_000 * GB, 16_000 * GB));
var separate = volumes.Report(archive);
Assert.Equal((9_000 * GB, true), (separate.Space!.FreeBytes, separate.FromSpacePath));
Assert.Equal(new VolumeFigures(120 * GB, 500 * GB), separate.Write);
Assert.Equal(VolumeSpace.FileSystemOf(space), separate.FileSystem);
// One volume read twice a moment apart: no second line.
volumes.Set(space, new VolumeFigures((120 * GB) - (8 << 20), 500 * GB));
Assert.Null(volumes.Report(archive).Write);
Assert.Equal(0, logger.Warnings);
}
[Test]
public void ASpacePathThatCannotBeMeasuredFallsBackWithOneWarning()
{
var root = context.CreateTempDirectory("fallback");
var archive = Directory.CreateDirectory(Path.Combine(root, "archive")).FullName;
var space = Path.Combine(root, "not-mapped");
var logger = new CountingLogger();
var volumes = new VolumeSpace(new ServerOptions { ArchivePath = archive, SpacePath = space }, logger);
for (var i = 0; i < 3; i++)
{
var report = volumes.Report(archive);
Assert.False(report.FromSpacePath);
Assert.True(Assert.NotNull(report.Space).TotalBytes > 0);
Assert.Null(report.Write);
}
Assert.Equal(1, logger.Warnings);
// Mapped later: measured again, and a later outage is warned about once more.
Directory.CreateDirectory(space);
Assert.True(volumes.Report(archive).FromSpacePath);
Directory.Delete(space);
volumes.Report(archive);
volumes.Report(archive);
Assert.Equal(2, logger.Warnings);
}
[Test]
public async Task TheStatsReportTheSpacePathAndTheWriteVolume()
{
var space = context.CreateTempDirectory("space");
await using var server = await TestServer.StartAsync(context, o => o.SpacePath = space, s =>
{
s.AddSingleton<FakeVolumeSpace>();
s.AddSingleton<VolumeSpace>(sp => sp.GetRequiredService<FakeVolumeSpace>());
});
var admin = await server.AdminAsync();
var stats = await admin.GetFromJsonAsync("/api/v1/archive/stats", ProtocolJson.Default.ArchiveStatsDto);
Assert.True(Assert.NotNull(stats).UsesSpacePath);
Assert.True(stats.CapacityBytes > 0 && stats.FreeBytes <= stats.CapacityBytes);
Assert.Equal(VolumeSpace.FileSystemOf(space), stats.FileSystem);
// Temporary folders share one volume: no write volume apart.
Assert.Null(stats.WriteFreeBytes);
// The archive's own volume (an Unraid share's cache) differs: the panel gets it apart.
server.Services.GetRequiredService<FakeVolumeSpace>().Set(server.Options.ArchivePath, new VolumeFigures(40 * GB, 250 * GB));
using var dashboard = await admin.GetAsync("/api/v1/server/dashboard");
var archive = (await dashboard.ReadAsync(ProtocolJson.Default.DashboardDto)).Archive;
Assert.Equal((40 * GB, 250 * GB, true), (archive.WriteFreeBytes, archive.WriteCapacityBytes, archive.UsesSpacePath));
Assert.NotEqual(40 * GB, archive.FreeBytes);
}
[Test]
public async Task AMissingSpacePathShowsTheArchiveVolume()
{
var root = context.CreateTempDirectory("unmapped");
await using var server = await TestServer.StartAsync(context, o => o.SpacePath = Path.Combine(root, "space"));
var admin = await server.AdminAsync();
for (var i = 0; i < 2; i++)
{
var stats = Assert.NotNull(await admin.GetFromJsonAsync("/api/v1/archive/stats", ProtocolJson.Default.ArchiveStatsDto));
Assert.False(stats.UsesSpacePath);
Assert.True(stats.CapacityBytes > 0);
Assert.Null(stats.WriteFreeBytes);
Assert.Equal(VolumeSpace.FileSystemOf(server.Options.ArchivePath), stats.FileSystem);
}
// Warned at start, not at every request.
var warnings = Directory.EnumerateFiles(server.Options.LogPath).Sum(log =>
{
using var reader = new StreamReader(new FileStream(log, FileMode.Open, FileAccess.Read, FileShare.ReadWrite));
return reader.ReadToEnd().Split('\n').Count(line => line.Contains("CHRONOS_SPACE_PATH", StringComparison.Ordinal));
});
Assert.Equal(1, warnings);
}
[Test]
public async Task TheDiagnosticsReportBothVolumes()
{
var space = context.CreateTempDirectory("space");
await using var server = await TestServer.StartAsync(context, o => o.SpacePath = space);
var admin = await server.AdminAsync();
using var response = await admin.GetAsync("/api/v1/server/diagnostics");
await response.ExpectAsync(HttpStatusCode.OK);
await using var body = await response.Content.ReadAsStreamAsync();
using var zip = new ZipArchive(body, ZipArchiveMode.Read);
using var reader = new StreamReader(Assert.NotNull(zip.GetEntry("environment.txt")).Open());
var report = await reader.ReadToEndAsync();
var fileSystem = Assert.NotNull(VolumeSpace.FileSystemOf(space));
Assert.Contains($"Archive {server.Options.ArchivePath} (", report);
Assert.Contains($"Space {space} (", report);
Assert.Contains($" GB, {fileSystem}", report);
}
[Test]
public async Task AFullArchiveVolumeRefusesTheCompletionUntilThereIsRoom()
{
const int chunk = 64 * 1024;
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = chunk, s =>
{
s.AddSingleton<FakeVolumeSpace>();
s.AddSingleton<VolumeSpace>(sp => sp.GetRequiredService<FakeVolumeSpace>());
});
var volumes = server.Services.GetRequiredService<FakeVolumeSpace>();
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var data = Uploads.Photo(chunk + 500, 71);
var start = await Uploads.StartAsync(editor, data, sessionKey: "roof");
var id = Assert.NotNull(start.UploadId);
for (var i = 0; i < 2; i++)
{
using var put = await Uploads.PutChunkAsync(editor, id, i, Uploads.Chunk(data, chunk, i));
await put.ExpectAsync(HttpStatusCode.OK);
}
// The upload volume (/data) has room, the archive's does not: nothing is staged, the upload keeps its chunks.
volumes.Set(server.Options.ArchivePath, new VolumeFigures(32L << 20, 500 * GB));
using (var full = await editor.PostAsync($"/api/v1/uploads/{id}/complete", null))
await full.ExpectErrorAsync(HttpStatusCode.InsufficientStorage, "insufficient_storage");
using (var state = await editor.GetAsync($"/api/v1/uploads/{id}"))
Assert.Equal("receiving", (await state.ReadAsync(ProtocolJson.Default.UploadState)).State);
Assert.True(server.Services.GetRequiredService<UploadStore>().Exists(id));
Assert.False(Directory.Exists(server.Options.StagingPath) && Directory.EnumerateFiles(server.Options.StagingPath).Any());
volumes.Set(server.Options.ArchivePath, new VolumeFigures(GB, 500 * GB));
Guid sequence;
using (var room = await editor.PostAsync($"/api/v1/uploads/{id}/complete", null))
{
await room.ExpectAsync(HttpStatusCode.OK);
sequence = (await room.ReadAsync(ProtocolJson.Default.CompleteUploadResponse)).SequenceId;
}
// A render sent into the archive is checked the same way.
var output = Uploads.Photo(3000, 72);
var request = new RenderUploadRequest(Guid.CreateVersion7(), "Roof.mp4", output.Length, Uploads.Sha(output), "4K HEVC", "hevc", 3840, 2160, 30, 10, null);
using var started = await editor.PostAsJsonAsync($"/api/v1/sequences/{sequence}/renders", request, ProtocolJson.Default.RenderUploadRequest);
await started.ExpectAsync(HttpStatusCode.Created);
var render = Assert.NotNull((await started.ReadAsync(ProtocolJson.Default.StartUploadResponse)).UploadId);
using (var put = await Uploads.PutChunkAsync(editor, render, 0, output)) await put.ExpectAsync(HttpStatusCode.OK);
volumes.Set(server.Options.ArchivePath, new VolumeFigures(0, 500 * GB));
using (var full = await editor.PostAsync($"/api/v1/sequences/{sequence}/renders/{render}/complete", null))
await full.ExpectErrorAsync(HttpStatusCode.InsufficientStorage, "insufficient_storage");
volumes.Set(server.Options.ArchivePath, null);
using (var room = await editor.PostAsync($"/api/v1/sequences/{sequence}/renders/{render}/complete", null))
await room.ExpectAsync(HttpStatusCode.OK);
}
[Test]
public void TheSpacePathComesFromTheFileOrTheEnvironment()
{
var root = context.CreateTempDirectory("space-option");
File.WriteAllText(Path.Combine(root, "server.json"), """{ "SpacePath": "/space" }""");
var file = ServerOptions.Load(new Dictionary<string, string?> { ["CHRONOS_DATA_PATH"] = root, ["CHRONOS_DB"] = "postgres" });
Assert.Equal("/space", file.SpacePath);
Assert.DoesNotContain("SpacePath", file.Locked);
var environment = ServerOptions.Load(new Dictionary<string, string?>
{
["CHRONOS_DATA_PATH"] = root,
["CHRONOS_DB"] = "postgres",
["CHRONOS_SPACE_PATH"] = "/mnt/array",
});
Assert.Equal("/mnt/array", environment.SpacePath);
Assert.Contains("SpacePath", environment.Locked);
Assert.Null(new ServerOptions().SpacePath);
}
/// <summary>Counts the warnings a <see cref="VolumeSpace"/> logs.</summary>
private sealed class CountingLogger : ILogger<VolumeSpace>
{
private int _warnings;
public int Warnings => Volatile.Read(ref _warnings);
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
public bool IsEnabled(LogLevel logLevel) => true;
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
{
if (logLevel == LogLevel.Warning) Interlocked.Increment(ref _warnings);
}
}
}