From d560b41b813a0902dab3823cc37c418a2086002f Mon Sep 17 00:00:00 2001 From: Alberto Balbo Date: Mon, 26 Oct 2026 21:58:27 +0100 Subject: [PATCH] feat(server): show the Unraid array's free space and check the archive volume before filing uploads On Unraid the archive is a user share (FUSE shfs), which reports the share's primary storage, often the cache, while the mover sends the photos to the array: the admin panel showed the cache's free space (owner review 2, R2). - New option SpacePath (server.json) / CHRONOS_SPACE_PATH: the admin panel shows that folder's volume; empty, or a folder that cannot be measured, falls back to the archive's volume (one warning, at start, not per request). - VolumeSpace measures folders (DriveInfo; the path root on Windows) and reads the filesystem type and mount from /proc/self/mountinfo on Linux (longest mount point on segment boundaries, octal escapes, optional fields, the top of stacked mounts), since DriveFormat calls an shfs share only "fuse". It is an injectable service, so tests simulate a full volume; the /data checks use it. - ArchiveStatsDto keeps FreeBytes/CapacityBytes for the shown volume and gains optional trailing members: the write volume's figures (when a space path is in use and they differ), the filesystem type and UsesSpacePath. - Admin panel (dashboard and archive): a tooltip on the storage figures names the volume and its filesystem and, on an shfs share without a space path, says they are the primary storage and to map the array view to /space; a "Write volume" line shows the archive's own volume when it differs. - Completing an upload, a render upload or a render node's result checks the archive volume for its size plus the 64 MiB reserve before the staging copy and answers 507 insufficient_storage, the upload still receiving. - Diagnostics list the archive and space volumes with filesystem and mount. - Unraid template: optional read-only "Array view" /space (default /mnt/user0/Timelapse) and "Space path" CHRONOS_SPACE_PATH=/space; compose and the releases README describe the same as an option. Co-Authored-By: Claude Opus 5.5 --- deploy/docker/docker-compose.yml | 5 + deploy/releases-repo/README.md | 6 + deploy/unraid/chronos.xml | 2 + i18n/en/Admin.resx | 6 + i18n/it/Admin.it.resx | 6 + src/Chronos.Protocol/Sequences.cs | 15 +- src/Chronos.Server/Admin/web/pages/archive.js | 8 +- .../Admin/web/pages/dashboard.js | 9 +- src/Chronos.Server/Admin/web/ui.js | 17 ++ src/Chronos.Server/Api/AdminEndpoints.cs | 4 +- src/Chronos.Server/Api/ArchiveEndpoints.cs | 34 +-- .../Api/DiagnosticsEndpoints.cs | 36 +-- .../Configuration/ServerOptions.cs | 11 + .../Hosting/ServerCommandLine.cs | 2 + src/Chronos.Server/Hosting/ServerHost.cs | 4 + src/Chronos.Server/Jobs/RenderNodeService.cs | 5 +- src/Chronos.Server/Storage/RenderUploads.cs | 5 +- src/Chronos.Server/Storage/UploadRules.cs | 13 - src/Chronos.Server/Storage/UploadService.cs | 30 +- src/Chronos.Server/Storage/VolumeSpace.cs | 181 +++++++++++ tests/Chronos.Server.Tests/Fakes.cs | 19 ++ .../Storage/VolumeSpaceTests.cs | 289 ++++++++++++++++++ 22 files changed, 641 insertions(+), 66 deletions(-) create mode 100644 src/Chronos.Server/Storage/VolumeSpace.cs create mode 100644 tests/Chronos.Server.Tests/Storage/VolumeSpaceTests.cs diff --git a/deploy/docker/docker-compose.yml b/deploy/docker/docker-compose.yml index 0f7fe1af..62fc0e4c 100644 --- a/deploy/docker/docker-compose.yml +++ b/deploy/docker/docker-compose.yml @@ -42,6 +42,9 @@ services: # CHRONOS_PRIORITY_QUEUE: "true" # Trace, Debug, Information, Warning or Error CHRONOS_LOG_LEVEL: Information + # The admin panel shows the archive volume's free space. An Unraid share (/mnt/user/) reports its primary + # storage, often the cache: map the share's array view to /space (volumes below) and show that one instead. + # CHRONOS_SPACE_PATH: /space PUID: ${PUID:-1000} PGID: ${PGID:-1000} TZ: ${TZ:-UTC} @@ -49,6 +52,8 @@ services: # The archive folder should belong to PUID:PGID; an empty or root-owned one is given to them at start (the folder # only, never its contents). - ${CHRONOS_ARCHIVE:-./archive}:/archive + # Optional, with CHRONOS_SPACE_PATH above: the same share seen on the array only, to show the array's free space. + # - /mnt/user0/Timelapse:/space:ro - ${CHRONOS_DATA:-./data}:/data postgres: diff --git a/deploy/releases-repo/README.md b/deploy/releases-repo/README.md index 80db0eb1..e79f6b27 100644 --- a/deploy/releases-repo/README.md +++ b/deploy/releases-repo/README.md @@ -79,6 +79,10 @@ admin panel shows under Settings so you can compare them. to sign in, run `docker login gitea.encke-hake.ts.net` with your Gitea account first. - Volumes: `/archive` (originals, sidecars, proxies, renders — the archive stays readable without Chronos) and `/data` (settings, TLS certificate, uploads in progress, logs). +- Free space: the admin panel shows the archive volume's, or with `CHRONOS_SPACE_PATH` that of another folder. An + Unraid share (`/mnt/user/`) reports its primary storage, often the cache, while the mover sends the photos to + the array: map the share's array view read-only (`/mnt/user0/:/space:ro`) and set `CHRONOS_SPACE_PATH: /space`. + Uploads always check the archive volume itself (`507` when it is full). - Upgrade: `docker compose pull && docker compose up -d`. Jobs in progress resume from their last checkpoint. - All settings: environment variables `CHRONOS_*` (see `docker-compose.yml`) or `/data/server.json`. Run `docker run --rm gitea.encke-hake.ts.net/balbosoft/chronos-server:latest --help` for the list. @@ -114,6 +118,8 @@ admin panel shows under Settings so you can compare them. archive share (default `/mnt/user/Timelapse`). The database host is the Unraid server's IP address (the PostgreSQL template publishes port 5432): container names resolve only when both containers are on the same custom Docker network, not on the default *bridge*. If you change the HTTPS port, enter the same port in *Public HTTPS port*. + *Array view* is the same share under `/mnt/user0` (default `/mnt/user0/Timelapse`): the admin panel then shows the + array's free space rather than the cache's. Change it with the archive share, or remove it and clear *Space path*. 4. Open the WebUI (`http://[IP]:8080/admin`). The template file is [`unraid/chronos.xml`](unraid/chronos.xml). diff --git a/deploy/unraid/chronos.xml b/deploy/unraid/chronos.xml index 38424327..a0f2c6f5 100644 --- a/deploy/unraid/chronos.xml +++ b/deploy/unraid/chronos.xml @@ -25,6 +25,7 @@ 8443 /mnt/user/Timelapse + /mnt/user0/Timelapse /mnt/user/appdata/chronos admin @@ -36,6 +37,7 @@ prefer auto + /space Chronos Server diff --git a/i18n/en/Admin.resx b/i18n/en/Admin.resx index 7de86a88..4b26aeef 100644 --- a/i18n/en/Admin.resx +++ b/i18n/en/Admin.resx @@ -83,6 +83,7 @@ {0} done today {0} failed {0} failed + File system: {0} {0} free Jobs up {0} · {1} · TLS {2} · {3}long @@ -105,11 +106,16 @@ running sequence sequences + Space of the folder set in CHRONOS_SPACE_PATH, shown instead of the archive's own volume Storage + Space of the volume that holds the archive + An Unraid share: these figures are its primary storage, often the cache, not the array the mover fills. Map the share's array view (/mnt/user0/<share>) to /space to see the array. frames per day · {0} total · peak {1}long Uploads · last 30 days {0} sequence without proxieslong {0} sequences without proxieslong + Write volume: {0} free + Free space of the archive's own volume, where uploads are written first: on Unraid the share's primary storage, often the cache The server did not answer; try again.long Adopt Deep analysis diff --git a/i18n/it/Admin.it.resx b/i18n/it/Admin.it.resx index 890c4f29..bb87d351 100644 --- a/i18n/it/Admin.it.resx +++ b/i18n/it/Admin.it.resx @@ -83,6 +83,7 @@ {0} completati oggi {0} fallito {0} falliti + File system: {0} {0} liberi Job attivo da {0} · {1} · TLS {2} · {3} @@ -105,11 +106,16 @@ in corso sequenza sequenze + Spazio della cartella indicata in CHRONOS_SPACE_PATH, mostrato al posto del volume dell'archivio Spazio + Spazio del volume che contiene l'archivio + Una condivisione Unraid: questi valori sono il suo storage primario, spesso la cache, non l'array che il mover riempie. Mappa la vista array della condivisione (/mnt/user0/<condivisione>) su /space per vedere l'array. foto al giorno · totale {0} · picco {1} Caricamenti · ultimi 30 giorni {0} sequenza senza proxy {0} sequenze senza proxy + Volume di scrittura: {0} liberi + Spazio libero del volume dell'archivio, dove i caricamenti vengono scritti prima: su Unraid lo storage primario della condivisione, spesso la cache Il server non ha risposto; riprova. Adozione Analisi completa diff --git a/src/Chronos.Protocol/Sequences.cs b/src/Chronos.Protocol/Sequences.cs index a2d44461..634dd337 100644 --- a/src/Chronos.Protocol/Sequences.cs +++ b/src/Chronos.Protocol/Sequences.cs @@ -111,6 +111,15 @@ public sealed record RenderDto( string? ProducedBy); /// Aggregates for the archive dashboard and the admin panel. +/// Free bytes of the space volume: the server's space path when one is set, else the archive's volume. +/// Size of the space volume. +/// +/// Free bytes of the archive's own volume, where uploads are written, when a space path is in use and its figures +/// differ (an Unraid share's primary storage, often the cache); null otherwise and from servers before owner review 2. +/// +/// Size of that write volume, with . +/// Filesystem type of the space volume (ext4, NTFS, fuse.shfs for an Unraid share); null when unknown. +/// The space volume is the server's space path, not the archive's volume. public sealed record ArchiveStatsDto( long Sequences, long Photos, @@ -125,7 +134,11 @@ public sealed record ArchiveStatsDto( IReadOnlyList ByCamera, IReadOnlyList ByFormat, int SequencesWithoutProxies, - int UnverifiedSequences); + int UnverifiedSequences, + long? WriteFreeBytes = null, + long? WriteCapacityBytes = null, + string? FileSystem = null, + bool UsesSpacePath = false); public sealed record KeyValueCount(string Key, long Count, long Bytes); diff --git a/src/Chronos.Server/Admin/web/pages/archive.js b/src/Chronos.Server/Admin/web/pages/archive.js index 9c5db2d9..c3caa124 100644 --- a/src/Chronos.Server/Admin/web/pages/archive.js +++ b/src/Chronos.Server/Admin/web/pages/archive.js @@ -1,6 +1,6 @@ // Archive (BRIEF §14.8): naming templates, proxy policy, scan/rebuild/verify/maintenance, statistics. import { t, plural, applyStrings } from "../i18n.js"; -import { escapeHtml, attr, formatBytes, formatNumber, bar, columns, toast, errorText, emptyState, markInvalid, clearInvalid, invalidText, escapeReverts } from "../ui.js"; +import { escapeHtml, attr, formatBytes, formatNumber, bar, storageTip, writeVolumeLine, columns, toast, errorText, emptyState, markInvalid, clearInvalid, invalidText, escapeReverts } from "../ui.js"; const PROXY_SIZES = [-1, 1280, 1920, 2560, 3840, 0]; const ACTIONS = ["scan", "rebuild", "verify", "maintenance"]; @@ -15,7 +15,9 @@ export async function render(root, { api, setMeta }) { return; } setMeta(settings.rootPath, { ltr: true }); + // The space volume's figures (the space path when the server has one), the write volume apart when it differs. const used = stats.capacityBytes > 0 ? (stats.capacityBytes - stats.freeBytes) / stats.capacityBytes : 0; + const spaceTip = attr(storageTip(stats)); const sizes = PROXY_SIZES.includes(settings.proxyLongEdge) ? PROXY_SIZES : [settings.proxyLongEdge, ...PROXY_SIZES]; const proxyOptions = sizes.map((size) => ``).join(""); const statsTable = (rows, labelId) => rows.length === 0 ? "" : `
${rows.map((r) => @@ -28,8 +30,8 @@ export async function render(root, { api, setMeta }) { ${escapeHtml(`${plural("Admin.Dashboard.PhotoCount", stats.photos)} · ${plural("Admin.Dashboard.CameraCount", stats.cameras)}`)}

${escapeHtml(formatBytes(stats.bytes))} ${escapeHtml(stats.firstYear ? `${stats.firstYear}–${stats.lastYear}` : "—")}
-

${escapeHtml(formatBytes(stats.freeBytes))} - ${bar(used)}${escapeHtml(t("Admin.Dashboard.Of", formatBytes(stats.capacityBytes)))}
+

${escapeHtml(formatBytes(stats.freeBytes))} + ${bar(used)}${escapeHtml(t("Admin.Dashboard.Of", formatBytes(stats.capacityBytes)))}${writeVolumeLine(stats)}

${formatNumber(stats.unverifiedSequences)}${escapeHtml(plural("Admin.Archive.UnverifiedUnit", stats.unverifiedSequences))} ${escapeHtml(plural("Admin.Dashboard.WithoutProxies", stats.sequencesWithoutProxies))}
diff --git a/src/Chronos.Server/Admin/web/pages/dashboard.js b/src/Chronos.Server/Admin/web/pages/dashboard.js index 259ce6b4..0bd4571d 100644 --- a/src/Chronos.Server/Admin/web/pages/dashboard.js +++ b/src/Chronos.Server/Admin/web/pages/dashboard.js @@ -1,6 +1,6 @@ // Dashboard (mockup Admin.html, BRIEF §14.8): storage, archive, jobs, render nodes, charts, activity, active jobs. import { t, plural, applyStrings, culture } from "../i18n.js"; -import { escapeHtml, attr, formatBytes, formatNumber, formatTime, formatDuration, bar, columns, toast, errorText, emptyState, redraw, fitColumns, PROXY_WARNING } from "../ui.js"; +import { escapeHtml, attr, formatBytes, formatNumber, formatTime, formatDuration, bar, storageTip, writeVolumeLine, columns, toast, errorText, emptyState, redraw, fitColumns, PROXY_WARNING } from "../ui.js"; import { kindLabel, nodeLabel, progressCell, stateCell, stateLabel, encoders } from "../jobs.js"; import { activityText } from "../audit.js"; @@ -30,8 +30,10 @@ export async function render(root, { api, setMeta, actions, onEvent }) { t(d.discovery ? "Admin.Dashboard.DiscoveryOn" : "Admin.Dashboard.DiscoveryOff"))); actions.innerHTML = d.proxyWarning ? PROXY_WARNING : ""; applyStrings(actions); + // The space volume's figures (the space path when the server has one), the write volume apart when it differs. const a = d.archive; const used = a.capacityBytes > 0 ? (a.capacityBytes - a.freeBytes) / a.capacityBytes : 0; + const spaceTip = attr(storageTip(a)); const online = d.nodes.filter((n) => n.state !== "offline"); const uploads = lastThirtyDays(d.uploads); const totalUploads = uploads.reduce((s, u) => s + u.value, 0); @@ -41,9 +43,10 @@ export async function render(root, { api, setMeta, actions, onEvent }) {

- ${escapeHtml(formatBytes(a.capacityBytes - a.freeBytes))}${escapeHtml(t("Admin.Dashboard.Of", formatBytes(a.capacityBytes)))} + ${escapeHtml(formatBytes(a.capacityBytes - a.freeBytes))}${escapeHtml(t("Admin.Dashboard.Of", formatBytes(a.capacityBytes)))} ${bar(used)} - ${escapeHtml(t("Admin.Dashboard.Free", formatBytes(a.freeBytes)))} · ${escapeHtml(t("Admin.Dashboard.Originals", formatBytes(a.bytes)))} + ${escapeHtml(t("Admin.Dashboard.Free", formatBytes(a.freeBytes)))} · ${escapeHtml(t("Admin.Dashboard.Originals", formatBytes(a.bytes)))} + ${writeVolumeLine(a)}

diff --git a/src/Chronos.Server/Admin/web/ui.js b/src/Chronos.Server/Admin/web/ui.js index bdad2168..081c320f 100644 --- a/src/Chronos.Server/Admin/web/ui.js +++ b/src/Chronos.Server/Admin/web/ui.js @@ -56,6 +56,23 @@ export function bar(fraction, muted = false) { ``; } +/** + * Tooltip of a storage tile's figures (archive stats): the volume they measure and its file system; on an Unraid share + * without a space volume, that they are the share's primary storage. + */ +export function storageTip(stats) { + const lines = [stats.usesSpacePath ? t("Admin.Dashboard.SpaceVolume.Tip") : t("Admin.Dashboard.Storage.Tip")]; + if (stats.fileSystem) lines.push(t("Admin.Dashboard.FileSystem", stats.fileSystem)); + if (stats.fileSystem === "fuse.shfs" && !stats.usesSpacePath) lines.push(t("Admin.Dashboard.UnraidShare.Tip")); + return lines.join("\n"); +} + +/** The write volume's free space as one more line of a storage tile, when the stats have it apart; "" otherwise. */ +export function writeVolumeLine(stats) { + if (stats.writeFreeBytes == null) return ""; + return `${escapeHtml(t("Admin.Dashboard.WriteVolume", formatBytes(stats.writeFreeBytes)))}`; +} + /** * Column chart named by label: values with labels under the bars and, when given, the value above each bar. Bars use * the accent colour, or the sun colour for upload activity. With edgeLabels the first and last labels align with the diff --git a/src/Chronos.Server/Api/AdminEndpoints.cs b/src/Chronos.Server/Api/AdminEndpoints.cs index 0c612b1c..3aa5e951 100644 --- a/src/Chronos.Server/Api/AdminEndpoints.cs +++ b/src/Chronos.Server/Api/AdminEndpoints.cs @@ -40,7 +40,7 @@ public static class AdminEndpoints } private static async Task DashboardAsync(HttpContext context, ChronosDatabase db, ServerArchive archive, ServerOptions options, ServerState state, - TimeProvider time) + TimeProvider time, VolumeSpace volumes) { var ct = context.RequestAborted; var now = time.GetUtcNow(); @@ -61,7 +61,7 @@ public static class AdminEndpoints options.TlsMode.ToString().ToLowerInvariant(), options.Discovery, users.Count, - await ArchiveEndpoints.StatsAsync(db, archive, ct), + await ArchiveEndpoints.StatsAsync(db, archive, volumes, ct), new JobCountsDto(counts.Queued, counts.Running, counts.Paused, counts.DoneSince, counts.FailedSince), options.Parallelism, options.PriorityQueue, diff --git a/src/Chronos.Server/Api/ArchiveEndpoints.cs b/src/Chronos.Server/Api/ArchiveEndpoints.cs index 30277ed2..1bfee04e 100644 --- a/src/Chronos.Server/Api/ArchiveEndpoints.cs +++ b/src/Chronos.Server/Api/ArchiveEndpoints.cs @@ -23,37 +23,31 @@ public static class ArchiveEndpoints archive.MapGet("/settings", async (HttpContext c, ServerArchive a, ChronosDatabase db, [FromServices] ArchiveService? archives) => Results.Json(Settings(archives is null ? a.Current : await a.RefreshAsync(archives, db, c.RequestAborted)), ProtocolJson.Default.ArchiveSettingsDto)); archive.MapPut("/settings", UpdateAsync); - archive.MapGet("/stats", async (HttpContext c, ChronosDatabase db, ServerArchive a) => - Results.Json(await StatsAsync(db, a, c.RequestAborted), ProtocolJson.Default.ArchiveStatsDto)); + archive.MapGet("/stats", async (HttpContext c, ChronosDatabase db, ServerArchive a, VolumeSpace volumes) => + Results.Json(await StatsAsync(db, a, volumes, c.RequestAborted), ProtocolJson.Default.ArchiveStatsDto)); foreach (var (path, kind) in new[] { ("scan", JobKind.Scan), ("rebuild", JobKind.Rebuild), ("verify", JobKind.Verify), ("maintenance", JobKind.Maintenance) }) { archive.MapPost("/" + path, (HttpContext c, Caller caller, ServerArchive a, JobEngineHost host, AuditLog audit) => EnqueueAsync(c, caller, a, host, audit, kind)); } } - /// Totals, distributions, free space and health of the archive (also shown on the dashboard). - public static async Task StatsAsync(ChronosDatabase db, ServerArchive archive, CancellationToken cancellationToken) + /// + /// Totals, distributions, free space and health of the archive (also shown on the dashboard). The free space is the + /// space volume's (), with the archive's own volume apart when they differ; an offline + /// volume reports no figures (0). + /// + public static async Task StatsAsync(ChronosDatabase db, ServerArchive archive, VolumeSpace volumes, CancellationToken cancellationToken) { var stats = await db.Sequences.GetStatsAsync(archive.Id, cancellationToken); var withoutProxies = await db.Sequences.CountAsync( new SequenceQuery { ArchiveId = archive.Id, WithoutStatus = SequenceStatus.ProxiesBuilt }, cancellationToken); - long free = 0, capacity = 0; - try - { - var drive = new DriveInfo(archive.Root); - free = drive.AvailableFreeSpace; - capacity = drive.TotalSize; - } - catch (Exception ex) when (ex is IOException or ArgumentException or UnauthorizedAccessException) - { - // An offline archive volume reports no space figures. - } + var storage = volumes.Report(archive.Root); return new ArchiveStatsDto( stats.Sequences, stats.Photos, stats.Bytes, - free, - capacity, + storage.Space?.FreeBytes ?? 0, + storage.Space?.TotalBytes ?? 0, stats.ByCamera.Count(c => c.Model is not null), stats.ByYear.Count > 0 ? stats.ByYear[0].Year : null, stats.ByYear.Count > 0 ? stats.ByYear[^1].Year : null, @@ -62,7 +56,11 @@ public static class ArchiveEndpoints [.. stats.ByCamera.Select(c => new KeyValueCount(string.Join(' ', new[] { c.Make, c.Model }.Where(v => !string.IsNullOrEmpty(v))), c.Photos, c.Bytes))], [.. stats.ByFormat.Select(f => new KeyValueCount(f.Format.ToString().ToLowerInvariant(), f.Photos, f.Bytes))], withoutProxies, - stats.Health.Unverified); + stats.Health.Unverified, + storage.Write?.FreeBytes, + storage.Write?.TotalBytes, + storage.FileSystem, + storage.FromSpacePath); } /// diff --git a/src/Chronos.Server/Api/DiagnosticsEndpoints.cs b/src/Chronos.Server/Api/DiagnosticsEndpoints.cs index 292d60f4..3d0e24a0 100644 --- a/src/Chronos.Server/Api/DiagnosticsEndpoints.cs +++ b/src/Chronos.Server/Api/DiagnosticsEndpoints.cs @@ -8,6 +8,7 @@ using Chronos.Data.Processing; using Chronos.Server.Configuration; using Chronos.Server.Jobs; using Chronos.Server.Security; +using Chronos.Server.Storage; namespace Chronos.Server.Api; @@ -25,11 +26,11 @@ public static class DiagnosticsEndpoints api.MapGet("/server/diagnostics", DownloadAsync).Requires(Permissions.ManageArchive); private static async Task DownloadAsync(HttpContext context, Caller caller, ChronosDatabase db, ServerOptions options, ServerState state, - AuditLog audit, TimeProvider time, IServiceProvider services) + AuditLog audit, TimeProvider time, VolumeSpace volumes, IServiceProvider services) { var ct = context.RequestAborted; var failures = (await db.Jobs.ListAsync(JobState.Failed, FailedJobs, null, ct)).Items; - var report = Report(options, state, time); + var report = Report(options, state, time, volumes); var settings = JsonSerializer.Serialize(options, ServerOptionsJson.Default.ServerOptions); var encoders = Encoders(services.GetService()); @@ -64,7 +65,7 @@ public static class DiagnosticsEndpoints return Results.File(file, "application/zip", name); } - private static string Report(ServerOptions options, ServerState state, TimeProvider time) + private static string Report(ServerOptions options, ServerState state, TimeProvider time, VolumeSpace volumes) { var sb = new StringBuilder(); sb.AppendLine(CultureInfo.InvariantCulture, $"Chronos Server {SystemEndpoints.Version} ({options.ServerName})"); @@ -76,8 +77,11 @@ public static class DiagnosticsEndpoints sb.AppendLine("SQLite compile options: " + string.Join(' ', Chronos.Data.Sqlite.SqliteDatabase.CompileOptions)); sb.AppendLine(CultureInfo.InvariantCulture, $"TLS {options.TlsMode} {state.TlsFingerprint}"); sb.AppendLine(CultureInfo.InvariantCulture, $"Jobs one at a time, parallelism {options.Parallelism}, priority queue {(options.PriorityQueue ? "on" : "off")}"); - sb.AppendLine(CultureInfo.InvariantCulture, $"Archive {options.ArchivePath} ({Space(options.ArchivePath)})"); - sb.AppendLine(CultureInfo.InvariantCulture, $"Data {options.DataPath} ({Space(options.DataPath)})"); + // The archive's volume is where uploads are written; the space path, when set, is what the admin panel shows. + sb.AppendLine(CultureInfo.InvariantCulture, $"Archive {options.ArchivePath} ({Space(volumes, options.ArchivePath)})"); + if (!string.IsNullOrWhiteSpace(options.SpacePath)) + sb.AppendLine(CultureInfo.InvariantCulture, $"Space {options.SpacePath} ({Space(volumes, options.SpacePath)})"); + sb.AppendLine(CultureInfo.InvariantCulture, $"Data {options.DataPath} ({Space(volumes, options.DataPath)})"); sb.AppendLine(CultureInfo.InvariantCulture, $"Container {File.Exists("/.dockerenv")}"); sb.AppendLine("Set by environment: " + string.Join(", ", options.Locked)); return sb.ToString(); @@ -98,19 +102,15 @@ public static class DiagnosticsEndpoints return sb.ToString(); } - private static string Space(string path) + /// "120 GB free of 500 GB, fuse.shfs mounted at /archive from shfs" (the mount on Linux only; "…, NTFS" elsewhere). + private static string Space(VolumeSpace volumes, string path) { - try - { - if (!Directory.Exists(path)) return "missing"; - var full = Path.GetFullPath(path); - // Windows wants a drive root; Unix accepts any directory and reports the volume it is mounted on. - var drive = new DriveInfo(OperatingSystem.IsWindows() ? Path.GetPathRoot(full)! : full); - return string.Create(CultureInfo.InvariantCulture, $"{drive.AvailableFreeSpace / (1L << 30)} GB free of {drive.TotalSize / (1L << 30)} GB"); - } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException) - { - return "free space unknown"; - } + if (!Directory.Exists(path)) return "missing"; + var figures = volumes.Measure(path) is { } v + ? string.Create(CultureInfo.InvariantCulture, $"{v.FreeBytes / (1L << 30)} GB free of {v.TotalBytes / (1L << 30)} GB") + : "free space unknown"; + var mount = VolumeSpace.MountOf(path); + var fileSystem = mount?.FileSystem ?? VolumeSpace.FileSystemOf(path) ?? "file system unknown"; + return mount is null ? $"{figures}, {fileSystem}" : $"{figures}, {fileSystem} mounted at {mount.MountPoint} from {mount.Source}"; } } diff --git a/src/Chronos.Server/Configuration/ServerOptions.cs b/src/Chronos.Server/Configuration/ServerOptions.cs index be950aa2..63ecd80b 100644 --- a/src/Chronos.Server/Configuration/ServerOptions.cs +++ b/src/Chronos.Server/Configuration/ServerOptions.cs @@ -36,6 +36,7 @@ public sealed class ServerOptions [ ("CHRONOS_SERVER_NAME", nameof(ServerName), (o, v) => o.ServerName = v), ("CHRONOS_ARCHIVE_PATH", nameof(ArchivePath), (o, v) => o.ArchivePath = v), + ("CHRONOS_SPACE_PATH", nameof(SpacePath), (o, v) => o.SpacePath = v), ("CHRONOS_ADMIN_USER", nameof(AdminUser), (o, v) => o.AdminUser = v), ("CHRONOS_ADMIN_PASSWORD", nameof(AdminPassword), (o, v) => o.AdminPassword = v), ("CHRONOS_DB", nameof(Database), (o, v) => o.ApplyDatabase(v)), @@ -80,6 +81,16 @@ public sealed class ServerOptions public string ServerName { get; set; } = "Chronos Server"; public string ArchivePath { get; set; } = OperatingSystem.IsWindows() ? @"C:\ChronosArchive" : "/archive"; + + /// + /// A folder whose volume's free space the admin panel shows instead of the archive's (owner review 2, R2; key + /// SpacePath, variable CHRONOS_SPACE_PATH). On Unraid the archive is a user share, which reports its + /// primary storage (often the cache) while the mover sends the photos to the array: the template maps the share's + /// array view (/mnt/user0/<share>) read-only to /space. Empty: the archive's volume; a folder that + /// cannot be measured falls back to it with one warning. Uploads always check the archive's own volume. + /// + public string? SpacePath { get; set; } + public string DataPath { get; set; } = OperatingSystem.IsWindows() ? Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), "Chronos Server") : "/data"; public string? AdminUser { get; set; } public string? AdminPassword { get; set; } diff --git a/src/Chronos.Server/Hosting/ServerCommandLine.cs b/src/Chronos.Server/Hosting/ServerCommandLine.cs index 40cc5c1c..a350c88d 100644 --- a/src/Chronos.Server/Hosting/ServerCommandLine.cs +++ b/src/Chronos.Server/Hosting/ServerCommandLine.cs @@ -37,6 +37,8 @@ public sealed record ServerCommandLine(ServerCommand Command, IReadOnlyList) mapped read-only to /space CHRONOS_PARALLELISM photos the running job processes at once (jobs run one at a time; default: half the cores); CHRONOS_WORKERS, its former name, still sets it CHRONOS_PRIORITY_QUEUE a job of a higher level pauses the running one, which resumes afterwards (true | false) diff --git a/src/Chronos.Server/Hosting/ServerHost.cs b/src/Chronos.Server/Hosting/ServerHost.cs index e600f17b..d4c11146 100644 --- a/src/Chronos.Server/Hosting/ServerHost.cs +++ b/src/Chronos.Server/Hosting/ServerHost.cs @@ -88,6 +88,8 @@ public static class ServerHost + "they could put a link in place of a folder and redirect the files the server moves. Let only those accounts change it (others may read it).", options.ArchivePath); } + // A space path that cannot be measured is warned about at start, not first when someone opens the dashboard. + if (!string.IsNullOrWhiteSpace(options.SpacePath)) app.Services.GetRequiredService().Report(archive.Root); if (options.FormerWorkersVariable) { app.Logger.LogWarning("CHRONOS_WORKERS is the former name of CHRONOS_PARALLELISM: jobs now run one at a time, and the value is how many " @@ -112,6 +114,8 @@ public static class ServerHost services.AddSingleton(database); services.AddSingleton(archive); services.TryAddSingleton(TimeProvider.System); + // Tests register their own first, to simulate a full volume. + services.TryAddSingleton(); services.ConfigureHttpJsonOptions(j => j.SerializerOptions.TypeInfoResolverChain.Insert(0, ProtocolJson.Default)); services.Configure(o => o.ThrowOnBadRequest = true); services.Configure(o => o.ShutdownTimeout = TimeSpan.FromSeconds(30)); diff --git a/src/Chronos.Server/Jobs/RenderNodeService.cs b/src/Chronos.Server/Jobs/RenderNodeService.cs index 5bb0d4cc..3e0e7e90 100644 --- a/src/Chronos.Server/Jobs/RenderNodeService.cs +++ b/src/Chronos.Server/Jobs/RenderNodeService.cs @@ -30,6 +30,7 @@ public sealed class RenderNodeService( RenderStore renders, JobEngineHost jobs, EventHub events, + VolumeSpace volumes, TimeProvider time) { public const string Idle = "idle"; @@ -135,7 +136,8 @@ public sealed class RenderNodeService( if (await db.Uploads.FindByJobAsync(jobId, cancellationToken) is { State: UploadState.Receiving } open && open.Sha256 == sha && open.Size == request.Size && store.Exists(open.Id)) return Started(open); - if (UploadRules.FreeBytes(options.UploadsPath) is { } free && free < request.Size) return Outcome.Fail(507, "insufficient_storage"); + if (volumes.Measure(options.UploadsPath) is { } uploadVolume && uploadVolume.FreeBytes < request.Size) + return Outcome.Fail(507, "insufficient_storage"); var now = time.GetUtcNow(); var upload = await db.Uploads.CreateAsync(new Upload @@ -166,6 +168,7 @@ public sealed class RenderNodeService( if (upload is not { State: UploadState.Receiving } || upload.RenderJson is null) return Outcome.Fail(409, "upload_state"); if (!upload.IsComplete) return Outcome.Fail(409, "incomplete", (upload.ChunkCount - upload.ReceivedCount).ToString(CultureInfo.InvariantCulture)); + if (!uploads.ArchiveHasRoom(upload)) return Outcome.Fail(507, "insufficient_storage"); if (!await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Receiving, UploadState.Committing, cancellationToken: cancellationToken)) return Outcome.Fail(409, "upload_state"); diff --git a/src/Chronos.Server/Storage/RenderUploads.cs b/src/Chronos.Server/Storage/RenderUploads.cs index 0725e642..02e2214d 100644 --- a/src/Chronos.Server/Storage/RenderUploads.cs +++ b/src/Chronos.Server/Storage/RenderUploads.cs @@ -23,6 +23,7 @@ public sealed class RenderUploads( UploadStore store, RenderStore renders, EventHub events, + VolumeSpace volumes, TimeProvider time) { public async Task> StartAsync(Caller caller, Guid sequenceId, RenderUploadRequest request, CancellationToken cancellationToken) @@ -46,7 +47,8 @@ public sealed class RenderUploads( if (await db.Uploads.FindReceivingAsync(caller.Id, sha, cancellationToken) is { } open && open.SequenceId == sequenceId && open.Size == request.Size && Read(open)?.RenderId == request.RenderId && store.Exists(open.Id)) return Started(open); - if (UploadRules.FreeBytes(options.UploadsPath) is { } free && free < request.Size) return Outcome.Fail(507, "insufficient_storage"); + if (volumes.Measure(options.UploadsPath) is { } uploadVolume && uploadVolume.FreeBytes < request.Size) + return Outcome.Fail(507, "insufficient_storage"); var now = time.GetUtcNow(); var upload = await db.Uploads.CreateAsync(new Upload @@ -77,6 +79,7 @@ public sealed class RenderUploads( if (upload.State != UploadState.Receiving) return Outcome.Fail(409, "upload_state"); if (!upload.IsComplete) return Outcome.Fail(409, "incomplete", (upload.ChunkCount - upload.ReceivedCount).ToString(CultureInfo.InvariantCulture)); + if (!uploads.ArchiveHasRoom(upload)) return Outcome.Fail(507, "insufficient_storage"); if (!await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Receiving, UploadState.Committing, cancellationToken: cancellationToken)) return Outcome.Fail(409, "upload_state"); diff --git a/src/Chronos.Server/Storage/UploadRules.cs b/src/Chronos.Server/Storage/UploadRules.cs index 5a876e67..716faf8f 100644 --- a/src/Chronos.Server/Storage/UploadRules.cs +++ b/src/Chronos.Server/Storage/UploadRules.cs @@ -66,17 +66,4 @@ public static class UploadRules /// The session key an upload without a sequence is grouped by: the client's, or one per user. public static string SessionKeyOf(string? sessionKey, Guid userId) => sessionKey ?? "user-" + userId.ToString("N"); - - /// Free bytes on the volume holding , or null when the platform cannot tell. - public static long? FreeBytes(string path) - { - try - { - return new DriveInfo(Path.GetFullPath(path)).AvailableFreeSpace; - } - catch (Exception ex) when (ex is IOException or ArgumentException or UnauthorizedAccessException) - { - return null; - } - } } diff --git a/src/Chronos.Server/Storage/UploadService.cs b/src/Chronos.Server/Storage/UploadService.cs index 22adb2f4..20e880a1 100644 --- a/src/Chronos.Server/Storage/UploadService.cs +++ b/src/Chronos.Server/Storage/UploadService.cs @@ -17,9 +17,10 @@ namespace Chronos.Server.Storage; /// /// Chunked, resumable, hash-verified frame uploads (BRIEF §14.4): start (short-circuit on a known SHA-256 held by an -/// intact frame, resume an unfinished upload, quota and disk checks), idempotent chunks, and completion that stages the -/// file on the archive volume, verifies its hash and hands it to , which repairs a missing or -/// damaged frame of that content (no quota is charged for a repair: the archive already counts those bytes). +/// intact frame, resume an unfinished upload, quota and disk checks), idempotent chunks, and completion that checks the +/// archive volume's free space, stages the file there, verifies its hash and hands it to , +/// which repairs a missing or damaged frame of that content (no quota is charged for a repair: the archive already +/// counts those bytes). /// public sealed class UploadService( ChronosDatabase db, @@ -29,11 +30,12 @@ public sealed class UploadService( IArchiveIngest ingest, SequenceJobs sequenceJobs, EventHub events, + VolumeSpace volumes, IHostApplicationLifetime lifetime, TimeProvider time, ILogger logger) { - /// Space kept free on the upload volume beyond the announced size. + /// Space kept free on the upload and archive volumes beyond an upload's size. private const long DiskReserve = 64L * 1024 * 1024; /// Card events one account may publish per minute: progress once a second leaves room for every start and end. @@ -74,7 +76,7 @@ public sealed class UploadService( if (caller.User.UsedBytes + pending + request.Size > caller.User.QuotaBytes) return Fail(403, "quota_exceeded", caller.User.QuotaBytes.ToString(CultureInfo.InvariantCulture)); } - if (UploadRules.FreeBytes(options.UploadsPath) is { } free && free < request.Size + DiskReserve) + if (volumes.Measure(options.UploadsPath) is { } uploadVolume && uploadVolume.FreeBytes < request.Size + DiskReserve) return Fail(507, "insufficient_storage"); var now = time.GetUtcNow(); @@ -165,6 +167,7 @@ public sealed class UploadService( if (upload.State != UploadState.Receiving) return Fail(409, "upload_state"); if (!upload.IsComplete) return Fail(409, "incomplete", (upload.ChunkCount - upload.ReceivedCount).ToString(CultureInfo.InvariantCulture)); + if (!ArchiveHasRoom(upload)) return Fail(507, "insufficient_storage"); if (!await db.Uploads.TryTransitionAsync(id, UploadState.Receiving, UploadState.Committing, cancellationToken: stopping)) return Fail(409, "upload_state"); @@ -240,9 +243,24 @@ public sealed class UploadService( return true; } + /// + /// The archive volume has room for and the reserve: completing it copies the file into the + /// staging folder there (). Checked before the upload leaves the receiving state, so a + /// refused completion (507) can be repeated once there is room. A volume that cannot be measured refuses nothing. + /// + public bool ArchiveHasRoom(Upload upload) + { + ArgumentNullException.ThrowIfNull(upload); + if (volumes.Measure(archive.Root) is not { } volume || volume.FreeBytes >= upload.Size + DiskReserve) return true; + logger.LogWarning("The archive volume {Archive} has {Free} bytes free: upload {Upload} ({File}, {Size} bytes) waits for room", + archive.Root, volume.FreeBytes, upload.Id, upload.FileName, upload.Size); + return false; + } + /// /// Copies a fully received upload to the archive staging folder and checks its SHA-256. On a mismatch the upload - /// fails, its files are deleted and null is returned; the client starts over. + /// fails, its files are deleted and null is returned; the client starts over. Callers check + /// first. /// public async Task StageVerifiedAsync(Upload upload, CancellationToken cancellationToken) { diff --git a/src/Chronos.Server/Storage/VolumeSpace.cs b/src/Chronos.Server/Storage/VolumeSpace.cs new file mode 100644 index 00000000..b705c524 --- /dev/null +++ b/src/Chronos.Server/Storage/VolumeSpace.cs @@ -0,0 +1,181 @@ +using System.Text; +using Chronos.Server.Configuration; + +namespace Chronos.Server.Storage; + +/// Free and total bytes of a volume. +public sealed record VolumeFigures(long FreeBytes, long TotalBytes); + +/// One mount of /proc/self/mountinfo: its mount point, filesystem type (fuse.shfs) and source. +public sealed record MountEntry(string MountPoint, string FileSystem, string Source); + +/// +/// The archive's storage as the admin panel shows it: the space volume's figures ( +/// when set and measurable, else the archive root) and filesystem, plus the archive root's own figures (the write +/// volume) when a space path is in use and they differ. +/// +public sealed record StorageReport(VolumeFigures? Space, string? FileSystem, bool FromSpacePath, VolumeFigures? Write); + +/// +/// Free space of the server's folders (owner review 2, R2). The archive root is where uploads are staged and filed; on +/// Unraid it is a user share (FUSE shfs) that reports the share's primary storage, often the cache, while the +/// mover sends the photos to the array. names a folder whose volume the admin panel +/// shows instead (the share seen on the array only). Tests replace to simulate a full volume. +/// +public class VolumeSpace(ServerOptions options, ILogger logger) +{ + /// Two reads of one volume a moment apart differ by what was written in between. + private const long SameVolumeSlack = 64L * 1024 * 1024; + + private const string MountInfo = "/proc/self/mountinfo"; + + /// 1 while the space path cannot be measured: it is warned about once, not at every request. + private int _spaceUnavailable; + + /// Free and total bytes of the volume holding ; null when the folder is missing or the platform cannot tell. + public virtual VolumeFigures? Measure(string path) + { + try + { + if (!Directory.Exists(path)) return null; + var drive = Drive(path); + return new VolumeFigures(drive.AvailableFreeSpace, drive.TotalSize); + } + catch (Exception ex) when (ex is IOException or ArgumentException or UnauthorizedAccessException) + { + return null; + } + } + + /// + /// The storage figures of the archive at : of the space path when one is set and can be + /// measured, else of the archive root (a space path that cannot be measured is logged once, until it can again). + /// + public StorageReport Report(string archiveRoot) + { + var write = Measure(archiveRoot); + var spacePath = options.SpacePath; + if (string.IsNullOrWhiteSpace(spacePath)) return new StorageReport(write, FileSystemOf(archiveRoot), FromSpacePath: false, Write: null); + if (Measure(spacePath) is not { } space) + { + if (Interlocked.Exchange(ref _spaceUnavailable, 1) == 0) + { + logger.LogWarning("The space path {SpacePath} (CHRONOS_SPACE_PATH) is missing or cannot be measured: the admin panel shows the free space " + + "of the archive {Archive} instead. Map the folder into the container, or clear the setting.", spacePath, archiveRoot); + } + return new StorageReport(write, FileSystemOf(archiveRoot), FromSpacePath: false, Write: null); + } + if (Interlocked.Exchange(ref _spaceUnavailable, 0) == 1) logger.LogInformation("The space path {SpacePath} can be measured again", spacePath); + var differs = write is not null && (write.TotalBytes != space.TotalBytes || Math.Abs(write.FreeBytes - space.FreeBytes) > SameVolumeSlack); + return new StorageReport(space, FileSystemOf(spacePath), FromSpacePath: true, differs ? write : null); + } + + /// + /// The type of the filesystem holding the folder : on Linux the mount's type (an Unraid user + /// share is fuse.shfs, which the kernel's own figures call only FUSE), elsewhere what the system reports + /// (NTFS). Null when the folder is missing or the type unknown. + /// + public static string? FileSystemOf(string path) + { + if (!Directory.Exists(path)) return null; + if (MountOf(path) is { } mount) return mount.FileSystem; + try + { + return Drive(path).DriveFormat; + } + catch (Exception ex) when (ex is IOException or ArgumentException or UnauthorizedAccessException) + { + return null; + } + } + + /// + /// The mount holding the folder , read from /proc/self/mountinfo; null off Linux, when + /// the folder is missing or the file cannot be read. + /// + public static MountEntry? MountOf(string path) + { + if (!OperatingSystem.IsLinux() || !Directory.Exists(path)) return null; + try + { + return FindMount(File.ReadAllText(MountInfo), Path.GetFullPath(path)); + } + catch (Exception ex) when (ex is IOException or ArgumentException or UnauthorizedAccessException) + { + return null; + } + } + + /// + /// The mount of (the text of /proc/self/mountinfo) that holds the absolute + /// : the longest mount point that is the path or one of its parent folders and, of mounts + /// stacked on that point, the last listed (the one on top). Null when no line covers it. + /// + public static MountEntry? FindMount(string mountInfo, string path) + { + ArgumentNullException.ThrowIfNull(mountInfo); + ArgumentNullException.ThrowIfNull(path); + var target = path.TrimEnd('/'); + if (target.Length == 0) target = "/"; + MountEntry? best = null; + foreach (var line in mountInfo.Split('\n')) + { + if (ParseMountLine(line) is not { } entry || !Covers(entry.MountPoint, target)) continue; + if (best is null || entry.MountPoint.Length >= best.MountPoint.Length) best = entry; + } + return best; + } + + /// + /// One mountinfo line (proc(5)): id parent major:minor root mount-point options [optional fields…] - type source + /// super-options, where fields write a space, tab, newline or backslash as an octal escape (\040). Null when + /// the line has another shape. + /// + public static MountEntry? ParseMountLine(string line) + { + ArgumentNullException.ThrowIfNull(line); + var fields = line.TrimEnd('\r').Split(' '); + if (fields.Length < 9) return null; + // The optional fields (none or more, "shared:5 master:1") end at a lone "-". + var separator = Array.IndexOf(fields, "-", 6); + if (separator < 0 || separator + 2 >= fields.Length) return null; + var mountPoint = Unescape(fields[4]); + if (!mountPoint.StartsWith('/')) return null; + return new MountEntry(mountPoint, Unescape(fields[separator + 1]), Unescape(fields[separator + 2])); + } + + /// Windows wants a drive root; Unix accepts any folder and reports the filesystem it is on. + private static DriveInfo Drive(string path) + { + var full = Path.GetFullPath(path); + return new DriveInfo(OperatingSystem.IsWindows() ? Path.GetPathRoot(full)! : full); + } + + /// is or one of its parent folders. + private static bool Covers(string mountPoint, string path) => + mountPoint == "/" + ? path.StartsWith('/') + : path.StartsWith(mountPoint, StringComparison.Ordinal) && (path.Length == mountPoint.Length || path[mountPoint.Length] == '/'); + + /// Decodes the kernel's \ooo escapes (it escapes only ASCII characters). + private static string Unescape(string field) + { + if (!field.Contains('\\', StringComparison.Ordinal)) return field; + var sb = new StringBuilder(field.Length); + for (var i = 0; i < field.Length; i++) + { + if (field[i] == '\\' && i + 3 < field.Length && IsOctal(field[i + 1]) && IsOctal(field[i + 2]) && IsOctal(field[i + 3])) + { + sb.Append((char)(((field[i + 1] - '0') * 64) + ((field[i + 2] - '0') * 8) + (field[i + 3] - '0'))); + i += 3; + } + else + { + sb.Append(field[i]); + } + } + return sb.ToString(); + } + + private static bool IsOctal(char c) => c is >= '0' and <= '7'; +} diff --git a/tests/Chronos.Server.Tests/Fakes.cs b/tests/Chronos.Server.Tests/Fakes.cs index d235fdf8..2d1a0adc 100644 --- a/tests/Chronos.Server.Tests/Fakes.cs +++ b/tests/Chronos.Server.Tests/Fakes.cs @@ -1,8 +1,10 @@ using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; using Chronos.Core.Model; using Chronos.Data; using Chronos.Data.Archiving; using Chronos.Data.Jobs; +using Chronos.Server.Configuration; using Chronos.Server.Storage; namespace Chronos.Server.Tests; @@ -99,6 +101,23 @@ public sealed class FakeAssetLocator : IFrameAssetLocator } } +/// Measures volumes for real, except the folders given figures of their own (a full archive, a separate array). +public sealed class FakeVolumeSpace(ServerOptions options, ILogger logger) : VolumeSpace(options, logger) +{ + private readonly ConcurrentDictionary _figures = new(StringComparer.Ordinal); + + /// Gives these figures; null measures it for real again. + public void Set(string path, VolumeFigures? figures) + { + if (figures is null) _figures.TryRemove(Key(path), out _); + else _figures[Key(path)] = figures; + } + + public override VolumeFigures? Measure(string path) => _figures.TryGetValue(Key(path), out var figures) ? figures : base.Measure(path); + + private static string Key(string path) => Path.TrimEndingDirectorySeparator(Path.GetFullPath(path)); +} + /// A clock tests move by hand (token expiry, heartbeat timeouts). public sealed class ManualTimeProvider : TimeProvider { diff --git a/tests/Chronos.Server.Tests/Storage/VolumeSpaceTests.cs b/tests/Chronos.Server.Tests/Storage/VolumeSpaceTests.cs new file mode 100644 index 00000000..60ee8f38 --- /dev/null +++ b/tests/Chronos.Server.Tests/Storage/VolumeSpaceTests.cs @@ -0,0 +1,289 @@ +using System.IO.Compression; +using System.Net; +using System.Net.Http.Json; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Chronos.Protocol; +using Chronos.Server.Configuration; +using Chronos.Server.Storage; +using Chronos.Testing; + +namespace Chronos.Server.Tests.Storage; + +/// Owner review 2 (R2): the free space of the Unraid array instead of the cache, and of the archive before an upload is filed. +[Category("server")] +public sealed class VolumeSpaceTests(TestContext context) +{ + private const long GB = 1_000_000_000; + + /// + /// /proc/self/mountinfo of a Chronos container on Unraid: the overlay root, the share bind-mounted at /archive (FUSE + /// shfs, whose root is the share's folder), its array view at /space, appdata at /data, a disk mounted inside the + /// archive under a name with a space, and two mounts stacked on /tmp. + /// + private const string Unraid = """ + 574 520 0:52 / / rw,relatime master:255 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/ABC:/var/lib/docker/overlay2/l/DEF,upperdir=/var/lib/docker/overlay2/1f/diff + 575 574 0:55 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw + 590 574 0:45 /Timelapse /archive rw,noatime - fuse.shfs shfs rw,user_id=0,group_id=0,default_permissions,allow_other + 591 574 0:46 /Timelapse /space ro,noatime - fuse.shfs shfs rw,user_id=0,group_id=0,default_permissions,allow_other + 592 574 0:31 /appdata/chronos /data rw,noatime shared:5 master:3 - btrfs /dev/nvme0n1p1 rw,ssd,space_cache=v2,subvolid=5,subvol=/ + 593 590 9:1 / /archive/Big\040Trip rw,noatime - xfs /dev/md1p1 rw,attr2,inode64 + 594 574 0:60 / /tmp rw - tmpfs tmpfs rw + 595 574 0:61 / /tmp rw - tmpfs scratch rw,size=1g + this line is not a mount + """; + + [Theory] + [Case("/archive", "/archive", "fuse.shfs")] + [Case("/archive/2026/Roof/IMG_0001.CR3", "/archive", "fuse.shfs")] + [Case("/archive/", "/archive", "fuse.shfs")] + [Case("/space", "/space", "fuse.shfs")] + [Case("/data/uploads", "/data", "btrfs")] + [Case("/archive/Big Trip/IMG_0002.CR3", "/archive/Big Trip", "xfs")] + [Case("/archive/Big", "/archive", "fuse.shfs")] + [Case("/archived", "/", "overlay")] + [Case("/", "/", "overlay")] + [Case("/app/Chronos.Server.dll", "/", "overlay")] + public void TheLongestMountPointHoldsAPath(string path, string mountPoint, string fileSystem) + { + var mount = Assert.NotNull(VolumeSpace.FindMount(Unraid, path)); + Assert.Equal((mountPoint, fileSystem), (mount.MountPoint, mount.FileSystem)); + } + + [Test] + public void MountInfoLinesAreReadField() + { + var share = Assert.NotNull(VolumeSpace.FindMount(Unraid, "/archive")); + Assert.Equal("shfs", share.Source); + // Of mounts stacked on one point, the last listed is on top. + Assert.Equal("scratch", Assert.NotNull(VolumeSpace.FindMount(Unraid, "/tmp/x")).Source); + // Optional fields, a Windows line end and escapes in any field. + var data = Assert.NotNull(VolumeSpace.ParseMountLine("592 574 0:31 / /mnt/My\\040Disk rw shared:5 master:3 propagate_from:2 - ext4 /dev/sd\\134b rw\r")); + Assert.Equal(("/mnt/My Disk", "ext4", "/dev/sd\\b"), (data.MountPoint, data.FileSystem, data.Source)); + Assert.Null(VolumeSpace.ParseMountLine("592 574 0:31 / /data rw shared:5 btrfs /dev/sdb1 rw")); + Assert.Null(VolumeSpace.ParseMountLine("")); + Assert.Null(VolumeSpace.FindMount("", "/archive")); + Assert.Null(VolumeSpace.FindMount("590 574 0:45 /Timelapse /archive rw - fuse.shfs shfs rw", "/data")); + } + + [Test] + public void TheFileSystemOfAFolderIsKnown() + { + var folder = context.CreateTempDirectory("fs"); + var fileSystem = Assert.NotNull(VolumeSpace.FileSystemOf(folder)); + Assert.NotEmpty(fileSystem); + if (OperatingSystem.IsLinux()) Assert.Equal(fileSystem, Assert.NotNull(VolumeSpace.MountOf(folder)).FileSystem); + else Assert.Null(VolumeSpace.MountOf(folder)); + Assert.Null(VolumeSpace.FileSystemOf(Path.Combine(folder, "missing"))); + } + + [Test] + public void TheSpacePathIsShownWithTheWriteVolumeApartWhenTheyDiffer() + { + var root = context.CreateTempDirectory("report"); + var archive = Directory.CreateDirectory(Path.Combine(root, "archive")).FullName; + var space = Directory.CreateDirectory(Path.Combine(root, "space")).FullName; + var options = new ServerOptions { ArchivePath = archive }; + var logger = new CountingLogger(); + var volumes = new FakeVolumeSpace(options, logger); + volumes.Set(archive, new VolumeFigures(120 * GB, 500 * GB)); + + // No space path: the archive's own volume. + var plain = volumes.Report(archive); + Assert.Equal((120 * GB, false, (VolumeFigures?)null), (plain.Space!.FreeBytes, plain.FromSpacePath, plain.Write)); + + // The cache and the array: the array is shown, the cache apart. + options.SpacePath = space; + volumes.Set(space, new VolumeFigures(9_000 * GB, 16_000 * GB)); + var separate = volumes.Report(archive); + Assert.Equal((9_000 * GB, true), (separate.Space!.FreeBytes, separate.FromSpacePath)); + Assert.Equal(new VolumeFigures(120 * GB, 500 * GB), separate.Write); + Assert.Equal(VolumeSpace.FileSystemOf(space), separate.FileSystem); + + // One volume read twice a moment apart: no second line. + volumes.Set(space, new VolumeFigures((120 * GB) - (8 << 20), 500 * GB)); + Assert.Null(volumes.Report(archive).Write); + Assert.Equal(0, logger.Warnings); + } + + [Test] + public void ASpacePathThatCannotBeMeasuredFallsBackWithOneWarning() + { + var root = context.CreateTempDirectory("fallback"); + var archive = Directory.CreateDirectory(Path.Combine(root, "archive")).FullName; + var space = Path.Combine(root, "not-mapped"); + var logger = new CountingLogger(); + var volumes = new VolumeSpace(new ServerOptions { ArchivePath = archive, SpacePath = space }, logger); + + for (var i = 0; i < 3; i++) + { + var report = volumes.Report(archive); + Assert.False(report.FromSpacePath); + Assert.True(Assert.NotNull(report.Space).TotalBytes > 0); + Assert.Null(report.Write); + } + Assert.Equal(1, logger.Warnings); + + // Mapped later: measured again, and a later outage is warned about once more. + Directory.CreateDirectory(space); + Assert.True(volumes.Report(archive).FromSpacePath); + Directory.Delete(space); + volumes.Report(archive); + volumes.Report(archive); + Assert.Equal(2, logger.Warnings); + } + + [Test] + public async Task TheStatsReportTheSpacePathAndTheWriteVolume() + { + var space = context.CreateTempDirectory("space"); + await using var server = await TestServer.StartAsync(context, o => o.SpacePath = space, s => + { + s.AddSingleton(); + s.AddSingleton(sp => sp.GetRequiredService()); + }); + var admin = await server.AdminAsync(); + var stats = await admin.GetFromJsonAsync("/api/v1/archive/stats", ProtocolJson.Default.ArchiveStatsDto); + Assert.True(Assert.NotNull(stats).UsesSpacePath); + Assert.True(stats.CapacityBytes > 0 && stats.FreeBytes <= stats.CapacityBytes); + Assert.Equal(VolumeSpace.FileSystemOf(space), stats.FileSystem); + // Temporary folders share one volume: no write volume apart. + Assert.Null(stats.WriteFreeBytes); + + // The archive's own volume (an Unraid share's cache) differs: the panel gets it apart. + server.Services.GetRequiredService().Set(server.Options.ArchivePath, new VolumeFigures(40 * GB, 250 * GB)); + using var dashboard = await admin.GetAsync("/api/v1/server/dashboard"); + var archive = (await dashboard.ReadAsync(ProtocolJson.Default.DashboardDto)).Archive; + Assert.Equal((40 * GB, 250 * GB, true), (archive.WriteFreeBytes, archive.WriteCapacityBytes, archive.UsesSpacePath)); + Assert.NotEqual(40 * GB, archive.FreeBytes); + } + + [Test] + public async Task AMissingSpacePathShowsTheArchiveVolume() + { + var root = context.CreateTempDirectory("unmapped"); + await using var server = await TestServer.StartAsync(context, o => o.SpacePath = Path.Combine(root, "space")); + var admin = await server.AdminAsync(); + for (var i = 0; i < 2; i++) + { + var stats = Assert.NotNull(await admin.GetFromJsonAsync("/api/v1/archive/stats", ProtocolJson.Default.ArchiveStatsDto)); + Assert.False(stats.UsesSpacePath); + Assert.True(stats.CapacityBytes > 0); + Assert.Null(stats.WriteFreeBytes); + Assert.Equal(VolumeSpace.FileSystemOf(server.Options.ArchivePath), stats.FileSystem); + } + // Warned at start, not at every request. + var warnings = Directory.EnumerateFiles(server.Options.LogPath).Sum(log => + { + using var reader = new StreamReader(new FileStream(log, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)); + return reader.ReadToEnd().Split('\n').Count(line => line.Contains("CHRONOS_SPACE_PATH", StringComparison.Ordinal)); + }); + Assert.Equal(1, warnings); + } + + [Test] + public async Task TheDiagnosticsReportBothVolumes() + { + var space = context.CreateTempDirectory("space"); + await using var server = await TestServer.StartAsync(context, o => o.SpacePath = space); + var admin = await server.AdminAsync(); + using var response = await admin.GetAsync("/api/v1/server/diagnostics"); + await response.ExpectAsync(HttpStatusCode.OK); + await using var body = await response.Content.ReadAsStreamAsync(); + using var zip = new ZipArchive(body, ZipArchiveMode.Read); + using var reader = new StreamReader(Assert.NotNull(zip.GetEntry("environment.txt")).Open()); + var report = await reader.ReadToEndAsync(); + var fileSystem = Assert.NotNull(VolumeSpace.FileSystemOf(space)); + Assert.Contains($"Archive {server.Options.ArchivePath} (", report); + Assert.Contains($"Space {space} (", report); + Assert.Contains($" GB, {fileSystem}", report); + } + + [Test] + public async Task AFullArchiveVolumeRefusesTheCompletionUntilThereIsRoom() + { + const int chunk = 64 * 1024; + await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = chunk, s => + { + s.AddSingleton(); + s.AddSingleton(sp => sp.GetRequiredService()); + }); + var volumes = server.Services.GetRequiredService(); + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + var data = Uploads.Photo(chunk + 500, 71); + var start = await Uploads.StartAsync(editor, data, sessionKey: "roof"); + var id = Assert.NotNull(start.UploadId); + for (var i = 0; i < 2; i++) + { + using var put = await Uploads.PutChunkAsync(editor, id, i, Uploads.Chunk(data, chunk, i)); + await put.ExpectAsync(HttpStatusCode.OK); + } + + // The upload volume (/data) has room, the archive's does not: nothing is staged, the upload keeps its chunks. + volumes.Set(server.Options.ArchivePath, new VolumeFigures(32L << 20, 500 * GB)); + using (var full = await editor.PostAsync($"/api/v1/uploads/{id}/complete", null)) + await full.ExpectErrorAsync(HttpStatusCode.InsufficientStorage, "insufficient_storage"); + using (var state = await editor.GetAsync($"/api/v1/uploads/{id}")) + Assert.Equal("receiving", (await state.ReadAsync(ProtocolJson.Default.UploadState)).State); + Assert.True(server.Services.GetRequiredService().Exists(id)); + Assert.False(Directory.Exists(server.Options.StagingPath) && Directory.EnumerateFiles(server.Options.StagingPath).Any()); + + volumes.Set(server.Options.ArchivePath, new VolumeFigures(GB, 500 * GB)); + Guid sequence; + using (var room = await editor.PostAsync($"/api/v1/uploads/{id}/complete", null)) + { + await room.ExpectAsync(HttpStatusCode.OK); + sequence = (await room.ReadAsync(ProtocolJson.Default.CompleteUploadResponse)).SequenceId; + } + + // A render sent into the archive is checked the same way. + var output = Uploads.Photo(3000, 72); + var request = new RenderUploadRequest(Guid.CreateVersion7(), "Roof.mp4", output.Length, Uploads.Sha(output), "4K HEVC", "hevc", 3840, 2160, 30, 10, null); + using var started = await editor.PostAsJsonAsync($"/api/v1/sequences/{sequence}/renders", request, ProtocolJson.Default.RenderUploadRequest); + await started.ExpectAsync(HttpStatusCode.Created); + var render = Assert.NotNull((await started.ReadAsync(ProtocolJson.Default.StartUploadResponse)).UploadId); + using (var put = await Uploads.PutChunkAsync(editor, render, 0, output)) await put.ExpectAsync(HttpStatusCode.OK); + volumes.Set(server.Options.ArchivePath, new VolumeFigures(0, 500 * GB)); + using (var full = await editor.PostAsync($"/api/v1/sequences/{sequence}/renders/{render}/complete", null)) + await full.ExpectErrorAsync(HttpStatusCode.InsufficientStorage, "insufficient_storage"); + volumes.Set(server.Options.ArchivePath, null); + using (var room = await editor.PostAsync($"/api/v1/sequences/{sequence}/renders/{render}/complete", null)) + await room.ExpectAsync(HttpStatusCode.OK); + } + + [Test] + public void TheSpacePathComesFromTheFileOrTheEnvironment() + { + var root = context.CreateTempDirectory("space-option"); + File.WriteAllText(Path.Combine(root, "server.json"), """{ "SpacePath": "/space" }"""); + var file = ServerOptions.Load(new Dictionary { ["CHRONOS_DATA_PATH"] = root, ["CHRONOS_DB"] = "postgres" }); + Assert.Equal("/space", file.SpacePath); + Assert.DoesNotContain("SpacePath", file.Locked); + var environment = ServerOptions.Load(new Dictionary + { + ["CHRONOS_DATA_PATH"] = root, + ["CHRONOS_DB"] = "postgres", + ["CHRONOS_SPACE_PATH"] = "/mnt/array", + }); + Assert.Equal("/mnt/array", environment.SpacePath); + Assert.Contains("SpacePath", environment.Locked); + Assert.Null(new ServerOptions().SpacePath); + } + + /// Counts the warnings a logs. + private sealed class CountingLogger : ILogger + { + private int _warnings; + + public int Warnings => Volatile.Read(ref _warnings); + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + if (logLevel == LogLevel.Warning) Interlocked.Increment(ref _warnings); + } + } +}