diff --git a/deploy/docker/docker-compose.yml b/deploy/docker/docker-compose.yml index 3ab41a3f..8ca8370d 100644 --- a/deploy/docker/docker-compose.yml +++ b/deploy/docker/docker-compose.yml @@ -44,6 +44,9 @@ services: # CHRONOS_PRIORITY_QUEUE: "true" # Trace, Debug, Information, Warning or Error CHRONOS_LOG_LEVEL: Information + # The admin panel shows the archive volume's free space. An Unraid share (/mnt/user/) reports its primary + # storage, often the cache: map the share's array view to /space (volumes below) and show that one instead. + # CHRONOS_SPACE_PATH: /space PUID: ${PUID:-1000} PGID: ${PGID:-1000} TZ: ${TZ:-UTC} @@ -51,6 +54,8 @@ services: # The archive folder should belong to PUID:PGID; an empty or root-owned one is given to them at start (the folder # only, never its contents). - ${CHRONOS_ARCHIVE:-./archive}:/archive + # Optional, with CHRONOS_SPACE_PATH above: the same share seen on the array only, to show the array's free space. + # - /mnt/user0/Timelapse:/space:ro - ${CHRONOS_DATA:-./data}:/data postgres: diff --git a/deploy/releases-repo/README.md b/deploy/releases-repo/README.md index 20db5bcf..7fffff58 100644 --- a/deploy/releases-repo/README.md +++ b/deploy/releases-repo/README.md @@ -80,6 +80,10 @@ admin panel shows under Settings so you can compare them. `docker login gitea.encke-hake.ts.net` with your Gitea account first. - Volumes: `/archive` (originals, sidecars, proxies, renders — the archive stays readable without Chronos) and `/data` (settings, TLS certificate, uploads in progress, logs). +- Free space: the admin panel shows the archive volume's, or with `CHRONOS_SPACE_PATH` that of another folder. An + Unraid share (`/mnt/user/`) reports its primary storage, often the cache, while the mover sends the photos to + the array: map the share's array view read-only (`/mnt/user0/:/space:ro`) and set `CHRONOS_SPACE_PATH: /space`. + Uploads always check the archive volume itself (`507` when it is full). - Upgrade: `docker compose pull && docker compose up -d`. Jobs in progress resume from their last checkpoint. - All settings: environment variables `CHRONOS_*` (see `docker-compose.yml`) or `/data/server.json`. Run `docker run --rm gitea.encke-hake.ts.net/balbosoft/chronos-server:latest --help` for the list. @@ -115,6 +119,8 @@ admin panel shows under Settings so you can compare them. archive share (default `/mnt/user/Timelapse`). The database host is the Unraid server's IP address (the PostgreSQL template publishes port 5432): container names resolve only when both containers are on the same custom Docker network, not on the default *bridge*. If you change the HTTPS port, enter the same port in *Public HTTPS port*. + *Array view* is the same share under `/mnt/user0` (default `/mnt/user0/Timelapse`): the admin panel then shows the + array's free space rather than the cache's. Change it with the archive share, or remove it and clear *Space path*. 4. Open the WebUI (`http://[IP]:8080/admin`). The template file is [`unraid/chronos.xml`](unraid/chronos.xml). diff --git a/deploy/unraid/chronos.xml b/deploy/unraid/chronos.xml index 38424327..a0f2c6f5 100644 --- a/deploy/unraid/chronos.xml +++ b/deploy/unraid/chronos.xml @@ -25,6 +25,7 @@ 8443 /mnt/user/Timelapse + /mnt/user0/Timelapse /mnt/user/appdata/chronos admin @@ -36,6 +37,7 @@ prefer auto + /space Chronos Server diff --git a/i18n/en/Admin.resx b/i18n/en/Admin.resx index 7de86a88..4b26aeef 100644 --- a/i18n/en/Admin.resx +++ b/i18n/en/Admin.resx @@ -83,6 +83,7 @@ {0} done today {0} failed {0} failed + File system: {0} {0} free Jobs up {0} · {1} · TLS {2} · {3}long @@ -105,11 +106,16 @@ running sequence sequences + Space of the folder set in CHRONOS_SPACE_PATH, shown instead of the archive's own volume Storage + Space of the volume that holds the archive + An Unraid share: these figures are its primary storage, often the cache, not the array the mover fills. Map the share's array view (/mnt/user0/<share>) to /space to see the array. frames per day · {0} total · peak {1}long Uploads · last 30 days {0} sequence without proxieslong {0} sequences without proxieslong + Write volume: {0} free + Free space of the archive's own volume, where uploads are written first: on Unraid the share's primary storage, often the cache The server did not answer; try again.long Adopt Deep analysis diff --git a/i18n/it/Admin.it.resx b/i18n/it/Admin.it.resx index 890c4f29..bb87d351 100644 --- a/i18n/it/Admin.it.resx +++ b/i18n/it/Admin.it.resx @@ -83,6 +83,7 @@ {0} completati oggi {0} fallito {0} falliti + File system: {0} {0} liberi Job attivo da {0} · {1} · TLS {2} · {3} @@ -105,11 +106,16 @@ in corso sequenza sequenze + Spazio della cartella indicata in CHRONOS_SPACE_PATH, mostrato al posto del volume dell'archivio Spazio + Spazio del volume che contiene l'archivio + Una condivisione Unraid: questi valori sono il suo storage primario, spesso la cache, non l'array che il mover riempie. Mappa la vista array della condivisione (/mnt/user0/<condivisione>) su /space per vedere l'array. foto al giorno · totale {0} · picco {1} Caricamenti · ultimi 30 giorni {0} sequenza senza proxy {0} sequenze senza proxy + Volume di scrittura: {0} liberi + Spazio libero del volume dell'archivio, dove i caricamenti vengono scritti prima: su Unraid lo storage primario della condivisione, spesso la cache Il server non ha risposto; riprova. Adozione Analisi completa diff --git a/src/Chronos.Protocol/Sequences.cs b/src/Chronos.Protocol/Sequences.cs index a2d44461..634dd337 100644 --- a/src/Chronos.Protocol/Sequences.cs +++ b/src/Chronos.Protocol/Sequences.cs @@ -111,6 +111,15 @@ public sealed record RenderDto( string? ProducedBy); /// Aggregates for the archive dashboard and the admin panel. +/// Free bytes of the space volume: the server's space path when one is set, else the archive's volume. +/// Size of the space volume. +/// +/// Free bytes of the archive's own volume, where uploads are written, when a space path is in use and its figures +/// differ (an Unraid share's primary storage, often the cache); null otherwise and from servers before owner review 2. +/// +/// Size of that write volume, with . +/// Filesystem type of the space volume (ext4, NTFS, fuse.shfs for an Unraid share); null when unknown. +/// The space volume is the server's space path, not the archive's volume. public sealed record ArchiveStatsDto( long Sequences, long Photos, @@ -125,7 +134,11 @@ public sealed record ArchiveStatsDto( IReadOnlyList ByCamera, IReadOnlyList ByFormat, int SequencesWithoutProxies, - int UnverifiedSequences); + int UnverifiedSequences, + long? WriteFreeBytes = null, + long? WriteCapacityBytes = null, + string? FileSystem = null, + bool UsesSpacePath = false); public sealed record KeyValueCount(string Key, long Count, long Bytes); diff --git a/src/Chronos.Server/Admin/web/pages/archive.js b/src/Chronos.Server/Admin/web/pages/archive.js index 9c5db2d9..c3caa124 100644 --- a/src/Chronos.Server/Admin/web/pages/archive.js +++ b/src/Chronos.Server/Admin/web/pages/archive.js @@ -1,6 +1,6 @@ // Archive (BRIEF §14.8): naming templates, proxy policy, scan/rebuild/verify/maintenance, statistics. import { t, plural, applyStrings } from "../i18n.js"; -import { escapeHtml, attr, formatBytes, formatNumber, bar, columns, toast, errorText, emptyState, markInvalid, clearInvalid, invalidText, escapeReverts } from "../ui.js"; +import { escapeHtml, attr, formatBytes, formatNumber, bar, storageTip, writeVolumeLine, columns, toast, errorText, emptyState, markInvalid, clearInvalid, invalidText, escapeReverts } from "../ui.js"; const PROXY_SIZES = [-1, 1280, 1920, 2560, 3840, 0]; const ACTIONS = ["scan", "rebuild", "verify", "maintenance"]; @@ -15,7 +15,9 @@ export async function render(root, { api, setMeta }) { return; } setMeta(settings.rootPath, { ltr: true }); + // The space volume's figures (the space path when the server has one), the write volume apart when it differs. const used = stats.capacityBytes > 0 ? (stats.capacityBytes - stats.freeBytes) / stats.capacityBytes : 0; + const spaceTip = attr(storageTip(stats)); const sizes = PROXY_SIZES.includes(settings.proxyLongEdge) ? PROXY_SIZES : [settings.proxyLongEdge, ...PROXY_SIZES]; const proxyOptions = sizes.map((size) => ``).join(""); const statsTable = (rows, labelId) => rows.length === 0 ? "" : `
${rows.map((r) => @@ -28,8 +30,8 @@ export async function render(root, { api, setMeta }) { ${escapeHtml(`${plural("Admin.Dashboard.PhotoCount", stats.photos)} · ${plural("Admin.Dashboard.CameraCount", stats.cameras)}`)}

${escapeHtml(formatBytes(stats.bytes))} ${escapeHtml(stats.firstYear ? `${stats.firstYear}–${stats.lastYear}` : "—")}
-

${escapeHtml(formatBytes(stats.freeBytes))} - ${bar(used)}${escapeHtml(t("Admin.Dashboard.Of", formatBytes(stats.capacityBytes)))}
+

${escapeHtml(formatBytes(stats.freeBytes))} + ${bar(used)}${escapeHtml(t("Admin.Dashboard.Of", formatBytes(stats.capacityBytes)))}${writeVolumeLine(stats)}

${formatNumber(stats.unverifiedSequences)}${escapeHtml(plural("Admin.Archive.UnverifiedUnit", stats.unverifiedSequences))} ${escapeHtml(plural("Admin.Dashboard.WithoutProxies", stats.sequencesWithoutProxies))}
diff --git a/src/Chronos.Server/Admin/web/pages/dashboard.js b/src/Chronos.Server/Admin/web/pages/dashboard.js index 259ce6b4..0bd4571d 100644 --- a/src/Chronos.Server/Admin/web/pages/dashboard.js +++ b/src/Chronos.Server/Admin/web/pages/dashboard.js @@ -1,6 +1,6 @@ // Dashboard (mockup Admin.html, BRIEF §14.8): storage, archive, jobs, render nodes, charts, activity, active jobs. import { t, plural, applyStrings, culture } from "../i18n.js"; -import { escapeHtml, attr, formatBytes, formatNumber, formatTime, formatDuration, bar, columns, toast, errorText, emptyState, redraw, fitColumns, PROXY_WARNING } from "../ui.js"; +import { escapeHtml, attr, formatBytes, formatNumber, formatTime, formatDuration, bar, storageTip, writeVolumeLine, columns, toast, errorText, emptyState, redraw, fitColumns, PROXY_WARNING } from "../ui.js"; import { kindLabel, nodeLabel, progressCell, stateCell, stateLabel, encoders } from "../jobs.js"; import { activityText } from "../audit.js"; @@ -30,8 +30,10 @@ export async function render(root, { api, setMeta, actions, onEvent }) { t(d.discovery ? "Admin.Dashboard.DiscoveryOn" : "Admin.Dashboard.DiscoveryOff"))); actions.innerHTML = d.proxyWarning ? PROXY_WARNING : ""; applyStrings(actions); + // The space volume's figures (the space path when the server has one), the write volume apart when it differs. const a = d.archive; const used = a.capacityBytes > 0 ? (a.capacityBytes - a.freeBytes) / a.capacityBytes : 0; + const spaceTip = attr(storageTip(a)); const online = d.nodes.filter((n) => n.state !== "offline"); const uploads = lastThirtyDays(d.uploads); const totalUploads = uploads.reduce((s, u) => s + u.value, 0); @@ -41,9 +43,10 @@ export async function render(root, { api, setMeta, actions, onEvent }) {

- ${escapeHtml(formatBytes(a.capacityBytes - a.freeBytes))}${escapeHtml(t("Admin.Dashboard.Of", formatBytes(a.capacityBytes)))} + ${escapeHtml(formatBytes(a.capacityBytes - a.freeBytes))}${escapeHtml(t("Admin.Dashboard.Of", formatBytes(a.capacityBytes)))} ${bar(used)} - ${escapeHtml(t("Admin.Dashboard.Free", formatBytes(a.freeBytes)))} · ${escapeHtml(t("Admin.Dashboard.Originals", formatBytes(a.bytes)))} + ${escapeHtml(t("Admin.Dashboard.Free", formatBytes(a.freeBytes)))} · ${escapeHtml(t("Admin.Dashboard.Originals", formatBytes(a.bytes)))} + ${writeVolumeLine(a)}

diff --git a/src/Chronos.Server/Admin/web/ui.js b/src/Chronos.Server/Admin/web/ui.js index bdad2168..081c320f 100644 --- a/src/Chronos.Server/Admin/web/ui.js +++ b/src/Chronos.Server/Admin/web/ui.js @@ -56,6 +56,23 @@ export function bar(fraction, muted = false) { ``; } +/** + * Tooltip of a storage tile's figures (archive stats): the volume they measure and its file system; on an Unraid share + * without a space volume, that they are the share's primary storage. + */ +export function storageTip(stats) { + const lines = [stats.usesSpacePath ? t("Admin.Dashboard.SpaceVolume.Tip") : t("Admin.Dashboard.Storage.Tip")]; + if (stats.fileSystem) lines.push(t("Admin.Dashboard.FileSystem", stats.fileSystem)); + if (stats.fileSystem === "fuse.shfs" && !stats.usesSpacePath) lines.push(t("Admin.Dashboard.UnraidShare.Tip")); + return lines.join("\n"); +} + +/** The write volume's free space as one more line of a storage tile, when the stats have it apart; "" otherwise. */ +export function writeVolumeLine(stats) { + if (stats.writeFreeBytes == null) return ""; + return `${escapeHtml(t("Admin.Dashboard.WriteVolume", formatBytes(stats.writeFreeBytes)))}`; +} + /** * Column chart named by label: values with labels under the bars and, when given, the value above each bar. Bars use * the accent colour, or the sun colour for upload activity. With edgeLabels the first and last labels align with the diff --git a/src/Chronos.Server/Api/AdminEndpoints.cs b/src/Chronos.Server/Api/AdminEndpoints.cs index 0c612b1c..3aa5e951 100644 --- a/src/Chronos.Server/Api/AdminEndpoints.cs +++ b/src/Chronos.Server/Api/AdminEndpoints.cs @@ -40,7 +40,7 @@ public static class AdminEndpoints } private static async Task DashboardAsync(HttpContext context, ChronosDatabase db, ServerArchive archive, ServerOptions options, ServerState state, - TimeProvider time) + TimeProvider time, VolumeSpace volumes) { var ct = context.RequestAborted; var now = time.GetUtcNow(); @@ -61,7 +61,7 @@ public static class AdminEndpoints options.TlsMode.ToString().ToLowerInvariant(), options.Discovery, users.Count, - await ArchiveEndpoints.StatsAsync(db, archive, ct), + await ArchiveEndpoints.StatsAsync(db, archive, volumes, ct), new JobCountsDto(counts.Queued, counts.Running, counts.Paused, counts.DoneSince, counts.FailedSince), options.Parallelism, options.PriorityQueue, diff --git a/src/Chronos.Server/Api/ArchiveEndpoints.cs b/src/Chronos.Server/Api/ArchiveEndpoints.cs index 30277ed2..1bfee04e 100644 --- a/src/Chronos.Server/Api/ArchiveEndpoints.cs +++ b/src/Chronos.Server/Api/ArchiveEndpoints.cs @@ -23,37 +23,31 @@ public static class ArchiveEndpoints archive.MapGet("/settings", async (HttpContext c, ServerArchive a, ChronosDatabase db, [FromServices] ArchiveService? archives) => Results.Json(Settings(archives is null ? a.Current : await a.RefreshAsync(archives, db, c.RequestAborted)), ProtocolJson.Default.ArchiveSettingsDto)); archive.MapPut("/settings", UpdateAsync); - archive.MapGet("/stats", async (HttpContext c, ChronosDatabase db, ServerArchive a) => - Results.Json(await StatsAsync(db, a, c.RequestAborted), ProtocolJson.Default.ArchiveStatsDto)); + archive.MapGet("/stats", async (HttpContext c, ChronosDatabase db, ServerArchive a, VolumeSpace volumes) => + Results.Json(await StatsAsync(db, a, volumes, c.RequestAborted), ProtocolJson.Default.ArchiveStatsDto)); foreach (var (path, kind) in new[] { ("scan", JobKind.Scan), ("rebuild", JobKind.Rebuild), ("verify", JobKind.Verify), ("maintenance", JobKind.Maintenance) }) { archive.MapPost("/" + path, (HttpContext c, Caller caller, ServerArchive a, JobEngineHost host, AuditLog audit) => EnqueueAsync(c, caller, a, host, audit, kind)); } } - /// Totals, distributions, free space and health of the archive (also shown on the dashboard). - public static async Task StatsAsync(ChronosDatabase db, ServerArchive archive, CancellationToken cancellationToken) + /// + /// Totals, distributions, free space and health of the archive (also shown on the dashboard). The free space is the + /// space volume's (), with the archive's own volume apart when they differ; an offline + /// volume reports no figures (0). + /// + public static async Task StatsAsync(ChronosDatabase db, ServerArchive archive, VolumeSpace volumes, CancellationToken cancellationToken) { var stats = await db.Sequences.GetStatsAsync(archive.Id, cancellationToken); var withoutProxies = await db.Sequences.CountAsync( new SequenceQuery { ArchiveId = archive.Id, WithoutStatus = SequenceStatus.ProxiesBuilt }, cancellationToken); - long free = 0, capacity = 0; - try - { - var drive = new DriveInfo(archive.Root); - free = drive.AvailableFreeSpace; - capacity = drive.TotalSize; - } - catch (Exception ex) when (ex is IOException or ArgumentException or UnauthorizedAccessException) - { - // An offline archive volume reports no space figures. - } + var storage = volumes.Report(archive.Root); return new ArchiveStatsDto( stats.Sequences, stats.Photos, stats.Bytes, - free, - capacity, + storage.Space?.FreeBytes ?? 0, + storage.Space?.TotalBytes ?? 0, stats.ByCamera.Count(c => c.Model is not null), stats.ByYear.Count > 0 ? stats.ByYear[0].Year : null, stats.ByYear.Count > 0 ? stats.ByYear[^1].Year : null, @@ -62,7 +56,11 @@ public static class ArchiveEndpoints [.. stats.ByCamera.Select(c => new KeyValueCount(string.Join(' ', new[] { c.Make, c.Model }.Where(v => !string.IsNullOrEmpty(v))), c.Photos, c.Bytes))], [.. stats.ByFormat.Select(f => new KeyValueCount(f.Format.ToString().ToLowerInvariant(), f.Photos, f.Bytes))], withoutProxies, - stats.Health.Unverified); + stats.Health.Unverified, + storage.Write?.FreeBytes, + storage.Write?.TotalBytes, + storage.FileSystem, + storage.FromSpacePath); } /// diff --git a/src/Chronos.Server/Api/DiagnosticsEndpoints.cs b/src/Chronos.Server/Api/DiagnosticsEndpoints.cs index 292d60f4..3d0e24a0 100644 --- a/src/Chronos.Server/Api/DiagnosticsEndpoints.cs +++ b/src/Chronos.Server/Api/DiagnosticsEndpoints.cs @@ -8,6 +8,7 @@ using Chronos.Data.Processing; using Chronos.Server.Configuration; using Chronos.Server.Jobs; using Chronos.Server.Security; +using Chronos.Server.Storage; namespace Chronos.Server.Api; @@ -25,11 +26,11 @@ public static class DiagnosticsEndpoints api.MapGet("/server/diagnostics", DownloadAsync).Requires(Permissions.ManageArchive); private static async Task DownloadAsync(HttpContext context, Caller caller, ChronosDatabase db, ServerOptions options, ServerState state, - AuditLog audit, TimeProvider time, IServiceProvider services) + AuditLog audit, TimeProvider time, VolumeSpace volumes, IServiceProvider services) { var ct = context.RequestAborted; var failures = (await db.Jobs.ListAsync(JobState.Failed, FailedJobs, null, ct)).Items; - var report = Report(options, state, time); + var report = Report(options, state, time, volumes); var settings = JsonSerializer.Serialize(options, ServerOptionsJson.Default.ServerOptions); var encoders = Encoders(services.GetService()); @@ -64,7 +65,7 @@ public static class DiagnosticsEndpoints return Results.File(file, "application/zip", name); } - private static string Report(ServerOptions options, ServerState state, TimeProvider time) + private static string Report(ServerOptions options, ServerState state, TimeProvider time, VolumeSpace volumes) { var sb = new StringBuilder(); sb.AppendLine(CultureInfo.InvariantCulture, $"Chronos Server {SystemEndpoints.Version} ({options.ServerName})"); @@ -76,8 +77,11 @@ public static class DiagnosticsEndpoints sb.AppendLine("SQLite compile options: " + string.Join(' ', Chronos.Data.Sqlite.SqliteDatabase.CompileOptions)); sb.AppendLine(CultureInfo.InvariantCulture, $"TLS {options.TlsMode} {state.TlsFingerprint}"); sb.AppendLine(CultureInfo.InvariantCulture, $"Jobs one at a time, parallelism {options.Parallelism}, priority queue {(options.PriorityQueue ? "on" : "off")}"); - sb.AppendLine(CultureInfo.InvariantCulture, $"Archive {options.ArchivePath} ({Space(options.ArchivePath)})"); - sb.AppendLine(CultureInfo.InvariantCulture, $"Data {options.DataPath} ({Space(options.DataPath)})"); + // The archive's volume is where uploads are written; the space path, when set, is what the admin panel shows. + sb.AppendLine(CultureInfo.InvariantCulture, $"Archive {options.ArchivePath} ({Space(volumes, options.ArchivePath)})"); + if (!string.IsNullOrWhiteSpace(options.SpacePath)) + sb.AppendLine(CultureInfo.InvariantCulture, $"Space {options.SpacePath} ({Space(volumes, options.SpacePath)})"); + sb.AppendLine(CultureInfo.InvariantCulture, $"Data {options.DataPath} ({Space(volumes, options.DataPath)})"); sb.AppendLine(CultureInfo.InvariantCulture, $"Container {File.Exists("/.dockerenv")}"); sb.AppendLine("Set by environment: " + string.Join(", ", options.Locked)); return sb.ToString(); @@ -98,19 +102,15 @@ public static class DiagnosticsEndpoints return sb.ToString(); } - private static string Space(string path) + /// "120 GB free of 500 GB, fuse.shfs mounted at /archive from shfs" (the mount on Linux only; "…, NTFS" elsewhere). + private static string Space(VolumeSpace volumes, string path) { - try - { - if (!Directory.Exists(path)) return "missing"; - var full = Path.GetFullPath(path); - // Windows wants a drive root; Unix accepts any directory and reports the volume it is mounted on. - var drive = new DriveInfo(OperatingSystem.IsWindows() ? Path.GetPathRoot(full)! : full); - return string.Create(CultureInfo.InvariantCulture, $"{drive.AvailableFreeSpace / (1L << 30)} GB free of {drive.TotalSize / (1L << 30)} GB"); - } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException) - { - return "free space unknown"; - } + if (!Directory.Exists(path)) return "missing"; + var figures = volumes.Measure(path) is { } v + ? string.Create(CultureInfo.InvariantCulture, $"{v.FreeBytes / (1L << 30)} GB free of {v.TotalBytes / (1L << 30)} GB") + : "free space unknown"; + var mount = VolumeSpace.MountOf(path); + var fileSystem = mount?.FileSystem ?? VolumeSpace.FileSystemOf(path) ?? "file system unknown"; + return mount is null ? $"{figures}, {fileSystem}" : $"{figures}, {fileSystem} mounted at {mount.MountPoint} from {mount.Source}"; } } diff --git a/src/Chronos.Server/Configuration/ServerOptions.cs b/src/Chronos.Server/Configuration/ServerOptions.cs index be950aa2..63ecd80b 100644 --- a/src/Chronos.Server/Configuration/ServerOptions.cs +++ b/src/Chronos.Server/Configuration/ServerOptions.cs @@ -36,6 +36,7 @@ public sealed class ServerOptions [ ("CHRONOS_SERVER_NAME", nameof(ServerName), (o, v) => o.ServerName = v), ("CHRONOS_ARCHIVE_PATH", nameof(ArchivePath), (o, v) => o.ArchivePath = v), + ("CHRONOS_SPACE_PATH", nameof(SpacePath), (o, v) => o.SpacePath = v), ("CHRONOS_ADMIN_USER", nameof(AdminUser), (o, v) => o.AdminUser = v), ("CHRONOS_ADMIN_PASSWORD", nameof(AdminPassword), (o, v) => o.AdminPassword = v), ("CHRONOS_DB", nameof(Database), (o, v) => o.ApplyDatabase(v)), @@ -80,6 +81,16 @@ public sealed class ServerOptions public string ServerName { get; set; } = "Chronos Server"; public string ArchivePath { get; set; } = OperatingSystem.IsWindows() ? @"C:\ChronosArchive" : "/archive"; + + /// + /// A folder whose volume's free space the admin panel shows instead of the archive's (owner review 2, R2; key + /// SpacePath, variable CHRONOS_SPACE_PATH). On Unraid the archive is a user share, which reports its + /// primary storage (often the cache) while the mover sends the photos to the array: the template maps the share's + /// array view (/mnt/user0/<share>) read-only to /space. Empty: the archive's volume; a folder that + /// cannot be measured falls back to it with one warning. Uploads always check the archive's own volume. + /// + public string? SpacePath { get; set; } + public string DataPath { get; set; } = OperatingSystem.IsWindows() ? Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), "Chronos Server") : "/data"; public string? AdminUser { get; set; } public string? AdminPassword { get; set; } diff --git a/src/Chronos.Server/Hosting/ServerCommandLine.cs b/src/Chronos.Server/Hosting/ServerCommandLine.cs index 40cc5c1c..a350c88d 100644 --- a/src/Chronos.Server/Hosting/ServerCommandLine.cs +++ b/src/Chronos.Server/Hosting/ServerCommandLine.cs @@ -37,6 +37,8 @@ public sealed record ServerCommandLine(ServerCommand Command, IReadOnlyList) mapped read-only to /space CHRONOS_PARALLELISM photos the running job processes at once (jobs run one at a time; default: half the cores); CHRONOS_WORKERS, its former name, still sets it CHRONOS_PRIORITY_QUEUE a job of a higher level pauses the running one, which resumes afterwards (true | false) diff --git a/src/Chronos.Server/Hosting/ServerHost.cs b/src/Chronos.Server/Hosting/ServerHost.cs index e600f17b..d4c11146 100644 --- a/src/Chronos.Server/Hosting/ServerHost.cs +++ b/src/Chronos.Server/Hosting/ServerHost.cs @@ -88,6 +88,8 @@ public static class ServerHost + "they could put a link in place of a folder and redirect the files the server moves. Let only those accounts change it (others may read it).", options.ArchivePath); } + // A space path that cannot be measured is warned about at start, not first when someone opens the dashboard. + if (!string.IsNullOrWhiteSpace(options.SpacePath)) app.Services.GetRequiredService().Report(archive.Root); if (options.FormerWorkersVariable) { app.Logger.LogWarning("CHRONOS_WORKERS is the former name of CHRONOS_PARALLELISM: jobs now run one at a time, and the value is how many " @@ -112,6 +114,8 @@ public static class ServerHost services.AddSingleton(database); services.AddSingleton(archive); services.TryAddSingleton(TimeProvider.System); + // Tests register their own first, to simulate a full volume. + services.TryAddSingleton(); services.ConfigureHttpJsonOptions(j => j.SerializerOptions.TypeInfoResolverChain.Insert(0, ProtocolJson.Default)); services.Configure(o => o.ThrowOnBadRequest = true); services.Configure(o => o.ShutdownTimeout = TimeSpan.FromSeconds(30)); diff --git a/src/Chronos.Server/Jobs/RenderNodeService.cs b/src/Chronos.Server/Jobs/RenderNodeService.cs index 5bb0d4cc..3e0e7e90 100644 --- a/src/Chronos.Server/Jobs/RenderNodeService.cs +++ b/src/Chronos.Server/Jobs/RenderNodeService.cs @@ -30,6 +30,7 @@ public sealed class RenderNodeService( RenderStore renders, JobEngineHost jobs, EventHub events, + VolumeSpace volumes, TimeProvider time) { public const string Idle = "idle"; @@ -135,7 +136,8 @@ public sealed class RenderNodeService( if (await db.Uploads.FindByJobAsync(jobId, cancellationToken) is { State: UploadState.Receiving } open && open.Sha256 == sha && open.Size == request.Size && store.Exists(open.Id)) return Started(open); - if (UploadRules.FreeBytes(options.UploadsPath) is { } free && free < request.Size) return Outcome.Fail(507, "insufficient_storage"); + if (volumes.Measure(options.UploadsPath) is { } uploadVolume && uploadVolume.FreeBytes < request.Size) + return Outcome.Fail(507, "insufficient_storage"); var now = time.GetUtcNow(); var upload = await db.Uploads.CreateAsync(new Upload @@ -166,6 +168,7 @@ public sealed class RenderNodeService( if (upload is not { State: UploadState.Receiving } || upload.RenderJson is null) return Outcome.Fail(409, "upload_state"); if (!upload.IsComplete) return Outcome.Fail(409, "incomplete", (upload.ChunkCount - upload.ReceivedCount).ToString(CultureInfo.InvariantCulture)); + if (!uploads.ArchiveHasRoom(upload)) return Outcome.Fail(507, "insufficient_storage"); if (!await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Receiving, UploadState.Committing, cancellationToken: cancellationToken)) return Outcome.Fail(409, "upload_state"); diff --git a/src/Chronos.Server/Storage/RenderUploads.cs b/src/Chronos.Server/Storage/RenderUploads.cs index 0725e642..02e2214d 100644 --- a/src/Chronos.Server/Storage/RenderUploads.cs +++ b/src/Chronos.Server/Storage/RenderUploads.cs @@ -23,6 +23,7 @@ public sealed class RenderUploads( UploadStore store, RenderStore renders, EventHub events, + VolumeSpace volumes, TimeProvider time) { public async Task> StartAsync(Caller caller, Guid sequenceId, RenderUploadRequest request, CancellationToken cancellationToken) @@ -46,7 +47,8 @@ public sealed class RenderUploads( if (await db.Uploads.FindReceivingAsync(caller.Id, sha, cancellationToken) is { } open && open.SequenceId == sequenceId && open.Size == request.Size && Read(open)?.RenderId == request.RenderId && store.Exists(open.Id)) return Started(open); - if (UploadRules.FreeBytes(options.UploadsPath) is { } free && free < request.Size) return Outcome.Fail(507, "insufficient_storage"); + if (volumes.Measure(options.UploadsPath) is { } uploadVolume && uploadVolume.FreeBytes < request.Size) + return Outcome.Fail(507, "insufficient_storage"); var now = time.GetUtcNow(); var upload = await db.Uploads.CreateAsync(new Upload @@ -77,6 +79,7 @@ public sealed class RenderUploads( if (upload.State != UploadState.Receiving) return Outcome.Fail(409, "upload_state"); if (!upload.IsComplete) return Outcome.Fail(409, "incomplete", (upload.ChunkCount - upload.ReceivedCount).ToString(CultureInfo.InvariantCulture)); + if (!uploads.ArchiveHasRoom(upload)) return Outcome.Fail(507, "insufficient_storage"); if (!await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Receiving, UploadState.Committing, cancellationToken: cancellationToken)) return Outcome.Fail(409, "upload_state"); diff --git a/src/Chronos.Server/Storage/UploadRules.cs b/src/Chronos.Server/Storage/UploadRules.cs index 5a876e67..716faf8f 100644 --- a/src/Chronos.Server/Storage/UploadRules.cs +++ b/src/Chronos.Server/Storage/UploadRules.cs @@ -66,17 +66,4 @@ public static class UploadRules /// The session key an upload without a sequence is grouped by: the client's, or one per user. public static string SessionKeyOf(string? sessionKey, Guid userId) => sessionKey ?? "user-" + userId.ToString("N"); - - /// Free bytes on the volume holding , or null when the platform cannot tell. - public static long? FreeBytes(string path) - { - try - { - return new DriveInfo(Path.GetFullPath(path)).AvailableFreeSpace; - } - catch (Exception ex) when (ex is IOException or ArgumentException or UnauthorizedAccessException) - { - return null; - } - } } diff --git a/src/Chronos.Server/Storage/UploadService.cs b/src/Chronos.Server/Storage/UploadService.cs index 22adb2f4..20e880a1 100644 --- a/src/Chronos.Server/Storage/UploadService.cs +++ b/src/Chronos.Server/Storage/UploadService.cs @@ -17,9 +17,10 @@ namespace Chronos.Server.Storage; /// /// Chunked, resumable, hash-verified frame uploads (BRIEF §14.4): start (short-circuit on a known SHA-256 held by an -/// intact frame, resume an unfinished upload, quota and disk checks), idempotent chunks, and completion that stages the -/// file on the archive volume, verifies its hash and hands it to , which repairs a missing or -/// damaged frame of that content (no quota is charged for a repair: the archive already counts those bytes). +/// intact frame, resume an unfinished upload, quota and disk checks), idempotent chunks, and completion that checks the +/// archive volume's free space, stages the file there, verifies its hash and hands it to , +/// which repairs a missing or damaged frame of that content (no quota is charged for a repair: the archive already +/// counts those bytes). /// public sealed class UploadService( ChronosDatabase db, @@ -29,11 +30,12 @@ public sealed class UploadService( IArchiveIngest ingest, SequenceJobs sequenceJobs, EventHub events, + VolumeSpace volumes, IHostApplicationLifetime lifetime, TimeProvider time, ILogger logger) { - /// Space kept free on the upload volume beyond the announced size. + /// Space kept free on the upload and archive volumes beyond an upload's size. private const long DiskReserve = 64L * 1024 * 1024; /// Card events one account may publish per minute: progress once a second leaves room for every start and end. @@ -74,7 +76,7 @@ public sealed class UploadService( if (caller.User.UsedBytes + pending + request.Size > caller.User.QuotaBytes) return Fail(403, "quota_exceeded", caller.User.QuotaBytes.ToString(CultureInfo.InvariantCulture)); } - if (UploadRules.FreeBytes(options.UploadsPath) is { } free && free < request.Size + DiskReserve) + if (volumes.Measure(options.UploadsPath) is { } uploadVolume && uploadVolume.FreeBytes < request.Size + DiskReserve) return Fail(507, "insufficient_storage"); var now = time.GetUtcNow(); @@ -165,6 +167,7 @@ public sealed class UploadService( if (upload.State != UploadState.Receiving) return Fail(409, "upload_state"); if (!upload.IsComplete) return Fail(409, "incomplete", (upload.ChunkCount - upload.ReceivedCount).ToString(CultureInfo.InvariantCulture)); + if (!ArchiveHasRoom(upload)) return Fail(507, "insufficient_storage"); if (!await db.Uploads.TryTransitionAsync(id, UploadState.Receiving, UploadState.Committing, cancellationToken: stopping)) return Fail(409, "upload_state"); @@ -240,9 +243,24 @@ public sealed class UploadService( return true; } + /// + /// The archive volume has room for and the reserve: completing it copies the file into the + /// staging folder there (). Checked before the upload leaves the receiving state, so a + /// refused completion (507) can be repeated once there is room. A volume that cannot be measured refuses nothing. + /// + public bool ArchiveHasRoom(Upload upload) + { + ArgumentNullException.ThrowIfNull(upload); + if (volumes.Measure(archive.Root) is not { } volume || volume.FreeBytes >= upload.Size + DiskReserve) return true; + logger.LogWarning("The archive volume {Archive} has {Free} bytes free: upload {Upload} ({File}, {Size} bytes) waits for room", + archive.Root, volume.FreeBytes, upload.Id, upload.FileName, upload.Size); + return false; + } + /// /// Copies a fully received upload to the archive staging folder and checks its SHA-256. On a mismatch the upload - /// fails, its files are deleted and null is returned; the client starts over. + /// fails, its files are deleted and null is returned; the client starts over. Callers check + /// first. /// public async Task StageVerifiedAsync(Upload upload, CancellationToken cancellationToken) { diff --git a/src/Chronos.Server/Storage/VolumeSpace.cs b/src/Chronos.Server/Storage/VolumeSpace.cs new file mode 100644 index 00000000..b705c524 --- /dev/null +++ b/src/Chronos.Server/Storage/VolumeSpace.cs @@ -0,0 +1,181 @@ +using System.Text; +using Chronos.Server.Configuration; + +namespace Chronos.Server.Storage; + +/// Free and total bytes of a volume. +public sealed record VolumeFigures(long FreeBytes, long TotalBytes); + +/// One mount of /proc/self/mountinfo: its mount point, filesystem type (fuse.shfs) and source. +public sealed record MountEntry(string MountPoint, string FileSystem, string Source); + +/// +/// The archive's storage as the admin panel shows it: the space volume's figures ( +/// when set and measurable, else the archive root) and filesystem, plus the archive root's own figures (the write +/// volume) when a space path is in use and they differ. +/// +public sealed record StorageReport(VolumeFigures? Space, string? FileSystem, bool FromSpacePath, VolumeFigures? Write); + +/// +/// Free space of the server's folders (owner review 2, R2). The archive root is where uploads are staged and filed; on +/// Unraid it is a user share (FUSE shfs) that reports the share's primary storage, often the cache, while the +/// mover sends the photos to the array. names a folder whose volume the admin panel +/// shows instead (the share seen on the array only). Tests replace to simulate a full volume. +/// +public class VolumeSpace(ServerOptions options, ILogger logger) +{ + /// Two reads of one volume a moment apart differ by what was written in between. + private const long SameVolumeSlack = 64L * 1024 * 1024; + + private const string MountInfo = "/proc/self/mountinfo"; + + /// 1 while the space path cannot be measured: it is warned about once, not at every request. + private int _spaceUnavailable; + + /// Free and total bytes of the volume holding ; null when the folder is missing or the platform cannot tell. + public virtual VolumeFigures? Measure(string path) + { + try + { + if (!Directory.Exists(path)) return null; + var drive = Drive(path); + return new VolumeFigures(drive.AvailableFreeSpace, drive.TotalSize); + } + catch (Exception ex) when (ex is IOException or ArgumentException or UnauthorizedAccessException) + { + return null; + } + } + + /// + /// The storage figures of the archive at : of the space path when one is set and can be + /// measured, else of the archive root (a space path that cannot be measured is logged once, until it can again). + /// + public StorageReport Report(string archiveRoot) + { + var write = Measure(archiveRoot); + var spacePath = options.SpacePath; + if (string.IsNullOrWhiteSpace(spacePath)) return new StorageReport(write, FileSystemOf(archiveRoot), FromSpacePath: false, Write: null); + if (Measure(spacePath) is not { } space) + { + if (Interlocked.Exchange(ref _spaceUnavailable, 1) == 0) + { + logger.LogWarning("The space path {SpacePath} (CHRONOS_SPACE_PATH) is missing or cannot be measured: the admin panel shows the free space " + + "of the archive {Archive} instead. Map the folder into the container, or clear the setting.", spacePath, archiveRoot); + } + return new StorageReport(write, FileSystemOf(archiveRoot), FromSpacePath: false, Write: null); + } + if (Interlocked.Exchange(ref _spaceUnavailable, 0) == 1) logger.LogInformation("The space path {SpacePath} can be measured again", spacePath); + var differs = write is not null && (write.TotalBytes != space.TotalBytes || Math.Abs(write.FreeBytes - space.FreeBytes) > SameVolumeSlack); + return new StorageReport(space, FileSystemOf(spacePath), FromSpacePath: true, differs ? write : null); + } + + /// + /// The type of the filesystem holding the folder : on Linux the mount's type (an Unraid user + /// share is fuse.shfs, which the kernel's own figures call only FUSE), elsewhere what the system reports + /// (NTFS). Null when the folder is missing or the type unknown. + /// + public static string? FileSystemOf(string path) + { + if (!Directory.Exists(path)) return null; + if (MountOf(path) is { } mount) return mount.FileSystem; + try + { + return Drive(path).DriveFormat; + } + catch (Exception ex) when (ex is IOException or ArgumentException or UnauthorizedAccessException) + { + return null; + } + } + + /// + /// The mount holding the folder , read from /proc/self/mountinfo; null off Linux, when + /// the folder is missing or the file cannot be read. + /// + public static MountEntry? MountOf(string path) + { + if (!OperatingSystem.IsLinux() || !Directory.Exists(path)) return null; + try + { + return FindMount(File.ReadAllText(MountInfo), Path.GetFullPath(path)); + } + catch (Exception ex) when (ex is IOException or ArgumentException or UnauthorizedAccessException) + { + return null; + } + } + + /// + /// The mount of (the text of /proc/self/mountinfo) that holds the absolute + /// : the longest mount point that is the path or one of its parent folders and, of mounts + /// stacked on that point, the last listed (the one on top). Null when no line covers it. + /// + public static MountEntry? FindMount(string mountInfo, string path) + { + ArgumentNullException.ThrowIfNull(mountInfo); + ArgumentNullException.ThrowIfNull(path); + var target = path.TrimEnd('/'); + if (target.Length == 0) target = "/"; + MountEntry? best = null; + foreach (var line in mountInfo.Split('\n')) + { + if (ParseMountLine(line) is not { } entry || !Covers(entry.MountPoint, target)) continue; + if (best is null || entry.MountPoint.Length >= best.MountPoint.Length) best = entry; + } + return best; + } + + /// + /// One mountinfo line (proc(5)): id parent major:minor root mount-point options [optional fields…] - type source + /// super-options, where fields write a space, tab, newline or backslash as an octal escape (\040). Null when + /// the line has another shape. + /// + public static MountEntry? ParseMountLine(string line) + { + ArgumentNullException.ThrowIfNull(line); + var fields = line.TrimEnd('\r').Split(' '); + if (fields.Length < 9) return null; + // The optional fields (none or more, "shared:5 master:1") end at a lone "-". + var separator = Array.IndexOf(fields, "-", 6); + if (separator < 0 || separator + 2 >= fields.Length) return null; + var mountPoint = Unescape(fields[4]); + if (!mountPoint.StartsWith('/')) return null; + return new MountEntry(mountPoint, Unescape(fields[separator + 1]), Unescape(fields[separator + 2])); + } + + /// Windows wants a drive root; Unix accepts any folder and reports the filesystem it is on. + private static DriveInfo Drive(string path) + { + var full = Path.GetFullPath(path); + return new DriveInfo(OperatingSystem.IsWindows() ? Path.GetPathRoot(full)! : full); + } + + /// is or one of its parent folders. + private static bool Covers(string mountPoint, string path) => + mountPoint == "/" + ? path.StartsWith('/') + : path.StartsWith(mountPoint, StringComparison.Ordinal) && (path.Length == mountPoint.Length || path[mountPoint.Length] == '/'); + + /// Decodes the kernel's \ooo escapes (it escapes only ASCII characters). + private static string Unescape(string field) + { + if (!field.Contains('\\', StringComparison.Ordinal)) return field; + var sb = new StringBuilder(field.Length); + for (var i = 0; i < field.Length; i++) + { + if (field[i] == '\\' && i + 3 < field.Length && IsOctal(field[i + 1]) && IsOctal(field[i + 2]) && IsOctal(field[i + 3])) + { + sb.Append((char)(((field[i + 1] - '0') * 64) + ((field[i + 2] - '0') * 8) + (field[i + 3] - '0'))); + i += 3; + } + else + { + sb.Append(field[i]); + } + } + return sb.ToString(); + } + + private static bool IsOctal(char c) => c is >= '0' and <= '7'; +} diff --git a/tests/Chronos.Server.Tests/Fakes.cs b/tests/Chronos.Server.Tests/Fakes.cs index d235fdf8..2d1a0adc 100644 --- a/tests/Chronos.Server.Tests/Fakes.cs +++ b/tests/Chronos.Server.Tests/Fakes.cs @@ -1,8 +1,10 @@ using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; using Chronos.Core.Model; using Chronos.Data; using Chronos.Data.Archiving; using Chronos.Data.Jobs; +using Chronos.Server.Configuration; using Chronos.Server.Storage; namespace Chronos.Server.Tests; @@ -99,6 +101,23 @@ public sealed class FakeAssetLocator : IFrameAssetLocator } } +/// Measures volumes for real, except the folders given figures of their own (a full archive, a separate array). +public sealed class FakeVolumeSpace(ServerOptions options, ILogger logger) : VolumeSpace(options, logger) +{ + private readonly ConcurrentDictionary _figures = new(StringComparer.Ordinal); + + /// Gives these figures; null measures it for real again. + public void Set(string path, VolumeFigures? figures) + { + if (figures is null) _figures.TryRemove(Key(path), out _); + else _figures[Key(path)] = figures; + } + + public override VolumeFigures? Measure(string path) => _figures.TryGetValue(Key(path), out var figures) ? figures : base.Measure(path); + + private static string Key(string path) => Path.TrimEndingDirectorySeparator(Path.GetFullPath(path)); +} + /// A clock tests move by hand (token expiry, heartbeat timeouts). public sealed class ManualTimeProvider : TimeProvider { diff --git a/tests/Chronos.Server.Tests/Storage/VolumeSpaceTests.cs b/tests/Chronos.Server.Tests/Storage/VolumeSpaceTests.cs new file mode 100644 index 00000000..60ee8f38 --- /dev/null +++ b/tests/Chronos.Server.Tests/Storage/VolumeSpaceTests.cs @@ -0,0 +1,289 @@ +using System.IO.Compression; +using System.Net; +using System.Net.Http.Json; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Chronos.Protocol; +using Chronos.Server.Configuration; +using Chronos.Server.Storage; +using Chronos.Testing; + +namespace Chronos.Server.Tests.Storage; + +/// Owner review 2 (R2): the free space of the Unraid array instead of the cache, and of the archive before an upload is filed. +[Category("server")] +public sealed class VolumeSpaceTests(TestContext context) +{ + private const long GB = 1_000_000_000; + + /// + /// /proc/self/mountinfo of a Chronos container on Unraid: the overlay root, the share bind-mounted at /archive (FUSE + /// shfs, whose root is the share's folder), its array view at /space, appdata at /data, a disk mounted inside the + /// archive under a name with a space, and two mounts stacked on /tmp. + /// + private const string Unraid = """ + 574 520 0:52 / / rw,relatime master:255 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/ABC:/var/lib/docker/overlay2/l/DEF,upperdir=/var/lib/docker/overlay2/1f/diff + 575 574 0:55 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw + 590 574 0:45 /Timelapse /archive rw,noatime - fuse.shfs shfs rw,user_id=0,group_id=0,default_permissions,allow_other + 591 574 0:46 /Timelapse /space ro,noatime - fuse.shfs shfs rw,user_id=0,group_id=0,default_permissions,allow_other + 592 574 0:31 /appdata/chronos /data rw,noatime shared:5 master:3 - btrfs /dev/nvme0n1p1 rw,ssd,space_cache=v2,subvolid=5,subvol=/ + 593 590 9:1 / /archive/Big\040Trip rw,noatime - xfs /dev/md1p1 rw,attr2,inode64 + 594 574 0:60 / /tmp rw - tmpfs tmpfs rw + 595 574 0:61 / /tmp rw - tmpfs scratch rw,size=1g + this line is not a mount + """; + + [Theory] + [Case("/archive", "/archive", "fuse.shfs")] + [Case("/archive/2026/Roof/IMG_0001.CR3", "/archive", "fuse.shfs")] + [Case("/archive/", "/archive", "fuse.shfs")] + [Case("/space", "/space", "fuse.shfs")] + [Case("/data/uploads", "/data", "btrfs")] + [Case("/archive/Big Trip/IMG_0002.CR3", "/archive/Big Trip", "xfs")] + [Case("/archive/Big", "/archive", "fuse.shfs")] + [Case("/archived", "/", "overlay")] + [Case("/", "/", "overlay")] + [Case("/app/Chronos.Server.dll", "/", "overlay")] + public void TheLongestMountPointHoldsAPath(string path, string mountPoint, string fileSystem) + { + var mount = Assert.NotNull(VolumeSpace.FindMount(Unraid, path)); + Assert.Equal((mountPoint, fileSystem), (mount.MountPoint, mount.FileSystem)); + } + + [Test] + public void MountInfoLinesAreReadField() + { + var share = Assert.NotNull(VolumeSpace.FindMount(Unraid, "/archive")); + Assert.Equal("shfs", share.Source); + // Of mounts stacked on one point, the last listed is on top. + Assert.Equal("scratch", Assert.NotNull(VolumeSpace.FindMount(Unraid, "/tmp/x")).Source); + // Optional fields, a Windows line end and escapes in any field. + var data = Assert.NotNull(VolumeSpace.ParseMountLine("592 574 0:31 / /mnt/My\\040Disk rw shared:5 master:3 propagate_from:2 - ext4 /dev/sd\\134b rw\r")); + Assert.Equal(("/mnt/My Disk", "ext4", "/dev/sd\\b"), (data.MountPoint, data.FileSystem, data.Source)); + Assert.Null(VolumeSpace.ParseMountLine("592 574 0:31 / /data rw shared:5 btrfs /dev/sdb1 rw")); + Assert.Null(VolumeSpace.ParseMountLine("")); + Assert.Null(VolumeSpace.FindMount("", "/archive")); + Assert.Null(VolumeSpace.FindMount("590 574 0:45 /Timelapse /archive rw - fuse.shfs shfs rw", "/data")); + } + + [Test] + public void TheFileSystemOfAFolderIsKnown() + { + var folder = context.CreateTempDirectory("fs"); + var fileSystem = Assert.NotNull(VolumeSpace.FileSystemOf(folder)); + Assert.NotEmpty(fileSystem); + if (OperatingSystem.IsLinux()) Assert.Equal(fileSystem, Assert.NotNull(VolumeSpace.MountOf(folder)).FileSystem); + else Assert.Null(VolumeSpace.MountOf(folder)); + Assert.Null(VolumeSpace.FileSystemOf(Path.Combine(folder, "missing"))); + } + + [Test] + public void TheSpacePathIsShownWithTheWriteVolumeApartWhenTheyDiffer() + { + var root = context.CreateTempDirectory("report"); + var archive = Directory.CreateDirectory(Path.Combine(root, "archive")).FullName; + var space = Directory.CreateDirectory(Path.Combine(root, "space")).FullName; + var options = new ServerOptions { ArchivePath = archive }; + var logger = new CountingLogger(); + var volumes = new FakeVolumeSpace(options, logger); + volumes.Set(archive, new VolumeFigures(120 * GB, 500 * GB)); + + // No space path: the archive's own volume. + var plain = volumes.Report(archive); + Assert.Equal((120 * GB, false, (VolumeFigures?)null), (plain.Space!.FreeBytes, plain.FromSpacePath, plain.Write)); + + // The cache and the array: the array is shown, the cache apart. + options.SpacePath = space; + volumes.Set(space, new VolumeFigures(9_000 * GB, 16_000 * GB)); + var separate = volumes.Report(archive); + Assert.Equal((9_000 * GB, true), (separate.Space!.FreeBytes, separate.FromSpacePath)); + Assert.Equal(new VolumeFigures(120 * GB, 500 * GB), separate.Write); + Assert.Equal(VolumeSpace.FileSystemOf(space), separate.FileSystem); + + // One volume read twice a moment apart: no second line. + volumes.Set(space, new VolumeFigures((120 * GB) - (8 << 20), 500 * GB)); + Assert.Null(volumes.Report(archive).Write); + Assert.Equal(0, logger.Warnings); + } + + [Test] + public void ASpacePathThatCannotBeMeasuredFallsBackWithOneWarning() + { + var root = context.CreateTempDirectory("fallback"); + var archive = Directory.CreateDirectory(Path.Combine(root, "archive")).FullName; + var space = Path.Combine(root, "not-mapped"); + var logger = new CountingLogger(); + var volumes = new VolumeSpace(new ServerOptions { ArchivePath = archive, SpacePath = space }, logger); + + for (var i = 0; i < 3; i++) + { + var report = volumes.Report(archive); + Assert.False(report.FromSpacePath); + Assert.True(Assert.NotNull(report.Space).TotalBytes > 0); + Assert.Null(report.Write); + } + Assert.Equal(1, logger.Warnings); + + // Mapped later: measured again, and a later outage is warned about once more. + Directory.CreateDirectory(space); + Assert.True(volumes.Report(archive).FromSpacePath); + Directory.Delete(space); + volumes.Report(archive); + volumes.Report(archive); + Assert.Equal(2, logger.Warnings); + } + + [Test] + public async Task TheStatsReportTheSpacePathAndTheWriteVolume() + { + var space = context.CreateTempDirectory("space"); + await using var server = await TestServer.StartAsync(context, o => o.SpacePath = space, s => + { + s.AddSingleton(); + s.AddSingleton(sp => sp.GetRequiredService()); + }); + var admin = await server.AdminAsync(); + var stats = await admin.GetFromJsonAsync("/api/v1/archive/stats", ProtocolJson.Default.ArchiveStatsDto); + Assert.True(Assert.NotNull(stats).UsesSpacePath); + Assert.True(stats.CapacityBytes > 0 && stats.FreeBytes <= stats.CapacityBytes); + Assert.Equal(VolumeSpace.FileSystemOf(space), stats.FileSystem); + // Temporary folders share one volume: no write volume apart. + Assert.Null(stats.WriteFreeBytes); + + // The archive's own volume (an Unraid share's cache) differs: the panel gets it apart. + server.Services.GetRequiredService().Set(server.Options.ArchivePath, new VolumeFigures(40 * GB, 250 * GB)); + using var dashboard = await admin.GetAsync("/api/v1/server/dashboard"); + var archive = (await dashboard.ReadAsync(ProtocolJson.Default.DashboardDto)).Archive; + Assert.Equal((40 * GB, 250 * GB, true), (archive.WriteFreeBytes, archive.WriteCapacityBytes, archive.UsesSpacePath)); + Assert.NotEqual(40 * GB, archive.FreeBytes); + } + + [Test] + public async Task AMissingSpacePathShowsTheArchiveVolume() + { + var root = context.CreateTempDirectory("unmapped"); + await using var server = await TestServer.StartAsync(context, o => o.SpacePath = Path.Combine(root, "space")); + var admin = await server.AdminAsync(); + for (var i = 0; i < 2; i++) + { + var stats = Assert.NotNull(await admin.GetFromJsonAsync("/api/v1/archive/stats", ProtocolJson.Default.ArchiveStatsDto)); + Assert.False(stats.UsesSpacePath); + Assert.True(stats.CapacityBytes > 0); + Assert.Null(stats.WriteFreeBytes); + Assert.Equal(VolumeSpace.FileSystemOf(server.Options.ArchivePath), stats.FileSystem); + } + // Warned at start, not at every request. + var warnings = Directory.EnumerateFiles(server.Options.LogPath).Sum(log => + { + using var reader = new StreamReader(new FileStream(log, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)); + return reader.ReadToEnd().Split('\n').Count(line => line.Contains("CHRONOS_SPACE_PATH", StringComparison.Ordinal)); + }); + Assert.Equal(1, warnings); + } + + [Test] + public async Task TheDiagnosticsReportBothVolumes() + { + var space = context.CreateTempDirectory("space"); + await using var server = await TestServer.StartAsync(context, o => o.SpacePath = space); + var admin = await server.AdminAsync(); + using var response = await admin.GetAsync("/api/v1/server/diagnostics"); + await response.ExpectAsync(HttpStatusCode.OK); + await using var body = await response.Content.ReadAsStreamAsync(); + using var zip = new ZipArchive(body, ZipArchiveMode.Read); + using var reader = new StreamReader(Assert.NotNull(zip.GetEntry("environment.txt")).Open()); + var report = await reader.ReadToEndAsync(); + var fileSystem = Assert.NotNull(VolumeSpace.FileSystemOf(space)); + Assert.Contains($"Archive {server.Options.ArchivePath} (", report); + Assert.Contains($"Space {space} (", report); + Assert.Contains($" GB, {fileSystem}", report); + } + + [Test] + public async Task AFullArchiveVolumeRefusesTheCompletionUntilThereIsRoom() + { + const int chunk = 64 * 1024; + await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = chunk, s => + { + s.AddSingleton(); + s.AddSingleton(sp => sp.GetRequiredService()); + }); + var volumes = server.Services.GetRequiredService(); + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + var data = Uploads.Photo(chunk + 500, 71); + var start = await Uploads.StartAsync(editor, data, sessionKey: "roof"); + var id = Assert.NotNull(start.UploadId); + for (var i = 0; i < 2; i++) + { + using var put = await Uploads.PutChunkAsync(editor, id, i, Uploads.Chunk(data, chunk, i)); + await put.ExpectAsync(HttpStatusCode.OK); + } + + // The upload volume (/data) has room, the archive's does not: nothing is staged, the upload keeps its chunks. + volumes.Set(server.Options.ArchivePath, new VolumeFigures(32L << 20, 500 * GB)); + using (var full = await editor.PostAsync($"/api/v1/uploads/{id}/complete", null)) + await full.ExpectErrorAsync(HttpStatusCode.InsufficientStorage, "insufficient_storage"); + using (var state = await editor.GetAsync($"/api/v1/uploads/{id}")) + Assert.Equal("receiving", (await state.ReadAsync(ProtocolJson.Default.UploadState)).State); + Assert.True(server.Services.GetRequiredService().Exists(id)); + Assert.False(Directory.Exists(server.Options.StagingPath) && Directory.EnumerateFiles(server.Options.StagingPath).Any()); + + volumes.Set(server.Options.ArchivePath, new VolumeFigures(GB, 500 * GB)); + Guid sequence; + using (var room = await editor.PostAsync($"/api/v1/uploads/{id}/complete", null)) + { + await room.ExpectAsync(HttpStatusCode.OK); + sequence = (await room.ReadAsync(ProtocolJson.Default.CompleteUploadResponse)).SequenceId; + } + + // A render sent into the archive is checked the same way. + var output = Uploads.Photo(3000, 72); + var request = new RenderUploadRequest(Guid.CreateVersion7(), "Roof.mp4", output.Length, Uploads.Sha(output), "4K HEVC", "hevc", 3840, 2160, 30, 10, null); + using var started = await editor.PostAsJsonAsync($"/api/v1/sequences/{sequence}/renders", request, ProtocolJson.Default.RenderUploadRequest); + await started.ExpectAsync(HttpStatusCode.Created); + var render = Assert.NotNull((await started.ReadAsync(ProtocolJson.Default.StartUploadResponse)).UploadId); + using (var put = await Uploads.PutChunkAsync(editor, render, 0, output)) await put.ExpectAsync(HttpStatusCode.OK); + volumes.Set(server.Options.ArchivePath, new VolumeFigures(0, 500 * GB)); + using (var full = await editor.PostAsync($"/api/v1/sequences/{sequence}/renders/{render}/complete", null)) + await full.ExpectErrorAsync(HttpStatusCode.InsufficientStorage, "insufficient_storage"); + volumes.Set(server.Options.ArchivePath, null); + using (var room = await editor.PostAsync($"/api/v1/sequences/{sequence}/renders/{render}/complete", null)) + await room.ExpectAsync(HttpStatusCode.OK); + } + + [Test] + public void TheSpacePathComesFromTheFileOrTheEnvironment() + { + var root = context.CreateTempDirectory("space-option"); + File.WriteAllText(Path.Combine(root, "server.json"), """{ "SpacePath": "/space" }"""); + var file = ServerOptions.Load(new Dictionary { ["CHRONOS_DATA_PATH"] = root, ["CHRONOS_DB"] = "postgres" }); + Assert.Equal("/space", file.SpacePath); + Assert.DoesNotContain("SpacePath", file.Locked); + var environment = ServerOptions.Load(new Dictionary + { + ["CHRONOS_DATA_PATH"] = root, + ["CHRONOS_DB"] = "postgres", + ["CHRONOS_SPACE_PATH"] = "/mnt/array", + }); + Assert.Equal("/mnt/array", environment.SpacePath); + Assert.Contains("SpacePath", environment.Locked); + Assert.Null(new ServerOptions().SpacePath); + } + + /// Counts the warnings a logs. + private sealed class CountingLogger : ILogger + { + private int _warnings; + + public int Warnings => Volatile.Read(ref _warnings); + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + if (logLevel == LogLevel.Warning) Interlocked.Increment(ref _warnings); + } + } +}