diff --git a/i18n/en/Admin.resx b/i18n/en/Admin.resx index 90626969..761e668e 100644 --- a/i18n/en/Admin.resx +++ b/i18n/en/Admin.resx @@ -155,6 +155,11 @@ All logs Open the audit log Archive + Archive incomplete + The server may not create the archive folder {0}. Let its account write there (in Docker, the owner of the /archive mount); uploads, deletions and conversions that need it fail until then. + The archive folder {0} could not be created; a file with that name may be in the way. The server log says why. + The archive folder {0} cannot be created: the drive or share is read-only (in Docker, check that /archive is not mounted read-only). + The drive or share of {0} is not available. Connect it; the folders are created when the dashboard is opened again. Storage by year {0} camera {0} cameras diff --git a/i18n/en/Archive.resx b/i18n/en/Archive.resx index 789763e9..3cb381b3 100644 --- a/i18n/en/Archive.resx +++ b/i18n/en/Archive.resx @@ -372,6 +372,10 @@ Retag… Thumbnails Verify + Chronos may not create the archive folder {0}: check the folder's permissions and whether the drive or share is read-only.long: error sentence; {0} folder path + The archive folder {0} could not be created; a file with that name may be in the way.long: error sentence; {0} folder path + The archive folder {0} cannot be created: the drive or share is read-only.long: error sentence; {0} folder path + The drive or share of {0} is not available: connect it and try again.long: error sentence; {0} folder path camera cameras {0:N0} format diff --git a/i18n/it/Admin.it.resx b/i18n/it/Admin.it.resx index 2fc3ffc1..d918b430 100644 --- a/i18n/it/Admin.it.resx +++ b/i18n/it/Admin.it.resx @@ -150,6 +150,11 @@ Tutti i log Apri il registro di controllo Archivio + Archivio incompleto + Il server non può creare la cartella dell'archivio {0}. Consenti al suo account di scriverci (in Docker, al proprietario del mount /archive); fino ad allora caricamenti, eliminazioni e conversioni che ne hanno bisogno non riescono. + Non è stato possibile creare la cartella dell'archivio {0}; forse un file con lo stesso nome la blocca. Il log del server dice perché. + Impossibile creare la cartella dell'archivio {0}: l'unità o la condivisione è di sola lettura (in Docker, controlla che /archive non sia montato in sola lettura). + L'unità o la condivisione di {0} non è disponibile. Collegala: le cartelle vengono create quando riapri la dashboard. Spazio per anno {0} fotocamera {0} fotocamere diff --git a/i18n/it/Archive.it.resx b/i18n/it/Archive.it.resx index 6d4a22f2..3fecf541 100644 --- a/i18n/it/Archive.it.resx +++ b/i18n/it/Archive.it.resx @@ -365,6 +365,10 @@ Tag… Miniature Verifica + Chronos non può creare la cartella dell'archivio {0}: controlla i permessi della cartella e se l'unità o la condivisione è di sola lettura. + Non è stato possibile creare la cartella dell'archivio {0}; forse un file con lo stesso nome la blocca. + Impossibile creare la cartella dell'archivio {0}: l'unità o la condivisione è di sola lettura. + L'unità o la condivisione di {0} non è disponibile: collegala e riprova. fotocamera fotocamere {0:N0} formato diff --git a/src/Chronos.App/Archive/ArchiveFailures.cs b/src/Chronos.App/Archive/ArchiveFailures.cs index fc4e484e..255e165e 100644 --- a/src/Chronos.App/Archive/ArchiveFailures.cs +++ b/src/Chronos.App/Archive/ArchiveFailures.cs @@ -20,7 +20,7 @@ public static class ArchiveFailures // any more (FrameTrash.RestoreAsync). Other IOExceptions and InvalidDataExceptions (a sidecar of another sequence) // carry technical text. LeaseUnavailableException or NewerFormatException or StaleRenamePlanException or NestedRenameException or StaleDeletePlanException - or SequenceInLibraryException or TrashMismatchException or Core.Project.ProjectTooLargeException => ex.Message, + or SequenceInLibraryException or TrashMismatchException or Core.Project.ProjectTooLargeException or ArchiveStructureException => ex.Message, UnauthorizedAccessException => Loc.L("Archive.Failed.Access"), DatabaseException => Loc.L("Archive.Failed.Index"), ArgumentException => Loc.L("Archive.Failed.Gone"), diff --git a/src/Chronos.App/Infrastructure/ArchiveSetup.cs b/src/Chronos.App/Infrastructure/ArchiveSetup.cs index 0f47c4a6..19fceae6 100644 --- a/src/Chronos.App/Infrastructure/ArchiveSetup.cs +++ b/src/Chronos.App/Infrastructure/ArchiveSetup.cs @@ -1,5 +1,6 @@ using System.IO; using Chronos.App.Archiving; +using Chronos.Core.Archiving; using Chronos.Core.Diagnostics; using Chronos.Data.Archiving; @@ -35,8 +36,14 @@ public sealed class EngineArchiveSetup(AppServices services) : IArchiveSetup var defaults = services.Settings.ArchiveDefaults(); var archive = await Task.Run(async () => { - Directory.CreateDirectory(root); - return await archives.AdoptAsync(root, defaults, cancellationToken).ConfigureAwait(false); + // A new archive is made with its whole layout (the folder itself included), or not at all; an existing one gets + // the folders it lacks. One that can be written but not completed is refused with the reason, before it becomes + // the archive of this PC; a read-only one opens as before, to be browsed. + var opened = await archives.AdoptAsync(root, defaults, cancellationToken).ConfigureAwait(false); + var paths = new ArchivePaths(root); + if (ArchiveStructure.Ensure(paths) is { IsComplete: false } structure && ArchiveVolume.Probe(paths) == ArchiveAvailability.Online) + throw new ArchiveStructureException(structure); + return opened; }, cancellationToken); services.Settings.LastArchivePath = root; services.SaveSettings(); diff --git a/src/Chronos.App/Sections/SettingsViewModel.cs b/src/Chronos.App/Sections/SettingsViewModel.cs index 2583d187..39604b3a 100644 --- a/src/Chronos.App/Sections/SettingsViewModel.cs +++ b/src/Chronos.App/Sections/SettingsViewModel.cs @@ -1180,9 +1180,11 @@ public sealed class SettingsViewModel : SectionViewModel { var opened = await Task.Run(async () => File.Exists(new ArchivePaths(root).ManifestPath) ? await archives.OpenAsync(root) : null); if (opened is not null) await Shell.Services.QueueScanAsync(opened, full: false); + // A folder that is no archive yet becomes one in place, with the archive's whole layout (as Welcome's Create). + else await Shell.Services.ArchiveSetup.CreateAsync(root); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException or NewerFormatException - or DatabaseException) + or DatabaseException or ArgumentException or InvalidOperationException) { Log.Warning("settings", $"The archive at {root} could not be opened", ex); Shell.Notify(Loc.L("Errors.Archive.Open"), ArchiveActions.Cause(ex), isError: true); diff --git a/src/Chronos.Core/Archiving/ArchiveLayout.cs b/src/Chronos.Core/Archiving/ArchiveLayout.cs index a5e23cc4..4c43b252 100644 --- a/src/Chronos.Core/Archiving/ArchiveLayout.cs +++ b/src/Chronos.Core/Archiving/ArchiveLayout.cs @@ -19,6 +19,15 @@ public static class ArchiveLayout /// Resumable plans of mass operations (rename). public const string JournalFolder = "journal"; + /// Deleted sequences, photos and renders inside the root , kept until Empty. + public const string TrashFolder = "trash"; + + /// The photo entries inside . + public const string TrashPhotosFolder = "photos"; + + /// The deleted renders inside (owner review 1, R9). + public const string TrashVideosFolder = "videos"; + public const string OriginalsFolder = "originals"; public const string ProxiesFolder = "proxies"; public const string ExportsFolder = "exports"; @@ -86,6 +95,9 @@ public sealed class ArchivePaths public string LocksDirectory => Path.Combine(MetaDirectory, ArchiveLayout.LocksFolder); public string StagingDirectory => Path.Combine(MetaDirectory, ArchiveLayout.StagingFolder); public string JournalDirectory => Path.Combine(MetaDirectory, ArchiveLayout.JournalFolder); + public string TrashDirectory => Path.Combine(MetaDirectory, ArchiveLayout.TrashFolder); + public string TrashPhotosDirectory => Path.Combine(TrashDirectory, ArchiveLayout.TrashPhotosFolder); + public string TrashVideosDirectory => Path.Combine(TrashDirectory, ArchiveLayout.TrashVideosFolder); /// Paths of the sequence stored in (forward slashes). public SequencePaths Sequence(string relativeFolder) => new(this, relativeFolder); diff --git a/src/Chronos.Core/Archiving/ArchiveStructure.cs b/src/Chronos.Core/Archiving/ArchiveStructure.cs new file mode 100644 index 00000000..8768d09c --- /dev/null +++ b/src/Chronos.Core/Archiving/ArchiveStructure.cs @@ -0,0 +1,134 @@ +using Chronos.Core.Localization; + +namespace Chronos.Core.Archiving; + +/// Why the folders of an archive's layout could not all be created (). +public enum ArchiveStructureProblem +{ + None, + + /// The drive or share of the archive is not there (unplugged, disconnected, not ready). + Unavailable, + + /// The account Chronos runs as may not create folders there. + Denied, + + /// The drive or share is read-only (write-protected disk, read-only mount). + ReadOnly, + + /// Any other failure, a file where a folder belongs included. + Failed, +} + +/// What found and did. +/// The archive root. +/// The folders created, parents first. +/// Why the layout is not complete; when it is. +/// The folder that could not be created; null when the layout is complete. +/// The system's own text of the failure, for the log; null when the layout is complete. +public sealed record ArchiveStructureReport(string Root, IReadOnlyList Created, ArchiveStructureProblem Problem, string? Folder, string? Detail) +{ + public bool IsComplete => Problem == ArchiveStructureProblem.None; + + /// The problem as one localized sentence naming the folder; null when the layout is complete. + public string? Message => Problem switch + { + ArchiveStructureProblem.None => null, + ArchiveStructureProblem.Unavailable => Loc.L("Archive.Structure.Unavailable", Folder ?? Root), + ArchiveStructureProblem.Denied => Loc.L("Archive.Structure.Denied", Folder ?? Root), + ArchiveStructureProblem.ReadOnly => Loc.L("Archive.Structure.ReadOnly", Folder ?? Root), + _ => Loc.L("Archive.Structure.Failed", Folder ?? Root), + }; +} + +/// The folders of an archive's layout could not all be created; the message is one localized sentence. +public sealed class ArchiveStructureException(ArchiveStructureReport report) : IOException(report.Message) +{ + public ArchiveStructureReport Report { get; } = report; +} + +/// +/// The folders every archive holds (BRIEF §7) whatever made it: the root's .chronos with locks, +/// journal, staging and trash, whose photos and videos areas hold the deleted photos and +/// renders. archive.json is the archive service's (it needs a name and an id); the folders of a sequence +/// (originals, proxies, exports, its .chronos) are made when the sequence is, and the +/// trash's .erasing and .erased only when Empty needs them. +/// +public static class ArchiveStructure +{ + private const int WriteProtected = unchecked((int)0x80070013); // ERROR_WRITE_PROTECT + private const int NotReady = unchecked((int)0x80070015); // ERROR_NOT_READY + private const int BadNetworkPath = unchecked((int)0x80070035); // ERROR_BAD_NETPATH + private const int NetworkNameDeleted = unchecked((int)0x80070040); // ERROR_NETNAME_DELETED + private const int BadNetworkName = unchecked((int)0x80070043); // ERROR_BAD_NET_NAME + private const int UnixReadOnlyFileSystem = 30; // EROFS, which .NET keeps as the IOException's HResult + private const int UnixNoDevice = 19; // ENODEV + + /// The layout's folders of , parents first, the root included. + public static IReadOnlyList Folders(ArchivePaths archive) + { + ArgumentNullException.ThrowIfNull(archive); + return [archive.Root, archive.MetaDirectory, archive.LocksDirectory, archive.JournalDirectory, archive.StagingDirectory, + archive.TrashDirectory, archive.TrashPhotosDirectory, archive.TrashVideosDirectory]; + } + + /// The layout's folders that are not there, parents first; nothing is created. + public static IReadOnlyList Missing(ArchivePaths archive) => [.. Folders(archive).Where(f => !Directory.Exists(f))]; + + /// + /// Creates the layout's folders that are missing, never touching what is there: an existing folder and every file + /// stay as they are, and a file where a folder belongs is reported, not replaced. Safe to run any number of times + /// and by several processes at once. Stops at the first folder that cannot be created and says why; never throws for + /// the file system. + /// + /// The archive root. + /// Whether a missing root is created (a new archive); otherwise it is reported unavailable. + /// Creates one folder (; tests replace it). + public static ArchiveStructureReport Ensure(ArchivePaths archive, bool createRoot = false, Action? createDirectory = null) + { + ArgumentNullException.ThrowIfNull(archive); + createDirectory ??= static path => Directory.CreateDirectory(path); + var created = new List(); + if (!Directory.Exists(archive.Root) && (!createRoot || !VolumeIsThere(archive.Root))) + return new(archive.Root, created, ArchiveStructureProblem.Unavailable, archive.Root, null); + foreach (var folder in Folders(archive)) + { + if (Directory.Exists(folder)) continue; + try + { + createDirectory(folder); + created.Add(folder); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return new(archive.Root, created, Classify(ex), folder, ex.Message); + } + } + return new(archive.Root, created, ArchiveStructureProblem.None, null, null); + } + + /// Why creating a folder failed, from the exception the file system threw. + public static ArchiveStructureProblem Classify(Exception ex) => ex switch + { + UnauthorizedAccessException => ArchiveStructureProblem.Denied, + DirectoryNotFoundException or DriveNotFoundException => ArchiveStructureProblem.Unavailable, + IOException { HResult: WriteProtected } => ArchiveStructureProblem.ReadOnly, + IOException { HResult: NotReady or BadNetworkPath or NetworkNameDeleted or BadNetworkName } => ArchiveStructureProblem.Unavailable, + IOException { HResult: UnixReadOnlyFileSystem } when !OperatingSystem.IsWindows() => ArchiveStructureProblem.ReadOnly, + IOException { HResult: UnixNoDevice } when !OperatingSystem.IsWindows() => ArchiveStructureProblem.Unavailable, + _ => ArchiveStructureProblem.Failed, + }; + + /// Whether the drive or share lives on is there. + private static bool VolumeIsThere(string root) + { + try + { + return Path.GetPathRoot(root) is { Length: > 0 } volume && Directory.Exists(volume); + } + catch (Exception ex) when (ex is ArgumentException or IOException or UnauthorizedAccessException) + { + return false; + } + } +} diff --git a/src/Chronos.Data/Archiving/ArchiveService.cs b/src/Chronos.Data/Archiving/ArchiveService.cs index 77b63291..74dad4f4 100644 --- a/src/Chronos.Data/Archiving/ArchiveService.cs +++ b/src/Chronos.Data/Archiving/ArchiveService.cs @@ -141,7 +141,8 @@ public sealed class ArchiveService : IJobHandlerProvider settings = WithUsableName(settings, paths); settings.Validate(); if (File.Exists(paths.ManifestPath)) throw new InvalidOperationException($"{paths.Root} already is a Chronos archive."); - Directory.CreateDirectory(paths.MetaDirectory); + var structure = EnsureStructure(paths, createRoot: true); + if (!structure.IsComplete) throw new ArchiveStructureException(structure); var manifest = new ArchiveManifest { Id = Guid.CreateVersion7(), @@ -164,7 +165,10 @@ public sealed class ArchiveService : IJobHandlerProvider { var paths = new ArchivePaths(root); var archive = await RegisterAsync(paths, ManifestStore.ReadArchive(paths.ManifestPath), cancellationToken).ConfigureAwait(false); - if (ArchiveVolume.Probe(paths) != ArchiveAvailability.Online) return archive; + var availability = ArchiveVolume.Probe(paths); + // Folders an older Chronos, another program or a copy left out are made again; a read-only archive only says which. + if (availability != ArchiveAvailability.Offline) EnsureStructure(paths); + if (availability != ArchiveAvailability.Online) return archive; try { await Renamer.ResumePendingAsync(archive, cancellationToken).ConfigureAwait(false); @@ -571,6 +575,21 @@ public sealed class ArchiveService : IJobHandlerProvider await Imports.PlanEraseAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), cardId, sourceRoot, progress, cancellationToken).ConfigureAwait(false); + /// + /// Creates the folders of the archive's layout that are missing (), logging what it + /// made and, as a warning, why the layout stays incomplete. + /// and run it; it never throws for the file system. + /// + public static ArchiveStructureReport EnsureStructure(ArchivePaths paths, bool createRoot = false) + { + ArgumentNullException.ThrowIfNull(paths); + var report = ArchiveStructure.Ensure(paths, createRoot); + if (report.Created.Count > 0) + Log.Info("archive", $"Archive folders created in {paths.Root}: {string.Join(", ", report.Created.Select(f => f == paths.Root ? "." : paths.ToRelative(f)))}"); + if (!report.IsComplete) Log.Warning("archive", $"The archive folders of {paths.Root} are incomplete: {report.Message} {report.Detail}".TrimEnd()); + return report; + } + /// /// with a name an archive can keep: a blank one becomes the folder's name (the root itself /// for a drive root), one longer than is shortened. diff --git a/src/Chronos.Data/Archiving/FrameTrash.cs b/src/Chronos.Data/Archiving/FrameTrash.cs index 27b76ac1..d1f496b3 100644 --- a/src/Chronos.Data/Archiving/FrameTrash.cs +++ b/src/Chronos.Data/Archiving/FrameTrash.cs @@ -110,7 +110,7 @@ public sealed record PhotoRestoreResult(Sequence Sequence, int Restored, IReadOn public sealed partial class FrameTrash { /// Folder of the photo entries inside the trash. - public const string FolderName = "photos"; + public const string FolderName = ArchiveLayout.TrashPhotosFolder; /// What of a photo entry starts with (the folder name follows). public const string EntryPrefix = "photos-"; diff --git a/src/Chronos.Data/Archiving/SequenceTrash.cs b/src/Chronos.Data/Archiving/SequenceTrash.cs index d80a0d6f..fcdd564b 100644 --- a/src/Chronos.Data/Archiving/SequenceTrash.cs +++ b/src/Chronos.Data/Archiving/SequenceTrash.cs @@ -119,7 +119,7 @@ public sealed class TrashMismatchException() : IOException(Loc.L("Archive.Error. public sealed partial class SequenceTrash { /// Folder of the trash inside the archive's .chronos. - public const string FolderName = "trash"; + public const string FolderName = ArchiveLayout.TrashFolder; private const string RecordExtension = ".json"; diff --git a/src/Chronos.Data/Videos/IVideoLibrary.cs b/src/Chronos.Data/Videos/IVideoLibrary.cs index 152998dc..57a2704d 100644 --- a/src/Chronos.Data/Videos/IVideoLibrary.cs +++ b/src/Chronos.Data/Videos/IVideoLibrary.cs @@ -192,7 +192,7 @@ public static class VideoTrashNames public const string Prefix = "videos-"; /// The folder of the videos area inside an archive's trash, which holds the entries and their records. - public const string FolderName = "videos"; + public const string FolderName = Chronos.Core.Archiving.ArchiveLayout.TrashVideosFolder; /// The entry name of render deleted at . public static string EntryName(Guid renderId, DateTimeOffset deletedAt) => diff --git a/src/Chronos.Protocol/Admin.cs b/src/Chronos.Protocol/Admin.cs index f27a0932..56e232a0 100644 --- a/src/Chronos.Protocol/Admin.cs +++ b/src/Chronos.Protocol/Admin.cs @@ -86,6 +86,7 @@ public sealed record UploadDayDto(DateOnly Day, long Frames, long Bytes); /// /// GET /api/v1/server/dashboard (admin): everything the dashboard page shows in one call (BRIEF §14.8). /// : TLS mode is proxy and this request did not come through a trusted proxy over HTTPS. +/// : a folder of the archive's layout is missing and could not be created; null when it is complete. /// public sealed record DashboardDto( string ServerName, @@ -103,4 +104,11 @@ public sealed record DashboardDto( IReadOnlyList Uploads, IReadOnlyList Activity, IReadOnlyList ActiveJobs, - bool ProxyWarning); + bool ProxyWarning, + ArchiveProblemDto? ArchiveProblem = null); + +/// +/// Why a folder of the archive's layout (BRIEF §7) could not be created: is unavailable (drive or +/// share not there), denied, readOnly or failed; is the folder. +/// +public sealed record ArchiveProblemDto(string Code, string Folder); diff --git a/src/Chronos.Server/Admin/web/pages/dashboard.js b/src/Chronos.Server/Admin/web/pages/dashboard.js index 0bd4571d..f297369c 100644 --- a/src/Chronos.Server/Admin/web/pages/dashboard.js +++ b/src/Chronos.Server/Admin/web/pages/dashboard.js @@ -6,6 +6,21 @@ import { activityText } from "../audit.js"; const SCAN_BUTTON = ``; +// Why a folder of the archive's layout could not be created (DashboardDto.ArchiveProblem), each with its tooltip. +const ARCHIVE_PROBLEM_TIPS = { + unavailable: "Admin.Dashboard.ArchiveIncomplete.Unavailable.Tip", + denied: "Admin.Dashboard.ArchiveIncomplete.Denied.Tip", + readOnly: "Admin.Dashboard.ArchiveIncomplete.ReadOnly.Tip", + failed: "Admin.Dashboard.ArchiveIncomplete.Failed.Tip", +}; + +/** The warning chip of an incomplete archive layout; empty when every folder is there. */ +function archiveProblemChip(problem) { + if (!problem) return ""; + const tip = t(ARCHIVE_PROBLEM_TIPS[problem.code] ?? ARCHIVE_PROBLEM_TIPS.failed, problem.folder); + return ``; +} + export async function render(root, { api, setMeta, actions, onEvent }) { let timer = null; const refresh = () => { @@ -28,7 +43,7 @@ export async function render(root, { api, setMeta, actions, onEvent }) { const tls = `Admin.Settings.Tls.${d.tlsMode.charAt(0).toUpperCase()}${d.tlsMode.slice(1)}`; setMeta(t("Admin.Dashboard.Meta", formatDuration(d.uptimeSeconds), d.database, t(tls), t(d.discovery ? "Admin.Dashboard.DiscoveryOn" : "Admin.Dashboard.DiscoveryOff"))); - actions.innerHTML = d.proxyWarning ? PROXY_WARNING : ""; + actions.innerHTML = (d.proxyWarning ? PROXY_WARNING : "") + archiveProblemChip(d.archiveProblem); applyStrings(actions); // The space volume's figures (the space path when the server has one), the write volume apart when it differs. const a = d.archive; diff --git a/src/Chronos.Server/Api/AdminEndpoints.cs b/src/Chronos.Server/Api/AdminEndpoints.cs index 3e504403..39b692cf 100644 --- a/src/Chronos.Server/Api/AdminEndpoints.cs +++ b/src/Chronos.Server/Api/AdminEndpoints.cs @@ -69,10 +69,20 @@ public static class AdminEndpoints [.. uploads.Select(u => new UploadDayDto(u.Day, u.Count, u.Bytes))], [.. activity.Items.Select(Dto.Audit)], [.. running.Items.Concat(queued.Items.OrderByDescending(j => j.Priority).ThenBy(j => j.Created)).Take(10).Select(Dto.Job)], - AdminPanel.BypassesProxy(context, options)), + AdminPanel.BypassesProxy(context, options), + ArchiveProblem(archive.CheckStructure())), ProtocolJson.Default.DashboardDto); } + /// The dashboard's warning of an incomplete archive layout; null when every folder is there. + private static ArchiveProblemDto? ArchiveProblem(Core.Archiving.ArchiveStructureReport report) => report.IsComplete ? null : new(report.Problem switch + { + Core.Archiving.ArchiveStructureProblem.Unavailable => "unavailable", + Core.Archiving.ArchiveStructureProblem.Denied => "denied", + Core.Archiving.ArchiveStructureProblem.ReadOnly => "readOnly", + _ => "failed", + }, report.Folder ?? report.Root); + private static async Task SaveSettingsAsync(HttpContext context, Caller caller, ServerSettingsDto request, ServerSettingsStore store, ServerOptions options, AuditLog audit, JobEngineHost jobs, SchedulerHost schedules) { diff --git a/src/Chronos.Server/Api/SystemEndpoints.cs b/src/Chronos.Server/Api/SystemEndpoints.cs index 3e484977..cec1f6c2 100644 --- a/src/Chronos.Server/Api/SystemEndpoints.cs +++ b/src/Chronos.Server/Api/SystemEndpoints.cs @@ -72,12 +72,16 @@ public static class SystemEndpoints }).AllowAnonymousCaller(); } - /// Liveness for the container health check: 503 while the database cannot be reached. - private static async Task HealthAsync(HttpContext context, ServerOptions options, ChronosDatabase db) + /// + /// Liveness for the container health check: 503 while the database cannot be reached. An incomplete archive layout is + /// reported in the checks without failing the check: the server still serves what it can. + /// + private static async Task HealthAsync(HttpContext context, ServerOptions options, ChronosDatabase db, Storage.ServerArchive archive) { var checks = new Dictionary(StringComparer.Ordinal) { - ["archive"] = Directory.Exists(options.ArchivePath) ? "ok" : "missing", + // "incomplete": a folder of the archive's layout could not be created (the dashboard says which and why). + ["archive"] = !Directory.Exists(options.ArchivePath) ? "missing" : archive.Structure is { IsComplete: false } ? "incomplete" : "ok", ["data"] = Directory.Exists(options.DataPath) ? "ok" : "missing", }; using var timeout = CancellationTokenSource.CreateLinkedTokenSource(context.RequestAborted); diff --git a/src/Chronos.Server/Hosting/ServerHost.cs b/src/Chronos.Server/Hosting/ServerHost.cs index dd4d2ea6..647e4175 100644 --- a/src/Chronos.Server/Hosting/ServerHost.cs +++ b/src/Chronos.Server/Hosting/ServerHost.cs @@ -87,6 +87,11 @@ public static class ServerHost + "they could put a link in place of a folder and redirect the files the server moves. Let only those accounts change it (others may read it).", options.ArchivePath); } + if (archive.Structure is { IsComplete: false } structure) + { + app.Logger.LogWarning("The archive's folders are incomplete: {Problem} Uploads, deletions and conversions that need the missing folder fail " + + "until it can be created; the dashboard shows the problem until then. {Detail}", structure.Message, structure.Detail ?? string.Empty); + } // A space path that cannot be measured is warned about at start, not first when someone opens the dashboard. if (!string.IsNullOrWhiteSpace(options.SpacePath)) app.Services.GetRequiredService().Report(archive.Root); if (options.FormerWorkersVariable) diff --git a/src/Chronos.Server/Storage/ServerArchive.cs b/src/Chronos.Server/Storage/ServerArchive.cs index d33fdf7e..ec1d3838 100644 --- a/src/Chronos.Server/Storage/ServerArchive.cs +++ b/src/Chronos.Server/Storage/ServerArchive.cs @@ -1,3 +1,4 @@ +using Chronos.Core.Archiving; using Chronos.Core.Model; using Chronos.Data; using Chronos.Data.Archiving; @@ -46,14 +47,36 @@ public sealed class ServerArchive(Archive archive) public string Root => _current.RootPath; + /// + /// What the last found of the archive's folders; null before the first. The dashboard and + /// the health check show an incomplete layout. + /// + public ArchiveStructureReport? Structure => _structure; + + private volatile ArchiveStructureReport? _structure; + + /// + /// Creates the folders of the archive's layout that are missing (BRIEF §7) and remembers what it found + /// (): at start, and each time the dashboard is read, so a share that comes back heals. + /// + public ArchiveStructureReport CheckStructure() + { + var report = ArchiveStructure.Ensure(new ArchivePaths(Root)); + _structure = report; + return report; + } + /// /// Opens the archive at the configured root through the archive engine: an existing Chronos archive is registered - /// (and relinked if it moved), any other folder becomes a new archive with the default templates. + /// (and relinked if it moved), any other folder becomes a new archive with the default templates. Either way the + /// folders of the archive's layout are made (). /// public static async Task OpenAsync(ChronosDatabase db, ServerOptions options, CancellationToken cancellationToken) { var root = Path.TrimEndingDirectorySeparator(Path.GetFullPath(options.ArchivePath)); var archive = await new ArchiveService(db).AdoptAsync(root, options.ServerName, cancellationToken).ConfigureAwait(false); - return new ServerArchive(archive); + var served = new ServerArchive(archive); + served.CheckStructure(); + return served; } } diff --git a/tests/Chronos.Core.Tests/Archiving/ArchiveStructureTests.cs b/tests/Chronos.Core.Tests/Archiving/ArchiveStructureTests.cs new file mode 100644 index 00000000..96defc3b --- /dev/null +++ b/tests/Chronos.Core.Tests/Archiving/ArchiveStructureTests.cs @@ -0,0 +1,180 @@ +using System.Security.AccessControl; +using System.Security.Principal; +using Chronos.Core.Archiving; +using Chronos.Core.Localization; +using Chronos.Testing; + +namespace Chronos.Core.Tests.Archiving; + +[Category("archive")] +public sealed class ArchiveStructureTests(TestContext context) +{ + private ArchivePaths NewArchive() => new(context.CreateTempDirectory("structure")); + + [Test] + public void AnEmptyFolderGetsTheWholeLayout() + { + var archive = NewArchive(); + + var report = ArchiveStructure.Ensure(archive); + + Assert.True(report.IsComplete); + Assert.Null(report.Message); + foreach (var folder in ArchiveStructure.Folders(archive)) Assert.True(Directory.Exists(folder), folder); + Assert.SequenceEqual(ArchiveStructure.Folders(archive).Skip(1), report.Created); + Assert.Empty(ArchiveStructure.Missing(archive)); + string[] expected = [".chronos", ".chronos/locks", ".chronos/journal", ".chronos/staging", ".chronos/trash", ".chronos/trash/photos", ".chronos/trash/videos"]; + Assert.SequenceEqual(expected, report.Created.Select(archive.ToRelative)); + } + + [Test] + public void AMissingRootIsCreatedOnlyForANewArchive() + { + var parent = context.CreateTempDirectory("parent"); + var archive = new ArchivePaths(Path.Combine(parent, "Archive")); + + var refused = ArchiveStructure.Ensure(archive); + Assert.Equal(ArchiveStructureProblem.Unavailable, refused.Problem); + Assert.Equal(archive.Root, refused.Folder); + Assert.False(Directory.Exists(archive.Root), "an archive that is not there is never made up"); + + var created = ArchiveStructure.Ensure(archive, createRoot: true); + Assert.True(created.IsComplete); + Assert.Equal(archive.Root, created.Created[0]); + Assert.Empty(ArchiveStructure.Missing(archive)); + } + + [Test] + public void AnExistingLayoutAndItsFilesAreLeftAsTheyAre() + { + var archive = NewArchive(); + Assert.True(ArchiveStructure.Ensure(archive).IsComplete); + // Files where an archive keeps them, and the user's own: none may change, nor their times. + string[] files = + [ + archive.ManifestPath, + Path.Combine(archive.TrashPhotosDirectory, "entry.json"), + Path.Combine(archive.StagingDirectory, "copy.tmp"), + Path.Combine(archive.Root, "2026", "Harbour", "originals", "IMG_0001.JPG"), + Path.Combine(archive.Root, "notes.txt"), + ]; + var written = new DateTime(2026, 1, 2, 3, 4, 5, DateTimeKind.Utc); + for (var i = 0; i < files.Length; i++) + { + Directory.CreateDirectory(Path.GetDirectoryName(files[i])!); + File.WriteAllBytes(files[i], [(byte)i, 1, 2, 3]); + File.SetLastWriteTimeUtc(files[i], written); + } + var folders = Directory.GetDirectories(archive.Root, "*", SearchOption.AllDirectories).Order(StringComparer.Ordinal).ToList(); + + var again = ArchiveStructure.Ensure(archive); + + Assert.True(again.IsComplete); + Assert.Empty(again.Created); + for (var i = 0; i < files.Length; i++) + { + Assert.BytesEqual([(byte)i, 1, 2, 3], File.ReadAllBytes(files[i])); + Assert.Equal(written, File.GetLastWriteTimeUtc(files[i])); + } + Assert.SequenceEqual(folders, Directory.GetDirectories(archive.Root, "*", SearchOption.AllDirectories).Order(StringComparer.Ordinal)); + } + + [Test] + public void OnlyTheMissingFoldersAreCreated() + { + var archive = NewArchive(); + Directory.CreateDirectory(archive.LocksDirectory); + File.WriteAllText(Path.Combine(archive.LocksDirectory, "lease"), "held"); + + var report = ArchiveStructure.Ensure(archive); + + Assert.True(report.IsComplete); + Assert.False(report.Created.Contains(archive.MetaDirectory) || report.Created.Contains(archive.LocksDirectory)); + Assert.True(report.Created.Contains(archive.TrashVideosDirectory)); + Assert.Equal("held", File.ReadAllText(Path.Combine(archive.LocksDirectory, "lease"))); + } + + [Test] + public void AFileWhereAFolderBelongsIsReportedNotReplaced() + { + var archive = NewArchive(); + Directory.CreateDirectory(archive.MetaDirectory); + File.WriteAllText(archive.TrashDirectory, "mine"); + + var report = ArchiveStructure.Ensure(archive); + + Assert.Equal(ArchiveStructureProblem.Failed, report.Problem); + Assert.Equal(archive.TrashDirectory, report.Folder); + Assert.Equal("mine", File.ReadAllText(archive.TrashDirectory)); + Assert.True(Directory.Exists(archive.StagingDirectory), "the folders before it are made"); + Assert.Equal(Loc.L("Archive.Structure.Failed", archive.TrashDirectory), report.Message); + Assert.Equal(report.Message, new ArchiveStructureException(report).Message); + } + + [Test] + public void EachFailureSaysWhy() + { + var archive = NewArchive(); + (Exception Thrown, ArchiveStructureProblem Expected, string Key)[] cases = + [ + (new UnauthorizedAccessException("denied"), ArchiveStructureProblem.Denied, "Archive.Structure.Denied"), + (new IOException("write protected", unchecked((int)0x80070013)), ArchiveStructureProblem.ReadOnly, "Archive.Structure.ReadOnly"), + (new IOException("not ready", unchecked((int)0x80070015)), ArchiveStructureProblem.Unavailable, "Archive.Structure.Unavailable"), + (new DirectoryNotFoundException("gone"), ArchiveStructureProblem.Unavailable, "Archive.Structure.Unavailable"), + (new IOException("disk full", unchecked((int)0x80070070)), ArchiveStructureProblem.Failed, "Archive.Structure.Failed"), + ]; + foreach (var (thrown, expected, key) in cases) + { + var report = ArchiveStructure.Ensure(archive, createDirectory: _ => throw thrown); + Assert.Equal(expected, report.Problem); + Assert.Equal(archive.MetaDirectory, report.Folder); + Assert.Equal(thrown.Message, report.Detail); + Assert.Empty(report.Created); + Assert.Equal(Loc.L(key, archive.MetaDirectory), report.Message); + } + Assert.False(Directory.Exists(archive.MetaDirectory)); + if (!OperatingSystem.IsWindows()) + Assert.Equal(ArchiveStructureProblem.ReadOnly, ArchiveStructure.Classify(new IOException("read-only file system", 30))); + } + + [Test] + public void AFolderChronosMayNotWriteReportsTheError() + { + var archive = NewArchive(); + if (OperatingSystem.IsWindows()) + { + var info = new DirectoryInfo(archive.Root); + var me = WindowsIdentity.GetCurrent().User!; + var deny = new FileSystemAccessRule(me, FileSystemRights.CreateDirectories | FileSystemRights.CreateFiles, AccessControlType.Deny); + var security = info.GetAccessControl(); + security.AddAccessRule(deny); + info.SetAccessControl(security); + try + { + var report = ArchiveStructure.Ensure(archive); + Assert.Equal(ArchiveStructureProblem.Denied, report.Problem); + Assert.Equal(archive.MetaDirectory, report.Folder); + Assert.False(Directory.Exists(archive.MetaDirectory)); + } + finally + { + security.RemoveAccessRule(deny); + info.SetAccessControl(security); + } + return; + } + File.SetUnixFileMode(archive.Root, UnixFileMode.UserRead | UnixFileMode.UserExecute); + try + { + var report = ArchiveStructure.Ensure(archive); + // An account that ignores permissions (root in a CI container) writes anyway; the cases above cover the mapping. + if (report.IsComplete) Assert.Inconclusive("this account writes into read-only folders"); + Assert.Equal(ArchiveStructureProblem.Denied, report.Problem); + Assert.Equal(archive.MetaDirectory, report.Folder); + } + finally + { + File.SetUnixFileMode(archive.Root, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + } + } +} diff --git a/tests/Chronos.Data.Tests/Archiving/ArchiveStructureServiceTests.cs b/tests/Chronos.Data.Tests/Archiving/ArchiveStructureServiceTests.cs new file mode 100644 index 00000000..0881567c --- /dev/null +++ b/tests/Chronos.Data.Tests/Archiving/ArchiveStructureServiceTests.cs @@ -0,0 +1,64 @@ +using Chronos.Core.Archiving; +using Chronos.Data.Archiving; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Archiving; + +/// Every archive the service creates, adopts or opens holds its whole layout (BRIEF §7, owner request 2026-11-06). +[Category("archive")] +public sealed class ArchiveStructureServiceTests(TestContext context) +{ + [Test] + public async Task ANewArchiveGetsItsWholeLayout() + { + await using var db = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("db"), "chronos.db")), context.Cancellation); + var service = new ArchiveService(db); + var root = Path.Combine(context.CreateTempDirectory("new"), "Chronos", "Archive"); + + var archive = await service.AdoptAsync(root, "Fresh", context.Cancellation); + + var paths = new ArchivePaths(archive.RootPath); + Assert.FileExists(paths.ManifestPath); + Assert.Empty(ArchiveStructure.Missing(paths)); + Assert.True(Directory.Exists(paths.TrashPhotosDirectory) && Directory.Exists(paths.TrashVideosDirectory)); + } + + [Test] + public async Task OpeningAnArchiveMakesTheFoldersItLacksAndTouchesNothingElse() + { + await using var fx = await ArchiveFixture.CreateAsync(context); + var paths = fx.Paths; + var kept = Path.Combine(paths.StagingDirectory, "copy.tmp"); + File.WriteAllText(kept, "staged"); + var manifest = File.ReadAllBytes(paths.ManifestPath); + // An archive an older Chronos or a copy left without its trash, journal and locks. + Directory.Delete(paths.TrashDirectory, recursive: true); + Directory.Delete(paths.JournalDirectory); + Directory.Delete(paths.LocksDirectory, recursive: true); + + await fx.Service.OpenAsync(paths.Root, fx.Ct); + + Assert.Empty(ArchiveStructure.Missing(paths)); + Assert.Equal("staged", File.ReadAllText(kept)); + Assert.BytesEqual(manifest, File.ReadAllBytes(paths.ManifestPath)); + } + + [Test] + public async Task AnArchiveThatCannotBeLaidOutIsNotCreated() + { + await using var db = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("db"), "chronos.db")), context.Cancellation); + var service = new ArchiveService(db); + var root = context.CreateTempDirectory("blocked"); + // A file where the archive's metadata folder belongs: it stays, and the archive is refused with the reason. + var meta = new ArchivePaths(root).MetaDirectory; + File.WriteAllText(meta, "not a folder"); + + var error = await Assert.ThrowsAsync(() => service.CreateAsync(root, "Blocked", cancellationToken: context.Cancellation)); + + Assert.Equal(ArchiveStructureProblem.Failed, error.Report.Problem); + Assert.Equal(meta, error.Report.Folder); + Assert.Equal(error.Report.Message, error.Message); + Assert.Equal("not a folder", File.ReadAllText(meta)); + Assert.Empty(await db.Archives.ListAsync(context.Cancellation)); + } +} diff --git a/tests/Chronos.Data.Tests/Archiving/FrameTrashTests.cs b/tests/Chronos.Data.Tests/Archiving/FrameTrashTests.cs index 50a29786..bb84b3f0 100644 --- a/tests/Chronos.Data.Tests/Archiving/FrameTrashTests.cs +++ b/tests/Chronos.Data.Tests/Archiving/FrameTrashTests.cs @@ -105,7 +105,7 @@ public abstract class FrameTrashTests(TestContext context) : DatabaseTest(contex Assert.SequenceEqual([frames[1].Id, frames[3].Id], plan.Frames.Select(f => f.Id)); Assert.Equal(frames[1].Size + frames[3].Size, plan.Bytes); Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id)); - Assert.False(Directory.Exists(FrameTrash.Directory(fx.Paths)), "a dry run moves nothing"); + Assert.False(Directory.EnumerateFileSystemEntries(FrameTrash.Directory(fx.Paths)).Any(), "a dry run moves nothing"); foreach (var frame in frames) Assert.FileExists(fx.Paths.FromRelative(frame.RelativePath)); } diff --git a/tests/Chronos.Data.Tests/Archiving/SequenceTrashTests.cs b/tests/Chronos.Data.Tests/Archiving/SequenceTrashTests.cs index 7d256be8..40402f0e 100644 --- a/tests/Chronos.Data.Tests/Archiving/SequenceTrashTests.cs +++ b/tests/Chronos.Data.Tests/Archiving/SequenceTrashTests.cs @@ -26,6 +26,11 @@ public abstract class SequenceTrashTests(TestContext context) : DatabaseTest(con return sequences; } + /// What the trash holds besides its photos and videos areas, which every archive has (empty here). + private static IEnumerable TrashContents(ArchivePaths paths) => + Directory.EnumerateFileSystemEntries(paths.TrashDirectory, "*", SearchOption.AllDirectories) + .Where(e => e != paths.TrashPhotosDirectory && e != paths.TrashVideosDirectory); + private static (int Files, long Bytes) OnDisk(string folder) { var files = new DirectoryInfo(folder).EnumerateFiles("*", SearchOption.AllDirectories).ToList(); @@ -109,7 +114,7 @@ public abstract class SequenceTrashTests(TestContext context) : DatabaseTest(con Assert.Equal(6, plan.Frames); Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id)); Assert.True(Directory.Exists(folder), "a dry run moves nothing"); - Assert.False(Directory.Exists(SequenceTrash.TrashDirectory(fx.Paths))); + Assert.Empty(TrashContents(fx.Paths)); } [Test] @@ -166,7 +171,7 @@ public abstract class SequenceTrashTests(TestContext context) : DatabaseTest(con Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id)); Assert.Equal(render.OutputPath, (await fx.Db.Renders.ListBySequenceAsync(victim.Id)).Single().OutputPath); Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)); - Assert.Empty(Directory.EnumerateFileSystemEntries(SequenceTrash.TrashDirectory(fx.Paths))); + Assert.Empty(TrashContents(fx.Paths)); Assert.Equal(0, (await fx.ScanAsync()).Indexed); // Deleted again: the trash takes it again under the same name. @@ -360,7 +365,7 @@ public abstract class SequenceTrashTests(TestContext context) : DatabaseTest(con Assert.True(Directory.Exists(folder), "the folder is back where it was"); Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id)); - Assert.Empty(Directory.EnumerateFileSystemEntries(SequenceTrash.TrashDirectory(fx.Paths))); + Assert.Empty(TrashContents(fx.Paths)); await HealSequencesAsync(fx, "DELETE"); Assert.Count(1, await fx.Service.DeleteAsync(await fx.Service.PlanDeleteAsync([victim.Id], null, fx.Ct), null, null, fx.Ct)); } diff --git a/tests/Chronos.Server.Tests/Hosting/ArchiveStructureStartupTests.cs b/tests/Chronos.Server.Tests/Hosting/ArchiveStructureStartupTests.cs new file mode 100644 index 00000000..061e1549 --- /dev/null +++ b/tests/Chronos.Server.Tests/Hosting/ArchiveStructureStartupTests.cs @@ -0,0 +1,78 @@ +using System.Net; +using System.Net.Http.Json; +using Microsoft.Extensions.DependencyInjection; +using Chronos.Core.Archiving; +using Chronos.Data; +using Chronos.Data.Archiving; +using Chronos.Protocol; +using Chronos.Server.Storage; +using Chronos.Testing; + +namespace Chronos.Server.Tests.Hosting; + +/// The server lays out its archive folder whole at start (owner request 2026-11-06) and says when it cannot. +[Category("server")] +public sealed class ArchiveStructureStartupTests(TestContext context) +{ + [Test] + public async Task StartingTheServerLaysOutTheArchiveFolder() + { + await using var server = await TestServer.StartAsync(context); + + var paths = new ArchivePaths(server.Options.ArchivePath); + Assert.FileExists(paths.ManifestPath); + Assert.Empty(ArchiveStructure.Missing(paths)); + Assert.True(Assert.NotNull(server.Services.GetRequiredService().Structure).IsComplete); + var health = Assert.NotNull(await server.Client.GetFromJsonAsync("/health", ProtocolJson.Default.HealthStatus)); + Assert.Equal("ok", health.Checks["archive"]); + } + + [Test] + public async Task AnArchiveMissingFoldersGetsThemAtStart() + { + var root = Path.Combine(context.CreateTempDirectory("existing"), "archive"); + await using (var db = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("db"), "chronos.db")), context.Cancellation)) + await new ArchiveService(db).CreateAsync(root, "Older", cancellationToken: context.Cancellation); + var paths = new ArchivePaths(root); + Directory.Delete(paths.TrashDirectory, recursive: true); + Directory.Delete(paths.StagingDirectory); + var notes = Path.Combine(root, "notes.txt"); + File.WriteAllText(notes, "mine"); + + await using var server = await TestServer.StartAsync(context, o => o.ArchivePath = root); + + Assert.Empty(ArchiveStructure.Missing(paths)); + Assert.Equal("mine", File.ReadAllText(notes)); + } + + [Test] + public async Task TheDashboardWarnsWhileAFolderCannotBeCreated() + { + await using var server = await TestServer.StartAsync(context); + var admin = await server.AdminAsync(); + var paths = new ArchivePaths(server.Options.ArchivePath); + Directory.Delete(paths.TrashVideosDirectory); + File.WriteAllText(paths.TrashVideosDirectory, "in the way"); + + var warned = await DashboardAsync(admin); + var problem = Assert.NotNull(warned.ArchiveProblem); + Assert.Equal(("failed", paths.TrashVideosDirectory), (problem.Code, problem.Folder)); + var health = Assert.NotNull(await server.Client.GetFromJsonAsync("/health", ProtocolJson.Default.HealthStatus)); + Assert.Equal(("ok", "incomplete"), (health.Status, health.Checks["archive"])); + Assert.Equal("in the way", File.ReadAllText(paths.TrashVideosDirectory)); + + // Once the file is out of the way, the next look at the dashboard makes the folder and the warning goes. + File.Delete(paths.TrashVideosDirectory); + Assert.Null((await DashboardAsync(admin)).ArchiveProblem); + Assert.True(Directory.Exists(paths.TrashVideosDirectory)); + health = Assert.NotNull(await server.Client.GetFromJsonAsync("/health", ProtocolJson.Default.HealthStatus)); + Assert.Equal("ok", health.Checks["archive"]); + } + + private static async Task DashboardAsync(HttpClient admin) + { + using var response = await admin.GetAsync("/api/v1/server/dashboard"); + await response.ExpectAsync(HttpStatusCode.OK); + return await response.ReadAsync(ProtocolJson.Default.DashboardDto); + } +}