feat: single setup, mass-pass progress, per-server homes, Desktop shares its library (owner review 5, v1.1.0) (#68)
CI / Build and test (push) Successful in 11m36s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 12s
Release / Build, test and publish (push) Successful in 14m56s
CI / PostgreSQL 16 integration (push) Successful in 10m11s
CI / PostgreSQL 17 integration (push) Successful in 10m15s
CI / Docker image (push) Successful in 45s
Release / Server image (push) Successful in 4m55s

Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
This commit was merged in pull request #68.
This commit is contained in:
Alby96 authored and Alby96 committed 2026-11-15 18:11:43 +01:00
1 parent 988068ee8e
commit 20d64af4a4
793 files changed
+23882 -2096

No files matched your search

@@ -92,21 +92,39 @@ public sealed class WorkspaceCleaner
{
private static readonly JobKind[] ProjectJobs = [JobKind.WorkspaceCopy, JobKind.WorkspaceRender, JobKind.WorkspaceSend];
private readonly WorkspaceSync _sync;
private readonly IReadOnlyList<WorkspaceSync> _syncs;
private readonly ChronosDatabase _db;
private readonly Func<BundleHome, IProjectHome?> _homes;
/// <param name="homes">The home of a workspace project; null for one this PC cannot reach now.</param>
public WorkspaceCleaner(WorkspaceSync sync, ChronosDatabase db, Func<BundleHome, IProjectHome?> homes)
: this([sync ?? throw new ArgumentNullException(nameof(sync))], db, homes)
{
ArgumentNullException.ThrowIfNull(sync);
}
/// <summary>
/// The clean-up of a workspace split into homes (owner review 5, <see cref="WorkspaceHomes"/>): one sync per home's store,
/// and one run over all of them, so the size limit counts every home's projects together.
/// </summary>
/// <param name="syncs">The sync of each home's store.</param>
/// <param name="homes">The home of a workspace project; null for one this PC cannot reach now.</param>
public WorkspaceCleaner(IReadOnlyList<WorkspaceSync> syncs, ChronosDatabase db, Func<BundleHome, IProjectHome?> homes)
{
ArgumentNullException.ThrowIfNull(syncs);
ArgumentNullException.ThrowIfNull(db);
ArgumentNullException.ThrowIfNull(homes);
_sync = sync;
if (syncs.Count == 0 || syncs.Any(s => s is null)) throw new ArgumentException("At least one sync, and none null.", nameof(syncs));
_syncs = syncs;
_db = db;
_homes = homes;
}
/// <summary>The sync of the store <paramref name="project"/> is in.</summary>
private WorkspaceSync SyncOf(WorkspaceProject project) =>
_syncs.FirstOrDefault(s => ReferenceEquals(s.Store, project.Store))
?? _syncs.FirstOrDefault(s => string.Equals(s.Store.Root, project.Store.Root, StringComparison.OrdinalIgnoreCase))
?? _syncs[0];
/// <summary>How long a project another writer holds is waited for; past it the project is kept (<see cref="WorkspaceKeep.Jobs"/>).</summary>
public TimeSpan LockWait { get; init; } = TimeSpan.FromSeconds(30);
@@ -120,7 +138,7 @@ public sealed class WorkspaceCleaner
ArgumentNullException.ThrowIfNull(options);
var candidates = new List<Candidate>();
var used = 0L;
foreach (var project in _sync.Store.List())
foreach (var project in _syncs.SelectMany(s => s.Store.List()))
{
cancellationToken.ThrowIfCancellationRequested();
var candidate = Candidate.Of(project, options);
@@ -251,7 +269,7 @@ public sealed class WorkspaceCleaner
WorkspaceStatus status;
try
{
status = await _sync.StatusAsync(id, home, cancellationToken).ConfigureAwait(false);
status = await SyncOf(project).StatusAsync(id, home, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is ProjectException or DatabaseException or IOException or InvalidDataException or LeaseUnavailableException)
{
@@ -288,7 +306,7 @@ public sealed class WorkspaceCleaner
{
// Asked again under the lease: the editor may have opened it, a save or a render may have come meanwhile.
if (options.IsOpen(id)) return (WorkspaceKeep.Open, 0, 0);
var local = await _sync.StatusAsync(id, null, cancellationToken).ConfigureAwait(false);
var local = await SyncOf(project).StatusAsync(id, null, cancellationToken).ConfigureAwait(false);
if (local.State != WorkspaceState.InSync) return (WorkspaceKeep.NotInSync, 0, 0);
if (ReadLedger(project) is not { } renders) return (WorkspaceKeep.Unreadable, 0, 0);
if (renders.Unsent.Count > 0) return (WorkspaceKeep.UnsentRenders, 0, 0);
@@ -116,7 +116,7 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
if (!PathsEqual(root, parameters.WorkspaceRoot) && await InsideALibraryAsync(root, ct).ConfigureAwait(false))
throw new InvalidOperationException(Loc.L("Errors.Workspace.InsideLibrary"));
parameters = parameters with { WorkspaceRoot = root };
var store = new WorkspaceStore(root);
var store = WorkspaceStore.ForJob(root);
var project = store.Find(parameters.ProjectId) ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.NotCheckedOut"));
if (project.ReadBundle().Workspace is not { Home: { IsArchive: true } home } || home.SequenceId != sequenceId)
throw new InvalidOperationException(Loc.L("Errors.Workspace.OtherHome"));
@@ -167,7 +167,7 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
{
try
{
if (new WorkspaceStore(root).Find(parameters.ProjectId) is { } project)
if (WorkspaceStore.ForJob(root).Find(parameters.ProjectId) is { } project)
{
await RemovePartialMediaAsync(job, project).ConfigureAwait(false);
await RemoveCancelledOriginalsAsync(job, project, parameters).ConfigureAwait(false);
@@ -401,6 +401,9 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
var originalsLeft = withOriginals ? RemainingOriginals(project, frames, heldOriginals, (f, _) => f.Size) : new long[frames.Count + 1];
EnsureSpace(parameters.WorkspaceRoot, remaining[0] + originalsLeft[0]);
var clock = System.Diagnostics.Stopwatch.StartNew();
// Proxy x of y, then original x of y, of the sequence.
var start = Math.Min(copied, frames.Count);
await context.ReportAsync(start, total, new JobDetail(JobStep.CopyProxy, row.DisplayTitle, start, frames.Count)).ConfigureAwait(false);
for (var i = 0; i < frames.Count; i++)
{
ct.ThrowIfCancellationRequested();
@@ -414,12 +417,13 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
}
else
{
await context.ReportAsync(i + 1, total).ConfigureAwait(false);
await context.ReportItemAsync(i + 1, total, i + 1, frames.Count).ConfigureAwait(false);
}
}
var originals = new OriginalsTally();
if (withOriginals)
{
await context.ReportAsync(frames.Count, total, new JobDetail(JobStep.CopyOriginal, row.DisplayTitle, 0, frames.Count)).ConfigureAwait(false);
for (var i = 0; i < frames.Count; i++)
{
ct.ThrowIfCancellationRequested();
@@ -434,7 +438,7 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
}
else
{
await context.ReportAsync(done, total).ConfigureAwait(false);
await context.ReportItemAsync(done, total, i + 1, frames.Count).ConfigureAwait(false);
}
}
}
@@ -492,10 +496,10 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
if (!PathsEqual(root, parameters.WorkspaceRoot) && await InsideALibraryAsync(root, ct).ConfigureAwait(false))
throw new InvalidOperationException(Loc.L("Errors.Workspace.InsideLibrary"));
parameters = parameters with { WorkspaceRoot = root };
var store = new WorkspaceStore(root);
var store = WorkspaceStore.ForJob(root);
var project = store.Find(parameters.ProjectId) ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.NotCheckedOut"));
if (project.ReadBundle().Workspace is not { Home: { Kind: BundleHome.ServerKind } home } || home.SequenceId != sequenceId
|| !string.Equals(home.Server, server, StringComparison.OrdinalIgnoreCase))
|| !(options?.IsSameServer(home.Server, server) ?? string.Equals(home.Server, server, StringComparison.OrdinalIgnoreCase)))
throw new InvalidOperationException(Loc.L("Errors.Workspace.OtherHome"));
try
{
@@ -545,6 +549,8 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
EnsureSpace(parameters.WorkspaceRoot, remaining[0] + originalsLeft[0]);
var clock = System.Diagnostics.Stopwatch.StartNew();
var options = new ParallelOptions { MaxDegreeOfParallelism = Downloads, CancellationToken = ct };
var resumed = Math.Min(copied, frames.Count);
await context.ReportAsync(resumed, total, new JobDetail(JobStep.CopyProxy, sequence.DisplayTitle, resumed, frames.Count)).ConfigureAwait(false);
for (var start = 0; start < frames.Count; start += DownloadBatch)
{
var end = Math.Min(frames.Count, start + DownloadBatch);
@@ -561,13 +567,14 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
}
else
{
await context.ReportAsync(end, total).ConfigureAwait(false);
await context.ReportItemAsync(end, total, end, frames.Count).ConfigureAwait(false);
}
}
foreach (var entry in entries.Where(e => e.File is not null)) fetched[entry.Id] = entry;
var originals = new OriginalsTally();
if (withOriginals)
{
await context.ReportAsync(frames.Count, total, new JobDetail(JobStep.CopyOriginal, sequence.DisplayTitle, 0, frames.Count)).ConfigureAwait(false);
for (var start = 0; start < frames.Count; start += DownloadBatch)
{
var end = Math.Min(frames.Count, start + DownloadBatch);
@@ -582,7 +589,7 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
}
else
{
await context.ReportAsync(frames.Count + end, total).ConfigureAwait(false);
await context.ReportItemAsync(frames.Count + end, total, end, frames.Count).ConfigureAwait(false);
}
}
}
@@ -0,0 +1,293 @@
using System.Text.Json;
using Chronos.Core.Diagnostics;
using Chronos.Core.IO;
using Chronos.Core.Project;
using Chronos.Data.Sync;
namespace Chronos.Data.Workspaces;
/// <summary>
/// What a server home's folder says about its server (<see cref="HomeFolders.ServerFile"/>, owner review 5, #66): the key it
/// is named by, the id, the name and last address it was reached at, its TLS fingerprint and when it was first and last seen.
/// Someone browsing the folders, or a workspace moved to another PC, knows from it what the folder is.
/// </summary>
public sealed record ServerHomeRecord(string Format, int FormatVersion, string Key, Guid? Id, string? Name, string? Address, string? TlsFingerprint,
DateTimeOffset FirstSeen, DateTimeOffset LastSeen)
{
public const string FormatName = "chronos-server-home";
public const int CurrentFormatVersion = 1;
/// <summary>A record of <paramref name="key"/> first and last seen <paramref name="now"/>.</summary>
public static ServerHomeRecord Of(string key, string? name, string? address, string? tlsFingerprint, DateTimeOffset now) =>
new(FormatName, CurrentFormatVersion, key, ServerIdentity.IdOf(key), name, address, tlsFingerprint, now, now);
/// <summary>
/// Reads the record in <paramref name="folder"/>; null when there is none or it cannot be read (it only describes the
/// folder: nothing depends on it).
/// </summary>
public static ServerHomeRecord? Read(string folder)
{
ArgumentException.ThrowIfNullOrWhiteSpace(folder);
var path = Path.Combine(folder, HomeFolders.ServerFile);
if (!File.Exists(path)) return null;
try
{
return JsonSerializer.Deserialize(AtomicFile.ReadAllText(path), WorkspaceJson.Default.ServerHomeRecord) is { Format: FormatName, Key.Length: > 0 } record
? record
: null;
}
catch (Exception ex) when (ex is JsonException or IOException or UnauthorizedAccessException)
{
Log.Debug("workspace", $"{path} cannot be read: {ex.Message}");
return null;
}
}
/// <summary>Writes <paramref name="record"/> into <paramref name="folder"/> (made when it is not there), whole and atomically.</summary>
public static void Write(string folder, ServerHomeRecord record)
{
ArgumentException.ThrowIfNullOrWhiteSpace(folder);
ArgumentNullException.ThrowIfNull(record);
Directory.CreateDirectory(folder);
AtomicFile.WriteAllText(Path.Combine(folder, HomeFolders.ServerFile), JsonSerializer.Serialize(record, WorkspaceJson.Default.ServerHomeRecord));
}
}
/// <summary>One home of a workspace (<see cref="WorkspaceHomes.Homes"/>): its folder and store, and for a server home its key and record.</summary>
/// <param name="Folder">The folder's name: <see cref="HomeFolders.Local"/>, a server folder (<see cref="HomeFolders.ServerFolder"/>), or empty for the projects still in the parent.</param>
/// <param name="ServerKey">The key the server home's folder is named by (from its record, else made from the folder's name); null for this PC's home.</param>
public sealed record WorkspaceHome(string Folder, WorkspaceStore Store, string? ServerKey, ServerHomeRecord? Server)
{
/// <summary>This PC's archive's home.</summary>
public bool IsLocal => Folder == HomeFolders.Local;
/// <summary>Projects 1.0 kept in the parent itself, not moved into their home yet (in use when the folders were sorted).</summary>
public bool IsLegacy => Folder.Length == 0;
}
/// <summary>
/// The desktop's workspace split into one store per home (owner review 5, #66): Settings › Workspace › Folder is the
/// parent, whose <c>workspace.json</c> is of format <see cref="ParentFormatVersion"/> with layout <see cref="HomesLayout"/>
/// (a 1.0 Chronos refuses it as newer, so it never shows the workspace empty), and each home keeps its projects in its own
/// <see cref="WorkspaceStore"/>: <c>&lt;parent&gt;\local</c> for this PC's archive and <c>&lt;parent&gt;\servers\&lt;folder&gt;</c>
/// for each server (<see cref="ServerIdentity.FolderName"/>, with a <see cref="ServerHomeRecord"/> beside its manifest). Each
/// is an ordinary store with its own manifest and <c>.trash</c>, so sync, copies, renders, sends, leases and the trash work
/// on it unchanged. A parent still in the 1.0 layout (projects side by side in it, a manifest of format 1) is used as it is,
/// one store for every home, until <see cref="Chronos.Data.Homes.HomeLayoutMigration"/> sorts it; projects that sorting could not move
/// (in use then) are still listed where they are. A new, empty parent is made split.
/// </summary>
public sealed class WorkspaceHomes
{
/// <summary>The format of a split parent's manifest (a store's is <see cref="WorkspaceStore.FormatVersion"/>).</summary>
public const int ParentFormatVersion = 2;
/// <summary>The layout a split parent's manifest names.</summary>
public const string HomesLayout = "homes";
private readonly Lock _gate = new();
private readonly Dictionary<string, WorkspaceStore> _stores = new(StringComparer.OrdinalIgnoreCase);
private readonly WorkspaceStore? _legacy;
private ServerIdentity _identity;
/// <summary>Opens the workspace at <paramref name="root"/>, making the folder, and a split manifest when it holds none and no project.</summary>
/// <param name="identity">The servers known by id and their aliases (<see cref="Identity"/>).</param>
/// <exception cref="InvalidDataException">The folder's <c>workspace.json</c> is not a workspace manifest, or a newer one.</exception>
/// <exception cref="IOException">The folder cannot be made or read.</exception>
public WorkspaceHomes(string root, ServerIdentity? identity = null)
{
ArgumentException.ThrowIfNullOrWhiteSpace(root);
_identity = identity ?? ServerIdentity.None;
Root = WorkspaceStore.FullRoot(root);
Directory.CreateDirectory(Root);
var path = Path.Combine(Root, WorkspaceStore.ManifestFile);
var manifest = File.Exists(path) ? WorkspaceStore.TryRead(path) : null;
if (manifest is { Layout: HomesLayout })
{
if (manifest.FormatVersion > ParentFormatVersion)
throw new InvalidDataException($"{path} uses format {manifest.FormatVersion}, newer than this Chronos ({ParentFormatVersion}).");
Manifest = manifest;
}
else if (manifest is null && !File.Exists(path) && !HoldsProjects(Root))
{
Manifest = CreateSplit(path);
}
else
{
// The 1.0 layout (or a manifest the store makes again or refuses): one store for every home until it is sorted.
_legacy = new WorkspaceStore(Root);
Manifest = _legacy.Manifest;
}
if (IsSplit && HoldsProjects(Root)) _legacy = new WorkspaceStore(Root, Manifest);
}
/// <summary>The parent folder (Settings › Workspace › Folder).</summary>
public string Root { get; }
/// <summary>The parent's manifest: a split one, or the 1.0 store's.</summary>
public WorkspaceManifest Manifest { get; }
/// <summary>The parent is split into homes; false while it is in the 1.0 layout (one store for every home).</summary>
public bool IsSplit => Manifest.Layout == HomesLayout;
/// <summary>The servers known by id and their aliases: a server home's folder is named by its canonical key.</summary>
public ServerIdentity Identity
{
get => Volatile.Read(ref _identity);
set => Volatile.Write(ref _identity, value ?? ServerIdentity.None);
}
/// <summary>The folder of this PC's home (whether it exists or not).</summary>
public string LocalRoot => HomeFolders.LocalPath(Root);
/// <summary>The folder of the home of server <paramref name="serverKey"/> (whether it exists or not).</summary>
public string ServerRoot(string serverKey)
{
ArgumentException.ThrowIfNullOrWhiteSpace(serverKey);
return Path.Combine(Root, HomeFolders.Servers, Identity.FolderName(serverKey));
}
/// <summary>The folder of the store a project of <paramref name="home"/> belongs in: its home's, or the parent while it is not split.</summary>
public string RootFor(BundleHome home)
{
ArgumentNullException.ThrowIfNull(home);
if (!IsSplit) return Root;
return home.Kind == BundleHome.ServerKind && home.Server is { Length: > 0 } server ? ServerRoot(server) : LocalRoot;
}
/// <summary>
/// The store a project of <paramref name="home"/> is checked out into: its home's (made with its manifest, and for a server
/// its <see cref="ServerHomeRecord"/>, when it is not there), or the parent's while it is not split.
/// </summary>
public WorkspaceStore StoreFor(BundleHome home)
{
ArgumentNullException.ThrowIfNull(home);
if (!IsSplit) return _legacy!;
var root = RootFor(home);
var store = Open(root);
if (home.Kind == BundleHome.ServerKind && home.Server is { Length: > 0 } server && ServerHomeRecord.Read(root) is null)
ServerHomeRecord.Write(root, ServerHomeRecord.Of(Identity.Canonical(server), null, ServerIdentity.IsIdKey(server) ? null : server, null, DateTimeOffset.UtcNow));
return store;
}
/// <summary>The store of this PC's home (the parent's while it is not split).</summary>
public WorkspaceStore Local => StoreFor(new BundleHome(BundleHome.ArchiveKind, Guid.Empty));
/// <summary>
/// Records what is known of server <paramref name="serverKey"/> in its home's <see cref="ServerHomeRecord"/> (made with the
/// home when it is not there; its first sighting kept); nothing while the parent is not split.
/// </summary>
public ServerHomeRecord? Describe(string serverKey, string? name, string? address, string? tlsFingerprint, DateTimeOffset now)
{
ArgumentException.ThrowIfNullOrWhiteSpace(serverKey);
if (!IsSplit) return null;
var root = ServerRoot(serverKey);
Open(root);
var key = Identity.Canonical(serverKey);
var known = ServerHomeRecord.Read(root);
var record = ServerHomeRecord.Of(key, name ?? known?.Name, address ?? known?.Address, tlsFingerprint ?? known?.TlsFingerprint, now) with
{
FirstSeen = known?.FirstSeen ?? now,
};
ServerHomeRecord.Write(root, record);
return record;
}
/// <summary>
/// Every home that has a folder (this PC's first, then the servers' by folder name), and the parent's projects not moved
/// into their home yet; while the parent is not split, the parent's store (and any home a sorting stopped half-way made).
/// </summary>
public IReadOnlyList<WorkspaceHome> Homes()
{
var homes = new List<WorkspaceHome>();
if (_legacy is not null) homes.Add(new WorkspaceHome(string.Empty, _legacy, null, null));
if (Directory.Exists(LocalRoot) && HasManifest(LocalRoot)) homes.Add(new WorkspaceHome(HomeFolders.Local, Open(LocalRoot), null, null));
var servers = Path.Combine(Root, HomeFolders.Servers);
if (Directory.Exists(servers))
{
foreach (var folder in Directory.EnumerateDirectories(servers).Order(StringComparer.Ordinal))
{
var name = Path.GetFileName(folder);
if (!HomeFolders.IsServerFolder(name) || !HasManifest(folder)) continue;
var record = ServerHomeRecord.Read(folder);
var key = record?.Key ?? (HomeFolders.IdOfFolder(name) is { } id ? ServerIdentity.IdKey(id) : null);
homes.Add(new WorkspaceHome(name, Open(folder), key, record));
}
}
return homes;
}
/// <summary>The stores of every home that has a folder (<see cref="Homes"/>).</summary>
public IReadOnlyList<WorkspaceStore> All() => [.. Homes().Select(h => h.Store)];
/// <summary>The projects of every home.</summary>
public IReadOnlyList<WorkspaceProject> List() => [.. All().SelectMany(s => s.List())];
/// <summary>The working states of every home's projects (one unreadable is left out, and logged).</summary>
public IReadOnlyList<(WorkspaceProject Project, ProjectBundle Bundle)> ListBundles() => [.. All().SelectMany(s => s.ListBundles())];
/// <summary>The project's folder in whichever home holds it; null when none does.</summary>
public WorkspaceProject? Find(Guid projectId)
{
foreach (var store in All())
{
if (store.Find(projectId) is { } project) return project;
}
return null;
}
/// <summary>The trash of every home (<see cref="WorkspaceTrash.List"/>), each listed with its own fingerprint.</summary>
/// <exception cref="IOException">A trash cannot be read.</exception>
public IReadOnlyList<WorkspaceTrashListing> Trash(CancellationToken cancellationToken = default) =>
[.. All().Select(s => WorkspaceTrash.List(s.Root, cancellationToken))];
/// <summary>The folder at <paramref name="root"/> is the parent of a workspace split into homes (its manifest says so).</summary>
public static bool IsSplitRoot(string root)
{
ArgumentException.ThrowIfNullOrWhiteSpace(root);
var path = Path.Combine(WorkspaceStore.FullRoot(root), WorkspaceStore.ManifestFile);
return File.Exists(path) && WorkspaceStore.TryRead(path) is { Layout: HomesLayout };
}
/// <summary>The folder holds a folder named by a project id with a working state (the 1.0 layout).</summary>
internal static bool HoldsProjects(string root) =>
Directory.Exists(root) && Directory.EnumerateDirectories(root).Any(d => Guid.TryParseExact(Path.GetFileName(d), "N", out _)
&& File.Exists(Path.Combine(d, WorkspaceProject.BundleFile)));
/// <summary>The manifest of a split parent, made now; another program's made at the same moment wins.</summary>
internal static WorkspaceManifest CreateSplit(string path)
{
var manifest = new WorkspaceManifest(WorkspaceStore.FormatName, ParentFormatVersion, Guid.CreateVersion7(), DateTimeOffset.UtcNow, HomesLayout);
var temp = AtomicFile.TempPathFor(path);
try
{
File.WriteAllText(temp, JsonSerializer.Serialize(manifest, WorkspaceJson.Default.WorkspaceManifest));
try
{
File.Move(temp, path, overwrite: false);
}
catch (IOException) when (File.Exists(path))
{
if (WorkspaceStore.TryRead(path) is { Layout: HomesLayout } first) return first;
throw new InvalidDataException($"{path} was made by another program meanwhile and is not a workspace of homes.");
}
}
finally
{
if (File.Exists(temp)) File.Delete(temp);
}
return manifest;
}
private static bool HasManifest(string folder) => File.Exists(Path.Combine(folder, WorkspaceStore.ManifestFile));
private WorkspaceStore Open(string root)
{
lock (_gate)
{
if (_stores.TryGetValue(root, out var store)) return store;
store = new WorkspaceStore(root);
_stores[root] = store;
return store;
}
}
}
+64 -9
View File
@@ -1,5 +1,6 @@
using System.Text.Json;
using Chronos.Core.Diagnostics;
using Chronos.Core.IO;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Core.Project;
@@ -30,10 +31,21 @@ namespace Chronos.Data.Workspaces;
/// <param name="CurrentRoot">
/// The workspace folder in use now (the desktop's Settings › Workspace): a copy, render or send queued before the workspace
/// moved finds its project there when the folder its parameters name no longer holds it; null where only that one counts.
/// A workspace split into homes (owner review 5, <see cref="WorkspaceHomes"/>) is searched in every home, so a job queued
/// before its folders were sorted, or before its server's home was re-keyed, finds its project in the home it went to.
/// </param>
/// <param name="SameServer">
/// Whether two server keys name one server (<see cref="ServerIdentity.Same"/> over the desktop's alias table, owner review
/// 5): a job queued under an address key the server was reached under works on a project whose home names its id key. Null
/// compares the keys ignoring case (<see cref="IsSameServer"/>).
/// </param>
public sealed record WorkspaceJobOptions(ProcessingOptions Processing, Func<bool> HoldTransfers, Func<string, string, Task>? Moved = null,
Func<string?>? CurrentRoot = null)
Func<string?>? CurrentRoot = null, Func<string?, string?, bool>? SameServer = null)
{
/// <summary>Both keys name one server (<see cref="SameServer"/>, else equal ignoring case; null equals only null).</summary>
public bool IsSameServer(string? a, string? b) =>
SameServer is { } same ? same(a, b) : string.Equals(a, b, StringComparison.OrdinalIgnoreCase);
/// <summary>The options of an engine without a processing service of its own: the archive engine's decoders, nothing held, no move followed.</summary>
public static WorkspaceJobOptions For(ArchiveService archives)
{
@@ -204,10 +216,16 @@ internal static class WorkspaceJobRoots
/// resource key) when no folder holds the project.
/// </summary>
/// <exception cref="InvalidOperationException">The workspace is being moved, or no folder holds the project.</exception>
/// <remarks>
/// A folder that is a home of a workspace split into homes (owner review 5: <c>&lt;parent&gt;\local</c>,
/// <c>&lt;parent&gt;\servers\&lt;folder&gt;</c>) is moved with its parent, whose journal counts for it; and a split parent is
/// searched in each of its homes (and in itself, for a project not moved into its home yet), so a job queued with the 1.0
/// folder, or with a server's home before it was re-keyed, finds the project in the home it went to.
/// </remarks>
public static string Resolve(string named, Guid projectId, Func<string?>? current, string missing)
{
ArgumentException.ThrowIfNullOrWhiteSpace(named);
var candidates = new List<string>();
var folders = new List<string>();
string? currentRoot = null;
try
{
@@ -220,23 +238,60 @@ internal static class WorkspaceJobRoots
foreach (var root in new[] { WorkspaceStore.FullRoot(named), currentRoot })
{
if (root is null) continue;
if (WorkspaceMover.HasJournal(root))
foreach (var folder in new[] { root, SplitParentOf(root) })
{
// Past its switch the move's target is the workspace, and what is left here is a source still to clear.
if (WorkspaceMover.ReadJournal(root) is not { Stage: WorkspaceMoveStage.Switched } journal) throw new InvalidOperationException(Loc.L("Errors.Workspace.Moving"));
candidates.Add(WorkspaceStore.FullRoot(journal.To));
continue;
if (folder is null) continue;
if (WorkspaceMover.HasJournal(folder))
{
// Past its switch the move's target is the workspace, and what is left here is a source still to clear.
if (WorkspaceMover.ReadJournal(folder) is not { Stage: WorkspaceMoveStage.Switched } journal)
throw new InvalidOperationException(Loc.L("Errors.Workspace.Moving"));
folders.Add(WorkspaceStore.FullRoot(journal.To));
continue;
}
folders.Add(folder);
}
candidates.Add(root);
}
var candidates = new List<string>();
foreach (var folder in folders.Distinct(StringComparer.OrdinalIgnoreCase))
{
candidates.Add(folder);
if (WorkspaceHomes.IsSplitRoot(folder)) candidates.AddRange(HomesOf(folder));
}
foreach (var root in candidates.Distinct(StringComparer.OrdinalIgnoreCase))
{
if (!File.Exists(Path.Combine(root, projectId.ToString("N"), WorkspaceProject.BundleFile))) continue;
if (WorkspaceMover.HasJournal(root)) throw new InvalidOperationException(Loc.L("Errors.Workspace.Moving"));
if (WorkspaceMover.HasJournal(root) || (SplitParentOf(root) is { } parent && WorkspaceMover.HasJournal(parent)))
throw new InvalidOperationException(Loc.L("Errors.Workspace.Moving"));
if (!string.Equals(root, WorkspaceStore.FullRoot(named), StringComparison.OrdinalIgnoreCase))
Log.Info("workspace", $"Project {projectId} is no longer in {named}: the job works in {root}, where the workspace is now");
return root;
}
throw new InvalidOperationException(Loc.L(missing));
}
/// <summary>The split parent <paramref name="root"/> is a home of (<c>local</c>, or a folder in <c>servers</c>); null for any other folder.</summary>
internal static string? SplitParentOf(string root)
{
var name = Path.GetFileName(root);
var up = Path.GetDirectoryName(root);
if (up is null) return null;
string? parent = name == HomeFolders.Local ? up
: HomeFolders.IsServerFolder(name) && Path.GetFileName(up) == HomeFolders.Servers ? Path.GetDirectoryName(up)
: null;
return parent is not null && WorkspaceHomes.IsSplitRoot(parent) ? WorkspaceStore.FullRoot(parent) : null;
}
/// <summary>The homes of the split parent <paramref name="parent"/> that have a folder: this PC's, then the servers'.</summary>
private static IEnumerable<string> HomesOf(string parent)
{
var local = HomeFolders.LocalPath(parent);
if (Directory.Exists(local)) yield return local;
var servers = Path.Combine(parent, HomeFolders.Servers);
if (!Directory.Exists(servers)) yield break;
foreach (var folder in Directory.EnumerateDirectories(servers).Order(StringComparer.Ordinal))
{
if (HomeFolders.IsServerFolder(Path.GetFileName(folder))) yield return folder;
}
}
}
@@ -38,12 +38,20 @@ internal sealed class WorkspaceMoveJobHandler(ChronosDatabase db, WorkspaceJobOp
var mover = Mover(parameters);
if (WorkspaceMover.ReadJournal(mover.From) is null && await db.Jobs.HasPendingAsync(WorkspaceKinds, cancellationToken: ct).ConfigureAwait(false))
throw new InvalidOperationException(Loc.L("Errors.Workspace.JobsRunning"));
// Entry n of m in the detail, as a copy across volumes or a move on one (its journal says which); no project title (the
// mover tells counts only).
JobStep? step = null;
ValueTask Progress(long done, long total, bool durable)
{
step ??= WorkspaceMover.ReadJournal(mover.From) is { SameVolume: false } ? JobStep.CopyOriginal : JobStep.Reorganize;
var detail = new JobDetail(step.Value, null, 0, 0, (int)Math.Min(done + 1, total), (int)total, JobPartUnit.Entry);
return durable
? context.CheckpointAsync(string.Create(CultureInfo.InvariantCulture, $"move:{done}"), done, total, detail)
: context.ReportAsync(done, total, detail);
}
try
{
await mover.MoveAsync(parameters.PlanFingerprint, options.Moved, (done, total, durable) => durable
? context.CheckpointAsync(string.Create(CultureInfo.InvariantCulture, $"move:{done}"), done, total)
: context.ReportAsync(done, total),
context.Log, ct).ConfigureAwait(false);
await mover.MoveAsync(parameters.PlanFingerprint, options.Moved, Progress, context.Log, ct).ConfigureAwait(false);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
+55 -5
View File
@@ -79,8 +79,9 @@ internal sealed partial class WorkspaceMoveJson : JsonSerializerContext;
/// <summary>
/// Moves a workspace folder with its projects (phase 5c, <see cref="Core.Model.JobKind.WorkspaceMove"/>). What moves: every
/// folder named by a project id, then <c>workspace.json</c>, then <c>.trash</c>, in that order; anything else in the folder
/// stays where it is. On one volume each entry is renamed into the target (<see cref="Directory.Move(string, string)"/>);
/// folder named by a project id, then the homes' folders of a workspace split into homes (<c>local</c>, <c>servers</c>,
/// owner review 5), then <c>workspace.json</c>, then <c>.trash</c>, in that order; anything else in the folder stays where it
/// is. On one volume each entry is renamed into the target (<see cref="Directory.Move(string, string)"/>);
/// across volumes each file is copied into a <c>.part</c> beside its place, hashed as it is read and read back to hash the
/// copy, and only once both SHA-256 agree renamed into place (with the source's time), so a file at its final name in the
/// target is always one verified. The source is the workspace until every entry is at the target: then the desktop is told
@@ -185,6 +186,21 @@ public sealed class WorkspaceMover
entries.Add(name);
(files, bytes) = (files + count, bytes + size);
}
// A workspace split into homes (owner review 5): each home's folder moves whole, its projects counted one by one.
foreach (var home in HomeEntries)
{
var folder = Path.Combine(From, home);
if (!Directory.Exists(folder)) continue;
var (count, size) = Read(folder, home, fingerprint);
foreach (var project in ProjectFolders(folder))
{
var listed = Listed(project);
projects.Add(new WorkspaceMoveFolder(Guid.ParseExact(Path.GetFileName(project), "N"), File.Exists(Path.Combine(project, WorkspaceProject.BundleFile)),
listed.Count, listed.Sum(f => new FileInfo(Path.Combine(project, f)).Length)));
}
entries.Add(home);
(files, bytes) = (files + count, bytes + size);
}
var manifest = Path.Combine(From, WorkspaceStore.ManifestFile);
if (File.Exists(manifest))
{
@@ -303,7 +319,7 @@ public sealed class WorkspaceMover
if (switched is not null) await switched(From, To).ConfigureAwait(false);
Cut?.Invoke("switched", To);
var left = Clean(journal);
log?.Invoke(Loc.L("Archive.Log.WorkspaceMoved", To, journal.Entries.Count(e => Guid.TryParseExact(e, "N", out _)), survey.Files));
log?.Invoke(Loc.L("Archive.Log.WorkspaceMoved", To, MovedProjects(journal), survey.Files));
if (left > 0) log?.Invoke(Loc.L("Archive.Log.WorkspaceMoveLeft", From, left));
Log.Info("workspace", $"The workspace moved from {From} to {To}" + (left > 0 ? $"; {left} entr(ies) of the old folder stay there" : string.Empty));
return (survey, resumed);
@@ -383,7 +399,7 @@ public sealed class WorkspaceMover
Cut?.Invoke("rename", source);
Rename(source, target);
Cut?.Invoke("moved", target);
if (Guid.TryParseExact(entry, "N", out _)) log?.Invoke(Loc.L("Archive.Log.WorkspaceMoveProject", Name(target)));
LogProjects(entry, target, log);
}
else if (!Exists(target))
{
@@ -435,7 +451,7 @@ public sealed class WorkspaceMover
if (durable) clock.Restart();
if (progress is not null) await progress(done, total, durable).ConfigureAwait(false);
}
if (Guid.TryParseExact(entry, "N", out _)) log?.Invoke(Loc.L("Archive.Log.WorkspaceMoveProject", Name(source)));
LogProjects(entry, source, log);
}
journal = journal with { Verified = [.. journal.Verified, entry] };
WriteJournal(journal);
@@ -600,6 +616,40 @@ public sealed class WorkspaceMover
private const string TrashFolder = ".trash";
/// <summary>The homes' folders of a workspace split into homes, which move whole after the 1.0 project folders and before the manifest.</summary>
private static readonly string[] HomeEntries = [HomeFolders.Local, HomeFolders.Servers];
/// <summary>The project folders in a home's folder: <c>local</c>'s own, or those of every server home in <c>servers</c>.</summary>
private static IEnumerable<string> ProjectFolders(string home)
{
var homes = Path.GetFileName(home) == HomeFolders.Servers
? Directory.EnumerateDirectories(home).Where(d => HomeFolders.IsServerFolder(Path.GetFileName(d))).Order(StringComparer.Ordinal)
: Enumerable.Repeat(home, 1);
foreach (var store in homes)
{
foreach (var folder in Directory.EnumerateDirectories(store).Order(StringComparer.Ordinal))
{
if (Guid.TryParseExact(Path.GetFileName(folder), "N", out _)) yield return folder;
}
}
}
/// <summary>The projects a move took: its 1.0 project folders, and the project folders of the homes it moved (now at the target).</summary>
private int MovedProjects(WorkspaceMoveJournal journal) =>
journal.Entries.Count(e => Guid.TryParseExact(e, "N", out _))
+ journal.Entries.Where(e => HomeEntries.Contains(e) && Directory.Exists(Path.Combine(To, e))).Sum(e => ProjectFolders(Path.Combine(To, e)).Count());
/// <summary>The log line of each project an entry moved: the entry's own, or those of a home's folder.</summary>
private static void LogProjects(string entry, string folder, Action<string>? log)
{
if (log is null) return;
if (Guid.TryParseExact(entry, "N", out _)) log(Loc.L("Archive.Log.WorkspaceMoveProject", Name(folder)));
else if (HomeEntries.Contains(entry) && Directory.Exists(folder))
{
foreach (var project in ProjectFolders(folder)) log(Loc.L("Archive.Log.WorkspaceMoveProject", Name(project)));
}
}
private static readonly TimeSpan CheckpointEvery = TimeSpan.FromSeconds(5);
private static StringComparison PathComparison => OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
@@ -50,7 +50,7 @@ internal sealed class WorkspaceRenderJobHandler(ChronosDatabase db, ArchiveServi
// made where it is not.
var root = WorkspaceJobRoots.Resolve(parameters.WorkspaceRoot, parameters.ProjectId, options.CurrentRoot, "Errors.Workspace.RenderProject");
parameters = parameters with { WorkspaceRoot = root };
var store = new WorkspaceStore(root);
var store = WorkspaceStore.ForJob(root);
var project = store.Find(parameters.ProjectId) ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.RenderProject"));
var inputs = await ReadInputsAsync(project, parameters, render, cancellation).ConfigureAwait(false);
var (sequence, frames, settings, versionId) = (inputs.Sequence, inputs.Frames, inputs.Settings, inputs.VersionId);
@@ -93,6 +93,8 @@ internal sealed class WorkspaceRenderJobHandler(ChronosDatabase db, ArchiveServi
{
string? writing = null;
IReadOnlyList<TargetOutcome> outcomes;
var reporter = new RenderJobHandler.RenderReporter(context, sequence.DisplayTitle, targets);
await reporter.PreparingAsync().ConfigureAwait(false);
try
{
await using var renderer = new SequenceRender(frames, projectSettings, inputs.Analysis, options.Processing, originals: true, companions: null);
@@ -101,12 +103,13 @@ internal sealed class WorkspaceRenderJobHandler(ChronosDatabase db, ArchiveServi
ValueTask Starting(int index, string path)
{
writing = path;
return context.CheckpointAsync(new RenderCheckpoint(done, index, path).ToJson(), context.Done, context.Total);
return reporter.StartingAsync(index, new RenderCheckpoint(done, index, path).ToJson());
}
async ValueTask Record(int index, ExportTarget target, RenderResult result)
{
// The output is complete: recorded even if the job is being cancelled, so it is never orphaned.
var id = RenderJobHandler.RenderId(context.Job.Id, index);
await reporter.VerifyingAsync(Path.GetFileName(result.OutputPath)).ConfigureAwait(false);
var entry = await EntryAsync(project, context.Job.Id, id, target, result, versionId, settingsHash).ConfigureAwait(false);
if (render.IsStill) entry = entry with { Fps = 0, DurationSeconds = 0, Kind = RenderKind.Still };
using (await project.LockAsync(LockWait, CancellationToken.None).ConfigureAwait(false))
@@ -135,7 +138,7 @@ internal sealed class WorkspaceRenderJobHandler(ChronosDatabase db, ArchiveServi
setup => names.Reserve(taken => SequenceRender.StillPath(projectSettings.Export, exports, SequenceRender.NameContext(sequence, target, setup), still,
name => Taken(taken, name))),
path => Starting(index, path),
context.ReportAsync,
reporter.ReportAsync,
cancellation).ConfigureAwait(false);
await Record(index, target, result).ConfigureAwait(false);
outcomes = [new TargetOutcome(target, result, null)];
@@ -154,7 +157,7 @@ internal sealed class WorkspaceRenderJobHandler(ChronosDatabase db, ArchiveServi
name => Taken(taken, name))),
Starting,
Record,
context.ReportAsync,
reporter.ReportAsync,
cancellation).ConfigureAwait(false);
}
if (renderer.StandIns.Count > 0)
@@ -53,13 +53,13 @@ internal sealed class WorkspaceSendJobHandler(ChronosDatabase db, ArchiveService
// Where the project is now (a workspace moved since the job was queued), never while it is being moved; nothing is
// made where it is not.
var root = WorkspaceJobRoots.Resolve(parameters.WorkspaceRoot, parameters.ProjectId, options.CurrentRoot, "Errors.Workspace.NotCheckedOut");
var project = new WorkspaceStore(root).Find(parameters.ProjectId)
var project = WorkspaceStore.ForJob(root).Find(parameters.ProjectId)
?? throw new InvalidOperationException(Loc.L("Errors.Workspace.NotCheckedOut"));
var workspace = project.ReadBundle().Workspace ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.NotCheckedOut"));
var home = workspace.Home;
if (home.SequenceId != sequenceId || (server is null
? !home.IsArchive
: home.Kind != BundleHome.ServerKind || !string.Equals(home.Server, server, StringComparison.OrdinalIgnoreCase)))
: home.Kind != BundleHome.ServerKind || !options.IsSameServer(home.Server, server)))
throw new InvalidOperationException(Loc.L("Errors.Workspace.OtherHome"));
var entries = await ToSendAsync(project, parameters.RenderIds, ct).ConfigureAwait(false);
@@ -115,6 +115,8 @@ internal sealed class WorkspaceSendJobHandler(ChronosDatabase db, ArchiveService
ct.ThrowIfCancellationRequested();
if (held) await WhileHeldAsync(context).ConfigureAwait(false);
var entry = entries[i];
await context.ReportAsync(i, entries.Count, new JobDetail(JobStep.Upload, Path.GetFileName(entry.File), 0, 0, i + 1, entries.Count, JobPartUnit.Render))
.ConfigureAwait(false);
var file = project.ContainedExportFile(entry.File);
var refusal = file is null ? Loc.L("Errors.Workspace.SendMissing", Path.GetFileName(entry.File ?? string.Empty)) : await CheckAsync(entry, file, folders: !held, ct).ConfigureAwait(false);
if (refusal is null)
+55 -2
View File
@@ -11,7 +11,12 @@ using Chronos.Data.Sync;
namespace Chronos.Data.Workspaces;
/// <summary>The manifest of a workspace folder (<c>workspace.json</c>).</summary>
public sealed record WorkspaceManifest(string Format, int FormatVersion, Guid Id, DateTimeOffset Created);
/// <param name="Layout">
/// <see cref="WorkspaceHomes.HomesLayout"/> in the manifest of format <see cref="WorkspaceHomes.ParentFormatVersion"/> of a
/// workspace split into homes (owner review 5, #66): the folder holds no projects of its own but one store per home; null
/// for a store.
/// </param>
public sealed record WorkspaceManifest(string Format, int FormatVersion, Guid Id, DateTimeOffset Created, string? Layout = null);
/// <summary>
/// A workspace's sync state of one project besides its versions (<c>sync.json</c>): a restore to record, a conflict to
@@ -133,13 +138,15 @@ public sealed record WorkspaceMoveParameters(string From, string To, string Plan
[JsonSerializable(typeof(WorkspaceSendParameters))]
[JsonSerializable(typeof(WorkspaceMoveParameters))]
[JsonSerializable(typeof(WorkspaceRenders))]
[JsonSerializable(typeof(ServerHomeRecord))]
internal sealed partial class WorkspaceJson : JsonSerializerContext;
/// <summary>
/// The workspace of this PC (owner review 2, R8): a folder on a fast disk where projects are edited, never inside the
/// library. <c>&lt;root&gt;/workspace.json</c> names it; each project has a folder named by its id
/// (<see cref="WorkspaceProject"/>). There is no index: the store lists the folders holding a <c>project.chronos</c>. The
/// desktop puts it in <c>%LOCALAPPDATA%\Chronos\workspace</c> unless set otherwise; the store takes any folder.
/// store takes any folder; the desktop's workspace (<c>%LOCALAPPDATA%\Chronos\workspace</c> unless set otherwise) is split
/// into one store per home since owner review 5 (<see cref="WorkspaceHomes"/>: <c>local</c>, <c>servers\&lt;id&gt;</c>).
/// </summary>
public sealed class WorkspaceStore
{
@@ -168,6 +175,31 @@ public sealed class WorkspaceStore
Manifest = Read(path);
}
/// <summary>
/// The projects a workspace split into homes still holds where 1.0 kept them (<c>&lt;parent&gt;/&lt;project id&gt;</c>: one in
/// use when the folders were sorted, owner review 5): a view of the parent with its manifest, which makes and writes nothing
/// (<see cref="WorkspaceHomes"/>).
/// </summary>
internal WorkspaceStore(string root, WorkspaceManifest manifest)
{
Root = FullRoot(root);
Manifest = manifest;
}
/// <summary>
/// The store a workspace job opens at <paramref name="root"/> (<see cref="WorkspaceJobRoots.Resolve"/>): a split parent,
/// which still holds a project its sorting into homes left in place (one in use then), opens as the view of those projects
/// (owner review 5); any other folder as <see cref="WorkspaceStore(string)"/> does.
/// </summary>
/// <exception cref="InvalidDataException">The folder's <c>workspace.json</c> is not a workspace manifest, or a newer one.</exception>
/// <exception cref="IOException">The folder cannot be made or read.</exception>
internal static WorkspaceStore ForJob(string root)
{
ArgumentException.ThrowIfNullOrWhiteSpace(root);
var full = FullRoot(root);
return TryRead(Path.Combine(full, ManifestFile)) is { Layout: WorkspaceHomes.HomesLayout } manifest ? new WorkspaceStore(full, manifest) : new WorkspaceStore(full);
}
public string Root { get; }
public WorkspaceManifest Manifest { get; }
@@ -319,6 +351,25 @@ public sealed class WorkspaceStore
}
}
/// <summary>The manifest of the folder <paramref name="path"/> names, of any format (a store's or a parent's of homes); null when it is not one.</summary>
internal static WorkspaceManifest? TryRead(string path)
{
try
{
return JsonSerializer.Deserialize(AtomicFile.ReadAllText(path), WorkspaceJson.Default.WorkspaceManifest) is { Format: FormatName } manifest && manifest.Id != Guid.Empty
? manifest
: null;
}
catch (Exception ex) when (ex is JsonException or IOException or UnauthorizedAccessException)
{
return null;
}
}
/// <summary>Writes <paramref name="manifest"/> whole to a temporary file and moves it into place.</summary>
internal static void Write(string path, WorkspaceManifest manifest) =>
AtomicFile.WriteAllText(path, JsonSerializer.Serialize(manifest, WorkspaceJson.Default.WorkspaceManifest));
private static WorkspaceManifest Read(string path)
{
WorkspaceManifest? manifest;
@@ -331,6 +382,8 @@ public sealed class WorkspaceStore
throw new InvalidDataException($"{path} is not a workspace manifest.", ex);
}
if (manifest is null || manifest.Format != FormatName || manifest.Id == Guid.Empty) throw new InvalidDataException($"{path} is not a workspace manifest.");
if (manifest.Layout == WorkspaceHomes.HomesLayout)
throw new InvalidDataException($"{path} is the manifest of a workspace split into homes, whose projects are in its home folders ({HomeFolders.Local}, {HomeFolders.Servers}).");
if (manifest.FormatVersion > FormatVersion)
throw new InvalidDataException($"{path} uses format {manifest.FormatVersion}, newer than this Chronos ({FormatVersion}).");
return manifest;