Files
Chronos/src/Chronos.App/Workspace/WorkspaceHost.Cleanup.cs
T
Alby96 20d64af4a4
CI / Build and test (push) Successful in 11m36s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 12s
Release / Build, test and publish (push) Successful in 14m56s
CI / PostgreSQL 16 integration (push) Successful in 10m11s
CI / PostgreSQL 17 integration (push) Successful in 10m15s
CI / Docker image (push) Successful in 45s
Release / Server image (push) Successful in 4m55s
feat: single setup, mass-pass progress, per-server homes, Desktop shares its library (owner review 5, v1.1.0) (#68)
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
2026-11-15 18:11:43 +01:00

373 lines
21 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using System.IO;
using Chronos.Core.Diagnostics;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Data;
using Chronos.Data.Archiving;
using Chronos.Data.Projects;
using Chronos.Data.Workspaces;
namespace Chronos.App.Workspace;
/// <summary>
/// What the workspace takes of its drive (Settings › Workspace, phase 5c): the projects' media (proxies and originals, which
/// the clean-up and the size limit may remove), the whole folder, and its trash (<see cref="WorkspaceHost.TrashFolder"/>).
/// </summary>
/// <param name="MediaBytes">The bytes of every project's <c>media/</c> folder.</param>
/// <param name="TotalBytes">The bytes of the whole workspace folder, its trash included.</param>
/// <param name="TrashBytes">The bytes of the workspace's trash.</param>
/// <param name="TrashItems">The entries of the workspace's trash (one per copy removed or originals set aside).</param>
public sealed record WorkspaceUsage(long MediaBytes, long TotalBytes, long TrashBytes, int TrashItems)
{
/// <summary>A workspace that holds nothing.</summary>
public static WorkspaceUsage None { get; } = new(0, 0, 0, 0);
}
/// <summary>One entry of the workspace's trash as Empty… lists it: its folder name, size, files and when it was put there.</summary>
public sealed record WorkspaceTrashItem(string Name, long Bytes, int Files, DateTimeOffset Trashed);
/// <summary>
/// The plan of Settings › Workspace › Trash › Empty… (phase 5c): every entry of the workspace's trash and their bytes, and a
/// fingerprint over their names, sizes and times. Emptying erases exactly these entries, and only while the fingerprint
/// still matches (the user types the number of entries to confirm); schedules and the size limit never empty the trash.
/// </summary>
public sealed record WorkspaceTrashPlan(string Folder, IReadOnlyList<WorkspaceTrashItem> Items, long Bytes, string Fingerprint)
{
/// <summary>Nothing to erase: Empty… is disabled.</summary>
public bool IsEmpty => Items.Count == 0;
/// <summary>
/// The trash of each home the plan lists (owner review 5, separate homes: <c>local\</c> and each <c>servers\&lt;id&gt;\</c>
/// keep their own), each with its own fingerprint; emptying erases each one only while all of them still match.
/// </summary>
public IReadOnlyList<WorkspaceTrashListing> Listings { get; init; } = [];
}
/// <summary>What emptying the workspace's trash erased.</summary>
public sealed record WorkspaceTrashResult(int Items, long Bytes);
/// <summary>
/// The workspace's clean-up side (phase 5c, P4): its usage, the clean-up of media (<see cref="WorkspaceCleaner"/>: Settings ›
/// Workspace › Clean after and Size limit, run by the <c>workspace-cleanup</c> schedule and, for the size limit, after each
/// copy of media), the plan and emptying of its trash, and what a local copy's removal keeps of renders not sent and
/// originals the home lost.
/// </summary>
public sealed partial class WorkspaceHost
{
private readonly SemaphoreSlim _cleaning = new(1, 1);
/// <summary>
/// The workspace project open in Edit now (any thread may ask); null when none is. The clean-up never removes the media of
/// that one. Unset (as in a test without the shell), no project counts as open.
/// </summary>
public Func<Guid?>? OpenProject { get; set; }
/// <summary>What the workspace takes of its drive (<see cref="WorkspaceUsage"/>), measured off the UI thread; none while its folder is not made.</summary>
public Task<WorkspaceUsage> UsageAsync(CancellationToken cancellationToken = default) => Task.Run(() =>
{
var root = Root;
if (!Directory.Exists(root)) return WorkspaceUsage.None;
var media = 0L;
foreach (var store in Homes.All())
{
if (!Directory.Exists(store.Root)) continue;
foreach (var folder in Directory.EnumerateDirectories(store.Root).Where(d => Guid.TryParseExact(Path.GetFileName(d), "N", out _)))
{
cancellationToken.ThrowIfCancellationRequested();
media += WorkspaceCleaner.Measure(Path.Combine(folder, WorkspaceProject.MediaFolder)).Bytes;
}
}
var trash = Homes.Trash(cancellationToken);
return new WorkspaceUsage(media, WorkspaceCleaner.Measure(root).Bytes, trash.Sum(t => t.Bytes), trash.Sum(t => t.Entries.Count));
}, cancellationToken);
/// <summary>The plan of emptying the trash of every home of the workspace (<see cref="WorkspaceTrashPlan"/>); nothing is erased.</summary>
public Task<WorkspaceTrashPlan> PlanTrashAsync(CancellationToken cancellationToken = default) => Task.Run(() =>
{
var listings = Homes.Trash(cancellationToken);
var folder = listings.Count == 1 ? listings[0].Folder : WorkspaceTrash.FolderOf(Root);
var fingerprint = string.Join('|', listings.Select(l => l.Folder + "=" + l.Fingerprint));
return new WorkspaceTrashPlan(folder, [.. listings.SelectMany(l => l.Entries).Select(e => new WorkspaceTrashItem(e.Name, e.Bytes, e.Files, e.Trashed))
.OrderBy(i => i.Trashed)], listings.Sum(l => l.Bytes), fingerprint)
{ Listings = listings };
}, cancellationToken);
/// <summary>
/// Erases the entries of <paramref name="plan"/> from the workspace's trash for good, after the user typed their number;
/// refused when the trash changed since the plan (<c>Errors.Workspace.TrashChanged</c>).
/// </summary>
/// <exception cref="WorkspaceException">The trash changed since the plan; nothing was erased.</exception>
/// <exception cref="IOException">An entry could not be erased (a file held open); the others were.</exception>
public Task<WorkspaceTrashResult> EmptyTrashAsync(WorkspaceTrashPlan plan, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(plan);
return Task.Run(() =>
{
EnsureNotMoving();
IReadOnlyList<WorkspaceTrashListing> listings = plan.Listings.Count > 0 ? plan.Listings
: [new WorkspaceTrashListing(plan.Folder, [.. plan.Items.Select(i => new WorkspaceTrashEntry(i.Name, i.Bytes, i.Files, i.Trashed))], plan.Bytes,
plan.Fingerprint)];
try
{
// Every trash is checked before any is emptied: a plan is erased whole or not at all.
foreach (var listing in listings)
{
var now = WorkspaceTrash.List(Path.GetDirectoryName(listing.Folder)!, cancellationToken);
if (!string.Equals(now.Fingerprint, listing.Fingerprint, StringComparison.Ordinal))
throw new WorkspaceTrashChangedException($"The trash {now.Folder} changed after it was listed.");
}
var (items, bytes) = (0, 0L);
foreach (var listing in listings.Where(l => !l.IsEmpty))
{
var (erased, freed) = WorkspaceTrash.Empty(Path.GetDirectoryName(listing.Folder)!, listing, cancellationToken);
items += erased;
bytes += freed;
}
return new WorkspaceTrashResult(items, bytes);
}
catch (WorkspaceTrashChangedException ex)
{
Log.Info("workspace", $"The workspace's trash was not emptied: {ex.Message}");
throw new WorkspaceException(Loc.L("Errors.Workspace.TrashChanged"), ex);
}
}, cancellationToken);
}
/// <summary>
/// One run of the workspace's clean-up (<see cref="WorkspaceCleaner"/>): with <paramref name="cleanAfter"/>, Settings ›
/// Workspace › Clean after, then Size limit (the <c>workspace-clean</c> step of the schedule); without, the size limit
/// alone, never taking <paramref name="justCopied"/> (after a copy of its media). Null when the workspace cannot be
/// cleaned now (the index not open, the folder being moved); a folder not made yet has nothing to clean. One run at a time.
/// </summary>
public async Task<WorkspaceCleanReport?> CleanAsync(bool cleanAfter, Guid? justCopied = null, CancellationToken cancellationToken = default)
{
if (_db() is not { } db || IsMoving || !IsEnabled) return null;
var root = Root;
if (!Directory.Exists(root)) return WorkspaceCleanReport.Nothing;
var options = new WorkspaceCleanOptions
{
CleanAfterDays = cleanAfter ? Math.Max(0, _settings.WorkspaceCleanAfterDays) : 0,
SizeLimitBytes = Math.Max(0, _settings.WorkspaceSizeLimitGb) * (1L << 30),
Opened = OpenedSnapshot(),
IsOpen = id => OpenProject?.Invoke() == id,
JustCopied = justCopied,
};
if (options is { CleanAfterDays: 0, SizeLimitBytes: 0 }) return WorkspaceCleanReport.Nothing;
await _cleaning.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
var cleaner = new WorkspaceCleaner(AllSyncs(), db, HomeOf) { LockWait = LockWait };
return await Task.Run(() => cleaner.CleanAsync(options, cancellationToken), cancellationToken).ConfigureAwait(false);
}
finally
{
_cleaning.Release();
}
}
/// <summary>
/// A job of the engine changed (any thread): once a copy of media into this workspace is done, the size limit runs, never
/// taking the project just copied. Nothing else follows from a job here.
/// </summary>
public void OnJobChanged(Job job)
{
ArgumentNullException.ThrowIfNull(job);
if (job is not { Kind: JobKind.WorkspaceCopy, State: JobState.Done } || _settings.WorkspaceSizeLimitGb <= 0) return;
if (WorkspaceJobs.CopyParameters(job) is not { } copied || !IsUnderRoot(copied.WorkspaceRoot)) return;
_ = Task.Run(async () =>
{
try
{
if (await CleanAsync(cleanAfter: false, copied.ProjectId).ConfigureAwait(false) is { Cleaned.Count: > 0 } report)
Log.Info("workspace", $"Size limit after the copy of project {copied.ProjectId}: {report.Cleaned.Count} project(s) cleaned, {report.Freed} bytes freed");
}
catch (Exception ex) when (ex is not OutOfMemoryException)
{
Log.Warning("workspace", $"The size limit after the copy of project {copied.ProjectId} did not run", ex);
}
});
}
/// <summary>The folder is the workspace's own or one of its homes (<c>local\</c>, <c>servers\&lt;id&gt;\</c>).</summary>
private bool IsUnderRoot(string folder)
{
if (string.IsNullOrWhiteSpace(folder)) return false;
var root = Path.TrimEndingDirectorySeparator(Path.GetFullPath(Root));
var full = Path.TrimEndingDirectorySeparator(Path.GetFullPath(folder));
return string.Equals(full, root, StringComparison.OrdinalIgnoreCase)
|| full.StartsWith(root + Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase);
}
/// <summary>When each project was opened last, as Settings hold it now (written on the UI thread: copied until a copy reads whole).</summary>
private IReadOnlyDictionary<Guid, DateTimeOffset> OpenedSnapshot()
{
for (var attempt = 0; ; attempt++)
{
try
{
return new Dictionary<Guid, DateTimeOffset>(_settings.WorkspaceOpened);
}
catch (InvalidOperationException) when (attempt < 5)
{
// Written while it was copied: copied again.
}
}
}
/// <summary>
/// What a local copy's removal plan counts besides its versions (<see cref="WorkspaceRemovalPlan.UnsentRenders"/>,
/// <see cref="WorkspaceRemovalPlan.LastOriginals"/>): renders of its ledger not sent, and originals the home no longer
/// holds (all of them while the home cannot be asked: they go to the trash then, never deleted).
/// </summary>
private async Task<(int UnsentRenders, int LastOriginals)> RemovalMediaAsync(WorkspaceProject project, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(project);
var unsent = WorkspaceCleaner.UnsentRenders(project);
var last = await LastOriginalsOfAsync(project, cancellationToken).ConfigureAwait(false);
return (unsent, last);
}
/// <summary>
/// A removal without Discard is refused while the project's ledger holds renders not sent: Send first queues their
/// sending home (unless one is queued or running already), and the copy goes once they are there.
/// </summary>
/// <exception cref="WorkspaceException">Renders wait to be sent.</exception>
private async Task EnsureRendersSentAsync(WorkspaceProject project, BundleWorkspace workspace, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(project);
ArgumentNullException.ThrowIfNull(workspace);
if (WorkspaceCleaner.UnsentRenders(project) == 0) return;
try
{
await QueueRenderSendAsync(project, workspace, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is DatabaseException or InvalidOperationException or IOException or InvalidDataException)
{
Log.Warning("workspace", $"The renders of project {project.ProjectId} could not be queued to be sent", ex);
}
throw new WorkspaceException(WorkspaceText.ForHome("Errors.Workspace.UnsentRenders", !workspace.Home.IsArchive));
}
/// <summary>
/// Before a removed copy's media are deleted, under the project's lease: every file of its folder of originals, named by
/// the media index or not (an index that cannot be read names none), hashed now and deleted only when the home's frames,
/// read now, hold a photo of that SHA-256; every other one moved into <paramref name="aside"/> (the copy's trash entry),
/// never deleted, and every one when the home cannot be asked (<see cref="WorkspaceOriginals.ClearOriginals"/>).
/// </summary>
/// <returns>The originals that could be neither deleted nor moved: they stay in the folder of originals.</returns>
private async Task<int> ClearOriginalsAsync(WorkspaceProject project, string aside, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(project);
ArgumentException.ThrowIfNullOrEmpty(aside);
if (!WorkspaceCleaner.HoldsOriginalFiles(project)) return 0;
// The working state went into the entry first: the home is read from there.
BundleHome? home = null;
try
{
home = ProjectBundle.Load(Path.Combine(aside, WorkspaceProject.BundleFile)).Workspace?.Home;
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Info("workspace", $"The home of removed project {project.ProjectId} is not known: every original it holds is kept ({ex.Message})");
}
var held = home is null ? null : await HomeFramesAsync(project, home, cancellationToken).ConfigureAwait(false);
var (deleted, trashed, _, left) = WorkspaceOriginals.ClearOriginals(project, held?.Frames, aside);
Log.Info("workspace", $"Originals of removed project {project.ProjectId}: {deleted} deleted (the home holds them), {trashed} kept in {aside}"
+ (left > 0 ? $", {left} left where they are (held open)" : string.Empty));
return left;
}
/// <summary>The frames of <paramref name="home"/> as it holds them now; null (logged) when it cannot be asked.</summary>
private async Task<HomeFrames?> HomeFramesAsync(WorkspaceProject project, BundleHome home, CancellationToken cancellationToken)
{
try
{
return HomeOf(home) is { } reached ? await reached.FramesAsync(cancellationToken).ConfigureAwait(false) : null;
}
catch (Exception ex) when (ex is HomeUnavailableException or ProjectException or DatabaseException or IOException or InvalidDataException)
{
// Not known: every original counts as the last one, and goes to the trash.
Log.Info("workspace", $"The home of project {project.ProjectId} could not be asked for its frames: {ex.Message}");
return null;
}
}
/// <summary>
/// What a removal plan counts of the originals that go to the trash rather than away: the files of the folder of
/// originals the media index does not name (every one when the index cannot be read), plus the frames whose recorded
/// original the home no longer holds as it is (every one while the home cannot be asked). The removal hashes each file
/// itself; this count is what the plan says, never what decides.
/// </summary>
private async Task<int> LastOriginalsOfAsync(WorkspaceProject project, CancellationToken cancellationToken)
{
if (!WorkspaceCleaner.HoldsOriginalFiles(project)) return 0;
var files = Directory.EnumerateFiles(project.OriginalsDirectory)
.Where(f => !f.EndsWith(".part", StringComparison.OrdinalIgnoreCase)).Select(Path.GetFileName).OfType<string>().ToList();
WorkspaceMedia? media;
try
{
media = project.ReadMedia();
}
catch (InvalidDataException ex)
{
Log.Info("workspace", $"The media index of project {project.ProjectId} cannot be read: every original counts as one to keep ({ex.Message})");
return files.Count;
}
var named = media?.Frames.Where(f => f.Original is not null).Select(f => Path.GetFileName(f.Original!)).ToHashSet(StringComparer.OrdinalIgnoreCase) ?? [];
var unnamed = files.Count(f => !named.Contains(f));
if (media is null || named.Count == 0) return unnamed;
HomeFrames? held = null;
try
{
held = await HomeFramesAsync(project, project.ReadBundle().Workspace!.Home, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Info("workspace", $"The home of project {project.ProjectId} is not known: {ex.Message}");
}
return unnamed + WorkspaceCleaner.LastOriginals(media, held).Count(f => files.Contains(Path.GetFileName(f.Original!), StringComparer.OrdinalIgnoreCase));
}
/// <summary>
/// Edit › Export › Send again: queues the sending home of every render of the project not sent yet
/// (<see cref="JobKind.WorkspaceSend"/>), unless one is queued, running or paused already.
/// </summary>
/// <returns>True when a send is queued or waits already; false when nothing waits to be sent.</returns>
/// <exception cref="WorkspaceException">The workspace is being moved, or the index or the engine is not open.</exception>
public Task<bool> SendRendersAsync(Guid projectId, CancellationToken cancellationToken = default) => Task.Run(async () =>
{
EnsureNotMoving();
if (_engine() is null || _db() is null) throw new WorkspaceException(Loc.L("Editor.Error.NoEngine"));
var project = Existing(projectId);
if (WorkspaceCleaner.UnsentRenders(project) == 0) return false;
await QueueRenderSendAsync(project, project.ReadBundle().Workspace!, cancellationToken).ConfigureAwait(false);
return true;
}, cancellationToken);
/// <summary>A send of the project's renders is queued, running or paused (Send again waits for it).</summary>
public async Task<bool> RenderSendPendingAsync(Guid projectId, Guid sequenceId, CancellationToken cancellationToken = default)
{
if (_db() is not { } db) return false;
foreach (var pending in await db.Jobs.ListPendingAsync(JobKind.WorkspaceSend, sequenceId, cancellationToken).ConfigureAwait(false))
{
if (WorkspaceJobs.SendParameters(pending)?.ProjectId == projectId) return true;
}
return false;
}
/// <summary>Queues the sending of the project's renders not sent yet (<see cref="JobKind.WorkspaceSend"/>), unless one is queued or running.</summary>
private async Task QueueRenderSendAsync(WorkspaceProject project, BundleWorkspace workspace, CancellationToken cancellationToken)
{
if (_engine() is not { } engine || _db() is not { } db) return;
foreach (var pending in await db.Jobs.ListPendingAsync(JobKind.WorkspaceSend, workspace.Home.SequenceId, cancellationToken).ConfigureAwait(false))
{
if (WorkspaceJobs.SendParameters(pending)?.ProjectId == project.ProjectId) return;
}
var title = project.ReadBundle().Sequence.Title ?? string.Empty;
var job = await engine.EnqueueAsync(WorkspaceJobs.Send(workspace.Home.SequenceId, title, project.Store.Root, project.ProjectId, [], workspace.Home.Server),
cancellationToken: cancellationToken).ConfigureAwait(false);
Log.Info("workspace", $"The renders of project {project.ProjectId} not sent yet are queued to be sent (job {job.Id})");
}
}