Files
Chronos/src/Chronos.Data/Workspaces/WorkspaceCleaner.cs
T
Alby96andClaude Opus 5.5 db389a54f2
CI / Build and test (push) Successful in 9m26s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 8s
Release / Build, test and publish (push) Successful in 14m31s
CI / PostgreSQL 16 integration (push) Successful in 9m53s
CI / PostgreSQL 17 integration (push) Successful in 9m22s
CI / Docker image (push) Successful in 44s
Release / Server image (push) Successful in 1m36s
feat: owner review 2 phase 5c: originals, local render, clean-up and the workspace settings (#18)
Originals in the workspace, local render and send home with provenance, the workspace clean-up, size limit and trash, moving the workspace folder, and the complete Settings › Workspace page (owner review 2, R8, phase 5c). One adversarial review round (35 findings, 32 fixed), a copy-cancel race fixed, 184 strings in all 38 languages.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
2026-11-04 20:03:52 +01:00

471 lines
25 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using Chronos.Core.Archiving;
using Chronos.Core.Diagnostics;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Data.Projects;
namespace Chronos.Data.Workspaces;
/// <summary>Why the workspace's clean-up kept the media of a project it would have cleaned (each reason is logged with the project).</summary>
public enum WorkspaceKeep
{
/// <summary>The project is open in Edit.</summary>
Open,
/// <summary>The project is not in step with its home: changes not saved, versions not sent or not taken, a conflict, a damaged version.</summary>
NotInSync,
/// <summary>The project's home cannot be asked now: whether it holds everything is not known.</summary>
Offline,
/// <summary>A copy, render or send of the project, or a move of the workspace, is queued, running or paused.</summary>
Jobs,
/// <summary>The project's ledger holds renders made here that its home does not have yet.</summary>
UnsentRenders,
/// <summary>The project's working state, media index or render ledger cannot be read.</summary>
Unreadable,
/// <summary>The project holds no media files to remove.</summary>
NoMedia,
/// <summary>The project's media were copied just now (the size limit after a copy never takes that one).</summary>
JustCopied,
}
/// <summary>
/// What a run of the workspace's clean-up applies (phase 5c, Settings › Workspace): Clean after (projects not opened for that
/// many days) and Size limit (the projects' folders above it, the least recently used first); 0 turns either off.
/// </summary>
public sealed record WorkspaceCleanOptions
{
/// <summary>Days a project may go unopened before its media are removed; 0 never removes them for their age.</summary>
public int CleanAfterDays { get; init; }
/// <summary>Bytes the projects' folders may take (the trash is not counted: only the user empties it); 0 sets no limit.</summary>
public long SizeLimitBytes { get; init; }
/// <summary>When each project was opened last (Settings' <c>WorkspaceOpened</c>); a project not listed counts from when its working state was saved.</summary>
public IReadOnlyDictionary<Guid, DateTimeOffset> Opened { get; init; } = new Dictionary<Guid, DateTimeOffset>();
/// <summary>Whether a project is open in Edit now (asked again under the project's lease, just before anything goes).</summary>
public Func<Guid, bool> IsOpen { get; init; } = _ => false;
/// <summary>The project whose media a copy just brought (the size limit after a copy): never cleaned by this run.</summary>
public Guid? JustCopied { get; init; }
/// <summary>The time the ages count from.</summary>
public DateTimeOffset Now { get; init; } = DateTimeOffset.UtcNow;
}
/// <summary>A project whose media the clean-up removed: the bytes freed and the originals set aside in the workspace's trash.</summary>
public sealed record WorkspaceCleanedProject(Guid ProjectId, string Name, bool IsMain, string SequenceTitle, long Freed, int Trashed);
/// <summary>A project the clean-up would have cleaned and kept, with why.</summary>
public sealed record WorkspaceKeptProject(Guid ProjectId, string Name, bool IsMain, string SequenceTitle, WorkspaceKeep Reason);
/// <summary>What a run of the workspace's clean-up did: the projects cleaned, those kept with why, and the projects' bytes before and after.</summary>
public sealed record WorkspaceCleanReport(IReadOnlyList<WorkspaceCleanedProject> Cleaned, IReadOnlyList<WorkspaceKeptProject> Kept, long UsedBefore, long UsedAfter)
{
/// <summary>A run over a workspace with nothing in it.</summary>
public static WorkspaceCleanReport Nothing { get; } = new([], [], 0, 0);
/// <summary>The bytes the run freed.</summary>
public long Freed => Cleaned.Sum(c => c.Freed);
/// <summary>The originals the run set aside in the workspace's trash (their home no longer holds them).</summary>
public int Trashed => Cleaned.Sum(c => c.Trashed);
}
/// <summary>
/// The workspace's clean-up (phase 5c, P4): removes the media of projects whose home holds everything they hold, so they are
/// copied again when next opened. A project is cleaned only when it is in step with its home, asked now (never while the
/// home is out of reach), not open in Edit, with no copy, render or send of it queued or running (nor a move of the
/// workspace), and no local render waiting to be sent. Cleaning removes <c>media/proxies</c> and <c>media/originals</c>
/// (the media index stays, its frames naming no file: <see cref="WorkspaceMedia.Cleaned"/>) and the renders of
/// <c>exports/</c> the home took; never the working state, the versions, the sync state, the deep report or a render not
/// sent. An original is deleted only when the home still holds the same frame with the same SHA-256; otherwise it is moved
/// to the workspace's trash (<c>.trash/&lt;id&gt;-&lt;ticks&gt;/originals</c>). The trash itself is never emptied here.
/// </summary>
public sealed class WorkspaceCleaner
{
private static readonly JobKind[] ProjectJobs = [JobKind.WorkspaceCopy, JobKind.WorkspaceRender, JobKind.WorkspaceSend];
private readonly WorkspaceSync _sync;
private readonly ChronosDatabase _db;
private readonly Func<BundleHome, IProjectHome?> _homes;
/// <param name="homes">The home of a workspace project; null for one this PC cannot reach now.</param>
public WorkspaceCleaner(WorkspaceSync sync, ChronosDatabase db, Func<BundleHome, IProjectHome?> homes)
{
ArgumentNullException.ThrowIfNull(sync);
ArgumentNullException.ThrowIfNull(db);
ArgumentNullException.ThrowIfNull(homes);
_sync = sync;
_db = db;
_homes = homes;
}
/// <summary>How long a project another writer holds is waited for; past it the project is kept (<see cref="WorkspaceKeep.Jobs"/>).</summary>
public TimeSpan LockWait { get; init; } = TimeSpan.FromSeconds(30);
/// <summary>
/// One run: the projects not opened for <see cref="WorkspaceCleanOptions.CleanAfterDays"/>, oldest first, then, while the
/// projects' folders take more than <see cref="WorkspaceCleanOptions.SizeLimitBytes"/>, the least recently used others.
/// Each project the policies name is cleaned or kept with its reason.
/// </summary>
public async Task<WorkspaceCleanReport> CleanAsync(WorkspaceCleanOptions options, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(options);
var candidates = new List<Candidate>();
var used = 0L;
foreach (var project in _sync.Store.List())
{
cancellationToken.ThrowIfCancellationRequested();
var candidate = Candidate.Of(project, options);
used += candidate.FolderBytes;
candidates.Add(candidate);
}
var before = used;
var cleaned = new List<WorkspaceCleanedProject>();
var kept = new List<WorkspaceKeptProject>();
var named = new HashSet<Guid>();
var moving = await _db.Jobs.HasPendingAsync([JobKind.WorkspaceMove], null, cancellationToken).ConfigureAwait(false);
async Task CleanOneAsync(Candidate candidate)
{
named.Add(candidate.Project.ProjectId);
var (reason, freed, trashed) = await TryCleanAsync(candidate, options, moving, cancellationToken).ConfigureAwait(false);
if (reason is { } why)
{
kept.Add(new WorkspaceKeptProject(candidate.Project.ProjectId, candidate.Name, candidate.IsMain, candidate.Title, why));
return;
}
cleaned.Add(new WorkspaceCleanedProject(candidate.Project.ProjectId, candidate.Name, candidate.IsMain, candidate.Title, freed, trashed));
used -= freed;
}
if (options.CleanAfterDays > 0)
{
var cutoff = options.Now - TimeSpan.FromDays(options.CleanAfterDays);
foreach (var candidate in candidates.Where(c => c.Targeted && c.LastUsed < cutoff).OrderBy(c => c.LastUsed).ToList())
await CleanOneAsync(candidate).ConfigureAwait(false);
}
if (options.SizeLimitBytes > 0)
{
foreach (var candidate in candidates.Where(c => c.Targeted && !named.Contains(c.Project.ProjectId)).OrderBy(c => c.LastUsed).ToList())
{
if (used <= options.SizeLimitBytes) break;
await CleanOneAsync(candidate).ConfigureAwait(false);
}
}
foreach (var project in cleaned)
Log.Info("workspace", $"Media of project {project.ProjectId} ('{project.Name}') removed by the clean-up: {project.Freed} bytes freed, {project.Trashed} original(s) kept in the trash");
foreach (var project in kept) Log.Info("workspace", $"Media of project {project.ProjectId} ('{project.Name}') kept by the clean-up: {project.Reason}");
return new WorkspaceCleanReport(cleaned, kept, before, Math.Max(0, used));
}
/// <summary>The frames of <paramref name="media"/> whose original the home no longer holds as it is here (all of them when the home's frames are not known).</summary>
public static IReadOnlyList<WorkspaceFrame> LastOriginals(WorkspaceMedia? media, HomeFrames? home)
{
if (media is null) return [];
var held = home?.Frames.ToDictionary(f => f.Id);
return [.. media.Frames.Where(f => f.HasOriginal && (held is null || !Holds(held, f)))];
static bool Holds(Dictionary<Guid, Frame> held, WorkspaceFrame frame) =>
frame.OriginalSha256 is { Length: > 0 } sha && held.TryGetValue(frame.Id, out var there) && (there.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0
&& string.Equals(there.Sha256, sha, StringComparison.OrdinalIgnoreCase);
}
/// <summary>
/// The local renders of the project its home does not have yet (its ledger's unsent entries); a ledger that cannot be
/// read counts the files of <c>exports/</c> besides it, at least one: whether they went home is not known.
/// </summary>
public static int UnsentRenders(WorkspaceProject project)
{
ArgumentNullException.ThrowIfNull(project);
try
{
return project.ReadRenders().Unsent.Count;
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Warning("workspace", $"The render ledger of project {project.ProjectId} cannot be read: its renders count as not sent", ex);
var files = Directory.Exists(project.ExportsDirectory)
? Directory.EnumerateFiles(project.ExportsDirectory, "*", SearchOption.AllDirectories)
.Count(f => !string.Equals(Path.GetFileName(f), WorkspaceProject.RendersFileName, StringComparison.OrdinalIgnoreCase))
: 0;
return Math.Max(1, files);
}
}
/// <summary>
/// The workspace holds the project's media to edit: a media index whose files the clean-up did not remove (the folders of
/// proxies or originals there say so at once; otherwise the index is read).
/// </summary>
public static bool HoldsMedia(WorkspaceProject project)
{
ArgumentNullException.ThrowIfNull(project);
if (!File.Exists(project.MediaIndexPath)) return false;
if (Directory.Exists(project.ProxiesDirectory) || Directory.Exists(project.OriginalsDirectory)) return true;
try
{
return project.ReadMedia() is { IsCleaned: false };
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
// The index is there: whatever it holds, opening it says so.
return true;
}
}
/// <summary>Files and bytes under <paramref name="folder"/>, leases left out; nothing when it is not there.</summary>
public static (int Files, long Bytes) Measure(string folder)
{
var (files, bytes) = (0, 0L);
if (!Directory.Exists(folder)) return (files, bytes);
foreach (var file in new DirectoryInfo(folder).EnumerateFiles("*", SearchOption.AllDirectories))
{
if (WorkspaceProject.IsLeaseFile(file.Name)) continue;
files++;
bytes += file.Length;
}
return (files, bytes);
}
/// <summary>The project cleaned, or why it is kept.</summary>
private async Task<(WorkspaceKeep? Kept, long Freed, int Trashed)> TryCleanAsync(Candidate candidate, WorkspaceCleanOptions options, bool moving,
CancellationToken cancellationToken)
{
var project = candidate.Project;
var id = project.ProjectId;
if (candidate.Unreadable || candidate.Bundle is not { Workspace: { } workspace }) return (WorkspaceKeep.Unreadable, 0, 0);
if (options.JustCopied == id) return (WorkspaceKeep.JustCopied, 0, 0);
if (options.IsOpen(id)) return (WorkspaceKeep.Open, 0, 0);
if (moving || await BusyAsync(id, workspace.Home.SequenceId, cancellationToken).ConfigureAwait(false)) return (WorkspaceKeep.Jobs, 0, 0);
if (candidate.CleanableBytes == 0) return (WorkspaceKeep.NoMedia, 0, 0);
if (ReadLedger(project) is not { } ledger) return (WorkspaceKeep.Unreadable, 0, 0);
if (ledger.Unsent.Count > 0) return (WorkspaceKeep.UnsentRenders, 0, 0);
if (_homes(workspace.Home) is not { } home) return (WorkspaceKeep.Offline, 0, 0);
WorkspaceStatus status;
try
{
status = await _sync.StatusAsync(id, home, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is ProjectException or DatabaseException or IOException or InvalidDataException or LeaseUnavailableException)
{
Log.Info("workspace", $"The home of project {id} could not be asked for the clean-up: {ex.Message}");
return (WorkspaceKeep.Offline, 0, 0);
}
if (status.Offline && status.State != WorkspaceState.Gone) return (WorkspaceKeep.Offline, 0, 0);
if (status.State != WorkspaceState.InSync) return (WorkspaceKeep.NotInSync, 0, 0);
// The home's frames, asked now: an original goes only when the home holds the same bytes. Every file of the folder of
// originals counts, named by the media index or not (one a removal could not move aside stays unnamed there).
HomeFrames? held = null;
if (HoldsOriginalFiles(project))
{
try
{
held = await home.FramesAsync(cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is HomeUnavailableException or ProjectException or DatabaseException or IOException)
{
Log.Info("workspace", $"The frames of project {id}'s home could not be read for the clean-up: {ex.Message}");
return (WorkspaceKeep.Offline, 0, 0);
}
}
IDisposable lease;
try
{
lease = await project.LockAsync(LockWait, cancellationToken).ConfigureAwait(false);
}
catch (LeaseUnavailableException)
{
return (WorkspaceKeep.Jobs, 0, 0);
}
using (lease)
{
// Asked again under the lease: the editor may have opened it, a save or a render may have come meanwhile.
if (options.IsOpen(id)) return (WorkspaceKeep.Open, 0, 0);
var local = await _sync.StatusAsync(id, null, cancellationToken).ConfigureAwait(false);
if (local.State != WorkspaceState.InSync) return (WorkspaceKeep.NotInSync, 0, 0);
if (ReadLedger(project) is not { } renders) return (WorkspaceKeep.Unreadable, 0, 0);
if (renders.Unsent.Count > 0) return (WorkspaceKeep.UnsentRenders, 0, 0);
WorkspaceMedia? media;
try
{
media = project.ReadMedia();
}
catch (InvalidDataException)
{
return (WorkspaceKeep.Unreadable, 0, 0);
}
if (media is null || media.IsCleaned) return (WorkspaceKeep.NoMedia, 0, 0);
var (freed, trashed) = Clear(project, media, renders, held, options.Now);
return (null, freed, trashed);
}
}
/// <summary>
/// Removes the project's media and its sent renders, under its lease: the media index rewritten with no file first (so
/// nothing ever names a file that is gone), then the sent renders (only files inside <c>exports/</c>), then the
/// originals, each hashed now and deleted only when the home holds the same bytes, else moved to the trash
/// (<see cref="WorkspaceOriginals.ClearOriginals"/>), and last the proxies. A file that cannot be deleted or moved (held
/// open) stays, and is logged.
/// </summary>
private static (long Freed, int Trashed) Clear(WorkspaceProject project, WorkspaceMedia media, WorkspaceRenders renders, HomeFrames? held, DateTimeOffset now)
{
var before = Cleanable(project, renders);
project.WriteMedia(media with
{
Frames = [.. media.Frames.Select(f => f with { File = null, FileSha256 = null, FileBytes = 0, FileKind = null, Original = null, OriginalSha256 = null, OriginalBytes = 0 })],
Cleaned = now,
});
var ledger = renders;
foreach (var entry in renders.Entries.Where(e => e.IsSent))
{
if (project.ContainedExportFile(entry.File) is not { } path)
{
Log.Warning("workspace", $"The ledger of project {project.ProjectId} names '{entry.File}' for render {entry.RenderId}, not a file of its exports: nothing is removed for it");
continue;
}
try
{
if (Directory.Exists(path)) Directory.Delete(path, recursive: true);
else if (File.Exists(path)) File.Delete(path);
ledger = ledger.Without(entry.RenderId);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("workspace", $"The sent render {path} could not be removed; it stays", ex);
}
}
if (!ReferenceEquals(ledger, renders)) project.WriteRenders(ledger);
var (trashed, trashedBytes) = (0, 0L);
if (Directory.Exists(project.OriginalsDirectory))
{
using var trash = WorkspaceTrash.Begin(project.Store.Root, project.ProjectId, now.UtcDateTime);
var cleared = WorkspaceOriginals.ClearOriginals(project, held?.Frames, trash.Path);
(trashed, trashedBytes) = (cleared.Trashed, cleared.TrashedBytes);
if (trash.Finish() is { } aside) Log.Info("workspace", $"{trashed} original(s) of project {project.ProjectId} its home does not hold as they are kept in {aside}");
if (cleared.Left > 0) Log.Warning("workspace", $"{cleared.Left} original(s) of project {project.ProjectId} could be neither deleted nor set aside; they stay");
}
try
{
if (Directory.Exists(project.ProxiesDirectory)) Directory.Delete(project.ProxiesDirectory, recursive: true);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("workspace", $"{project.ProxiesDirectory} could not be removed whole; what is left stays", ex);
}
var after = Cleanable(project, ledger);
return (Math.Max(0, before - after - trashedBytes), trashed);
}
/// <summary>The project's folder of originals holds a file (an original, named by the media index or not, or a part of one).</summary>
public static bool HoldsOriginalFiles(WorkspaceProject project)
{
ArgumentNullException.ThrowIfNull(project);
try
{
return Directory.Exists(project.OriginalsDirectory) && Directory.EnumerateFiles(project.OriginalsDirectory, "*", SearchOption.AllDirectories).Any();
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// Not known: counted as holding some, so the home is asked.
return true;
}
}
/// <summary>What a clean-up may remove of the project: its proxies, its originals and the renders its home took.</summary>
private static long Cleanable(WorkspaceProject project, WorkspaceRenders renders) =>
Measure(project.ProxiesDirectory).Bytes + Measure(project.OriginalsDirectory).Bytes + SentBytes(project, renders);
/// <summary>The bytes of the renders of <paramref name="renders"/> the home took that are still in <c>exports/</c> (an image sequence's folder counts its files).</summary>
private static long SentBytes(WorkspaceProject project, WorkspaceRenders renders) =>
renders.Entries.Where(e => e.IsSent).Select(e => project.ContainedExportFile(e.File)).OfType<string>()
.Sum(path => File.Exists(path) ? new FileInfo(path).Length : Measure(path).Bytes);
/// <summary>The project's ledger; null (logged) when it cannot be read.</summary>
private static WorkspaceRenders? ReadLedger(WorkspaceProject project)
{
try
{
return project.ReadRenders();
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Warning("workspace", $"The render ledger of project {project.ProjectId} cannot be read: the clean-up keeps the project", ex);
return null;
}
}
/// <summary>A copy, render or send of the project is queued, running or paused (one whose parameters do not read counts too).</summary>
private async Task<bool> BusyAsync(Guid projectId, Guid sequenceId, CancellationToken cancellationToken)
{
foreach (var kind in ProjectJobs)
{
foreach (var job in await _db.Jobs.ListPendingAsync(kind, sequenceId, cancellationToken).ConfigureAwait(false))
{
var named = kind switch
{
JobKind.WorkspaceCopy => WorkspaceJobs.CopyParameters(job)?.ProjectId,
JobKind.WorkspaceRender => WorkspaceJobs.RenderParameters(job)?.ProjectId,
_ => WorkspaceJobs.SendParameters(job)?.ProjectId,
};
if (named is null || named == projectId) return true;
}
}
return false;
}
/// <summary>A project of the workspace as the clean-up weighs it.</summary>
private sealed record Candidate(WorkspaceProject Project, ProjectBundle? Bundle, WorkspaceMedia? Media, bool Unreadable, string Name, bool IsMain, string Title,
DateTimeOffset LastUsed, long FolderBytes, long CleanableBytes)
{
/// <summary>The policies may name it: media a copy wrote (cleaned ones are done), or a copy that cannot be read holding media files.</summary>
public bool Targeted => Media is { IsCleaned: false } || (Unreadable && CleanableBytes > 0);
public static Candidate Of(WorkspaceProject project, WorkspaceCleanOptions options)
{
var folder = Measure(project.Folder).Bytes;
var media = Measure(project.ProxiesDirectory).Bytes + Measure(project.OriginalsDirectory).Bytes;
var saved = File.Exists(project.BundlePath) ? new DateTimeOffset(File.GetLastWriteTimeUtc(project.BundlePath), TimeSpan.Zero) : DateTimeOffset.MinValue;
ProjectBundle bundle;
try
{
bundle = project.ReadBundle();
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
return new Candidate(project, null, null, true, project.ProjectId.ToString("N"), false, string.Empty, Used(saved), folder, media);
}
var workspace = bundle.Workspace!;
var title = bundle.Sequence.Title is { Length: > 0 } t ? t : bundle.Sequence.Folder;
WorkspaceMedia? index;
var unreadable = false;
try
{
index = project.ReadMedia();
}
catch (InvalidDataException)
{
(index, unreadable) = (null, true);
}
var sent = 0L;
try
{
sent = SentBytes(project, project.ReadRenders());
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
// Weighed by its media alone: cleaning it is refused while its ledger cannot be read.
}
return new Candidate(project, bundle, index, unreadable, workspace.Name, workspace.IsMain, title, Used(bundle.Saved), folder, media + sent);
DateTimeOffset Used(DateTimeOffset fallback) => options.Opened.TryGetValue(project.ProjectId, out var opened) ? opened : fallback;
}
}
}