using Chronos.Core.Archiving;
using Chronos.Core.Diagnostics;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Data.Projects;
namespace Chronos.Data.Workspaces;
/// Why the workspace's clean-up kept the media of a project it would have cleaned (each reason is logged with the project).
public enum WorkspaceKeep
{
/// The project is open in Edit.
Open,
/// The project is not in step with its home: changes not saved, versions not sent or not taken, a conflict, a damaged version.
NotInSync,
/// The project's home cannot be asked now: whether it holds everything is not known.
Offline,
/// A copy, render or send of the project, or a move of the workspace, is queued, running or paused.
Jobs,
/// The project's ledger holds renders made here that its home does not have yet.
UnsentRenders,
/// The project's working state, media index or render ledger cannot be read.
Unreadable,
/// The project holds no media files to remove.
NoMedia,
/// The project's media were copied just now (the size limit after a copy never takes that one).
JustCopied,
}
///
/// What a run of the workspace's clean-up applies (phase 5c, Settings › Workspace): Clean after (projects not opened for that
/// many days) and Size limit (the projects' folders above it, the least recently used first); 0 turns either off.
///
public sealed record WorkspaceCleanOptions
{
/// Days a project may go unopened before its media are removed; 0 never removes them for their age.
public int CleanAfterDays { get; init; }
/// Bytes the projects' folders may take (the trash is not counted: only the user empties it); 0 sets no limit.
public long SizeLimitBytes { get; init; }
/// When each project was opened last (Settings' WorkspaceOpened); a project not listed counts from when its working state was saved.
public IReadOnlyDictionary Opened { get; init; } = new Dictionary();
/// Whether a project is open in Edit now (asked again under the project's lease, just before anything goes).
public Func IsOpen { get; init; } = _ => false;
/// The project whose media a copy just brought (the size limit after a copy): never cleaned by this run.
public Guid? JustCopied { get; init; }
/// The time the ages count from.
public DateTimeOffset Now { get; init; } = DateTimeOffset.UtcNow;
}
/// A project whose media the clean-up removed: the bytes freed and the originals set aside in the workspace's trash.
public sealed record WorkspaceCleanedProject(Guid ProjectId, string Name, bool IsMain, string SequenceTitle, long Freed, int Trashed);
/// A project the clean-up would have cleaned and kept, with why.
public sealed record WorkspaceKeptProject(Guid ProjectId, string Name, bool IsMain, string SequenceTitle, WorkspaceKeep Reason);
/// What a run of the workspace's clean-up did: the projects cleaned, those kept with why, and the projects' bytes before and after.
public sealed record WorkspaceCleanReport(IReadOnlyList Cleaned, IReadOnlyList Kept, long UsedBefore, long UsedAfter)
{
/// A run over a workspace with nothing in it.
public static WorkspaceCleanReport Nothing { get; } = new([], [], 0, 0);
/// The bytes the run freed.
public long Freed => Cleaned.Sum(c => c.Freed);
/// The originals the run set aside in the workspace's trash (their home no longer holds them).
public int Trashed => Cleaned.Sum(c => c.Trashed);
}
///
/// The workspace's clean-up (phase 5c, P4): removes the media of projects whose home holds everything they hold, so they are
/// copied again when next opened. A project is cleaned only when it is in step with its home, asked now (never while the
/// home is out of reach), not open in Edit, with no copy, render or send of it queued or running (nor a move of the
/// workspace), and no local render waiting to be sent. Cleaning removes media/proxies and media/originals
/// (the media index stays, its frames naming no file: ) and the renders of
/// exports/ the home took; never the working state, the versions, the sync state, the deep report or a render not
/// sent. An original is deleted only when the home still holds the same frame with the same SHA-256; otherwise it is moved
/// to the workspace's trash (.trash/<id>-<ticks>/originals). The trash itself is never emptied here.
///
public sealed class WorkspaceCleaner
{
private static readonly JobKind[] ProjectJobs = [JobKind.WorkspaceCopy, JobKind.WorkspaceRender, JobKind.WorkspaceSend];
private readonly WorkspaceSync _sync;
private readonly ChronosDatabase _db;
private readonly Func _homes;
/// The home of a workspace project; null for one this PC cannot reach now.
public WorkspaceCleaner(WorkspaceSync sync, ChronosDatabase db, Func homes)
{
ArgumentNullException.ThrowIfNull(sync);
ArgumentNullException.ThrowIfNull(db);
ArgumentNullException.ThrowIfNull(homes);
_sync = sync;
_db = db;
_homes = homes;
}
/// How long a project another writer holds is waited for; past it the project is kept ().
public TimeSpan LockWait { get; init; } = TimeSpan.FromSeconds(30);
///
/// One run: the projects not opened for , oldest first, then, while the
/// projects' folders take more than , the least recently used others.
/// Each project the policies name is cleaned or kept with its reason.
///
public async Task CleanAsync(WorkspaceCleanOptions options, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(options);
var candidates = new List();
var used = 0L;
foreach (var project in _sync.Store.List())
{
cancellationToken.ThrowIfCancellationRequested();
var candidate = Candidate.Of(project, options);
used += candidate.FolderBytes;
candidates.Add(candidate);
}
var before = used;
var cleaned = new List();
var kept = new List();
var named = new HashSet();
var moving = await _db.Jobs.HasPendingAsync([JobKind.WorkspaceMove], null, cancellationToken).ConfigureAwait(false);
async Task CleanOneAsync(Candidate candidate)
{
named.Add(candidate.Project.ProjectId);
var (reason, freed, trashed) = await TryCleanAsync(candidate, options, moving, cancellationToken).ConfigureAwait(false);
if (reason is { } why)
{
kept.Add(new WorkspaceKeptProject(candidate.Project.ProjectId, candidate.Name, candidate.IsMain, candidate.Title, why));
return;
}
cleaned.Add(new WorkspaceCleanedProject(candidate.Project.ProjectId, candidate.Name, candidate.IsMain, candidate.Title, freed, trashed));
used -= freed;
}
if (options.CleanAfterDays > 0)
{
var cutoff = options.Now - TimeSpan.FromDays(options.CleanAfterDays);
foreach (var candidate in candidates.Where(c => c.Targeted && c.LastUsed < cutoff).OrderBy(c => c.LastUsed).ToList())
await CleanOneAsync(candidate).ConfigureAwait(false);
}
if (options.SizeLimitBytes > 0)
{
foreach (var candidate in candidates.Where(c => c.Targeted && !named.Contains(c.Project.ProjectId)).OrderBy(c => c.LastUsed).ToList())
{
if (used <= options.SizeLimitBytes) break;
await CleanOneAsync(candidate).ConfigureAwait(false);
}
}
foreach (var project in cleaned)
Log.Info("workspace", $"Media of project {project.ProjectId} ('{project.Name}') removed by the clean-up: {project.Freed} bytes freed, {project.Trashed} original(s) kept in the trash");
foreach (var project in kept) Log.Info("workspace", $"Media of project {project.ProjectId} ('{project.Name}') kept by the clean-up: {project.Reason}");
return new WorkspaceCleanReport(cleaned, kept, before, Math.Max(0, used));
}
/// The frames of whose original the home no longer holds as it is here (all of them when the home's frames are not known).
public static IReadOnlyList LastOriginals(WorkspaceMedia? media, HomeFrames? home)
{
if (media is null) return [];
var held = home?.Frames.ToDictionary(f => f.Id);
return [.. media.Frames.Where(f => f.HasOriginal && (held is null || !Holds(held, f)))];
static bool Holds(Dictionary held, WorkspaceFrame frame) =>
frame.OriginalSha256 is { Length: > 0 } sha && held.TryGetValue(frame.Id, out var there) && (there.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0
&& string.Equals(there.Sha256, sha, StringComparison.OrdinalIgnoreCase);
}
///
/// The local renders of the project its home does not have yet (its ledger's unsent entries); a ledger that cannot be
/// read counts the files of exports/ besides it, at least one: whether they went home is not known.
///
public static int UnsentRenders(WorkspaceProject project)
{
ArgumentNullException.ThrowIfNull(project);
try
{
return project.ReadRenders().Unsent.Count;
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Warning("workspace", $"The render ledger of project {project.ProjectId} cannot be read: its renders count as not sent", ex);
var files = Directory.Exists(project.ExportsDirectory)
? Directory.EnumerateFiles(project.ExportsDirectory, "*", SearchOption.AllDirectories)
.Count(f => !string.Equals(Path.GetFileName(f), WorkspaceProject.RendersFileName, StringComparison.OrdinalIgnoreCase))
: 0;
return Math.Max(1, files);
}
}
///
/// The workspace holds the project's media to edit: a media index whose files the clean-up did not remove (the folders of
/// proxies or originals there say so at once; otherwise the index is read).
///
public static bool HoldsMedia(WorkspaceProject project)
{
ArgumentNullException.ThrowIfNull(project);
if (!File.Exists(project.MediaIndexPath)) return false;
if (Directory.Exists(project.ProxiesDirectory) || Directory.Exists(project.OriginalsDirectory)) return true;
try
{
return project.ReadMedia() is { IsCleaned: false };
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
// The index is there: whatever it holds, opening it says so.
return true;
}
}
/// Files and bytes under , leases left out; nothing when it is not there.
public static (int Files, long Bytes) Measure(string folder)
{
var (files, bytes) = (0, 0L);
if (!Directory.Exists(folder)) return (files, bytes);
foreach (var file in new DirectoryInfo(folder).EnumerateFiles("*", SearchOption.AllDirectories))
{
if (WorkspaceProject.IsLeaseFile(file.Name)) continue;
files++;
bytes += file.Length;
}
return (files, bytes);
}
/// The project cleaned, or why it is kept.
private async Task<(WorkspaceKeep? Kept, long Freed, int Trashed)> TryCleanAsync(Candidate candidate, WorkspaceCleanOptions options, bool moving,
CancellationToken cancellationToken)
{
var project = candidate.Project;
var id = project.ProjectId;
if (candidate.Unreadable || candidate.Bundle is not { Workspace: { } workspace }) return (WorkspaceKeep.Unreadable, 0, 0);
if (options.JustCopied == id) return (WorkspaceKeep.JustCopied, 0, 0);
if (options.IsOpen(id)) return (WorkspaceKeep.Open, 0, 0);
if (moving || await BusyAsync(id, workspace.Home.SequenceId, cancellationToken).ConfigureAwait(false)) return (WorkspaceKeep.Jobs, 0, 0);
if (candidate.CleanableBytes == 0) return (WorkspaceKeep.NoMedia, 0, 0);
if (ReadLedger(project) is not { } ledger) return (WorkspaceKeep.Unreadable, 0, 0);
if (ledger.Unsent.Count > 0) return (WorkspaceKeep.UnsentRenders, 0, 0);
if (_homes(workspace.Home) is not { } home) return (WorkspaceKeep.Offline, 0, 0);
WorkspaceStatus status;
try
{
status = await _sync.StatusAsync(id, home, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is ProjectException or DatabaseException or IOException or InvalidDataException or LeaseUnavailableException)
{
Log.Info("workspace", $"The home of project {id} could not be asked for the clean-up: {ex.Message}");
return (WorkspaceKeep.Offline, 0, 0);
}
if (status.Offline && status.State != WorkspaceState.Gone) return (WorkspaceKeep.Offline, 0, 0);
if (status.State != WorkspaceState.InSync) return (WorkspaceKeep.NotInSync, 0, 0);
// The home's frames, asked now: an original goes only when the home holds the same bytes. Every file of the folder of
// originals counts, named by the media index or not (one a removal could not move aside stays unnamed there).
HomeFrames? held = null;
if (HoldsOriginalFiles(project))
{
try
{
held = await home.FramesAsync(cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is HomeUnavailableException or ProjectException or DatabaseException or IOException)
{
Log.Info("workspace", $"The frames of project {id}'s home could not be read for the clean-up: {ex.Message}");
return (WorkspaceKeep.Offline, 0, 0);
}
}
IDisposable lease;
try
{
lease = await project.LockAsync(LockWait, cancellationToken).ConfigureAwait(false);
}
catch (LeaseUnavailableException)
{
return (WorkspaceKeep.Jobs, 0, 0);
}
using (lease)
{
// Asked again under the lease: the editor may have opened it, a save or a render may have come meanwhile.
if (options.IsOpen(id)) return (WorkspaceKeep.Open, 0, 0);
var local = await _sync.StatusAsync(id, null, cancellationToken).ConfigureAwait(false);
if (local.State != WorkspaceState.InSync) return (WorkspaceKeep.NotInSync, 0, 0);
if (ReadLedger(project) is not { } renders) return (WorkspaceKeep.Unreadable, 0, 0);
if (renders.Unsent.Count > 0) return (WorkspaceKeep.UnsentRenders, 0, 0);
WorkspaceMedia? media;
try
{
media = project.ReadMedia();
}
catch (InvalidDataException)
{
return (WorkspaceKeep.Unreadable, 0, 0);
}
if (media is null || media.IsCleaned) return (WorkspaceKeep.NoMedia, 0, 0);
var (freed, trashed) = Clear(project, media, renders, held, options.Now);
return (null, freed, trashed);
}
}
///
/// Removes the project's media and its sent renders, under its lease: the media index rewritten with no file first (so
/// nothing ever names a file that is gone), then the sent renders (only files inside exports/), then the
/// originals, each hashed now and deleted only when the home holds the same bytes, else moved to the trash
/// (), and last the proxies. A file that cannot be deleted or moved (held
/// open) stays, and is logged.
///
private static (long Freed, int Trashed) Clear(WorkspaceProject project, WorkspaceMedia media, WorkspaceRenders renders, HomeFrames? held, DateTimeOffset now)
{
var before = Cleanable(project, renders);
project.WriteMedia(media with
{
Frames = [.. media.Frames.Select(f => f with { File = null, FileSha256 = null, FileBytes = 0, FileKind = null, Original = null, OriginalSha256 = null, OriginalBytes = 0 })],
Cleaned = now,
});
var ledger = renders;
foreach (var entry in renders.Entries.Where(e => e.IsSent))
{
if (project.ContainedExportFile(entry.File) is not { } path)
{
Log.Warning("workspace", $"The ledger of project {project.ProjectId} names '{entry.File}' for render {entry.RenderId}, not a file of its exports: nothing is removed for it");
continue;
}
try
{
if (Directory.Exists(path)) Directory.Delete(path, recursive: true);
else if (File.Exists(path)) File.Delete(path);
ledger = ledger.Without(entry.RenderId);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("workspace", $"The sent render {path} could not be removed; it stays", ex);
}
}
if (!ReferenceEquals(ledger, renders)) project.WriteRenders(ledger);
var (trashed, trashedBytes) = (0, 0L);
if (Directory.Exists(project.OriginalsDirectory))
{
using var trash = WorkspaceTrash.Begin(project.Store.Root, project.ProjectId, now.UtcDateTime);
var cleared = WorkspaceOriginals.ClearOriginals(project, held?.Frames, trash.Path);
(trashed, trashedBytes) = (cleared.Trashed, cleared.TrashedBytes);
if (trash.Finish() is { } aside) Log.Info("workspace", $"{trashed} original(s) of project {project.ProjectId} its home does not hold as they are kept in {aside}");
if (cleared.Left > 0) Log.Warning("workspace", $"{cleared.Left} original(s) of project {project.ProjectId} could be neither deleted nor set aside; they stay");
}
try
{
if (Directory.Exists(project.ProxiesDirectory)) Directory.Delete(project.ProxiesDirectory, recursive: true);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("workspace", $"{project.ProxiesDirectory} could not be removed whole; what is left stays", ex);
}
var after = Cleanable(project, ledger);
return (Math.Max(0, before - after - trashedBytes), trashed);
}
/// The project's folder of originals holds a file (an original, named by the media index or not, or a part of one).
public static bool HoldsOriginalFiles(WorkspaceProject project)
{
ArgumentNullException.ThrowIfNull(project);
try
{
return Directory.Exists(project.OriginalsDirectory) && Directory.EnumerateFiles(project.OriginalsDirectory, "*", SearchOption.AllDirectories).Any();
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// Not known: counted as holding some, so the home is asked.
return true;
}
}
/// What a clean-up may remove of the project: its proxies, its originals and the renders its home took.
private static long Cleanable(WorkspaceProject project, WorkspaceRenders renders) =>
Measure(project.ProxiesDirectory).Bytes + Measure(project.OriginalsDirectory).Bytes + SentBytes(project, renders);
/// The bytes of the renders of the home took that are still in exports/ (an image sequence's folder counts its files).
private static long SentBytes(WorkspaceProject project, WorkspaceRenders renders) =>
renders.Entries.Where(e => e.IsSent).Select(e => project.ContainedExportFile(e.File)).OfType()
.Sum(path => File.Exists(path) ? new FileInfo(path).Length : Measure(path).Bytes);
/// The project's ledger; null (logged) when it cannot be read.
private static WorkspaceRenders? ReadLedger(WorkspaceProject project)
{
try
{
return project.ReadRenders();
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Warning("workspace", $"The render ledger of project {project.ProjectId} cannot be read: the clean-up keeps the project", ex);
return null;
}
}
/// A copy, render or send of the project is queued, running or paused (one whose parameters do not read counts too).
private async Task BusyAsync(Guid projectId, Guid sequenceId, CancellationToken cancellationToken)
{
foreach (var kind in ProjectJobs)
{
foreach (var job in await _db.Jobs.ListPendingAsync(kind, sequenceId, cancellationToken).ConfigureAwait(false))
{
var named = kind switch
{
JobKind.WorkspaceCopy => WorkspaceJobs.CopyParameters(job)?.ProjectId,
JobKind.WorkspaceRender => WorkspaceJobs.RenderParameters(job)?.ProjectId,
_ => WorkspaceJobs.SendParameters(job)?.ProjectId,
};
if (named is null || named == projectId) return true;
}
}
return false;
}
/// A project of the workspace as the clean-up weighs it.
private sealed record Candidate(WorkspaceProject Project, ProjectBundle? Bundle, WorkspaceMedia? Media, bool Unreadable, string Name, bool IsMain, string Title,
DateTimeOffset LastUsed, long FolderBytes, long CleanableBytes)
{
/// The policies may name it: media a copy wrote (cleaned ones are done), or a copy that cannot be read holding media files.
public bool Targeted => Media is { IsCleaned: false } || (Unreadable && CleanableBytes > 0);
public static Candidate Of(WorkspaceProject project, WorkspaceCleanOptions options)
{
var folder = Measure(project.Folder).Bytes;
var media = Measure(project.ProxiesDirectory).Bytes + Measure(project.OriginalsDirectory).Bytes;
var saved = File.Exists(project.BundlePath) ? new DateTimeOffset(File.GetLastWriteTimeUtc(project.BundlePath), TimeSpan.Zero) : DateTimeOffset.MinValue;
ProjectBundle bundle;
try
{
bundle = project.ReadBundle();
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
return new Candidate(project, null, null, true, project.ProjectId.ToString("N"), false, string.Empty, Used(saved), folder, media);
}
var workspace = bundle.Workspace!;
var title = bundle.Sequence.Title is { Length: > 0 } t ? t : bundle.Sequence.Folder;
WorkspaceMedia? index;
var unreadable = false;
try
{
index = project.ReadMedia();
}
catch (InvalidDataException)
{
(index, unreadable) = (null, true);
}
var sent = 0L;
try
{
sent = SentBytes(project, project.ReadRenders());
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
// Weighed by its media alone: cleaning it is refused while its ledger cannot be read.
}
return new Candidate(project, bundle, index, unreadable, workspace.Name, workspace.IsMain, title, Used(bundle.Saved), folder, media + sent);
DateTimeOffset Used(DateTimeOffset fallback) => options.Opened.TryGetValue(project.ProjectId, out var opened) ? opened : fallback;
}
}
}