feat: owner review 2 phase 5c: originals, local render, clean-up and the workspace settings (#18)
CI / Build and test (push) Successful in 9m26s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 8s
Release / Build, test and publish (push) Successful in 14m31s
CI / PostgreSQL 16 integration (push) Successful in 9m53s
CI / PostgreSQL 17 integration (push) Successful in 9m22s
CI / Docker image (push) Successful in 44s
Release / Server image (push) Successful in 1m36s

Originals in the workspace, local render and send home with provenance, the workspace clean-up, size limit and trash, moving the workspace folder, and the complete Settings › Workspace page (owner review 2, R8, phase 5c). One adversarial review round (35 findings, 32 fixed), a copy-cancel race fixed, 184 strings in all 38 languages.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
This commit was merged in pull request #18.
This commit is contained in:
Alby96andClaude Opus 5.5 authored and Alby96 committed 2026-11-04 20:03:52 +01:00
1 parent 81c303f484
commit db389a54f2
364 files changed
+17417 -595

No files matched your search

@@ -0,0 +1,470 @@
using Chronos.Core.Archiving;
using Chronos.Core.Diagnostics;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Data.Projects;
namespace Chronos.Data.Workspaces;
/// <summary>Why the workspace's clean-up kept the media of a project it would have cleaned (each reason is logged with the project).</summary>
public enum WorkspaceKeep
{
/// <summary>The project is open in Edit.</summary>
Open,
/// <summary>The project is not in step with its home: changes not saved, versions not sent or not taken, a conflict, a damaged version.</summary>
NotInSync,
/// <summary>The project's home cannot be asked now: whether it holds everything is not known.</summary>
Offline,
/// <summary>A copy, render or send of the project, or a move of the workspace, is queued, running or paused.</summary>
Jobs,
/// <summary>The project's ledger holds renders made here that its home does not have yet.</summary>
UnsentRenders,
/// <summary>The project's working state, media index or render ledger cannot be read.</summary>
Unreadable,
/// <summary>The project holds no media files to remove.</summary>
NoMedia,
/// <summary>The project's media were copied just now (the size limit after a copy never takes that one).</summary>
JustCopied,
}
/// <summary>
/// What a run of the workspace's clean-up applies (phase 5c, Settings › Workspace): Clean after (projects not opened for that
/// many days) and Size limit (the projects' folders above it, the least recently used first); 0 turns either off.
/// </summary>
public sealed record WorkspaceCleanOptions
{
/// <summary>Days a project may go unopened before its media are removed; 0 never removes them for their age.</summary>
public int CleanAfterDays { get; init; }
/// <summary>Bytes the projects' folders may take (the trash is not counted: only the user empties it); 0 sets no limit.</summary>
public long SizeLimitBytes { get; init; }
/// <summary>When each project was opened last (Settings' <c>WorkspaceOpened</c>); a project not listed counts from when its working state was saved.</summary>
public IReadOnlyDictionary<Guid, DateTimeOffset> Opened { get; init; } = new Dictionary<Guid, DateTimeOffset>();
/// <summary>Whether a project is open in Edit now (asked again under the project's lease, just before anything goes).</summary>
public Func<Guid, bool> IsOpen { get; init; } = _ => false;
/// <summary>The project whose media a copy just brought (the size limit after a copy): never cleaned by this run.</summary>
public Guid? JustCopied { get; init; }
/// <summary>The time the ages count from.</summary>
public DateTimeOffset Now { get; init; } = DateTimeOffset.UtcNow;
}
/// <summary>A project whose media the clean-up removed: the bytes freed and the originals set aside in the workspace's trash.</summary>
public sealed record WorkspaceCleanedProject(Guid ProjectId, string Name, bool IsMain, string SequenceTitle, long Freed, int Trashed);
/// <summary>A project the clean-up would have cleaned and kept, with why.</summary>
public sealed record WorkspaceKeptProject(Guid ProjectId, string Name, bool IsMain, string SequenceTitle, WorkspaceKeep Reason);
/// <summary>What a run of the workspace's clean-up did: the projects cleaned, those kept with why, and the projects' bytes before and after.</summary>
public sealed record WorkspaceCleanReport(IReadOnlyList<WorkspaceCleanedProject> Cleaned, IReadOnlyList<WorkspaceKeptProject> Kept, long UsedBefore, long UsedAfter)
{
/// <summary>A run over a workspace with nothing in it.</summary>
public static WorkspaceCleanReport Nothing { get; } = new([], [], 0, 0);
/// <summary>The bytes the run freed.</summary>
public long Freed => Cleaned.Sum(c => c.Freed);
/// <summary>The originals the run set aside in the workspace's trash (their home no longer holds them).</summary>
public int Trashed => Cleaned.Sum(c => c.Trashed);
}
/// <summary>
/// The workspace's clean-up (phase 5c, P4): removes the media of projects whose home holds everything they hold, so they are
/// copied again when next opened. A project is cleaned only when it is in step with its home, asked now (never while the
/// home is out of reach), not open in Edit, with no copy, render or send of it queued or running (nor a move of the
/// workspace), and no local render waiting to be sent. Cleaning removes <c>media/proxies</c> and <c>media/originals</c>
/// (the media index stays, its frames naming no file: <see cref="WorkspaceMedia.Cleaned"/>) and the renders of
/// <c>exports/</c> the home took; never the working state, the versions, the sync state, the deep report or a render not
/// sent. An original is deleted only when the home still holds the same frame with the same SHA-256; otherwise it is moved
/// to the workspace's trash (<c>.trash/&lt;id&gt;-&lt;ticks&gt;/originals</c>). The trash itself is never emptied here.
/// </summary>
public sealed class WorkspaceCleaner
{
private static readonly JobKind[] ProjectJobs = [JobKind.WorkspaceCopy, JobKind.WorkspaceRender, JobKind.WorkspaceSend];
private readonly WorkspaceSync _sync;
private readonly ChronosDatabase _db;
private readonly Func<BundleHome, IProjectHome?> _homes;
/// <param name="homes">The home of a workspace project; null for one this PC cannot reach now.</param>
public WorkspaceCleaner(WorkspaceSync sync, ChronosDatabase db, Func<BundleHome, IProjectHome?> homes)
{
ArgumentNullException.ThrowIfNull(sync);
ArgumentNullException.ThrowIfNull(db);
ArgumentNullException.ThrowIfNull(homes);
_sync = sync;
_db = db;
_homes = homes;
}
/// <summary>How long a project another writer holds is waited for; past it the project is kept (<see cref="WorkspaceKeep.Jobs"/>).</summary>
public TimeSpan LockWait { get; init; } = TimeSpan.FromSeconds(30);
/// <summary>
/// One run: the projects not opened for <see cref="WorkspaceCleanOptions.CleanAfterDays"/>, oldest first, then, while the
/// projects' folders take more than <see cref="WorkspaceCleanOptions.SizeLimitBytes"/>, the least recently used others.
/// Each project the policies name is cleaned or kept with its reason.
/// </summary>
public async Task<WorkspaceCleanReport> CleanAsync(WorkspaceCleanOptions options, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(options);
var candidates = new List<Candidate>();
var used = 0L;
foreach (var project in _sync.Store.List())
{
cancellationToken.ThrowIfCancellationRequested();
var candidate = Candidate.Of(project, options);
used += candidate.FolderBytes;
candidates.Add(candidate);
}
var before = used;
var cleaned = new List<WorkspaceCleanedProject>();
var kept = new List<WorkspaceKeptProject>();
var named = new HashSet<Guid>();
var moving = await _db.Jobs.HasPendingAsync([JobKind.WorkspaceMove], null, cancellationToken).ConfigureAwait(false);
async Task CleanOneAsync(Candidate candidate)
{
named.Add(candidate.Project.ProjectId);
var (reason, freed, trashed) = await TryCleanAsync(candidate, options, moving, cancellationToken).ConfigureAwait(false);
if (reason is { } why)
{
kept.Add(new WorkspaceKeptProject(candidate.Project.ProjectId, candidate.Name, candidate.IsMain, candidate.Title, why));
return;
}
cleaned.Add(new WorkspaceCleanedProject(candidate.Project.ProjectId, candidate.Name, candidate.IsMain, candidate.Title, freed, trashed));
used -= freed;
}
if (options.CleanAfterDays > 0)
{
var cutoff = options.Now - TimeSpan.FromDays(options.CleanAfterDays);
foreach (var candidate in candidates.Where(c => c.Targeted && c.LastUsed < cutoff).OrderBy(c => c.LastUsed).ToList())
await CleanOneAsync(candidate).ConfigureAwait(false);
}
if (options.SizeLimitBytes > 0)
{
foreach (var candidate in candidates.Where(c => c.Targeted && !named.Contains(c.Project.ProjectId)).OrderBy(c => c.LastUsed).ToList())
{
if (used <= options.SizeLimitBytes) break;
await CleanOneAsync(candidate).ConfigureAwait(false);
}
}
foreach (var project in cleaned)
Log.Info("workspace", $"Media of project {project.ProjectId} ('{project.Name}') removed by the clean-up: {project.Freed} bytes freed, {project.Trashed} original(s) kept in the trash");
foreach (var project in kept) Log.Info("workspace", $"Media of project {project.ProjectId} ('{project.Name}') kept by the clean-up: {project.Reason}");
return new WorkspaceCleanReport(cleaned, kept, before, Math.Max(0, used));
}
/// <summary>The frames of <paramref name="media"/> whose original the home no longer holds as it is here (all of them when the home's frames are not known).</summary>
public static IReadOnlyList<WorkspaceFrame> LastOriginals(WorkspaceMedia? media, HomeFrames? home)
{
if (media is null) return [];
var held = home?.Frames.ToDictionary(f => f.Id);
return [.. media.Frames.Where(f => f.HasOriginal && (held is null || !Holds(held, f)))];
static bool Holds(Dictionary<Guid, Frame> held, WorkspaceFrame frame) =>
frame.OriginalSha256 is { Length: > 0 } sha && held.TryGetValue(frame.Id, out var there) && (there.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0
&& string.Equals(there.Sha256, sha, StringComparison.OrdinalIgnoreCase);
}
/// <summary>
/// The local renders of the project its home does not have yet (its ledger's unsent entries); a ledger that cannot be
/// read counts the files of <c>exports/</c> besides it, at least one: whether they went home is not known.
/// </summary>
public static int UnsentRenders(WorkspaceProject project)
{
ArgumentNullException.ThrowIfNull(project);
try
{
return project.ReadRenders().Unsent.Count;
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Warning("workspace", $"The render ledger of project {project.ProjectId} cannot be read: its renders count as not sent", ex);
var files = Directory.Exists(project.ExportsDirectory)
? Directory.EnumerateFiles(project.ExportsDirectory, "*", SearchOption.AllDirectories)
.Count(f => !string.Equals(Path.GetFileName(f), WorkspaceProject.RendersFileName, StringComparison.OrdinalIgnoreCase))
: 0;
return Math.Max(1, files);
}
}
/// <summary>
/// The workspace holds the project's media to edit: a media index whose files the clean-up did not remove (the folders of
/// proxies or originals there say so at once; otherwise the index is read).
/// </summary>
public static bool HoldsMedia(WorkspaceProject project)
{
ArgumentNullException.ThrowIfNull(project);
if (!File.Exists(project.MediaIndexPath)) return false;
if (Directory.Exists(project.ProxiesDirectory) || Directory.Exists(project.OriginalsDirectory)) return true;
try
{
return project.ReadMedia() is { IsCleaned: false };
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
// The index is there: whatever it holds, opening it says so.
return true;
}
}
/// <summary>Files and bytes under <paramref name="folder"/>, leases left out; nothing when it is not there.</summary>
public static (int Files, long Bytes) Measure(string folder)
{
var (files, bytes) = (0, 0L);
if (!Directory.Exists(folder)) return (files, bytes);
foreach (var file in new DirectoryInfo(folder).EnumerateFiles("*", SearchOption.AllDirectories))
{
if (WorkspaceProject.IsLeaseFile(file.Name)) continue;
files++;
bytes += file.Length;
}
return (files, bytes);
}
/// <summary>The project cleaned, or why it is kept.</summary>
private async Task<(WorkspaceKeep? Kept, long Freed, int Trashed)> TryCleanAsync(Candidate candidate, WorkspaceCleanOptions options, bool moving,
CancellationToken cancellationToken)
{
var project = candidate.Project;
var id = project.ProjectId;
if (candidate.Unreadable || candidate.Bundle is not { Workspace: { } workspace }) return (WorkspaceKeep.Unreadable, 0, 0);
if (options.JustCopied == id) return (WorkspaceKeep.JustCopied, 0, 0);
if (options.IsOpen(id)) return (WorkspaceKeep.Open, 0, 0);
if (moving || await BusyAsync(id, workspace.Home.SequenceId, cancellationToken).ConfigureAwait(false)) return (WorkspaceKeep.Jobs, 0, 0);
if (candidate.CleanableBytes == 0) return (WorkspaceKeep.NoMedia, 0, 0);
if (ReadLedger(project) is not { } ledger) return (WorkspaceKeep.Unreadable, 0, 0);
if (ledger.Unsent.Count > 0) return (WorkspaceKeep.UnsentRenders, 0, 0);
if (_homes(workspace.Home) is not { } home) return (WorkspaceKeep.Offline, 0, 0);
WorkspaceStatus status;
try
{
status = await _sync.StatusAsync(id, home, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is ProjectException or DatabaseException or IOException or InvalidDataException or LeaseUnavailableException)
{
Log.Info("workspace", $"The home of project {id} could not be asked for the clean-up: {ex.Message}");
return (WorkspaceKeep.Offline, 0, 0);
}
if (status.Offline && status.State != WorkspaceState.Gone) return (WorkspaceKeep.Offline, 0, 0);
if (status.State != WorkspaceState.InSync) return (WorkspaceKeep.NotInSync, 0, 0);
// The home's frames, asked now: an original goes only when the home holds the same bytes. Every file of the folder of
// originals counts, named by the media index or not (one a removal could not move aside stays unnamed there).
HomeFrames? held = null;
if (HoldsOriginalFiles(project))
{
try
{
held = await home.FramesAsync(cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is HomeUnavailableException or ProjectException or DatabaseException or IOException)
{
Log.Info("workspace", $"The frames of project {id}'s home could not be read for the clean-up: {ex.Message}");
return (WorkspaceKeep.Offline, 0, 0);
}
}
IDisposable lease;
try
{
lease = await project.LockAsync(LockWait, cancellationToken).ConfigureAwait(false);
}
catch (LeaseUnavailableException)
{
return (WorkspaceKeep.Jobs, 0, 0);
}
using (lease)
{
// Asked again under the lease: the editor may have opened it, a save or a render may have come meanwhile.
if (options.IsOpen(id)) return (WorkspaceKeep.Open, 0, 0);
var local = await _sync.StatusAsync(id, null, cancellationToken).ConfigureAwait(false);
if (local.State != WorkspaceState.InSync) return (WorkspaceKeep.NotInSync, 0, 0);
if (ReadLedger(project) is not { } renders) return (WorkspaceKeep.Unreadable, 0, 0);
if (renders.Unsent.Count > 0) return (WorkspaceKeep.UnsentRenders, 0, 0);
WorkspaceMedia? media;
try
{
media = project.ReadMedia();
}
catch (InvalidDataException)
{
return (WorkspaceKeep.Unreadable, 0, 0);
}
if (media is null || media.IsCleaned) return (WorkspaceKeep.NoMedia, 0, 0);
var (freed, trashed) = Clear(project, media, renders, held, options.Now);
return (null, freed, trashed);
}
}
/// <summary>
/// Removes the project's media and its sent renders, under its lease: the media index rewritten with no file first (so
/// nothing ever names a file that is gone), then the sent renders (only files inside <c>exports/</c>), then the
/// originals, each hashed now and deleted only when the home holds the same bytes, else moved to the trash
/// (<see cref="WorkspaceOriginals.ClearOriginals"/>), and last the proxies. A file that cannot be deleted or moved (held
/// open) stays, and is logged.
/// </summary>
private static (long Freed, int Trashed) Clear(WorkspaceProject project, WorkspaceMedia media, WorkspaceRenders renders, HomeFrames? held, DateTimeOffset now)
{
var before = Cleanable(project, renders);
project.WriteMedia(media with
{
Frames = [.. media.Frames.Select(f => f with { File = null, FileSha256 = null, FileBytes = 0, FileKind = null, Original = null, OriginalSha256 = null, OriginalBytes = 0 })],
Cleaned = now,
});
var ledger = renders;
foreach (var entry in renders.Entries.Where(e => e.IsSent))
{
if (project.ContainedExportFile(entry.File) is not { } path)
{
Log.Warning("workspace", $"The ledger of project {project.ProjectId} names '{entry.File}' for render {entry.RenderId}, not a file of its exports: nothing is removed for it");
continue;
}
try
{
if (Directory.Exists(path)) Directory.Delete(path, recursive: true);
else if (File.Exists(path)) File.Delete(path);
ledger = ledger.Without(entry.RenderId);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("workspace", $"The sent render {path} could not be removed; it stays", ex);
}
}
if (!ReferenceEquals(ledger, renders)) project.WriteRenders(ledger);
var (trashed, trashedBytes) = (0, 0L);
if (Directory.Exists(project.OriginalsDirectory))
{
using var trash = WorkspaceTrash.Begin(project.Store.Root, project.ProjectId, now.UtcDateTime);
var cleared = WorkspaceOriginals.ClearOriginals(project, held?.Frames, trash.Path);
(trashed, trashedBytes) = (cleared.Trashed, cleared.TrashedBytes);
if (trash.Finish() is { } aside) Log.Info("workspace", $"{trashed} original(s) of project {project.ProjectId} its home does not hold as they are kept in {aside}");
if (cleared.Left > 0) Log.Warning("workspace", $"{cleared.Left} original(s) of project {project.ProjectId} could be neither deleted nor set aside; they stay");
}
try
{
if (Directory.Exists(project.ProxiesDirectory)) Directory.Delete(project.ProxiesDirectory, recursive: true);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("workspace", $"{project.ProxiesDirectory} could not be removed whole; what is left stays", ex);
}
var after = Cleanable(project, ledger);
return (Math.Max(0, before - after - trashedBytes), trashed);
}
/// <summary>The project's folder of originals holds a file (an original, named by the media index or not, or a part of one).</summary>
public static bool HoldsOriginalFiles(WorkspaceProject project)
{
ArgumentNullException.ThrowIfNull(project);
try
{
return Directory.Exists(project.OriginalsDirectory) && Directory.EnumerateFiles(project.OriginalsDirectory, "*", SearchOption.AllDirectories).Any();
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// Not known: counted as holding some, so the home is asked.
return true;
}
}
/// <summary>What a clean-up may remove of the project: its proxies, its originals and the renders its home took.</summary>
private static long Cleanable(WorkspaceProject project, WorkspaceRenders renders) =>
Measure(project.ProxiesDirectory).Bytes + Measure(project.OriginalsDirectory).Bytes + SentBytes(project, renders);
/// <summary>The bytes of the renders of <paramref name="renders"/> the home took that are still in <c>exports/</c> (an image sequence's folder counts its files).</summary>
private static long SentBytes(WorkspaceProject project, WorkspaceRenders renders) =>
renders.Entries.Where(e => e.IsSent).Select(e => project.ContainedExportFile(e.File)).OfType<string>()
.Sum(path => File.Exists(path) ? new FileInfo(path).Length : Measure(path).Bytes);
/// <summary>The project's ledger; null (logged) when it cannot be read.</summary>
private static WorkspaceRenders? ReadLedger(WorkspaceProject project)
{
try
{
return project.ReadRenders();
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Warning("workspace", $"The render ledger of project {project.ProjectId} cannot be read: the clean-up keeps the project", ex);
return null;
}
}
/// <summary>A copy, render or send of the project is queued, running or paused (one whose parameters do not read counts too).</summary>
private async Task<bool> BusyAsync(Guid projectId, Guid sequenceId, CancellationToken cancellationToken)
{
foreach (var kind in ProjectJobs)
{
foreach (var job in await _db.Jobs.ListPendingAsync(kind, sequenceId, cancellationToken).ConfigureAwait(false))
{
var named = kind switch
{
JobKind.WorkspaceCopy => WorkspaceJobs.CopyParameters(job)?.ProjectId,
JobKind.WorkspaceRender => WorkspaceJobs.RenderParameters(job)?.ProjectId,
_ => WorkspaceJobs.SendParameters(job)?.ProjectId,
};
if (named is null || named == projectId) return true;
}
}
return false;
}
/// <summary>A project of the workspace as the clean-up weighs it.</summary>
private sealed record Candidate(WorkspaceProject Project, ProjectBundle? Bundle, WorkspaceMedia? Media, bool Unreadable, string Name, bool IsMain, string Title,
DateTimeOffset LastUsed, long FolderBytes, long CleanableBytes)
{
/// <summary>The policies may name it: media a copy wrote (cleaned ones are done), or a copy that cannot be read holding media files.</summary>
public bool Targeted => Media is { IsCleaned: false } || (Unreadable && CleanableBytes > 0);
public static Candidate Of(WorkspaceProject project, WorkspaceCleanOptions options)
{
var folder = Measure(project.Folder).Bytes;
var media = Measure(project.ProxiesDirectory).Bytes + Measure(project.OriginalsDirectory).Bytes;
var saved = File.Exists(project.BundlePath) ? new DateTimeOffset(File.GetLastWriteTimeUtc(project.BundlePath), TimeSpan.Zero) : DateTimeOffset.MinValue;
ProjectBundle bundle;
try
{
bundle = project.ReadBundle();
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
return new Candidate(project, null, null, true, project.ProjectId.ToString("N"), false, string.Empty, Used(saved), folder, media);
}
var workspace = bundle.Workspace!;
var title = bundle.Sequence.Title is { Length: > 0 } t ? t : bundle.Sequence.Folder;
WorkspaceMedia? index;
var unreadable = false;
try
{
index = project.ReadMedia();
}
catch (InvalidDataException)
{
(index, unreadable) = (null, true);
}
var sent = 0L;
try
{
sent = SentBytes(project, project.ReadRenders());
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
// Weighed by its media alone: cleaning it is refused while its ledger cannot be read.
}
return new Candidate(project, bundle, index, unreadable, workspace.Name, workspace.IsMain, title, Used(bundle.Saved), folder, media + sent);
DateTimeOffset Used(DateTimeOffset fallback) => options.Opened.TryGetValue(project.ProjectId, out var opened) ? opened : fallback;
}
}
}
@@ -19,60 +19,6 @@ using SequenceManifest = Chronos.Protocol.SequenceManifest;
namespace Chronos.Data.Workspaces;
/// <summary>Builds the jobs of the workspace (owner review 2, R8), and their handlers for the desktop's engine.</summary>
public static class WorkspaceJobs
{
/// <summary>
/// <see cref="JobKind.WorkspaceCopy"/>: the media of workspace project <paramref name="projectId"/> (checked out first,
/// <see cref="WorkspaceSync.CheckoutAsync"/>) copied from its home, the sequence of this PC's archive.
/// </summary>
public static ArchiveJobRequest Copy(Sequence sequence, string workspaceRoot, Guid projectId, string media = BundleMedia.Proxies)
{
ArgumentNullException.ThrowIfNull(sequence);
ArgumentException.ThrowIfNullOrWhiteSpace(workspaceRoot);
return new ArchiveJobRequest(JobKind.WorkspaceCopy, [sequence.Id], sequence.DisplayTitle,
JsonSerializer.Serialize(new WorkspaceCopyParameters(workspaceRoot, projectId, media), WorkspaceJson.Default.WorkspaceCopyParameters), JobPriority.User);
}
/// <summary>
/// <see cref="JobKind.WorkspaceCopy"/> of a project whose home is a server (phase 5b): the media of workspace project
/// <paramref name="projectId"/> (checked out first) downloaded from sequence <paramref name="sequenceId"/> of
/// <paramref name="server"/> (<see cref="SyncService.ServerKey"/>), into the workspace alone. The job's target is the
/// server's sequence, as for a sync download.
/// </summary>
public static ArchiveJobRequest CopyFromServer(Guid sequenceId, string title, string server, string workspaceRoot, Guid projectId, string media = BundleMedia.Proxies)
{
ArgumentException.ThrowIfNullOrWhiteSpace(server);
ArgumentException.ThrowIfNullOrWhiteSpace(workspaceRoot);
return new ArchiveJobRequest(JobKind.WorkspaceCopy, [sequenceId], title,
JsonSerializer.Serialize(new WorkspaceCopyParameters(workspaceRoot, projectId, media, server), WorkspaceJson.Default.WorkspaceCopyParameters), JobPriority.User);
}
/// <summary>
/// The handlers of the workspace's jobs for a desktop's engine (never a server's: the job writes into a folder its
/// parameters name); copies from a server fail without a connection to it.
/// </summary>
public static IReadOnlyList<IJobHandler> CreateHandlers(ChronosDatabase db, ArchiveService archives) => CreateHandlers(db, archives, null);
/// <param name="servers">The connection a copy from a server's sequence downloads through (the desktop's signed-in session).</param>
public static IReadOnlyList<IJobHandler> CreateHandlers(ChronosDatabase db, ArchiveService archives, ISyncConnector? servers) =>
[new WorkspaceCopyJobHandler(db, archives, servers)];
/// <summary>
/// What a first copy of a sequence's proxies writes into a workspace, as the copy reckons it before it starts (the
/// archive's proxies' sizes, an estimate for those it makes), and the free bytes on <paramref name="workspaceRoot"/>'s
/// drive (null when it cannot be measured): "Open in workspace" shows both. The copy keeps <see cref="CopySpaceReserve"/>
/// free besides.
/// </summary>
/// <exception cref="InvalidOperationException">The index has no such sequence or archive.</exception>
public static Task<(long Bytes, long? Free)> CopyFactsAsync(ChronosDatabase db, ArchiveService archives, Guid sequenceId, string workspaceRoot,
CancellationToken cancellationToken = default) =>
new WorkspaceCopyJobHandler(db, archives).FactsAsync(sequenceId, workspaceRoot, cancellationToken);
/// <summary>Bytes the copy leaves free on the workspace's drive besides what it copies.</summary>
public static long CopySpaceReserve => WorkspaceCopyJobHandler.SpaceReserve;
}
/// <summary>
/// <see cref="JobKind.WorkspaceCopy"/> for a sequence of this PC's archive (owner review 2, R8): its proxies into the
/// workspace project's <c>media/proxies/&lt;frame id&gt;.jpg</c>, the archive's own when current, else made here from the
@@ -91,7 +37,14 @@ public static class WorkspaceJobs
/// removes the media it wrote itself, once it has stopped writing and under the project's lease, unless another copy of the
/// project goes on with them (<see cref="RemovePartialMediaAsync"/>); one cancelled while paused or failed, which no handler
/// runs, has them removed the same way (<see cref="CancelledAsync"/>).
/// Originals come in phase 5c: asked for, the job fails saying so.
/// With <see cref="BundleMedia.Originals"/> (phase 5c) the proxies are copied first as above, then each frame's original into
/// <c>media/originals/&lt;frame id&gt;-&lt;sha8&gt;&lt;ext&gt;</c> (<see cref="WorkspaceProject.OriginalName"/>): streamed into a
/// <c>.part</c> file and hashed as it is written, the SHA-256 checked against the home's (recorded as copied when the home has
/// none), the file read back after its rename; one already there with the same SHA-256 is kept. A frame whose original the
/// home lacks (gone, flagged missing or damaged, refused by a server, not matching its SHA-256 twice) keeps its proxy: the
/// copy finishes saying how many. The space reckoned adds the originals not held yet; the checkpoint of that stage is
/// <c>originals:&lt;frame&gt;:&lt;frames version&gt;:&lt;folder&gt;</c>. Originals the committed media no longer name are deleted
/// only when the home holds a frame of the same SHA-256, else kept in the workspace's trash (<see cref="WorkspaceOriginals"/>).
/// A project whose home is a server (phase 5b, <see cref="WorkspaceJobs.CopyFromServer"/>) has its media downloaded from
/// the server through the desktop's session (<see cref="ISyncConnector"/>), never through this PC's archive: the frames as
/// of one frames version with the aliases of the server's conversions (<c>GET …/projects/frames</c>), each frame's proxy as
@@ -100,10 +53,19 @@ public static class WorkspaceJobs
/// arrives and checked against the file written, four at a time; then the server's deep report. A file the editor's media
/// or this run hold of the same photo is hashed again and kept; the editor's file is never written over, as above. The
/// space, the passes, the checkpoint and a cancelled first copy go as for a copy from this PC's archive (the server's
/// folder of the sequence in the checkpoint). Like the sync's downloads it is not held on a metered connection.
/// folder of the sequence in the checkpoint). On a metered connection it waits before it downloads anything while
/// <see cref="WorkspaceJobOptions.HoldTransfers"/> says so (the desktop pauses it meanwhile).
/// A copy queued before the workspace moved works where the workspace is now, and none runs while the workspace is being
/// moved (<see cref="WorkspaceJobRoots"/>). A copy of originals the user cancelled over media a copy finished before
/// removes the originals it wrote that the media do not name, each deleted only when the home holds its bytes
/// (<see cref="RemoveCancelledOriginalsAsync"/>).
/// </summary>
internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService archives, ISyncConnector? servers = null) : ICancelledJobHandler
internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService archives, ISyncConnector? servers = null, WorkspaceJobOptions? options = null)
: ICancelledJobHandler
{
/// <summary>How often a copy held on a metered connection looks again.</summary>
internal static TimeSpan HoldPoll { get; set; } = TimeSpan.FromSeconds(2);
private const int PageSize = 1000;
private const int Passes = 3;
@@ -135,8 +97,7 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
{
var ct = context.Cancellation;
var parameters = Parameters(context.Job);
if (parameters.Media == BundleMedia.Originals) throw new NotSupportedException(Loc.L("Errors.Workspace.OriginalsLater"));
if (parameters.Media != BundleMedia.Proxies) throw new InvalidDataException(Loc.L("Errors.Workspace.Parameters"));
if (parameters.Media is not (BundleMedia.Proxies or BundleMedia.Originals)) throw new InvalidDataException(Loc.L("Errors.Workspace.Parameters"));
if (parameters.Server is { Length: > 0 } server)
{
await RunFromServerAsync(context, parameters, server).ConfigureAwait(false);
@@ -149,7 +110,13 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
// R8), and a workspace inside one would put its copies among that library's own files.
if (await InsideALibraryAsync(parameters.WorkspaceRoot, ct).ConfigureAwait(false))
throw new InvalidOperationException(Loc.L("Errors.Workspace.InsideLibrary"));
var store = new WorkspaceStore(parameters.WorkspaceRoot);
// Where the project is now (a workspace moved since the job was queued), never while it is being moved; nothing is
// made where it is not.
var root = WorkspaceJobRoots.Resolve(parameters.WorkspaceRoot, parameters.ProjectId, options?.CurrentRoot, "Errors.Workspace.NotCheckedOut");
if (!PathsEqual(root, parameters.WorkspaceRoot) && await InsideALibraryAsync(root, ct).ConfigureAwait(false))
throw new InvalidOperationException(Loc.L("Errors.Workspace.InsideLibrary"));
parameters = parameters with { WorkspaceRoot = root };
var store = new WorkspaceStore(root);
var project = store.Find(parameters.ProjectId) ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.NotCheckedOut"));
if (project.ReadBundle().Workspace is not { Home: { IsArchive: true } home } || home.SequenceId != sequenceId)
throw new InvalidOperationException(Loc.L("Errors.Workspace.OtherHome"));
@@ -162,6 +129,7 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
{
// Stopped writing: what a first copy the user cancelled wrote goes (a pause or a shutdown resumes it).
await RemovePartialMediaAsync(context.Job, project).ConfigureAwait(false);
await RemoveCancelledOriginalsAsync(context.Job, project, parameters).ConfigureAwait(false);
throw;
}
}
@@ -183,13 +151,27 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
{
return Task.CompletedTask;
}
// Opening a store makes its folder: one gone stays gone.
if (!Directory.Exists(Path.Combine(parameters.WorkspaceRoot, parameters.ProjectId.ToString("N")))) return Task.CompletedTask;
string root;
try
{
// Where the project is now; opening a store makes its folder, so one gone stays gone. A workspace being moved is
// left as it is: the next copy checks every file it finds.
root = WorkspaceJobRoots.Resolve(parameters.WorkspaceRoot, parameters.ProjectId, options?.CurrentRoot, "Errors.Workspace.NotCheckedOut");
}
catch (InvalidOperationException ex)
{
Log.Info("workspace", $"The media of cancelled copy {job.Id} are left as they are: {ex.Message}");
return Task.CompletedTask;
}
_ = Task.Run(async () =>
{
try
{
if (new WorkspaceStore(parameters.WorkspaceRoot).Find(parameters.ProjectId) is { } project) await RemovePartialMediaAsync(job, project).ConfigureAwait(false);
if (new WorkspaceStore(root).Find(parameters.ProjectId) is { } project)
{
await RemovePartialMediaAsync(job, project).ConfigureAwait(false);
await RemoveCancelledOriginalsAsync(job, project, parameters).ConfigureAwait(false);
}
}
catch (Exception ex) when (ex is not OutOfMemoryException)
{
@@ -241,6 +223,83 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
static bool Clearable(WorkspaceProject project) => project.Exists && !File.Exists(project.MediaIndexPath) && Directory.Exists(project.MediaDirectory);
}
/// <summary>Waits while transfers with a server are held (a metered connection), saying so once.</summary>
private async Task WhileHeldAsync(JobContext context)
{
if (options?.HoldTransfers is not { } held || !held()) return;
context.Log(Loc.L("Archive.Log.WorkspaceCopyHeld"));
while (held()) await Task.Delay(HoldPoll, context.Cancellation).ConfigureAwait(false);
}
/// <summary>
/// A copy of originals the user cancelled over media a copy finished before (<c>media.json</c> there, so
/// <see cref="RemovePartialMediaAsync"/> keeps them): the originals it wrote that those media do not name go, under the
/// project's lease, unless another copy of the project goes on with them; each is hashed and deleted only when the home's
/// frames, read now, hold its bytes, else kept in the workspace's trash. What a download left half done goes. A home that
/// cannot be asked leaves the whole files where they are (the next copy reuses them, a clean-up hashes them).
/// </summary>
private async Task RemoveCancelledOriginalsAsync(Job job, WorkspaceProject project, WorkspaceCopyParameters parameters)
{
if (parameters.Media != BundleMedia.Originals) return;
try
{
if (!project.Exists || !File.Exists(project.MediaIndexPath) || !Directory.Exists(project.OriginalsDirectory)) return;
if (!await StillToClearAsync(job, project).ConfigureAwait(false)) return;
var home = await HomeFramesOrNullAsync(job, parameters).ConfigureAwait(false);
using (await project.LockExistingAsync(LockWait, CancellationToken.None).ConfigureAwait(false))
{
if (!await StillToClearAsync(job, project).ConfigureAwait(false)) return;
lock (WritersGate)
{
// A copy of this program that started since the jobs were read: its files stay.
if (Writers.ContainsKey(project.Folder)) return;
}
if (TryReadMedia(project) is not { } media || !Directory.Exists(project.OriginalsDirectory)) return;
var named = media.Frames.Where(f => f.Original is not null).Select(f => Path.GetFileName(f.Original!)).ToHashSet(StringComparer.OrdinalIgnoreCase);
var staging = Path.Combine(project.OriginalsDirectory, OriginalsStaging);
if (Directory.Exists(staging)) Directory.Delete(staging, recursive: true);
var written = Directory.EnumerateFiles(project.OriginalsDirectory).Where(f => !named.Contains(Path.GetFileName(f))).ToList();
foreach (var part in written.Where(IsPart)) TryDelete(part);
var whole = written.Where(f => !IsPart(f)).ToList();
if (home is null)
{
if (whole.Count > 0) Log.Info("workspace", $"The home of project {project.ProjectId} could not be asked: the {whole.Count} original(s) cancelled copy {job.Id} wrote stay");
return;
}
var trashed = WorkspaceOriginals.DeleteOrTrash(project, whole, home);
if (!Directory.EnumerateFileSystemEntries(project.OriginalsDirectory).Any()) Directory.Delete(project.OriginalsDirectory);
if (whole.Count > 0) Log.Info("workspace", $"The {whole.Count} original(s) cancelled copy {job.Id} of project {project.ProjectId} wrote were removed ({trashed} kept in the trash)");
}
}
catch (DirectoryNotFoundException)
{
// The project's folder went meanwhile (its removal): nothing is left to clear.
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or DatabaseException or LeaseUnavailableException)
{
Log.Warning("workspace", $"The originals of a cancelled copy of project {project.ProjectId} could not be removed", ex);
}
}
/// <summary>The frames of the copy's home as they are now (this PC's index, or the server's); null when they cannot be read.</summary>
private async Task<IReadOnlyList<Frame>?> HomeFramesOrNullAsync(Job job, WorkspaceCopyParameters parameters)
{
if (job.TargetIds is not [var sequenceId]) return null;
try
{
if (parameters.Server is not { Length: > 0 } server) return (await FramesAsync(sequenceId, CancellationToken.None).ConfigureAwait(false)).Frames;
if (servers is null) return null;
using var remote = await servers.ConnectAsync(server, CancellationToken.None).ConfigureAwait(false);
if (remote is null) return null;
return (await ServerFramesAsync(remote, sequenceId, CancellationToken.None).ConfigureAwait(false)).Frames;
}
catch (Exception ex) when (ex is not OutOfMemoryException)
{
Log.Info("workspace", $"The home of cancelled copy {job.Id} could not be asked for its frames: {ex.Message}");
return null;
}
}
/// <summary>The cancelled copy still reads so, and no other copy of the project into this workspace waits, runs or is paused.</summary>
private async Task<bool> StillToClearAsync(Job job, WorkspaceProject project)
{
@@ -313,6 +372,7 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
// Read only: a read-only archive is copied from like any other.
if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(archive.RootPath, ArchiveAvailability.Offline);
var (copied, checkpointVersion, checkpointFolder) = Parse(context.Checkpoint);
var withOriginals = parameters.Media == BundleMedia.Originals;
for (var pass = 1; ; pass++)
{
var (row, frames) = await FramesAsync(sequenceId, ct).ConfigureAwait(false);
@@ -328,15 +388,18 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
// Files kept from an earlier run are this frames version's and this folder's only (a conversion keeps a frame's id,
// not its photo; a sequence moved meanwhile may have left its photos' names to another sequence's).
if ((checkpointVersion is { } version && version != row.FramesVersion) || (checkpointFolder is { } read && read != row.Folder)) copied = 0;
var total = (long)frames.Count;
var total = (long)frames.Count * (withOriginals ? 2 : 1);
var tally = new Tally();
var entries = new WorkspaceFrame[frames.Count];
// The media the editor reads until this copy commits: their files stay until then (RemoveStale).
var current = TryReadMedia(project)?.Frames.Where(f => f.File is not null).ToDictionary(f => f.Id) ?? [];
var held = TryReadMedia(project)?.Frames ?? [];
var current = held.Where(f => f.File is not null).ToDictionary(f => f.Id);
var heldOriginals = held.Where(f => f.Original is not null).ToDictionary(f => f.Id);
Directory.CreateDirectory(project.ProxiesDirectory);
// The workspace is on this PC's own drive, with the index, the settings and the logs: the copy never fills it.
var remaining = Remaining(project, folder, frames, policy);
EnsureSpace(parameters.WorkspaceRoot, remaining[0]);
var originalsLeft = withOriginals ? RemainingOriginals(project, frames, heldOriginals, (f, _) => f.Size) : new long[frames.Count + 1];
EnsureSpace(parameters.WorkspaceRoot, remaining[0] + originalsLeft[0]);
var clock = System.Diagnostics.Stopwatch.StartNew();
for (var i = 0; i < frames.Count; i++)
{
@@ -346,7 +409,7 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
{
await context.CheckpointAsync(Checkpoint(Stage.Copy, i + 1, row.FramesVersion, row.Folder), i + 1, total).ConfigureAwait(false);
// Another program may take the space meanwhile.
EnsureSpace(parameters.WorkspaceRoot, remaining[i + 1]);
EnsureSpace(parameters.WorkspaceRoot, remaining[i + 1] + originalsLeft[0]);
clock.Restart();
}
else
@@ -354,6 +417,27 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
await context.ReportAsync(i + 1, total).ConfigureAwait(false);
}
}
var originals = new OriginalsTally();
if (withOriginals)
{
for (var i = 0; i < frames.Count; i++)
{
ct.ThrowIfCancellationRequested();
entries[i] = await CopyOriginalAsync(project, paths, frames[i], entries[i], heldOriginals.GetValueOrDefault(frames[i].Id),
originals, ct).ConfigureAwait(false);
var done = frames.Count + i + 1;
if (clock.Elapsed >= CheckpointEvery)
{
await context.CheckpointAsync(Checkpoint(Stage.Originals, i + 1, row.FramesVersion, row.Folder), done, total).ConfigureAwait(false);
EnsureSpace(parameters.WorkspaceRoot, originalsLeft[i + 1]);
clock.Restart();
}
else
{
await context.ReportAsync(done, total).ConfigureAwait(false);
}
}
}
var analysis = CopyAnalysis(folder, project, frames.Count);
var (after, now) = await FramesAsync(sequenceId, ct).ConfigureAwait(false);
var moved = after.Folder != row.Folder || !SequenceFolders.Holds(paths.Sequence(after.Folder), sequenceId) || Renamed(frames, now, entries);
@@ -376,15 +460,19 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
}
}
var aliases = await HomePhotos.AliasesAsync(db, sequenceId, ct).ConfigureAwait(false);
var media = new WorkspaceMedia(WorkspaceMedia.FormatName, WorkspaceMedia.CurrentFormatVersion, BundleMedia.Proxies, sequenceId, row.FramesVersion, entries,
var media = new WorkspaceMedia(WorkspaceMedia.FormatName, WorkspaceMedia.CurrentFormatVersion, parameters.Media, sequenceId, row.FramesVersion, entries,
DateTimeOffset.UtcNow, analysis, aliases.Count > 0 ? aliases : null);
// The working state is the editor's meanwhile: it is held only while the media index and the positions move.
using (await project.LockAsync(LockWait, ct).ConfigureAwait(false))
{
// The commit point: a cancel that came while the copy waited (a query it did not stop, the lease) wins here,
// so a cancelled copy never commits and its clean-up removes what it wrote.
ct.ThrowIfCancellationRequested();
Commit(project, project.ReadBundle(), media, row.Folder, row.DisplayTitle);
RemoveStale(project, media);
RemoveStale(project, media, frames);
}
context.Log(Loc.L("Archive.Log.WorkspaceCopied", row.DisplayTitle, tally.Copied, tally.Made, tally.Kept, tally.Missing));
if (withOriginals) originals.LogTo(context, "Archive.Log.WorkspaceOriginals", row.DisplayTitle);
await context.CheckpointAsync(Checkpoint(Stage.Done, frames.Count, row.FramesVersion, row.Folder), total, total).ConfigureAwait(false);
return;
}
@@ -400,13 +488,19 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
if (context.Job.TargetIds is not [var sequenceId]) throw new InvalidDataException(Loc.L("Errors.Workspace.Parameters"));
// Asked before the store is opened, which makes its folder (as for a copy from this PC's archive).
if (await InsideALibraryAsync(parameters.WorkspaceRoot, ct).ConfigureAwait(false)) throw new InvalidOperationException(Loc.L("Errors.Workspace.InsideLibrary"));
var store = new WorkspaceStore(parameters.WorkspaceRoot);
var root = WorkspaceJobRoots.Resolve(parameters.WorkspaceRoot, parameters.ProjectId, options?.CurrentRoot, "Errors.Workspace.NotCheckedOut");
if (!PathsEqual(root, parameters.WorkspaceRoot) && await InsideALibraryAsync(root, ct).ConfigureAwait(false))
throw new InvalidOperationException(Loc.L("Errors.Workspace.InsideLibrary"));
parameters = parameters with { WorkspaceRoot = root };
var store = new WorkspaceStore(root);
var project = store.Find(parameters.ProjectId) ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.NotCheckedOut"));
if (project.ReadBundle().Workspace is not { Home: { Kind: BundleHome.ServerKind } home } || home.SequenceId != sequenceId
|| !string.Equals(home.Server, server, StringComparison.OrdinalIgnoreCase))
throw new InvalidOperationException(Loc.L("Errors.Workspace.OtherHome"));
try
{
// Nothing is downloaded while transfers with a server are held (a metered connection).
await WhileHeldAsync(context).ConfigureAwait(false);
using (WritingMedia(project))
{
using var remote = (servers is null ? null : await servers.ConnectAsync(server, ct).ConfigureAwait(false))
@@ -418,6 +512,7 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
{
// Stopped writing: what a first copy the user cancelled wrote goes (a pause or a shutdown resumes it).
await RemovePartialMediaAsync(context.Job, project).ConfigureAwait(false);
await RemoveCancelledOriginalsAsync(context.Job, project, parameters).ConfigureAwait(false);
throw;
}
}
@@ -426,6 +521,7 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
CancellationToken ct)
{
var (copied, checkpointVersion, checkpointFolder) = Parse(context.Checkpoint);
var withOriginals = parameters.Media == BundleMedia.Originals;
// What this run downloaded, pass after pass: kept where a frame still shows the same photo.
var fetched = new Dictionary<Guid, WorkspaceFrame>();
for (var pass = 1; ; pass++)
@@ -433,15 +529,20 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
var (sequence, frames, assets, aliases, framesVersion, spec) = await ServerFramesAsync(remote, sequenceId, ct).ConfigureAwait(false);
var folder = sequence.Folder ?? string.Empty;
if ((checkpointVersion is { } version && version != framesVersion) || (checkpointFolder is { } read && read != folder)) copied = 0;
var total = (long)frames.Count;
var total = (long)frames.Count * (withOriginals ? 2 : 1);
var tally = new Tally();
var entries = new WorkspaceFrame[frames.Count];
// The media the editor reads until this copy commits: their files stay until then (RemoveStale).
var current = TryReadMedia(project)?.Frames.Where(f => f.File is not null).ToDictionary(f => f.Id) ?? [];
var held = TryReadMedia(project)?.Frames ?? [];
var current = held.Where(f => f.File is not null).ToDictionary(f => f.Id);
var heldOriginals = held.Where(f => f.Original is not null).ToDictionary(f => f.Id);
Directory.CreateDirectory(project.ProxiesDirectory);
// The workspace is on this PC's own drive: the copy never fills it (the server's files estimated at its proxy size).
var remaining = RemainingDownloads(project, frames, assets, spec);
EnsureSpace(parameters.WorkspaceRoot, remaining[0]);
var originalsLeft = withOriginals
? RemainingOriginals(project, frames, heldOriginals, (f, _) => assets.GetValueOrDefault(f.Id) is { } asset ? asset.Size : 0)
: new long[frames.Count + 1];
EnsureSpace(parameters.WorkspaceRoot, remaining[0] + originalsLeft[0]);
var clock = System.Diagnostics.Stopwatch.StartNew();
var options = new ParallelOptions { MaxDegreeOfParallelism = Downloads, CancellationToken = ct };
for (var start = 0; start < frames.Count; start += DownloadBatch)
@@ -455,7 +556,7 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
{
await context.CheckpointAsync(Checkpoint(Stage.Copy, end, framesVersion, folder), end, total).ConfigureAwait(false);
// Another program may take the space meanwhile.
EnsureSpace(parameters.WorkspaceRoot, remaining[end]);
EnsureSpace(parameters.WorkspaceRoot, remaining[end] + originalsLeft[0]);
clock.Restart();
}
else
@@ -464,6 +565,27 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
}
}
foreach (var entry in entries.Where(e => e.File is not null)) fetched[entry.Id] = entry;
var originals = new OriginalsTally();
if (withOriginals)
{
for (var start = 0; start < frames.Count; start += DownloadBatch)
{
var end = Math.Min(frames.Count, start + DownloadBatch);
await Parallel.ForEachAsync(Enumerable.Range(start, end - start), options, async (i, token) =>
entries[i] = await DownloadOriginalAsync(remote, project, frames[i], assets.GetValueOrDefault(frames[i].Id), entries[i],
heldOriginals.GetValueOrDefault(frames[i].Id), originals, token).ConfigureAwait(false)).ConfigureAwait(false);
if (clock.Elapsed >= CheckpointEvery || end == frames.Count)
{
await context.CheckpointAsync(Checkpoint(Stage.Originals, end, framesVersion, folder), frames.Count + end, total).ConfigureAwait(false);
EnsureSpace(parameters.WorkspaceRoot, originalsLeft[end]);
clock.Restart();
}
else
{
await context.ReportAsync(frames.Count + end, total).ConfigureAwait(false);
}
}
}
var analysis = await CopyServerAnalysisAsync(remote, sequenceId, project, frames.Count, ct).ConfigureAwait(false);
var after = await remote.GetSequenceAsync(sequenceId, ct).ConfigureAwait(false);
if (after.FramesVersion != framesVersion && pass < Passes)
@@ -476,15 +598,19 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
continue;
}
// Frames still arriving are copied as they were read (Update media takes the rest).
var media = new WorkspaceMedia(WorkspaceMedia.FormatName, WorkspaceMedia.CurrentFormatVersion, BundleMedia.Proxies, sequenceId, framesVersion, entries,
var media = new WorkspaceMedia(WorkspaceMedia.FormatName, WorkspaceMedia.CurrentFormatVersion, parameters.Media, sequenceId, framesVersion, entries,
DateTimeOffset.UtcNow, analysis, aliases.Count > 0 ? aliases : null);
// The working state is the editor's meanwhile: it is held only while the media index and the positions move.
using (await project.LockAsync(LockWait, ct).ConfigureAwait(false))
{
// The commit point: a cancel that came while the copy waited (a query it did not stop, the lease) wins here,
// so a cancelled copy never commits and its clean-up removes what it wrote.
ct.ThrowIfCancellationRequested();
Commit(project, project.ReadBundle(), media, folder, sequence.DisplayTitle);
RemoveStale(project, media);
RemoveStale(project, media, frames);
}
context.Log(Loc.L("Archive.Log.WorkspaceDownloaded", sequence.DisplayTitle, tally.Copied, tally.Made, tally.Kept, tally.Missing));
if (withOriginals) originals.LogTo(context, "Archive.Log.WorkspaceOriginalsDownloaded", sequence.DisplayTitle);
await context.CheckpointAsync(Checkpoint(Stage.Done, frames.Count, framesVersion, folder), total, total).ConfigureAwait(false);
return;
}
@@ -626,6 +752,166 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
entry is { FileSha256: { } held } && string.Equals(held, sha, StringComparison.OrdinalIgnoreCase) ? entry.FileKind ?? ProxyCache.Proxy : null;
}
/// <summary>
/// A frame's original from this PC's archive into <c>media/originals/</c> (phase 5c), beside its proxy in
/// <paramref name="entry"/>: kept when the workspace holds it already with the same SHA-256, else streamed into a
/// <c>.part</c> file hashed as it is written (never held in memory), checked against the home's SHA-256, renamed and read
/// back. A frame flagged missing or damaged at the home, or whose file is gone, keeps its proxy alone; so does one whose
/// original twice does not match the SHA-256 the home has of it (the part is removed, the copy goes on).
/// </summary>
private static async Task<WorkspaceFrame> CopyOriginalAsync(WorkspaceProject project, ArchivePaths paths, Frame frame, WorkspaceFrame entry,
WorkspaceFrame? held, OriginalsTally tally, CancellationToken cancellationToken)
{
var extension = OriginalExtension(frame);
var expected = string.IsNullOrEmpty(frame.Sha256) ? null : frame.Sha256.ToLowerInvariant();
if ((frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0) return tally.Missed(entry, frame.Index);
if (await HeldOriginalAsync(project, frame, expected, extension, held, cancellationToken).ConfigureAwait(false) is { } kept)
{
tally.Keep();
return entry with { Original = kept.Name, OriginalSha256 = kept.Sha256, OriginalBytes = kept.Bytes };
}
var source = paths.FromRelative(frame.RelativePath);
if (!File.Exists(source)) return tally.Missed(entry, frame.Index);
Directory.CreateDirectory(project.OriginalsDirectory);
var part = Path.Combine(project.OriginalsDirectory, frame.Id.ToString("N") + ".part");
for (var attempt = 1; ; attempt++)
{
string sha;
long bytes;
try
{
await using var input = new FileStream(source, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete, StreamBuffer,
FileOptions.Asynchronous | FileOptions.SequentialScan);
await using var output = new FileStream(part, FileMode.Create, FileAccess.Write, FileShare.None, StreamBuffer, FileOptions.Asynchronous);
using var hashing = new HashingStream(output);
await input.CopyToAsync(hashing, StreamBuffer, cancellationToken).ConfigureAwait(false);
output.Flush(flushToDisk: true);
(sha, bytes) = (hashing.Sha256(), hashing.Written);
}
catch (FileNotFoundException)
{
// Gone while it was read (a Reorganize): the frame keeps its proxy, the pass's check of the folder decides the rest.
TryDelete(part);
return tally.Missed(entry, frame.Index);
}
catch (OperationCanceledException)
{
// A part is never resumed from (a source is read whole): it goes, so a cancel leaves nothing half written.
TryDelete(part);
throw;
}
if (expected is not null && sha != expected)
{
TryDelete(part);
if (attempt == 1) continue;
Log.Warning("workspace", Loc.L("Errors.Workspace.OriginalMismatch", source));
return tally.Missed(entry, frame.Index);
}
var name = WorkspaceProject.OriginalName(frame.Id, sha, extension);
var target = project.MediaFile(name);
File.Move(part, target, overwrite: true);
if (string.Equals(await ContentHash.Sha256Async(target, cancellationToken: cancellationToken).ConfigureAwait(false), sha, StringComparison.OrdinalIgnoreCase))
{
tally.Copy();
return entry with { Original = name, OriginalSha256 = sha, OriginalBytes = bytes };
}
TryDelete(target);
if (attempt == 2) throw new IOException(Loc.L("Errors.Workspace.CopyMismatch", target));
}
}
/// <summary>
/// A frame's original from a server into <c>media/originals/</c> (phase 5c): kept when held already, else downloaded with
/// <see cref="SyncTransfers.FetchOriginalAsync"/> (resumable, checked against the server's SHA-256, two attempts) into a
/// staging folder beside the originals, then renamed into place. A frame the server has no original of (flagged missing
/// or damaged, 404), one this account may not read (403), or one that does not match twice keeps its proxy alone.
/// </summary>
private static async Task<WorkspaceFrame> DownloadOriginalAsync(ISyncRemote remote, WorkspaceProject project, Frame frame, ManifestFrame? asset, WorkspaceFrame entry,
WorkspaceFrame? held, OriginalsTally tally, CancellationToken cancellationToken)
{
if (asset is null || (frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0 || (((FrameFlags)asset.Flags) & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0)
return tally.Missed(entry, frame.Index);
var extension = Path.GetExtension(asset.FileName);
var expected = string.IsNullOrEmpty(asset.Sha256) ? null : asset.Sha256.ToLowerInvariant();
if (await HeldOriginalAsync(project, frame, expected, extension, held, cancellationToken).ConfigureAwait(false) is { } kept)
{
tally.Keep();
return entry with { Original = kept.Name, OriginalSha256 = kept.Sha256, OriginalBytes = kept.Bytes };
}
var staging = Path.Combine(project.OriginalsDirectory, OriginalsStaging);
Directory.CreateDirectory(staging);
string path, sha;
try
{
(path, sha) = await SyncTransfers.FetchOriginalAsync(remote, asset, staging, cancellationToken).ConfigureAwait(false);
}
catch (ChronosApiException ex) when (ex.Status is HttpStatusCode.NotFound or HttpStatusCode.Forbidden)
{
return tally.Missed(entry, frame.Index);
}
catch (InvalidDataException ex)
{
Log.Warning("workspace", $"The original of frame {frame.Id} did not match the server's SHA-256", ex);
return tally.Missed(entry, frame.Index);
}
var name = WorkspaceProject.OriginalName(frame.Id, sha, extension);
var target = project.MediaFile(name);
var bytes = new FileInfo(path).Length;
File.Move(path, target, overwrite: true);
tally.Copy();
return entry with { Original = name, OriginalSha256 = sha, OriginalBytes = bytes };
}
/// <summary>
/// The original the workspace holds of <paramref name="frame"/>, hashed again: the one the media name for it (the same
/// photo, of the SHA-256 the home has when it has one), else the file a copy of <paramref name="expected"/> takes; null when
/// none holds it (a file that no longer matches is copied anew).
/// </summary>
private static async Task<(string Name, string Sha256, long Bytes)?> HeldOriginalAsync(WorkspaceProject project, Frame frame, string? expected, string extension,
WorkspaceFrame? held, CancellationToken cancellationToken)
{
var candidates = new List<(string Name, string? Sha)>();
if (held is { Original: { } heldName, OriginalSha256: { } heldSha } && SamePhoto(held, frame) && (expected is null || string.Equals(heldSha, expected, StringComparison.OrdinalIgnoreCase)))
candidates.Add((heldName, heldSha.ToLowerInvariant()));
if (expected is not null) candidates.Add((WorkspaceProject.OriginalName(frame.Id, expected, extension), expected));
foreach (var (name, sha) in candidates)
{
var path = project.MediaFile(name);
if (!File.Exists(path)) continue;
var found = await ContentHash.Sha256Async(path, cancellationToken: cancellationToken).ConfigureAwait(false);
if (string.Equals(found, sha, StringComparison.OrdinalIgnoreCase)) return (name, found.ToLowerInvariant(), new FileInfo(path).Length);
}
return null;
}
/// <summary>
/// The bytes a copy of the originals still writes from each frame on (index <c>i</c>: frames <c>i</c> to the last; one
/// more entry, 0): <paramref name="size"/> of each frame whose original the workspace does not hold yet under the name a
/// copy gives it, nothing for one the media name already, or flagged missing.
/// </summary>
private static long[] RemainingOriginals(WorkspaceProject project, List<Frame> frames, Dictionary<Guid, WorkspaceFrame> held, Func<Frame, int, long> size)
{
var remaining = new long[frames.Count + 1];
for (var i = frames.Count - 1; i >= 0; i--)
{
var frame = frames[i];
var there = (held.GetValueOrDefault(frame.Id) is { Original: { } name } && File.Exists(project.MediaFile(name)))
|| (!string.IsNullOrEmpty(frame.Sha256) && File.Exists(project.MediaFile(WorkspaceProject.OriginalName(frame.Id, frame.Sha256, OriginalExtension(frame)))));
var bytes = there || (frame.Flags & FrameFlags.Missing) != 0 ? 0 : Math.Max(0, size(frame, i));
remaining[i] = remaining[i + 1] + bytes;
}
return remaining;
}
/// <summary>The extension of a frame's original as its path at the home has it (else its file name), which decoders go by.</summary>
private static string OriginalExtension(Frame frame) => Path.GetExtension(frame.RelativePath) is { Length: > 0 } extension ? extension : Path.GetExtension(frame.FileName);
/// <summary>The folder beside the originals a download from a server stages them in (resumable parts), emptied by the commit.</summary>
private const string OriginalsStaging = ".staging";
/// <summary>The buffer an original is streamed through.</summary>
private const int StreamBuffer = 1 << 20;
/// <summary>
/// The bytes a copy from a server still writes from each frame on (index <c>i</c>: frames <c>i</c> to the last; one more
/// entry, 0): nothing for a frame the workspace holds a file of already or the server has neither proxy nor thumbnail of,
@@ -678,6 +964,9 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
}
/// <summary>Whether <paramref name="folder"/> is an archive's root of this index or lies inside one.</summary>
private static bool PathsEqual(string a, string b) =>
string.Equals(Path.TrimEndingDirectorySeparator(Path.GetFullPath(a)), Path.TrimEndingDirectorySeparator(Path.GetFullPath(b)), OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal);
private async Task<bool> InsideALibraryAsync(string folder, CancellationToken cancellationToken)
{
var path = Plain(Path.GetFullPath(folder));
@@ -839,7 +1128,7 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
private static void Commit(WorkspaceProject project, ProjectBundle bundle, WorkspaceMedia media, string folder, string title)
{
var old = TryReadMedia(project);
var workspace = bundle.Workspace! with { Media = new BundleMedia(BundleMedia.Proxies, media.FramesVersion) };
var workspace = bundle.Workspace! with { Media = new BundleMedia(media.Kind, media.FramesVersion) };
var sequence = bundle.Sequence with { Folder = folder, Title = title, Frames = media.Frames.Count };
var working = bundle.Project;
// Without the media index the positions were made against, they are taken as naming these frames.
@@ -868,8 +1157,36 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
/// <summary>
/// Proxies of frames the sequence no longer has, and downloads a stopped copy from a server left half done: regenerable
/// copies, removed so the media folder holds the frames listed.
/// copies, removed so the media folder holds the frames listed. Media of kind originals (phase 5c) clear their folder
/// alike, but an original is never erased on a guess: one the media no longer name is deleted only when the home's
/// frames (<paramref name="home"/>, as this copy read them) hold a photo of its SHA-256, else it goes to the workspace's
/// trash (<see cref="WorkspaceOriginals.DeleteOrTrash"/>). A copy of the proxies alone leaves the originals as they are.
/// </summary>
private static void RemoveStale(WorkspaceProject project, WorkspaceMedia media, IReadOnlyList<Frame> home)
{
RemoveStale(project, media);
if (!media.HoldsOriginals || !Directory.Exists(project.OriginalsDirectory)) return;
var staging = Path.Combine(project.OriginalsDirectory, OriginalsStaging);
if (Directory.Exists(staging))
{
try
{
Directory.Delete(staging, recursive: true);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("workspace", $"{staging} could not be removed", ex);
}
}
var originals = media.Frames.Where(f => f.Original is not null).Select(f => Path.GetFileName(f.Original!)).ToHashSet(StringComparer.OrdinalIgnoreCase);
var stale = Directory.EnumerateFiles(project.OriginalsDirectory).Where(f => !originals.Contains(Path.GetFileName(f))).ToList();
foreach (var part in stale.Where(IsPart)) TryDelete(part);
WorkspaceOriginals.DeleteOrTrash(project, [.. stale.Where(f => !IsPart(f))], home);
}
private static bool IsPart(string file) => file.EndsWith(".part", StringComparison.OrdinalIgnoreCase);
/// <summary>The proxies' folder cleared as <see cref="RemoveStale(WorkspaceProject, WorkspaceMedia, IReadOnlyList{Frame})"/> says.</summary>
private static void RemoveStale(WorkspaceProject project, WorkspaceMedia media)
{
var listed = media.Frames.Where(f => f.File is not null).Select(f => Path.GetFileName(f.File!)).ToHashSet(StringComparer.OrdinalIgnoreCase);
@@ -943,6 +1260,18 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
return (Remaining([], paths.Sequence(row.Folder), frames, policy)[0], FreeBytes(workspaceRoot));
}
/// <summary>
/// The bytes a copy of the sequence's originals writes (phase 5c): every frame's original the archive has, or, for a
/// workspace project that holds some already (<paramref name="project"/>), only those it lacks.
/// </summary>
internal async Task<long> OriginalsBytesAsync(Guid sequenceId, WorkspaceProject? project, CancellationToken cancellationToken)
{
var (_, frames) = await FramesAsync(sequenceId, cancellationToken).ConfigureAwait(false);
if (project is null || !File.Exists(project.MediaIndexPath)) return frames.Where(f => (f.Flags & FrameFlags.Missing) == 0).Sum(f => Math.Max(0, f.Size));
var held = (TryReadMedia(project)?.Frames ?? []).Where(f => f.Original is not null).ToDictionary(f => f.Id);
return RemainingOriginals(project, frames, held, (f, _) => f.Size)[0];
}
/// <summary>A proxy's size when the archive has none to measure: some 1.5 MB at 1920 px, scaled by the area at the copy's long edge.</summary>
private static long Estimate(Frame frame, ProxyPolicy policy)
{
@@ -1034,6 +1363,9 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
{
Copy,
Done,
/// <summary>The proxies are copied; the originals are (phase 5c).</summary>
Originals,
}
private static string Checkpoint(Stage stage, int frame, int framesVersion, string folder) =>
@@ -1048,11 +1380,46 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
private static (int Copied, int? FramesVersion, string? Folder) Parse(string? checkpoint)
{
var parts = checkpoint?.Split(':', 4);
if (parts is not ([_, _] or [_, _, _] or [_, _, _, _]) || !Enum.TryParse<Stage>(parts[0], ignoreCase: true, out var stage) || stage != Stage.Copy
if (parts is not ([_, _] or [_, _, _] or [_, _, _, _]) || !Enum.TryParse<Stage>(parts[0], ignoreCase: true, out var stage) || stage is not (Stage.Copy or Stage.Originals)
|| !int.TryParse(parts[1], NumberStyles.None, CultureInfo.InvariantCulture, out var next))
return (0, null, null);
int? version = parts.Length >= 3 && int.TryParse(parts[2], NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture, out var v) ? v : null;
return (next, version, parts.Length == 4 ? parts[3] : null);
return (stage == Stage.Originals ? int.MaxValue : next, version, parts.Length == 4 ? parts[3] : null);
}
/// <summary>What a copy of the originals did with each frame (phase 5c), counted from the workers that download them at once.</summary>
private sealed class OriginalsTally
{
/// <summary>Frames named one by one in the job's log when they have no original; the count says the rest.</summary>
private const int Named = 20;
private readonly System.Collections.Concurrent.ConcurrentQueue<(int Index, string Name)> _missed = new();
private int _copied;
private int _kept;
public int Copied => Volatile.Read(ref _copied);
public int Kept => Volatile.Read(ref _kept);
public int Missing => _missed.Count;
public void Copy() => Interlocked.Increment(ref _copied);
public void Keep() => Interlocked.Increment(ref _kept);
/// <summary>The frame keeps its proxy alone: <paramref name="entry"/> with no original.</summary>
public WorkspaceFrame Missed(WorkspaceFrame entry, int index)
{
_missed.Enqueue((index, entry.FileName));
return entry with { Original = null, OriginalSha256 = null, OriginalBytes = 0 };
}
/// <summary>Says in the job's log what was copied (<paramref name="key"/>), and names the first frames without an original.</summary>
public void LogTo(JobContext context, string key, string title)
{
foreach (var (_, name) in _missed.OrderBy(m => m.Index).Take(Named)) context.Log(Loc.L("Archive.Log.WorkspaceOriginalMissing", title, name));
context.Log(Loc.L(key, title, Copied, Kept, Missing));
}
}
/// <summary>What a copy did with each frame, counted from the workers that download them at once.</summary>
@@ -0,0 +1,242 @@
using System.Text.Json;
using Chronos.Core.Diagnostics;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Data.Archiving;
using Chronos.Data.Archiving.Jobs;
using Chronos.Data.Jobs;
using Chronos.Data.Processing;
using Chronos.Data.Sync;
namespace Chronos.Data.Workspaces;
/// <summary>
/// What the workspace's job handlers take from the desktop besides the index and the archive engine (phase 5c).
/// </summary>
/// <param name="Processing">
/// Decoders, encoders and tuning of a local render (<see cref="JobKind.WorkspaceRender"/>): the desktop's
/// <see cref="ProcessingService.Options"/>.
/// </param>
/// <param name="HoldTransfers">
/// True while transfers with a server are held (Settings › Server › Pause metered transfers on a metered connection): a
/// <see cref="JobKind.WorkspaceCopy"/> or <see cref="JobKind.WorkspaceSend"/> naming a server waits before it downloads or
/// uploads anything then. Never holds a copy from or a send to this PC's archive.
/// </param>
/// <param name="Moved">
/// Told once a <see cref="JobKind.WorkspaceMove"/> verified every file at its target (from, to): the desktop names the new
/// folder in Settings › Workspace and refreshes the workspace; null where nothing follows the folder.
/// </param>
/// <param name="CurrentRoot">
/// The workspace folder in use now (the desktop's Settings › Workspace): a copy, render or send queued before the workspace
/// moved finds its project there when the folder its parameters name no longer holds it; null where only that one counts.
/// </param>
public sealed record WorkspaceJobOptions(ProcessingOptions Processing, Func<bool> HoldTransfers, Func<string, string, Task>? Moved = null,
Func<string?>? CurrentRoot = null)
{
/// <summary>The options of an engine without a processing service of its own: the archive engine's decoders, nothing held, no move followed.</summary>
public static WorkspaceJobOptions For(ArchiveService archives)
{
ArgumentNullException.ThrowIfNull(archives);
return new WorkspaceJobOptions(new ProcessingOptions { Archive = new ArchiveServiceOptions { Decoder = archives.Decoder } }, () => false);
}
}
/// <summary>Builds the jobs of the workspace (owner review 2, R8), and their handlers for the desktop's engine.</summary>
public static class WorkspaceJobs
{
/// <summary>
/// <see cref="JobKind.WorkspaceCopy"/>: the media of workspace project <paramref name="projectId"/> (checked out first,
/// <see cref="WorkspaceSync.CheckoutAsync"/>) copied from its home, the sequence of this PC's archive.
/// </summary>
public static ArchiveJobRequest Copy(Sequence sequence, string workspaceRoot, Guid projectId, string media = BundleMedia.Proxies)
{
ArgumentNullException.ThrowIfNull(sequence);
ArgumentException.ThrowIfNullOrWhiteSpace(workspaceRoot);
return new ArchiveJobRequest(JobKind.WorkspaceCopy, [sequence.Id], sequence.DisplayTitle,
JsonSerializer.Serialize(new WorkspaceCopyParameters(workspaceRoot, projectId, media), WorkspaceJson.Default.WorkspaceCopyParameters), JobPriority.User);
}
/// <summary>
/// <see cref="JobKind.WorkspaceCopy"/> of a project whose home is a server (phase 5b): the media of workspace project
/// <paramref name="projectId"/> (checked out first) downloaded from sequence <paramref name="sequenceId"/> of
/// <paramref name="server"/> (<see cref="SyncService.ServerKey"/>), into the workspace alone. The job's target is the
/// server's sequence, as for a sync download.
/// </summary>
public static ArchiveJobRequest CopyFromServer(Guid sequenceId, string title, string server, string workspaceRoot, Guid projectId, string media = BundleMedia.Proxies)
{
ArgumentException.ThrowIfNullOrWhiteSpace(server);
ArgumentException.ThrowIfNullOrWhiteSpace(workspaceRoot);
return new ArchiveJobRequest(JobKind.WorkspaceCopy, [sequenceId], title,
JsonSerializer.Serialize(new WorkspaceCopyParameters(workspaceRoot, projectId, media, server), WorkspaceJson.Default.WorkspaceCopyParameters), JobPriority.User);
}
/// <summary>
/// <see cref="JobKind.WorkspaceRender"/> (phase 5c): workspace project <paramref name="projectId"/> rendered on this PC from
/// the originals its media hold, into its <c>exports/</c>, as <paramref name="renderJson"/> says (Protocol
/// <c>RenderJobParameters</c> with the portable settings: what <see cref="ProcessingJobs"/> writes for a render), each
/// output recorded with <paramref name="projectVersionId"/> in the project's ledger and sent home after. The target is the
/// home's sequence (this PC's archive, or <paramref name="server"/>'s, <see cref="SyncService.ServerKey"/>), so the
/// editor follows it as it follows a render of that sequence.
/// </summary>
public static ArchiveJobRequest Render(Guid sequenceId, string title, string workspaceRoot, Guid projectId, Guid? projectVersionId, string renderJson,
string? server = null)
{
ArgumentException.ThrowIfNullOrWhiteSpace(workspaceRoot);
ArgumentException.ThrowIfNullOrWhiteSpace(renderJson);
return new ArchiveJobRequest(JobKind.WorkspaceRender, [sequenceId], title,
JsonSerializer.Serialize(new WorkspaceRenderParameters(workspaceRoot, projectId, projectVersionId, renderJson, server),
WorkspaceJson.Default.WorkspaceRenderParameters), JobPriority.User);
}
/// <summary>
/// <see cref="JobKind.WorkspaceSend"/> (phase 5c): the local renders <paramref name="renderIds"/> of workspace project
/// <paramref name="projectId"/> (every unsent one of its ledger when empty) sent to its home, the sequence of this PC's
/// archive or of <paramref name="server"/> (<see cref="SyncService.ServerKey"/>), with their project and version.
/// </summary>
public static ArchiveJobRequest Send(Guid sequenceId, string title, string workspaceRoot, Guid projectId, IReadOnlyList<Guid> renderIds, string? server = null,
JobPriority priority = JobPriority.User)
{
ArgumentException.ThrowIfNullOrWhiteSpace(workspaceRoot);
ArgumentNullException.ThrowIfNull(renderIds);
return new ArchiveJobRequest(JobKind.WorkspaceSend, [sequenceId], title,
JsonSerializer.Serialize(new WorkspaceSendParameters(workspaceRoot, projectId, [.. renderIds], server), WorkspaceJson.Default.WorkspaceSendParameters),
priority);
}
/// <summary>
/// <see cref="JobKind.WorkspaceMove"/> (phase 5c): the workspace folder <paramref name="from"/> moved with its projects to
/// <paramref name="to"/>, as the plan of fingerprint <paramref name="planFingerprint"/> the user confirmed showed it. The
/// job has no target; its label is the folder it moves to.
/// </summary>
public static ArchiveJobRequest Move(string from, string to, string planFingerprint)
{
ArgumentException.ThrowIfNullOrWhiteSpace(from);
ArgumentException.ThrowIfNullOrWhiteSpace(to);
ArgumentException.ThrowIfNullOrWhiteSpace(planFingerprint);
return new ArchiveJobRequest(JobKind.WorkspaceMove, [], to,
JsonSerializer.Serialize(new WorkspaceMoveParameters(from, to, planFingerprint), WorkspaceJson.Default.WorkspaceMoveParameters), JobPriority.User);
}
/// <summary>The parameters of a <see cref="JobKind.WorkspaceRender"/> job; null when it has none that read.</summary>
public static WorkspaceRenderParameters? RenderParameters(Job job) => Read(job, JobKind.WorkspaceRender, WorkspaceJson.Default.WorkspaceRenderParameters);
/// <summary>The parameters of a <see cref="JobKind.WorkspaceSend"/> job; null when it has none that read.</summary>
public static WorkspaceSendParameters? SendParameters(Job job) => Read(job, JobKind.WorkspaceSend, WorkspaceJson.Default.WorkspaceSendParameters);
/// <summary>The parameters of a <see cref="JobKind.WorkspaceMove"/> job; null when it has none that read.</summary>
public static WorkspaceMoveParameters? MoveParameters(Job job) => Read(job, JobKind.WorkspaceMove, WorkspaceJson.Default.WorkspaceMoveParameters);
/// <summary>The parameters of a <see cref="JobKind.WorkspaceCopy"/> job; null when it has none that read.</summary>
public static WorkspaceCopyParameters? CopyParameters(Job job) => Read(job, JobKind.WorkspaceCopy, WorkspaceJson.Default.WorkspaceCopyParameters);
/// <summary>
/// The handlers of the workspace's jobs for a desktop's engine (never a server's: the job writes into a folder its
/// parameters name); copies from a server fail without a connection to it.
/// </summary>
public static IReadOnlyList<IJobHandler> CreateHandlers(ChronosDatabase db, ArchiveService archives) => CreateHandlers(db, archives, null);
/// <param name="servers">The connection a copy from a server's sequence downloads through (the desktop's signed-in session).</param>
public static IReadOnlyList<IJobHandler> CreateHandlers(ChronosDatabase db, ArchiveService archives, ISyncConnector? servers) =>
CreateHandlers(db, archives, servers, WorkspaceJobOptions.For(archives));
/// <summary>
/// The handlers of every workspace job (phase 5c): the copy of a project's media, its local render, the sending of its
/// renders home and the move of the workspace folder.
/// </summary>
/// <param name="servers">The connection a copy from, or a send to, a server's sequence goes through (the desktop's signed-in session).</param>
/// <param name="options">The desktop's processing options, its hold on transfers and what follows a moved folder.</param>
public static IReadOnlyList<IJobHandler> CreateHandlers(ChronosDatabase db, ArchiveService archives, ISyncConnector? servers, WorkspaceJobOptions options)
{
ArgumentNullException.ThrowIfNull(db);
ArgumentNullException.ThrowIfNull(archives);
ArgumentNullException.ThrowIfNull(options);
return
[
new WorkspaceCopyJobHandler(db, archives, servers, options),
new WorkspaceRenderJobHandler(db, archives, options),
new WorkspaceSendJobHandler(db, archives, servers, options),
new WorkspaceMoveJobHandler(db, options),
];
}
/// <summary>
/// What a first copy of a sequence's proxies writes into a workspace, as the copy reckons it before it starts (the
/// archive's proxies' sizes, an estimate for those it makes), and the free bytes on <paramref name="workspaceRoot"/>'s
/// drive (null when it cannot be measured): "Open in workspace" shows both. The copy keeps <see cref="CopySpaceReserve"/>
/// free besides.
/// </summary>
/// <exception cref="InvalidOperationException">The index has no such sequence or archive.</exception>
public static Task<(long Bytes, long? Free)> CopyFactsAsync(ChronosDatabase db, ArchiveService archives, Guid sequenceId, string workspaceRoot,
CancellationToken cancellationToken = default) =>
new WorkspaceCopyJobHandler(db, archives).FactsAsync(sequenceId, workspaceRoot, cancellationToken);
/// <summary>Bytes the copy leaves free on the workspace's drive besides what it copies.</summary>
public static long CopySpaceReserve => WorkspaceCopyJobHandler.SpaceReserve;
private static T? Read<T>(Job job, JobKind kind, System.Text.Json.Serialization.Metadata.JsonTypeInfo<T> info) where T : class
{
ArgumentNullException.ThrowIfNull(job);
if (job.Kind != kind || job.ParametersJson is not { Length: > 0 } json) return null;
try
{
return JsonSerializer.Deserialize(json, info);
}
catch (JsonException)
{
return null;
}
}
}
/// <summary>
/// Where a copy, render or send of a workspace project works (phase 5c): the folder its parameters name, or the one the
/// workspace moved to since. Nothing of the workspace is written while it is being moved, and nothing is made where the
/// project is not (opening a store makes its folder).
/// </summary>
internal static class WorkspaceJobRoots
{
/// <summary>
/// The workspace folder that holds <paramref name="projectId"/> for a job queued with <paramref name="named"/>: that folder
/// when it holds the project; else the target of a move of it past its switch, or the folder in use now
/// (<paramref name="current"/>), when either holds it. Refused while a move of either folder has not switched yet
/// (<c>Errors.Workspace.Moving</c>: the job fails and can be retried once the move ends); <paramref name="missing"/> (a
/// resource key) when no folder holds the project.
/// </summary>
/// <exception cref="InvalidOperationException">The workspace is being moved, or no folder holds the project.</exception>
public static string Resolve(string named, Guid projectId, Func<string?>? current, string missing)
{
ArgumentException.ThrowIfNullOrWhiteSpace(named);
var candidates = new List<string>();
string? currentRoot = null;
try
{
currentRoot = current?.Invoke() is { Length: > 0 } root ? WorkspaceStore.FullRoot(root) : null;
}
catch (Exception ex) when (ex is ArgumentException or IOException or NotSupportedException)
{
Log.Debug("workspace", $"The workspace folder in use cannot be read: {ex.Message}");
}
foreach (var root in new[] { WorkspaceStore.FullRoot(named), currentRoot })
{
if (root is null) continue;
if (WorkspaceMover.HasJournal(root))
{
// Past its switch the move's target is the workspace, and what is left here is a source still to clear.
if (WorkspaceMover.ReadJournal(root) is not { Stage: WorkspaceMoveStage.Switched } journal) throw new InvalidOperationException(Loc.L("Errors.Workspace.Moving"));
candidates.Add(WorkspaceStore.FullRoot(journal.To));
continue;
}
candidates.Add(root);
}
foreach (var root in candidates.Distinct(StringComparer.OrdinalIgnoreCase))
{
if (!File.Exists(Path.Combine(root, projectId.ToString("N"), WorkspaceProject.BundleFile))) continue;
if (WorkspaceMover.HasJournal(root)) throw new InvalidOperationException(Loc.L("Errors.Workspace.Moving"));
if (!string.Equals(root, WorkspaceStore.FullRoot(named), StringComparison.OrdinalIgnoreCase))
Log.Info("workspace", $"Project {projectId} is no longer in {named}: the job works in {root}, where the workspace is now");
return root;
}
throw new InvalidOperationException(Loc.L(missing));
}
}
+62 -5
View File
@@ -17,6 +17,13 @@ namespace Chronos.Data.Workspaces;
/// What the file is when a server sent it (phase 5b): <c>proxy</c>, or <c>thumb</c>, a thumbnail standing in while the server
/// had no proxy of the frame, which a later copy replaces once it has one; null for a proxy (copies from this PC's archive).
/// </param>
/// <param name="Original">
/// The frame's original the workspace holds, relative to the media folder (<see cref="WorkspaceProject.OriginalName"/>:
/// "originals/&lt;frame id&gt;-&lt;sha8&gt;&lt;ext&gt;", phase 5c), in media of kind <see cref="BundleMedia.Originals"/>; null
/// when it holds none (proxies only, or the home had no original of the frame: the proxy stands in for editing).
/// </param>
/// <param name="OriginalSha256">The original's SHA-256 as copied and checked (lower case); null with <paramref name="Original"/>.</param>
/// <param name="OriginalBytes">The original's size as copied; 0 with no original.</param>
public sealed record WorkspaceFrame(
Guid Id,
int Index,
@@ -46,7 +53,10 @@ public sealed record WorkspaceFrame(
string? File,
string? FileSha256,
long FileBytes,
string? FileKind = null)
string? FileKind = null,
string? Original = null,
string? OriginalSha256 = null,
long OriginalBytes = 0)
{
/// <summary>The record of a home's frame, with the file the workspace holds for it (<paramref name="fileKind"/>: what a server sent).</summary>
public static WorkspaceFrame Of(Frame frame, string? file, string? fileSha256, long fileBytes, string? fileKind = null)
@@ -62,6 +72,24 @@ public sealed record WorkspaceFrame(
[JsonIgnore]
public bool IsStandIn => FileKind == Sync.ProxyCache.Thumb;
/// <summary>The workspace holds the frame's original (media of kind <see cref="BundleMedia.Originals"/>).</summary>
[JsonIgnore]
public bool HasOriginal => Original is not null;
/// <summary>
/// The frame as a render from the workspace's originals reads it (phase 5c): its file is <see cref="Original"/>, and
/// it is flagged missing while the workspace holds no original for it (a render refuses such media first).
/// </summary>
public Frame ToOriginalFrame(Guid sequenceId)
{
var frame = ToFrame(sequenceId);
return frame with
{
RelativePath = Original ?? FileName,
Flags = Original is null ? Flags | FrameFlags.Missing : Flags,
};
}
/// <summary>The frame as the processing engine reads it; flagged missing while the workspace holds no file for it.</summary>
public Frame ToFrame(Guid sequenceId) => new()
{
@@ -97,16 +125,34 @@ public sealed record WorkspaceFrame(
/// <summary>
/// What a workspace project holds of its sequence's photos (<c>media/media.json</c>, owner review 2, R8): the kind
/// (<see cref="BundleMedia.Proxies"/>; originals in phase 5c), the sequence and its frames version at the home when they
/// were copied, and every frame in index order. Written last by a copy, so a copy cut short leaves the previous index.
/// (<see cref="BundleMedia.Proxies"/>, or from phase 5c <see cref="BundleMedia.Originals"/>: the proxies plus each frame's
/// original, <see cref="WorkspaceFrame.Original"/>), the sequence and its frames version at the home when they were
/// copied, and every frame in index order. Written last by a copy, so a copy cut short leaves the previous index. An index
/// of phase 5b has no original fields: it reads as proxies only. One that holds originals or was cleaned is written as
/// format 2 (<see cref="WrittenFormatVersion"/>), which a Chronos before phase 5c refuses as newer and reads as media
/// removed (copied again) rather than as proxies whose files are all missing; proxies alone stay format 1, which it reads.
/// </summary>
/// <param name="Aliases">The encodings of the sequence's photos a conversion joined at the home (R7, its tombstones of reason
/// "replaced") when the media were copied: a version parked on a photo before its file was converted still finds its frame.</param>
/// <param name="Cleaned">
/// When the workspace's clean-up removed the files of these media (phase 5c, <see cref="WorkspaceCleaner"/>): the frames stay,
/// none names a file, and the project reads as <see cref="WorkspaceState.MediaRemoved"/> until a copy brings them again (a
/// copy writes a new index, without it). Null for media a copy wrote.
/// </param>
public sealed record WorkspaceMedia(string Format, int FormatVersion, string Kind, Guid SequenceId, int FramesVersion, IReadOnlyList<WorkspaceFrame> Frames,
DateTimeOffset Copied, string? AnalysisSha256 = null, IReadOnlyList<PhotoAlias>? Aliases = null)
DateTimeOffset Copied, string? AnalysisSha256 = null, IReadOnlyList<PhotoAlias>? Aliases = null, DateTimeOffset? Cleaned = null)
{
public const string FormatName = "chronos-workspace-media";
public const int CurrentFormatVersion = 1;
/// <summary>The newest format this Chronos reads (2: media with originals, or cleaned).</summary>
public const int CurrentFormatVersion = 2;
/// <summary>The format of media of proxies only that a copy wrote, which a Chronos before phase 5c reads too.</summary>
public const int ProxiesFormatVersion = 1;
/// <summary>The format these media are written in: 2 when they hold originals (or name one) or were cleaned, else 1.</summary>
[JsonIgnore]
public int WrittenFormatVersion => HoldsOriginals || IsCleaned || Frames.Any(f => f.HasOriginal) ? CurrentFormatVersion : ProxiesFormatVersion;
/// <summary>The frames' photos in index order, as project positions name them, with the encodings conversions joined.</summary>
[JsonIgnore]
@@ -115,6 +161,17 @@ public sealed record WorkspaceMedia(string Format, int FormatVersion, string Kin
/// <summary>The frames held, as the processing engine reads them.</summary>
public IReadOnlyList<Frame> ToFrames() => [.. Frames.Select(f => f.ToFrame(SequenceId))];
/// <summary>The frames over the originals the workspace holds (<see cref="WorkspaceFrame.ToOriginalFrame"/>), as a local render reads them.</summary>
public IReadOnlyList<Frame> ToOriginalFrames() => [.. Frames.Select(f => f.ToOriginalFrame(SequenceId))];
/// <summary>The clean-up removed the files of these media (<see cref="Cleaned"/>): nothing to edit until they are copied again.</summary>
[JsonIgnore]
public bool IsCleaned => Cleaned is not null;
/// <summary>Media of kind <see cref="BundleMedia.Originals"/>: the proxies plus the originals the frames name.</summary>
[JsonIgnore]
public bool HoldsOriginals => Kind == BundleMedia.Originals;
/// <summary>
/// These media with their photos named as <paramref name="home"/> names them now (owner review 2, R8): a frame copied
/// before the home hashed its photo (a folder adopted, verified since) takes the SHA-256 the home's frame of that id has
@@ -0,0 +1,89 @@
using System.Globalization;
using Chronos.Core.Diagnostics;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Data.Jobs;
namespace Chronos.Data.Workspaces;
/// <summary>
/// <see cref="JobKind.WorkspaceMove"/> (phase 5c, <see cref="WorkspaceJobs.Move"/>): the workspace folder moved with its
/// projects, after the plan the user confirmed (its fingerprint checked again): journaled in <c>&lt;from&gt;/.move.json</c>,
/// moved per project on one volume, copied and hash-verified on both sides across volumes, a project's source deleted only
/// once every file of it is verified, resumable from the journal; at the end <see cref="WorkspaceJobOptions.Moved"/> is told.
/// On failure the source stays the workspace and unverified partial copies are removed (<see cref="WorkspaceMover"/>).
/// A move that starts afresh is refused while another job of the workspace waits, runs or is paused (they write into the
/// folder that moves). Cancelled by the user, while it runs or while it is paused or failed, what it did is undone as
/// after a failure, unless the target already became the workspace; a pause or the program closing keeps the journal,
/// from which the next run goes on. The checkpoint carries the progress only: the journal and the files' names say what
/// is done (a file reaches its final name in the target only once verified).
/// </summary>
internal sealed class WorkspaceMoveJobHandler(ChronosDatabase db, WorkspaceJobOptions options) : ICancelledJobHandler
{
/// <summary>The other jobs of the workspace, which write into the folder a move takes away.</summary>
private static readonly JobKind[] WorkspaceKinds = [JobKind.WorkspaceCopy, JobKind.WorkspaceRender, JobKind.WorkspaceSend];
public JobKind Kind => JobKind.WorkspaceMove;
/// <summary>Tests: the mover the job runs (one volume or not, a crash, a damaged copy); null makes the real one.</summary>
internal Func<WorkspaceMoveParameters, WorkspaceMover>? Movers { get; init; }
public async Task RunAsync(JobContext context)
{
ArgumentNullException.ThrowIfNull(context);
var ct = context.Cancellation;
var parameters = WorkspaceJobs.MoveParameters(context.Job) is { From.Length: > 0, To.Length: > 0, PlanFingerprint.Length: > 0 } read
? read
: throw new InvalidDataException(Loc.L("Errors.Workspace.JobParameters"));
var mover = Mover(parameters);
if (WorkspaceMover.ReadJournal(mover.From) is null && await db.Jobs.HasPendingAsync(WorkspaceKinds, cancellationToken: ct).ConfigureAwait(false))
throw new InvalidOperationException(Loc.L("Errors.Workspace.JobsRunning"));
try
{
await mover.MoveAsync(parameters.PlanFingerprint, options.Moved, (done, total, durable) => durable
? context.CheckpointAsync(string.Create(CultureInfo.InvariantCulture, $"move:{done}"), done, total)
: context.ReportAsync(done, total),
context.Log, ct).ConfigureAwait(false);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
// Stopped: the user's cancel undoes what was done; a pause, a yield or the program closing goes on from the journal.
if (await db.Jobs.GetAsync(context.Job.Id, CancellationToken.None).ConfigureAwait(false) is { State: JobState.Cancelled }) Undo(mover, context.Job);
throw;
}
}
/// <summary>A move cancelled while no handler ran it (paused, failed): what it did is undone, unless the target is the workspace already.</summary>
public Task CancelledAsync(Job job, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(job);
if (WorkspaceJobs.MoveParameters(job) is not { From.Length: > 0, To.Length: > 0 } parameters) return Task.CompletedTask;
_ = Task.Run(() =>
{
try
{
Undo(Mover(parameters), job);
}
catch (Exception ex) when (ex is not OutOfMemoryException)
{
// Nobody waits for this task: the journal stays, and the move can be taken up.
Log.Warning("workspace", $"Cancelled move {job.Id} could not be undone", ex);
}
}, CancellationToken.None);
return Task.CompletedTask;
}
private WorkspaceMover Mover(WorkspaceMoveParameters parameters) => Movers?.Invoke(parameters) ?? new WorkspaceMover(parameters.From, parameters.To);
private static void Undo(WorkspaceMover mover, Job job)
{
try
{
if (!mover.Abandon()) Log.Info("workspace", $"Move {job.Id} was cancelled after {mover.To} became the workspace; it is not undone");
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("workspace", $"Cancelled move {job.Id} could not be undone; its journal stays to take it up", ex);
}
}
}
@@ -0,0 +1,713 @@
using System.Globalization;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using Chronos.Core.Diagnostics;
using Chronos.Core.IO;
using Chronos.Core.Localization;
namespace Chronos.Data.Workspaces;
/// <summary>Why the workspace folder cannot move to a folder (<see cref="WorkspaceMoveSurvey.Refusal"/>).</summary>
public enum WorkspaceMoveRefusal
{
/// <summary>The target is the workspace folder itself or lies inside it (<c>Errors.Workspace.MoveSameFolder</c>).</summary>
SameFolder,
/// <summary>The target holds a workspace (its manifest, its trash, a move's journal) or a folder of the same name as one moved (<c>Errors.Workspace.MoveTargetNotEmpty</c>).</summary>
TargetNotEmpty,
/// <summary>The target's drive lacks the room for a copy across drives (<c>Errors.Workspace.MoveNoRoom</c>).</summary>
NoRoom,
}
/// <summary>A folder of a project the workspace holds, as a move surveys it.</summary>
/// <param name="IsProject">It holds a working state (<c>project.chronos</c>); a folder that does not (a leftover) moves all the same.</param>
public sealed record WorkspaceMoveFolder(Guid ProjectId, bool IsProject, int Files, long Bytes);
/// <summary>
/// What moving a workspace folder <see cref="From"/> to <see cref="To"/> would take, read without writing anything: the
/// entries that move (every folder named by a project id, the manifest, the trash), their files and bytes (leases left
/// out), the target drive's free bytes, whether both lie on one volume, why it is refused, and a fingerprint of the files
/// read (their relative names, sizes and times), which the move checks again before it starts.
/// </summary>
/// <param name="Free">The target drive's free bytes; null when they cannot be measured (a share), which is not refused.</param>
/// <param name="Missing">With <see cref="WorkspaceMoveRefusal.NoRoom"/>, the bytes the target's drive lacks; 0 otherwise.</param>
public sealed record WorkspaceMoveSurvey(string From, string To, IReadOnlyList<WorkspaceMoveFolder> Projects, IReadOnlyList<string> Entries, int Files, long Bytes,
long? Free, bool SameVolume, WorkspaceMoveRefusal? Refusal, long Missing, string Fingerprint)
{
public bool CanMove => Refusal is null;
/// <summary>The refusal as one localized sentence; null when the move can go.</summary>
public string? RefusalText => Refusal switch
{
WorkspaceMoveRefusal.SameFolder => Loc.L("Errors.Workspace.MoveSameFolder", To),
WorkspaceMoveRefusal.TargetNotEmpty => Loc.L("Errors.Workspace.MoveTargetNotEmpty", To),
WorkspaceMoveRefusal.NoRoom => Loc.L("Errors.Workspace.MoveNoRoom", (Missing / 1e9).ToString("0.0", Loc.Culture)),
_ => null,
};
}
/// <summary>How far a move of the workspace folder went, as its journal records it.</summary>
public enum WorkspaceMoveStage
{
/// <summary>The entries are being moved (one volume) or copied and verified (across volumes); the source is the workspace.</summary>
Moving,
/// <summary>Every entry is verified at the target and the desktop was told: the target is the workspace; what is left of the source goes.</summary>
Switched,
}
/// <summary>
/// The journal of a move of the workspace folder (<c>&lt;from&gt;/.move.json</c>, <see cref="WorkspaceMover.JournalFile"/>):
/// where it goes, the plan's fingerprint, whether it renames folders on one volume, the entries it moves in their order,
/// those copied and verified at the target already (across volumes), and how far it went. Written whole and atomically
/// before every step that depends on it, so a move stopped at any moment is taken up, or undone, from it.
/// </summary>
public sealed record WorkspaceMoveJournal(string Format, int FormatVersion, string From, string To, string Fingerprint, bool SameVolume,
IReadOnlyList<string> Entries, IReadOnlyList<string> Verified, WorkspaceMoveStage Stage, DateTimeOffset Started)
{
public const string FormatName = "chronos-workspace-move";
public const int CurrentFormatVersion = 1;
}
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
UseStringEnumConverter = true, WriteIndented = true)]
[JsonSerializable(typeof(WorkspaceMoveJournal))]
internal sealed partial class WorkspaceMoveJson : JsonSerializerContext;
/// <summary>
/// Moves a workspace folder with its projects (phase 5c, <see cref="Core.Model.JobKind.WorkspaceMove"/>). What moves: every
/// folder named by a project id, then <c>workspace.json</c>, then <c>.trash</c>, in that order; anything else in the folder
/// stays where it is. On one volume each entry is renamed into the target (<see cref="Directory.Move(string, string)"/>);
/// across volumes each file is copied into a <c>.part</c> beside its place, hashed as it is read and read back to hash the
/// copy, and only once both SHA-256 agree renamed into place (with the source's time), so a file at its final name in the
/// target is always one verified. The source is the workspace until every entry is at the target: then the desktop is told
/// (the switch), and only after it does the source of a copy go. The journal (<see cref="WorkspaceMoveJournal"/>) is
/// written in the source before the first step and at every entry: a move stopped (paused, the program closed, the PC
/// down) goes on from it, trusting the files at their final names and the entries it records as verified; a move that
/// fails before the switch is undone (<see cref="Abandon"/>): renamed entries go back, copies at the target are removed,
/// the journal goes. After the switch a source entry goes only when each of its files still matches its copy at the target
/// (name, size and time, as verified); one that changed since is kept and logged, and one that cannot be removed (a file held
/// open) stays with the journal, so the move taken up later clears it: neither fails a move whose target is the workspace.
/// A rename refused because the folders lie on two volumes after all (a volume mounted in a folder) turns the move into a
/// copy before anything was renamed.
/// </summary>
public sealed class WorkspaceMover
{
/// <summary>The journal of a move under way, in the workspace folder it leaves.</summary>
public const string JournalFile = ".move.json";
/// <summary>Where a file copied across volumes is written until its hash is checked.</summary>
internal const string PartSuffix = ".part";
private const int BufferSize = 1 << 20;
private const int DeleteAttempts = 5;
/// <param name="from">The workspace folder in use.</param>
/// <param name="to">The folder it moves to (made when it is not there).</param>
public WorkspaceMover(string from, string to)
{
ArgumentException.ThrowIfNullOrWhiteSpace(from);
ArgumentException.ThrowIfNullOrWhiteSpace(to);
From = WorkspaceStore.FullRoot(from);
To = WorkspaceStore.FullRoot(to);
}
public string From { get; }
public string To { get; }
/// <summary>Free bytes on the volume of a folder (the host's measure); null takes the drive's (<see cref="DriveInfo"/>).</summary>
public Func<string, long?>? FreeSpace { get; init; }
/// <summary>Whether both folders lie on one volume; null decides from their drives (tests set it to try the copy on one disk).</summary>
public bool? SameVolume { get; init; }
/// <summary>Bytes a copy across volumes leaves free on the target's drive besides what it copies (as a copy into the workspace keeps).</summary>
public long Reserve { get; init; } = WorkspaceCopyJobHandler.SpaceReserve;
/// <summary>
/// Called at named steps with the path concerned (tests: a crash, a file damaged on its way): <c>verify</c> (a copy's
/// <c>.part</c>, written and about to be read back), <c>copied</c> (a file at its final name), <c>moved</c> (an entry
/// renamed into the target), <c>rename</c> (an entry about to be renamed into the target), <c>switched</c> (the desktop told).
/// </summary>
internal Action<string, string>? Cut { get; init; }
/// <summary>The journal of a move under way out of <paramref name="root"/>; null when there is none, or it cannot be read (logged).</summary>
public static WorkspaceMoveJournal? ReadJournal(string root)
{
ArgumentException.ThrowIfNullOrWhiteSpace(root);
var path = Path.Combine(WorkspaceStore.FullRoot(root), JournalFile);
if (!File.Exists(path)) return null;
try
{
if (JsonSerializer.Deserialize(AtomicFile.ReadAllText(path), WorkspaceMoveJson.Default.WorkspaceMoveJournal) is
{ Format: WorkspaceMoveJournal.FormatName, From.Length: > 0, To.Length: > 0, Entries: not null, Verified: not null } journal
&& journal.FormatVersion <= WorkspaceMoveJournal.CurrentFormatVersion)
return journal;
Log.Warning("workspace", $"{path} is not a journal of a move this Chronos reads");
}
catch (Exception ex) when (ex is JsonException or IOException or UnauthorizedAccessException)
{
Log.Warning("workspace", $"{path} cannot be read", ex);
}
return null;
}
/// <summary>
/// True when <paramref name="root"/> holds a move's journal, readable or not: the folder is being moved, or a move of it
/// stopped before it ended, and nothing may write there until it is taken up or undone.
/// </summary>
public static bool HasJournal(string root) => File.Exists(Path.Combine(WorkspaceStore.FullRoot(root), JournalFile));
private string JournalPath => Path.Combine(From, JournalFile);
/// <summary>What the move would take and whether it can go (<see cref="WorkspaceMoveSurvey"/>); nothing is written.</summary>
/// <exception cref="IOException">The workspace folder cannot be read.</exception>
public WorkspaceMoveSurvey Survey()
{
var comparison = PathComparison;
var projects = new List<WorkspaceMoveFolder>();
var entries = new List<string>();
var (files, bytes) = (0, 0L);
var fingerprint = new StringBuilder();
fingerprint.Append(From).Append('\n').Append(To).Append('\n');
if (Directory.Exists(From))
{
foreach (var directory in Directory.EnumerateDirectories(From).Order(StringComparer.Ordinal))
{
var name = Path.GetFileName(directory);
if (!Guid.TryParseExact(name, "N", out var id)) continue;
var (count, size) = Read(directory, name, fingerprint);
projects.Add(new WorkspaceMoveFolder(id, File.Exists(Path.Combine(directory, WorkspaceProject.BundleFile)), count, size));
entries.Add(name);
(files, bytes) = (files + count, bytes + size);
}
var manifest = Path.Combine(From, WorkspaceStore.ManifestFile);
if (File.Exists(manifest))
{
var info = new FileInfo(manifest);
entries.Add(WorkspaceStore.ManifestFile);
Line(fingerprint, WorkspaceStore.ManifestFile, info);
(files, bytes) = (files + 1, bytes + info.Length);
}
var trash = Path.Combine(From, TrashFolder);
if (Directory.Exists(trash))
{
var (count, size) = Read(trash, TrashFolder, fingerprint);
entries.Add(TrashFolder);
(files, bytes) = (files + count, bytes + size);
}
}
var same = SameVolume ?? OnOneVolume(From, To);
var free = FreeBytes(To);
WorkspaceMoveRefusal? refusal = null;
long missing = 0;
if (string.Equals(From, To, comparison) || To.StartsWith(From + Path.DirectorySeparatorChar, comparison))
{
refusal = WorkspaceMoveRefusal.SameFolder;
}
else if (File.Exists(To) || TargetHolds(entries))
{
refusal = WorkspaceMoveRefusal.TargetNotEmpty;
}
else if (!same && free is { } room && room < bytes + Reserve)
{
refusal = WorkspaceMoveRefusal.NoRoom;
missing = bytes + Reserve - room;
}
return new WorkspaceMoveSurvey(From, To, projects, entries, files, bytes, free, same, refusal, missing,
Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(fingerprint.ToString()))));
}
/// <summary>
/// Moves the workspace, or takes up the move its journal records: the plan of <paramref name="fingerprint"/> is checked
/// again first (a move taken up has checked it), every entry is moved or copied and verified, <paramref name="switched"/>
/// is told (from, to) once all are at the target, then what is left of the source goes and the journal with it.
/// <paramref name="progress"/> hears the files done and their total, with true where the job should keep it as its
/// checkpoint; <paramref name="log"/> the lines of the job's log. A failure before the switch undoes what was done
/// (<see cref="Abandon"/>) and is thrown; a cancel (a pause, the program closing) leaves the journal for the next run.
/// </summary>
/// <returns>The survey of the move as it began, and whether it was taken up from a journal.</returns>
/// <exception cref="InvalidOperationException">
/// The workspace changed since its plan (<c>Errors.Workspace.MoveChanged</c>), the move is refused now (the target filled,
/// the room went), another move's journal is there (<c>Errors.Workspace.Moving</c>), or a copy did not match its source
/// (<c>Errors.Workspace.MoveVerify</c>).
/// </exception>
/// <exception cref="IOException">A file or folder could not be read, written or moved.</exception>
public async Task<(WorkspaceMoveSurvey Survey, bool Resumed)> MoveAsync(string fingerprint, Func<string, string, Task>? switched = null,
Func<long, long, bool, ValueTask>? progress = null, Action<string>? log = null, CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(fingerprint);
var survey = Survey();
var journal = ReadJournal(From);
var resumed = journal is not null;
if (journal is null)
{
if (HasJournal(From)) throw new InvalidOperationException(Loc.L("Errors.Workspace.Moving"));
if (!string.Equals(survey.Fingerprint, fingerprint, StringComparison.Ordinal)) throw new InvalidOperationException(Loc.L("Errors.Workspace.MoveChanged"));
if (survey.RefusalText is { } refused) throw new InvalidOperationException(refused);
journal = new WorkspaceMoveJournal(WorkspaceMoveJournal.FormatName, WorkspaceMoveJournal.CurrentFormatVersion, From, To, fingerprint, survey.SameVolume,
survey.Entries, [], WorkspaceMoveStage.Moving, DateTimeOffset.UtcNow);
Directory.CreateDirectory(To);
WriteJournal(journal);
Log.Info("workspace", $"Moving the workspace {From} to {To} ({survey.Entries.Count} entries, {survey.Files} files, {survey.Bytes} bytes, "
+ (survey.SameVolume ? "renamed" : "copied") + ")");
}
else
{
if (!string.Equals(journal.To, To, PathComparison) || !string.Equals(journal.From, From, PathComparison))
throw new InvalidOperationException(Loc.L("Errors.Workspace.Moving"));
log?.Invoke(Loc.L("Archive.Log.WorkspaceMoveResumed", JournalPath));
Log.Info("workspace", $"The move of the workspace {From} to {To} is taken up from its journal ({journal.Stage})");
}
if (journal.Stage == WorkspaceMoveStage.Moving)
{
try
{
if (journal.SameVolume)
{
try
{
journal = await RenameAsync(journal, progress, log, cancellationToken).ConfigureAwait(false);
}
catch (IOException ex) when (IsCrossVolume(ex) && NothingRenamed(journal))
{
// Two volumes under one drive letter (a volume mounted in a folder): the files are copied and verified.
Log.Info("workspace", $"{From} and {To} lie on two volumes after all: the move copies and verifies the files ({ex.Message})");
if (FreeBytes(To) is { } room && room < survey.Bytes + Reserve)
throw new InvalidOperationException(Loc.L("Errors.Workspace.MoveNoRoom", ((survey.Bytes + Reserve - room) / 1e9).ToString("0.0", Loc.Culture)), ex);
journal = journal with { SameVolume = false };
WriteJournal(journal);
journal = await CopyAsync(journal, survey, progress, log, cancellationToken).ConfigureAwait(false);
}
}
else
{
journal = await CopyAsync(journal, survey, progress, log, cancellationToken).ConfigureAwait(false);
}
}
catch (Exception ex) when (ex is not OperationCanceledException || !cancellationToken.IsCancellationRequested)
{
Log.Warning("workspace", $"The move of the workspace {From} to {To} failed; it is undone", ex);
TryAbandon();
throw;
}
journal = journal with { Stage = WorkspaceMoveStage.Switched };
WriteJournal(journal);
}
// From here on the target is the workspace: told again when a run taken up after the switch did not see its answer.
if (switched is not null) await switched(From, To).ConfigureAwait(false);
Cut?.Invoke("switched", To);
var left = Clean(journal);
log?.Invoke(Loc.L("Archive.Log.WorkspaceMoved", To, journal.Entries.Count(e => Guid.TryParseExact(e, "N", out _)), survey.Files));
if (left > 0) log?.Invoke(Loc.L("Archive.Log.WorkspaceMoveLeft", From, left));
Log.Info("workspace", $"The workspace moved from {From} to {To}" + (left > 0 ? $"; {left} entr(ies) of the old folder stay there" : string.Empty));
return (survey, resumed);
}
/// <summary>A move or rename refused because source and target lie on two volumes (Windows <c>ERROR_NOT_SAME_DEVICE</c>, Unix <c>EXDEV</c>).</summary>
private static bool IsCrossVolume(IOException ex) => (ex.HResult & 0xFFFF) == (OperatingSystem.IsWindows() ? 0x11 : 18);
/// <summary>No entry of the journal was renamed into the target yet.</summary>
private bool NothingRenamed(WorkspaceMoveJournal journal) => journal.Entries.All(e => !Exists(Path.Combine(To, e)));
/// <summary>
/// Undoes a move its journal records as not switched yet, so the source stays the workspace: on one volume the entries
/// renamed into the target go back; across volumes the copies it made at the target are removed (what cannot be removed
/// is logged: the source is whole). The journal then goes. A move that switched is not undone (false): the target is the
/// workspace, and the move is taken up to finish clearing the source. True when there was nothing left to undo, or it was.
/// </summary>
/// <exception cref="IOException">An entry renamed into the target could not go back; the journal stays, and the move can be taken up.</exception>
public bool Abandon()
{
var journal = ReadJournal(From);
if (journal is null) return true;
if (journal.Stage != WorkspaceMoveStage.Moving) return false;
foreach (var entry in journal.Entries.Reverse())
{
var source = Path.Combine(From, entry);
var target = Path.Combine(To, entry);
if (journal.SameVolume)
{
if (Exists(target) && !Exists(source)) Rename(target, source);
continue;
}
try
{
// Refused when the target held the entry: whatever is there is this move's copy.
if (Directory.Exists(target)) Directory.Delete(target, recursive: true);
else if (File.Exists(target)) File.Delete(target);
var part = target + PartSuffix;
if (File.Exists(part)) File.Delete(part);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("workspace", $"The copy {target} of an undone move could not be removed", ex);
}
}
File.Delete(JournalPath);
TryDeleteEmpty(To);
Log.Info("workspace", $"The move of the workspace {From} to {To} was undone; the workspace stays in {From}");
return true;
}
private void TryAbandon()
{
try
{
Abandon();
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("workspace", $"The move of the workspace {From} to {To} could not be undone; its journal stays to take it up", ex);
}
}
/// <summary>One volume: each entry renamed into the target, an entry already there (a run that stopped after it) taken as moved.</summary>
private async Task<WorkspaceMoveJournal> RenameAsync(WorkspaceMoveJournal journal, Func<long, long, bool, ValueTask>? progress, Action<string>? log,
CancellationToken cancellationToken)
{
var moved = 0;
foreach (var entry in journal.Entries)
{
cancellationToken.ThrowIfCancellationRequested();
var source = Path.Combine(From, entry);
var target = Path.Combine(To, entry);
if (Exists(source))
{
if (Exists(target)) throw new IOException(Loc.L("Errors.Workspace.MoveTargetNotEmpty", To));
Cut?.Invoke("rename", source);
Rename(source, target);
Cut?.Invoke("moved", target);
if (Guid.TryParseExact(entry, "N", out _)) log?.Invoke(Loc.L("Archive.Log.WorkspaceMoveProject", Name(target)));
}
else if (!Exists(target))
{
// Gone from both sides: nothing to move (the trash emptied meanwhile cannot happen while the move holds the workspace).
Log.Warning("workspace", $"{entry} of the workspace {From} is neither there nor at {To}");
}
moved++;
if (progress is not null) await progress(moved, journal.Entries.Count, true).ConfigureAwait(false);
}
return journal;
}
/// <summary>
/// Across volumes: each entry's files copied and verified (<see cref="CopyFileAsync"/>), the entry recorded as verified in
/// the journal once all of its files are; nothing of the source is touched.
/// </summary>
private async Task<WorkspaceMoveJournal> CopyAsync(WorkspaceMoveJournal journal, WorkspaceMoveSurvey survey, Func<long, long, bool, ValueTask>? progress,
Action<string>? log, CancellationToken cancellationToken)
{
var total = (long)survey.Files;
long done = 0;
var clock = System.Diagnostics.Stopwatch.StartNew();
foreach (var entry in journal.Entries)
{
cancellationToken.ThrowIfCancellationRequested();
var source = Path.Combine(From, entry);
var target = Path.Combine(To, entry);
if (journal.Verified.Contains(entry, StringComparer.Ordinal))
{
done += Directory.Exists(source) ? Listed(source).Count : File.Exists(source) ? 1 : 0;
continue;
}
if (File.Exists(source))
{
await CopyFileAsync(source, target, cancellationToken).ConfigureAwait(false);
done++;
}
else if (Directory.Exists(source))
{
Directory.CreateDirectory(target);
foreach (var folder in new DirectoryInfo(source).EnumerateDirectories("*", SearchOption.AllDirectories))
Directory.CreateDirectory(Path.Combine(target, Path.GetRelativePath(source, folder.FullName)));
foreach (var relative in Listed(source))
{
cancellationToken.ThrowIfCancellationRequested();
await CopyFileAsync(Path.Combine(source, relative), Path.Combine(target, relative), cancellationToken).ConfigureAwait(false);
done++;
var durable = clock.Elapsed >= CheckpointEvery;
if (durable) clock.Restart();
if (progress is not null) await progress(done, total, durable).ConfigureAwait(false);
}
if (Guid.TryParseExact(entry, "N", out _)) log?.Invoke(Loc.L("Archive.Log.WorkspaceMoveProject", Name(source)));
}
journal = journal with { Verified = [.. journal.Verified, entry] };
WriteJournal(journal);
if (progress is not null) await progress(done, total, true).ConfigureAwait(false);
}
return journal;
}
/// <summary>
/// One file across volumes: a file at its final name with the source's size was verified by an earlier run and stays;
/// otherwise the source is streamed into <c>&lt;target&gt;.part</c> and hashed as it is read, the copy is read back and
/// hashed, and only when both agree is it renamed into place, with the source's time.
/// </summary>
/// <exception cref="InvalidOperationException">The copy does not match its source (<c>Errors.Workspace.MoveVerify</c>); the <c>.part</c> is removed.</exception>
private async Task CopyFileAsync(string source, string target, CancellationToken cancellationToken)
{
var info = new FileInfo(source);
if (File.Exists(target) && new FileInfo(target).Length == info.Length) return;
var part = target + PartSuffix;
try
{
string sha;
using (var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256))
{
var buffer = new byte[BufferSize];
await using (var input = new FileStream(source, FileMode.Open, FileAccess.Read, FileShare.Read, BufferSize, FileOptions.Asynchronous | FileOptions.SequentialScan))
await using (var output = new FileStream(part, FileMode.Create, FileAccess.Write, FileShare.None, BufferSize, FileOptions.Asynchronous))
{
int read;
while ((read = await input.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) > 0)
{
hash.AppendData(buffer, 0, read);
await output.WriteAsync(buffer.AsMemory(0, read), cancellationToken).ConfigureAwait(false);
}
await output.FlushAsync(cancellationToken).ConfigureAwait(false);
output.Flush(flushToDisk: true);
}
sha = Convert.ToHexStringLower(hash.GetHashAndReset());
}
Cut?.Invoke("verify", part);
if (!string.Equals(await Sha256Async(part, cancellationToken).ConfigureAwait(false), sha, StringComparison.Ordinal))
throw new InvalidOperationException(Loc.L("Errors.Workspace.MoveVerify", LongPath.ToPortableRelative(From, source)));
File.Move(part, target, overwrite: true);
File.SetLastWriteTimeUtc(target, info.LastWriteTimeUtc);
Cut?.Invoke("copied", target);
}
catch
{
TryDelete(part);
throw;
}
}
/// <summary>
/// After the switch: across volumes every entry of the source goes, folders first, the manifest last, each only while
/// every file of it still matches its copy at the target (<see cref="StillAsCopied"/>): one that changed since it was
/// verified is kept, and logged, never deleted. Then the journal, and the source folder when nothing else is left in it.
/// An entry that cannot be removed (a file held open past a few attempts) stays, and the journal with it naming only
/// such entries, so the move taken up clears them.
/// </summary>
/// <returns>The entries of the source left where they are.</returns>
private int Clean(WorkspaceMoveJournal journal)
{
var stuck = new List<string>();
var kept = 0;
if (!journal.SameVolume)
{
foreach (var entry in journal.Entries.OrderBy(e => e == WorkspaceStore.ManifestFile ? 1 : 0))
{
var source = Path.Combine(From, entry);
if (!Exists(source)) continue;
if (!StillAsCopied(source, Path.Combine(To, entry)))
{
kept++;
Log.Warning("workspace", $"{source} changed after it was copied to {To}: it is kept where it is, never deleted");
continue;
}
for (var attempt = 1; ; attempt++)
{
try
{
if (Directory.Exists(source)) Directory.Delete(source, recursive: true);
else if (File.Exists(source)) File.Delete(source);
break;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
if (attempt >= DeleteAttempts)
{
Log.Warning("workspace", $"{source} could not be removed after the move: it stays, and the move taken up removes it", ex);
stuck.Add(entry);
break;
}
// An indexer or an antivirus reading a file a moment: tried again shortly.
Thread.Sleep(100 * attempt);
}
}
}
}
if (stuck.Count > 0)
{
WriteJournal(journal with { Entries = stuck, Verified = [.. journal.Verified.Where(stuck.Contains)] });
return stuck.Count + kept;
}
File.Delete(JournalPath);
TryDeleteEmpty(From);
return kept;
}
/// <summary>Clock steps a target's file system may round a file's time to (FAT keeps two seconds).</summary>
private static readonly TimeSpan TimeSlack = TimeSpan.FromSeconds(2);
/// <summary>
/// Every file of the source entry (a file, or a folder's files, leases left out) is at the target with the same relative
/// name and size and the time the copy gave it (the source's, within the target's rounding): nothing came or changed in
/// the source since it was verified.
/// </summary>
private static bool StillAsCopied(string source, string target)
{
try
{
if (File.Exists(source)) return Matches(new FileInfo(source), new FileInfo(target));
return Listed(source).All(relative => Matches(new FileInfo(Path.Combine(source, relative)), new FileInfo(Path.Combine(target, relative))));
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Debug("workspace", $"{source} could not be compared with {target}: {ex.Message}");
return false;
}
static bool Matches(FileInfo from, FileInfo to) =>
to.Exists && from.Length == to.Length && (from.LastWriteTimeUtc - to.LastWriteTimeUtc).Duration() <= TimeSlack;
}
/// <summary>A folder's files relative to it, in ordinal order, the leases left out (they belong to whoever holds the project there).</summary>
private static List<string> Listed(string folder) =>
[.. new DirectoryInfo(folder).EnumerateFiles("*", SearchOption.AllDirectories).Where(f => !WorkspaceProject.IsLeaseFile(f.Name))
.Select(f => Path.GetRelativePath(folder, f.FullName)).Order(StringComparer.Ordinal)];
/// <summary>Files and bytes of an entry, each file's name, size and time added to the fingerprint.</summary>
private static (int Files, long Bytes) Read(string folder, string name, StringBuilder fingerprint)
{
var (files, bytes) = (0, 0L);
foreach (var relative in Listed(folder))
{
var info = new FileInfo(Path.Combine(folder, relative));
Line(fingerprint, name + "/" + relative.Replace('\\', '/'), info);
files++;
bytes += info.Length;
}
return (files, bytes);
}
private static void Line(StringBuilder fingerprint, string name, FileInfo info) =>
fingerprint.Append(name).Append('|').Append(info.Length.ToString(CultureInfo.InvariantCulture)).Append('|')
.Append(info.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture)).Append('\n');
/// <summary>The target holds a workspace's own files, or a folder or file named as one of the entries that move.</summary>
private bool TargetHolds(IEnumerable<string> entries) =>
Directory.Exists(To) && (Exists(Path.Combine(To, WorkspaceStore.ManifestFile)) || Exists(Path.Combine(To, TrashFolder)) || Exists(Path.Combine(To, JournalFile))
|| entries.Any(e => Exists(Path.Combine(To, e))));
private const string TrashFolder = ".trash";
private static readonly TimeSpan CheckpointEvery = TimeSpan.FromSeconds(5);
private static StringComparison PathComparison => OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
private static bool Exists(string path) => Directory.Exists(path) || File.Exists(path);
private static void Rename(string source, string target)
{
if (Directory.Exists(source)) Directory.Move(source, target);
else File.Move(source, target);
}
/// <summary>The name a project's working state gives it, for the log; its folder's name when it cannot be read.</summary>
private static string Name(string folder)
{
var bundle = Path.Combine(folder, WorkspaceProject.BundleFile);
try
{
if (File.Exists(bundle) && Core.Project.ProjectBundle.Load(bundle).Workspace?.Name is { Length: > 0 } name) return name;
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException or JsonException)
{
Log.Debug("workspace", $"{bundle} names no project: {ex.Message}");
}
return Path.GetFileName(folder);
}
private void WriteJournal(WorkspaceMoveJournal journal) =>
AtomicFile.WriteAllText(JournalPath, JsonSerializer.Serialize(journal, WorkspaceMoveJson.Default.WorkspaceMoveJournal));
/// <summary>Free bytes on the volume of <paramref name="folder"/> (its nearest folder that exists); null when they cannot be measured.</summary>
private long? FreeBytes(string folder)
{
var existing = Nearest(folder);
if (existing is null) return null;
if (FreeSpace is { } measure) return measure(existing);
try
{
// Windows wants a drive root; Unix accepts any directory and reports the volume it is mounted on.
return new DriveInfo(OperatingSystem.IsWindows() ? Path.GetPathRoot(existing)! : existing).AvailableFreeSpace;
}
catch (Exception ex) when (ex is ArgumentException or IOException or UnauthorizedAccessException)
{
return null;
}
}
/// <summary>
/// Both folders on one volume: on Windows the same drive; on Unix the same mount point. A doubt reads as two volumes,
/// whose copy is the safe way (a rename across volumes would fail).
/// </summary>
private static bool OnOneVolume(string from, string to)
{
try
{
if (Nearest(to) is not { } target) return false;
if (OperatingSystem.IsWindows())
return string.Equals(Path.GetPathRoot(from), Path.GetPathRoot(target), StringComparison.OrdinalIgnoreCase)
&& !Path.GetPathRoot(from)!.StartsWith(@"\\", StringComparison.Ordinal);
var mounts = DriveInfo.GetDrives().Select(d => Path.TrimEndingDirectorySeparator(d.RootDirectory.FullName)).ToList();
return string.Equals(Mount(from, mounts), Mount(target, mounts), StringComparison.Ordinal);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException)
{
return false;
}
static string? Mount(string path, List<string> mounts) =>
mounts.Where(m => path == m || path.StartsWith(m.EndsWith('/') ? m : m + "/", StringComparison.Ordinal)).MaxBy(m => m.Length);
}
/// <summary><paramref name="folder"/> or its nearest parent that exists; null when none does.</summary>
private static string? Nearest(string folder)
{
for (var current = folder; !string.IsNullOrEmpty(current); current = Path.GetDirectoryName(current))
{
if (Directory.Exists(current)) return current;
}
return null;
}
private static async Task<string> Sha256Async(string path, CancellationToken cancellationToken)
{
await using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, BufferSize, FileOptions.Asynchronous | FileOptions.SequentialScan);
return Convert.ToHexStringLower(await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false));
}
private static void TryDelete(string path)
{
try
{
File.Delete(path);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Debug("workspace", $"{path} stays: {ex.Message}");
}
}
private static void TryDeleteEmpty(string folder)
{
try
{
if (Directory.Exists(folder) && !Directory.EnumerateFileSystemEntries(folder).Any()) Directory.Delete(folder);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Debug("workspace", $"{folder} stays: {ex.Message}");
}
}
}
@@ -0,0 +1,263 @@
using System.Globalization;
using System.Security.Cryptography;
using System.Text;
using Chronos.Core.Diagnostics;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Data.Archiving;
using Chronos.Data.Projects;
namespace Chronos.Data.Workspaces;
/// <summary>
/// What "Proxies only" (phase 5c) does to a workspace project that holds its originals: <paramref name="Originals"/> files
/// of <paramref name="Bytes"/> the home holds with the same SHA-256, deleted; <paramref name="ToTrash"/> files of
/// <paramref name="TrashBytes"/> it does not (gone there, damaged, or another photo now), moved to the workspace's trash.
/// <paramref name="Fingerprint"/> names the files and the home's answer the plan read: an apply refuses a plan the
/// project or the home changed since.
/// </summary>
public sealed record WorkspaceProxiesOnlyPlan(Guid ProjectId, int Originals, long Bytes, int ToTrash, long TrashBytes, string Fingerprint)
{
/// <summary>Nothing to remove: the project holds no originals.</summary>
public bool IsEmpty => Originals == 0 && ToTrash == 0;
}
/// <summary>
/// The originals of a workspace project (phase 5c): what a copy of them writes, and their removal. An original is never
/// erased on a guess: a file is deleted only when the home holds a photo of its SHA-256, read from the file at that moment,
/// that the home has not flagged missing or damaged; any other goes to the workspace's trash
/// (<c>&lt;root&gt;/.trash/&lt;project id&gt;-&lt;ticks&gt;/originals/</c>, which the clean-up empties).
/// </summary>
public static class WorkspaceOriginals
{
/// <summary>The workspace's trash folder, under its root (the App's <c>WorkspaceHost.TrashFolder</c>).</summary>
public const string TrashFolder = ".trash";
/// <summary>
/// The bytes a copy of the sequence's originals writes from this PC's archive: all the archive has, or only those
/// <paramref name="project"/> lacks when it holds some already.
/// </summary>
public static Task<long> ArchiveOriginalsBytesAsync(ChronosDatabase db, ArchiveService archives, Guid sequenceId, WorkspaceProject? project = null,
CancellationToken cancellationToken = default) =>
new WorkspaceCopyJobHandler(db, archives).OriginalsBytesAsync(sequenceId, project, cancellationToken);
/// <summary>
/// What "Proxies only" removes from <paramref name="project"/>, each original hashed now and weighed against
/// <paramref name="home"/> (the home's frames as read just before): nothing when it holds none.
/// </summary>
/// <exception cref="InvalidDataException">The media index is damaged or newer.</exception>
public static WorkspaceProxiesOnlyPlan PlanProxiesOnly(WorkspaceProject project, HomeFrames home)
{
ArgumentNullException.ThrowIfNull(project);
ArgumentNullException.ThrowIfNull(home);
var files = Files(project);
var held = HeldAtHome(home.Frames);
var fingerprint = new StringBuilder(home.FramesVersion.ToString(CultureInfo.InvariantCulture));
int deleted = 0, trashed = 0;
long deletedBytes = 0, trashedBytes = 0;
foreach (var file in files)
{
var (sha, bytes) = Hash(file);
var atHome = sha is not null && held.Contains(sha);
if (atHome) (deleted, deletedBytes) = (deleted + 1, deletedBytes + bytes);
else (trashed, trashedBytes) = (trashed + 1, trashedBytes + bytes);
fingerprint.Append('|').Append(Path.GetFileName(file)).Append(':').Append(sha).Append(':').Append(atHome ? 'd' : 't');
}
var digest = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(fingerprint.ToString())));
return new WorkspaceProxiesOnlyPlan(project.ProjectId, deleted, deletedBytes, trashed, trashedBytes, digest);
}
/// <summary>
/// Applies a plan of <see cref="PlanProxiesOnly"/> under the project's lease: planned again, refused when anything changed
/// (<c>Errors.Workspace.MediaChanged</c>); then the media index and the project file say proxies first (so a stop
/// half way leaves media that name no original that is gone), and last each original is deleted or trashed as the plan
/// says, hashed again as it goes.
/// </summary>
/// <returns>The files moved to the trash.</returns>
/// <exception cref="InvalidOperationException">The project or the home changed since the plan.</exception>
/// <exception cref="Chronos.Core.Archiving.LeaseUnavailableException">Another writer holds the project.</exception>
public static async Task<int> ApplyProxiesOnlyAsync(WorkspaceProject project, WorkspaceProxiesOnlyPlan plan, HomeFrames home, TimeSpan lockWait,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(project);
ArgumentNullException.ThrowIfNull(plan);
ArgumentNullException.ThrowIfNull(home);
using (await project.LockAsync(lockWait, cancellationToken).ConfigureAwait(false))
{
var now = PlanProxiesOnly(project, home);
if (now.ProjectId != plan.ProjectId || now.Fingerprint != plan.Fingerprint) throw new InvalidOperationException(Loc.L("Errors.Workspace.MediaChanged"));
var media = project.ReadMedia();
var files = Files(project);
if (media is not null)
{
project.WriteMedia(media with
{
Kind = BundleMedia.Proxies,
Frames = [.. media.Frames.Select(f => f with { Original = null, OriginalSha256 = null, OriginalBytes = 0 })],
});
}
var bundle = project.ReadBundle();
if (bundle.Workspace is { } workspace && workspace.Media.Kind != BundleMedia.Proxies)
project.WriteBundle(bundle with { Workspace = workspace with { Media = workspace.Media with { Kind = BundleMedia.Proxies } } });
var trashed = DeleteOrTrash(project, files, home.Frames);
var staging = Path.Combine(project.OriginalsDirectory, StagingFolder);
if (Directory.Exists(staging)) TryDeleteFolder(staging);
if (Directory.Exists(project.OriginalsDirectory) && !Directory.EnumerateFileSystemEntries(project.OriginalsDirectory).Any()) TryDeleteFolder(project.OriginalsDirectory);
Log.Info("workspace", $"Originals of project {project.ProjectId} removed: {plan.Originals} deleted, {trashed} kept in {TrashFolder}");
return trashed;
}
}
/// <summary>
/// Deletes each of <paramref name="files"/> (originals of <paramref name="project"/>) whose SHA-256, read now, is that of a
/// photo <paramref name="home"/> holds, not flagged missing or damaged; moves the others to the workspace's trash (one
/// entry, filled under its staging name and named once whole). A file that can be neither deleted nor moved (held open)
/// stays where it is, and is logged.
/// </summary>
/// <returns>The files moved to the trash.</returns>
internal static int DeleteOrTrash(WorkspaceProject project, IReadOnlyList<string> files, IReadOnlyList<Frame> home)
{
if (files.Count == 0) return 0;
using var trash = WorkspaceTrash.Begin(project.Store.Root, project.ProjectId, DateTime.UtcNow);
return DeleteOrTrash(project, files, home, Path.Combine(trash.Path, WorkspaceProject.OriginalsFolder)).Trashed;
}
/// <summary>
/// Clears the project's folder of originals for good: every file in it, whether the media index names it or not, hashed
/// now and deleted only when <paramref name="home"/> (the home's frames as just read; null when they are not known) holds
/// a photo of that SHA-256 not flagged missing or damaged, else moved into <paramref name="trashEntry"/>'s
/// <c>originals</c> folder (made when needed). The parts a copy left (<c>.part</c>, <c>.staging</c>: downloads not
/// finished) are deleted, and the folder goes once nothing is left in it. The caller holds the project's lease.
/// </summary>
/// <returns>
/// The files deleted, those moved to the trash with their bytes, and those that stay where they are (neither deleted nor
/// moved: held open), which the folder keeps.
/// </returns>
public static (int Deleted, int Trashed, long TrashedBytes, int Left) ClearOriginals(WorkspaceProject project, IReadOnlyList<Frame>? home, string trashEntry)
{
ArgumentNullException.ThrowIfNull(project);
ArgumentException.ThrowIfNullOrEmpty(trashEntry);
var folder = project.OriginalsDirectory;
if (!Directory.Exists(folder)) return (0, 0, 0, 0);
var staging = Path.Combine(folder, StagingFolder);
var files = new List<string>();
var parts = new List<string>();
foreach (var file in Directory.EnumerateFiles(folder, "*", SearchOption.AllDirectories))
{
if (Inside(staging, file)) continue;
(file.EndsWith(".part", StringComparison.OrdinalIgnoreCase) ? parts : files).Add(file);
}
var (deleted, trashed, trashedBytes, left) = DeleteOrTrash(project, files, home ?? [], Path.Combine(trashEntry, WorkspaceProject.OriginalsFolder));
foreach (var part in parts)
{
try
{
File.Delete(part);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("workspace", $"{part} could not be removed", ex);
}
}
if (Directory.Exists(staging)) TryDeleteFolder(staging);
RemoveEmptyFolders(folder);
return (deleted, trashed, trashedBytes, left);
}
/// <summary>The folder of originals a copy from a server downloads into until a file is whole.</summary>
internal const string StagingFolder = ".staging";
private static (int Deleted, int Trashed, long TrashedBytes, int Left) DeleteOrTrash(WorkspaceProject project, IReadOnlyList<string> files, IReadOnlyList<Frame> home,
string trash)
{
var held = HeldAtHome(home);
int deleted = 0, trashed = 0, left = 0;
long trashedBytes = 0;
foreach (var file in files)
{
try
{
var (sha, bytes) = Hash(file);
if (sha is not null && held.Contains(sha))
{
File.Delete(file);
deleted++;
continue;
}
var target = Path.Combine(trash, Path.GetRelativePath(project.OriginalsDirectory, file));
Directory.CreateDirectory(Path.GetDirectoryName(target)!);
// Never over another file of the entry: one that cannot go there stays where it is.
File.Move(file, target, overwrite: false);
trashed++;
trashedBytes += bytes;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
left++;
Log.Warning("workspace", $"{file} could not be removed; it stays where it is", ex);
}
}
return (deleted, trashed, trashedBytes, left);
}
private static bool Inside(string folder, string path) =>
path.StartsWith(Path.TrimEndingDirectorySeparator(folder) + Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase);
/// <summary>Removes the empty folders under <paramref name="folder"/>, and <paramref name="folder"/> itself when it is left empty.</summary>
private static void RemoveEmptyFolders(string folder)
{
try
{
foreach (var sub in Directory.EnumerateDirectories(folder, "*", SearchOption.AllDirectories).OrderByDescending(d => d.Length).ToList())
{
if (!Directory.EnumerateFileSystemEntries(sub).Any()) Directory.Delete(sub);
}
if (!Directory.EnumerateFileSystemEntries(folder).Any()) Directory.Delete(folder);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Debug("workspace", $"{folder} stays: {ex.Message}");
}
}
/// <summary>The original files in the project's folder of originals (never a part a copy writes, nor its staging folder).</summary>
private static List<string> Files(WorkspaceProject project)
{
if (!Directory.Exists(project.OriginalsDirectory)) return [];
return [.. Directory.EnumerateFiles(project.OriginalsDirectory)
.Where(f => !f.EndsWith(".part", StringComparison.OrdinalIgnoreCase))
.Order(StringComparer.OrdinalIgnoreCase)];
}
/// <summary>The SHA-256 (lower case) of the photos the home holds, flagged neither missing nor damaged.</summary>
private static HashSet<string> HeldAtHome(IReadOnlyList<Frame> home) =>
home.Where(f => !string.IsNullOrEmpty(f.Sha256) && (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0)
.Select(f => f.Sha256!.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
private static (string? Sha256, long Bytes) Hash(string file)
{
try
{
using var stream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 20, FileOptions.SequentialScan);
return (Convert.ToHexStringLower(SHA256.HashData(stream)), stream.Length);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("workspace", $"{file} could not be read", ex);
return (null, 0);
}
}
private static void TryDeleteFolder(string folder)
{
try
{
Directory.Delete(folder, recursive: true);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("workspace", $"{folder} could not be removed", ex);
}
}
}
@@ -0,0 +1,302 @@
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using Chronos.Core.Analysis.Deep;
using Chronos.Core.Archiving;
using Chronos.Core.Export;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Core.Render;
using Chronos.Data.Archiving;
using Chronos.Data.Jobs;
using Chronos.Data.Processing;
using Chronos.Data.Processing.Jobs;
using SequenceFrames = Chronos.Data.Processing.SequenceFrames;
namespace Chronos.Data.Workspaces;
/// <summary>
/// <see cref="JobKind.WorkspaceRender"/> (phase 5c, <see cref="WorkspaceJobs.Render"/>): a workspace project rendered on this
/// PC from the originals its media hold, as <see cref="Processing.Jobs.RenderJobHandler"/> renders an archive's sequence,
/// into the project's <c>exports/</c> (each output written by the encoder under a hidden temporary name, then renamed), each
/// output hashed and recorded in the project's ledger (<see cref="WorkspaceRenders"/>, its id
/// <c>RenderJobHandler.RenderId(jobId, index)</c>), then a <see cref="JobKind.WorkspaceSend"/> queued of every render of the
/// project not sent yet (this run's, an earlier run's of this job, and those whose send failed before), whenever this job
/// ever recorded one that is not sent: a resumed or retried run queues it too. A render queued before the workspace moved
/// works where the workspace is now, and none runs while it is being moved (<see cref="WorkspaceJobRoots"/>).
/// Media without an original for every placed frame are refused. The project's lease is held while the inputs are read
/// and while each ledger entry is written, not for the whole render: the editor keeps saving its working state meanwhile.
/// The settings rendered are the ones the job embeds (the editor's portable settings), else the version it names from
/// <c>versions/</c>; the version is recorded only when <c>versions/</c> holds it with the content hash of what rendered.
/// Checkpoint: the entries recorded and the output being written; a resumed job removes that output's leftovers (never one
/// the ledger names) and renders only what is not recorded yet.
/// </summary>
internal sealed class WorkspaceRenderJobHandler(ChronosDatabase db, ArchiveService archives, WorkspaceJobOptions options) : IJobHandler
{
private static readonly TimeSpan LockWait = TimeSpan.FromMinutes(2);
public JobKind Kind => JobKind.WorkspaceRender;
public async Task RunAsync(JobContext context)
{
ArgumentNullException.ThrowIfNull(context);
// The index and the archive engine are not read: everything a local render needs is in the workspace project.
_ = (db, archives);
var cancellation = context.Cancellation;
var parameters = WorkspaceJobs.RenderParameters(context.Job) ?? throw new InvalidDataException(Loc.L("Errors.Workspace.JobParameters"));
var (render, export) = ProcessingJobs.ReadRender(parameters.RenderJson);
// Where the project is now (a workspace moved since the job was queued), never while it is being moved; nothing is
// made where it is not.
var root = WorkspaceJobRoots.Resolve(parameters.WorkspaceRoot, parameters.ProjectId, options.CurrentRoot, "Errors.Workspace.RenderProject");
parameters = parameters with { WorkspaceRoot = root };
var store = new WorkspaceStore(root);
var project = store.Find(parameters.ProjectId) ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.RenderProject"));
var inputs = await ReadInputsAsync(project, parameters, render, cancellation).ConfigureAwait(false);
var (sequence, frames, settings, versionId) = (inputs.Sequence, inputs.Frames, inputs.Settings, inputs.VersionId);
// A local render's outputs stay in the project's exports, from where they are sent home.
var projectSettings = ProjectStore.Parse(settings) with { Export = export with { Folder = null } };
var settingsJson = projectSettings.ToJson();
var settingsHash = ProjectContent.Hash(inputs.Portable);
var exports = project.ExportsDirectory;
Directory.CreateDirectory(exports);
using var names = new OutputNames();
var all = RenderJobHandler.Targets(projectSettings.Export, render.PresetName);
var resume = RenderCheckpoint.Parse(context.Checkpoint);
var done = resume.Done.ToList();
var ledger = project.ReadRenders();
for (var i = 0; i < all.Count; i++)
{
if (!done.Contains(i) && ledger.Find(RenderJobHandler.RenderId(context.Job.Id, i)) is not null) done.Add(i);
}
if (resume is { Pending: { } pending, PendingOutput: { } chosen } && !done.Contains(pending) && Inside(exports, chosen) && names.TryReserve(chosen))
{
Discard(chosen, keepOutput: ledger.Entries.Any(e => SamePath(project.ExportFile(e.File), chosen)));
names.Release(chosen);
}
var targets = all.Select((t, i) => (i, t)).Where(t => !done.Contains(t.i)).ToList();
// What an earlier run of this job recorded and did not send (it stopped before queueing the send, or that failed).
var recorded = ledger.Entries.Where(e => e.JobId == context.Job.Id && e.IsUnsent).Select(e => e.RenderId).ToList();
if (targets.Count == 0)
{
await QueueSendAsync(context, parameters, sequence, recorded).ConfigureAwait(false);
return;
}
if (targets.Count > 0)
{
string? writing = null;
IReadOnlyList<TargetOutcome> outcomes;
try
{
await using var renderer = new SequenceRender(frames, projectSettings, inputs.Analysis, options.Processing, originals: true, companions: null);
outcomes = await renderer.RenderAllAsync(
targets,
(target, setup) => names.Reserve(taken => ExportNaming.OutputPath(projectSettings.Export, exports, SequenceRender.NameContext(sequence, target, setup),
name => taken(name) || ledger.Entries.Any(e => SamePath(project.ExportFile(e.File), name)))),
(index, path) =>
{
writing = path;
return context.CheckpointAsync(new RenderCheckpoint(done, index, path).ToJson(), context.Done, context.Total);
},
async (index, target, result) =>
{
// The output is complete: recorded even if the job is being cancelled, so it is never orphaned.
var id = RenderJobHandler.RenderId(context.Job.Id, index);
var entry = await EntryAsync(project, context.Job.Id, id, target, result, versionId, settingsHash).ConfigureAwait(false);
using (await project.LockAsync(LockWait, CancellationToken.None).ConfigureAwait(false))
{
ledger = project.ReadRenders().With(entry);
project.WriteRenders(ledger);
}
done.Add(index);
if (!recorded.Contains(id)) recorded.Add(id);
writing = null;
await context.CheckpointAsync(new RenderCheckpoint(done, null, null).ToJson(), context.Done, context.Total).ConfigureAwait(false);
context.Log(Loc.L("Export.Log.Rendered", target.PresetName, Path.GetFileName(result.OutputPath)));
foreach (var remark in SequenceRender.Remarks(result)) context.Log(remark);
},
context.ReportAsync,
cancellation).ConfigureAwait(false);
if (renderer.StandIns.Count > 0)
context.Log(Loc.L("Export.Log.StandIns", string.Join(", ", renderer.StandIns.Select(i => (i + 1).ToString(Loc.Culture)))));
}
catch (OperationCanceledException) when (cancellation.IsCancellationRequested)
{
// What the encoder was writing goes; every output recorded stays, and a resumed job renders the rest.
if (writing is { } partial) Discard(partial, keepOutput: true);
await QueueSendAsync(context, parameters, sequence, recorded).ConfigureAwait(false);
throw;
}
var failed = outcomes.Where(o => o.Error is not null).ToList();
foreach (var outcome in failed) context.Log(Loc.L("Export.Log.Failed", outcome.Target.PresetName, outcome.Error));
await QueueSendAsync(context, parameters, sequence, recorded).ConfigureAwait(false);
if (failed.Count > 0)
throw new InvalidOperationException(failed.Count == 1 ? failed[0].Error : Loc.L("Errors.Render.Failed", failed.Count, outcomes.Count));
}
}
/// <summary>What the render reads, taken under the project's lease so the media, the versions and the project file agree.</summary>
private sealed record Inputs(Sequence Sequence, SequenceFrames Frames, string Settings, string Portable, Guid? VersionId, DeepAnalysisReport? Analysis);
/// <summary>
/// The sequence over the workspace's originals, the settings to render (placed on its frames) and the version they are,
/// read under the project's lease.
/// </summary>
/// <exception cref="InvalidOperationException">
/// The project is gone (<c>Errors.Workspace.RenderProject</c>), holds no originals (<c>Errors.Workspace.OriginalsMissing</c>),
/// lacks the original of a frame the render places (<c>Errors.Workspace.RenderNoOriginals</c>, with their number), or the
/// version the job names is not in <c>versions/</c> (<c>Errors.Workspace.RenderVersion</c>).
/// </exception>
private static async Task<Inputs> ReadInputsAsync(WorkspaceProject project, WorkspaceRenderParameters parameters, Protocol.RenderJobParameters render,
CancellationToken cancellationToken)
{
using (await project.LockAsync(LockWait, cancellationToken).ConfigureAwait(false))
{
if (!File.Exists(project.BundlePath)) throw new InvalidOperationException(Loc.L("Errors.Workspace.RenderProject"));
var bundle = project.ReadBundle();
var workspace = bundle.Workspace!;
var media = project.ReadMedia();
if (media is not { HoldsOriginals: true })
throw new InvalidOperationException(Loc.L(workspace.Home.IsArchive ? "Errors.Workspace.OriginalsMissing" : "Errors.Workspace.OriginalsMissing.Server"));
var versions = project.Versions();
var named = parameters.ProjectVersionId ?? render.ProjectVersionId;
var version = named is { } id ? versions.FirstOrDefault(v => v.Id == id) : null;
string portable;
if (render.SettingsJson is { Length: > 0 } embedded) portable = embedded;
else if (named is not null) portable = version?.SettingsJson ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.RenderVersion"));
else portable = versions.Count > 0 ? versions[^1].SettingsJson : ProjectContent.ToPortable(bundle.Project, media.Photos);
// Recorded only for the settings it holds (a version is never named for settings it did not render).
Guid? versionId = version is not null && ProjectContent.Verifies(portable, version.ContentHash) ? version.Id : null;
var settings = portable.Contains("pendingFrames", StringComparison.OrdinalIgnoreCase) ? Place(portable, media.Photos) : portable;
var placed = ProjectStore.Parse(settings);
var excluded = placed.ExcludedFrames.ToHashSet();
var lacking = media.Frames.Count(f => f.Original is null && !excluded.Contains(f.Index) && (f.Flags & SequenceFrames.NotRendered) == 0);
if (lacking > 0) throw new InvalidOperationException(Loc.L("Errors.Workspace.RenderNoOriginals", lacking));
var frames = media.ToOriginalFrames();
var files = media.Frames.Select(f => f.Original is { } original ? project.MediaFile(original) : null).ToList();
var sequence = SequenceOf(bundle, media, frames);
return new Inputs(sequence, new SequenceFrames(sequence, frames, files), settings, portable, versionId, Analysis(project, frames.Count));
}
}
/// <summary>The home's sequence as the project file and the media name it (the workspace keeps no index row of it).</summary>
private static Sequence SequenceOf(ProjectBundle bundle, WorkspaceMedia media, IReadOnlyList<Frame> frames)
{
var times = frames.Where(f => f.CapturedAt is not null).Select(f => f.CapturedAt!.Value).Order().ToList();
var home = bundle.Workspace!.Home;
return new Sequence
{
Id = home.SequenceId,
ArchiveId = bundle.Archive.Id,
Title = bundle.Sequence.Title,
Folder = bundle.Sequence.Folder,
FrameCount = frames.Count,
FirstCapture = times.Count > 0 ? times[0] : null,
LastCapture = times.Count > 0 ? times[^1] : null,
FramesVersion = media.FramesVersion,
Origin = home.IsArchive ? SequenceOrigin.Local : SequenceOrigin.Server,
};
}
/// <summary>The deep report the copy brought from the home, when it describes these frames; null otherwise.</summary>
private static DeepAnalysisReport? Analysis(WorkspaceProject project, int frameCount)
{
if (!File.Exists(project.AnalysisPath)) return null;
try
{
var report = DeepAnalysisReport.Load(project.AnalysisPath);
return report.FrameCount == frameCount ? report : null;
}
catch (Exception ex) when (ex is IOException or InvalidDataException or JsonException or UnauthorizedAccessException)
{
Core.Diagnostics.Log.Warning("workspace", $"{project.AnalysisPath} cannot be read", ex);
return null;
}
}
/// <summary>Portable settings placed on the media's frames; others as they are.</summary>
private static string Place(string portable, FramePhotos photos)
{
try
{
return ProjectContent.Place(portable, photos);
}
catch (JsonException)
{
// Not a project: reading it says so.
return portable;
}
}
/// <summary>The ledger entry of a finished output: hashed as it is on the disk now.</summary>
private static async Task<WorkspaceRenderEntry> EntryAsync(WorkspaceProject project, Guid jobId, Guid renderId, ExportTarget target, RenderResult result,
Guid? versionId, string settingsHash)
{
var sha = await HashAsync(result.OutputPath).ConfigureAwait(false);
var file = Path.GetRelativePath(project.ExportsDirectory, result.OutputPath).Replace('\\', '/');
return new WorkspaceRenderEntry(renderId, jobId, file, sha, result.Bytes, target.PresetName, result.Codec.ToString(), result.Width, result.Height,
result.FrameRate.Value, result.DurationSeconds, project.ProjectId, versionId, settingsHash, DateTimeOffset.UtcNow);
}
/// <summary>
/// The SHA-256 of an output: of the file, or, for an image sequence's folder, of its files' relative paths and SHA-256 in
/// order (a send checks the folder the same way).
/// </summary>
internal static async Task<string> HashAsync(string output)
{
if (File.Exists(output)) return await ContentHash.Sha256Async(output, cancellationToken: CancellationToken.None).ConfigureAwait(false);
var listing = new StringBuilder();
foreach (var file in Directory.EnumerateFiles(output, "*", SearchOption.AllDirectories).Order(StringComparer.Ordinal))
{
var sha = await ContentHash.Sha256Async(file, cancellationToken: CancellationToken.None).ConfigureAwait(false);
listing.Append(Path.GetRelativePath(output, file).Replace('\\', '/')).Append(' ').Append(sha).Append('\n');
}
return Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(listing.ToString())));
}
/// <summary>
/// Queues the send home of every render of the project not sent yet (an empty list of ids: the send reads the ledger when
/// it runs), whatever stops the job now, once this job recorded one that is not sent (they are done).
/// </summary>
private static async Task QueueSendAsync(JobContext context, WorkspaceRenderParameters parameters, Sequence sequence, List<Guid> recorded)
{
if (recorded.Count == 0) return;
var send = WorkspaceJobs.Send(sequence.Id, sequence.DisplayTitle, parameters.WorkspaceRoot, parameters.ProjectId, [], parameters.Server,
context.Job.Priority);
await context.EnqueueFollowUpAsync(send.Kind, send.Targets, send.Label, send.ParametersJson, send.Priority, CancellationToken.None).ConfigureAwait(false);
}
private static bool Inside(string folder, string path)
{
var relative = Path.GetRelativePath(folder, path);
return !relative.StartsWith("..", StringComparison.Ordinal) && !Path.IsPathRooted(relative);
}
private static bool SamePath(string a, string b) =>
string.Equals(Path.GetFullPath(a), Path.GetFullPath(b), OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal);
/// <summary>
/// Removes what an interrupted attempt left of <paramref name="output"/>: the encoder's hidden temporary file or folder,
/// and an output completed but never recorded (it renders again under the same name) unless <paramref name="keepOutput"/>.
/// </summary>
private static void Discard(string output, bool keepOutput)
{
var parent = Path.GetDirectoryName(output);
if (parent is null || !Directory.Exists(parent)) return;
var leftovers = Directory.EnumerateFileSystemEntries(parent, "." + Path.GetFileName(output) + ".*.tmp").ToList();
if (!keepOutput) leftovers.Add(output);
foreach (var leftover in leftovers)
{
try
{
if (Directory.Exists(leftover)) Directory.Delete(leftover, recursive: true);
else if (File.Exists(leftover)) File.Delete(leftover);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Core.Diagnostics.Log.Warning("workspace", $"{leftover} could not be removed", ex);
}
}
}
}
@@ -0,0 +1,136 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using Chronos.Core.IO;
using Chronos.Core.Model;
namespace Chronos.Data.Workspaces;
/// <summary>
/// One local render of a workspace project (phase 5c), as its ledger (<see cref="WorkspaceRenders"/>) records it: the output
/// a <see cref="JobKind.WorkspaceRender"/> wrote into the project's <c>exports/</c>, its SHA-256 and size as written, what it
/// is (preset, codec, size, rate, length), the project and version it was rendered from, and whether it went home
/// (<see cref="JobKind.WorkspaceSend"/>). The file stays in <c>exports/</c> after it is sent, until a clean-up removes it.
/// </summary>
/// <param name="RenderId">
/// The render's id, the same at the home once sent (<c>RenderJobHandler.RenderId(jobId, index)</c>): a send repeated after a
/// cut finds the home holding it already, and nothing is filed twice.
/// </param>
/// <param name="JobId">The <see cref="JobKind.WorkspaceRender"/> job that wrote it.</param>
/// <param name="File">The output, relative to the project's <c>exports/</c> folder (forward slashes; <see cref="WorkspaceProject.ExportFile"/>).</param>
/// <param name="Sha256">The output's SHA-256 (lower case) as it was written: a send hashes the file again and refuses another.</param>
/// <param name="Size">The output's size in bytes.</param>
/// <param name="ProjectVersionId">The project version rendered; null for a working state no version names.</param>
/// <param name="SettingsHash">The content hash of the portable settings rendered (<c>ProjectContent</c>); null when not known.</param>
/// <param name="Rendered">When the output was finished.</param>
/// <param name="State"><see cref="Pending"/>, <see cref="SentState"/> or <see cref="Failed"/> (a later Chronos may write others: read as pending).</param>
/// <param name="Sent">When the home took it; null until then.</param>
/// <param name="Error">Why the last send failed (one localized sentence); null otherwise.</param>
public sealed record WorkspaceRenderEntry(
Guid RenderId,
Guid JobId,
string File,
string Sha256,
long Size,
string PresetName,
string? Codec,
int Width,
int Height,
double Fps,
double DurationSeconds,
Guid ProjectId,
Guid? ProjectVersionId,
string? SettingsHash,
DateTimeOffset Rendered,
string State = WorkspaceRenderEntry.Pending,
DateTimeOffset? Sent = null,
string? Error = null)
{
/// <summary>Written, not at the home yet.</summary>
public const string Pending = "pending";
/// <summary>The home holds it (a render row of the home's sequence with this id).</summary>
public const string SentState = "sent";
/// <summary>The last send failed (<see cref="Error"/> says why); sent again by the next send.</summary>
public const string Failed = "failed";
/// <summary>The home holds this render.</summary>
[JsonIgnore]
public bool IsSent => State == SentState;
/// <summary>Not at the home yet: pending, failed, or a state this Chronos does not know. A clean-up never removes its file.</summary>
[JsonIgnore]
public bool IsUnsent => !IsSent;
/// <summary>This entry as taken by the home at <paramref name="at"/>.</summary>
public WorkspaceRenderEntry AsSent(DateTimeOffset at) => this with { State = SentState, Sent = at, Error = null };
/// <summary>This entry after a send that failed for <paramref name="error"/>; it stays to be sent again.</summary>
public WorkspaceRenderEntry AsFailed(string error) => this with { State = Failed, Error = error };
}
/// <summary>
/// The ledger of a workspace project's local renders (<c>exports/renders.json</c>, phase 5c): every output a
/// <see cref="JobKind.WorkspaceRender"/> finished, oldest first, one entry per render id. Read whole and written whole and
/// atomically (temporary file, flush, rename), always under the project's lease; a missing ledger is an empty one.
/// </summary>
public sealed record WorkspaceRenders(string Format, int FormatVersion, IReadOnlyList<WorkspaceRenderEntry> Entries)
{
public const string FormatName = "chronos-workspace-renders";
public const int CurrentFormatVersion = 1;
/// <summary>A ledger without entries.</summary>
public static WorkspaceRenders Empty { get; } = new(FormatName, CurrentFormatVersion, []);
/// <summary>The entries not at the home yet (<see cref="WorkspaceRenderEntry.IsUnsent"/>), oldest first.</summary>
[JsonIgnore]
public IReadOnlyList<WorkspaceRenderEntry> Unsent => [.. Entries.Where(e => e.IsUnsent)];
/// <summary>The entry of <paramref name="renderId"/>; null when the ledger has none.</summary>
public WorkspaceRenderEntry? Find(Guid renderId) => Entries.FirstOrDefault(e => e.RenderId == renderId);
/// <summary>This ledger with <paramref name="entry"/> in place of the entry of its render id, or appended when it has none.</summary>
public WorkspaceRenders With(WorkspaceRenderEntry entry)
{
ArgumentNullException.ThrowIfNull(entry);
var entries = Entries.ToList();
var at = entries.FindIndex(e => e.RenderId == entry.RenderId);
if (at >= 0) entries[at] = entry;
else entries.Add(entry);
return this with { Entries = entries };
}
/// <summary>This ledger without the entry of <paramref name="renderId"/> (a sent file a clean-up removed, a discarded render).</summary>
public WorkspaceRenders Without(Guid renderId) => Entries.Any(e => e.RenderId == renderId) ? this with { Entries = [.. Entries.Where(e => e.RenderId != renderId)] } : this;
/// <summary>The ledger at <paramref name="path"/>; <see cref="Empty"/> when there is none.</summary>
/// <exception cref="InvalidDataException">The file is no ledger, is damaged, or a newer Chronos wrote it.</exception>
/// <exception cref="IOException">The file cannot be read.</exception>
public static WorkspaceRenders Read(string path)
{
ArgumentException.ThrowIfNullOrEmpty(path);
if (!System.IO.File.Exists(path)) return Empty;
WorkspaceRenders? renders;
try
{
renders = JsonSerializer.Deserialize(AtomicFile.ReadAllText(path), WorkspaceJson.Default.WorkspaceRenders);
}
catch (JsonException ex)
{
throw new InvalidDataException($"{path} cannot be read.", ex);
}
if (renders is null || renders.Format != FormatName || renders.Entries is null) throw new InvalidDataException($"{path} is not a renders ledger.");
if (renders.FormatVersion > CurrentFormatVersion) throw new InvalidDataException($"{path} is newer than this Chronos.");
return renders;
}
/// <summary>Writes <paramref name="renders"/> to <paramref name="path"/> whole and atomically, making its folder when needed.</summary>
/// <exception cref="IOException">The file cannot be written; the ledger there stays as it was.</exception>
public static void Write(string path, WorkspaceRenders renders)
{
ArgumentException.ThrowIfNullOrEmpty(path);
ArgumentNullException.ThrowIfNull(renders);
if (Path.GetDirectoryName(Path.GetFullPath(path)) is { Length: > 0 } folder) Directory.CreateDirectory(folder);
AtomicFile.WriteAllText(path, JsonSerializer.Serialize(renders with { Format = FormatName, FormatVersion = CurrentFormatVersion }, WorkspaceJson.Default.WorkspaceRenders));
}
}
@@ -0,0 +1,384 @@
using System.Net;
using Chronos.Core.Archiving;
using Chronos.Core.Diagnostics;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Data.Archiving;
using Chronos.Data.Jobs;
using Chronos.Data.Projects;
using Chronos.Data.Sync;
using Chronos.Protocol;
using Chronos.Protocol.Client;
namespace Chronos.Data.Workspaces;
/// <summary>
/// <see cref="JobKind.WorkspaceSend"/> (phase 5c, <see cref="WorkspaceJobs.Send"/>): the local renders of a workspace project
/// sent home with their project and version, per unsent ledger entry (<see cref="WorkspaceRenders"/>; the ids the job names,
/// or every unsent one): the file hashed again (one missing or changed since it was rendered is refused, marked failed in the
/// ledger and left where it is), the version it names pushed first when the home lacks it (a push that fails fails the job,
/// the entry left as it was; a version the push replayed under a new id, after the home moved on, is found again by the
/// content hash of what rendered, else the render goes with its project alone, and the entry says so from then on), then
/// filed in this PC's archive (copied into the archive's staging folder, read back against
/// its SHA-256, and filed under the sequence's lease as the sync files a server's exports,
/// <see cref="SyncTransfers.FileRenderAsync"/>, after checking the project and version are the sequence's) or uploaded to its
/// server (<see cref="RenderUploadRequest"/> with <c>ProjectId</c> and <c>ProjectVersionId</c>, which the server checks the
/// same way), and the entry marked sent. The file stays in <c>exports/</c> until a clean-up removes it.
/// Idempotent by render id: a send repeated after a cut finds the home holding the render already and only marks it sent; a
/// render id the home files under another sequence is refused. The ledger is read and written under the project's lease,
/// each entry read again before it is written, so a render finished meanwhile is kept.
/// A send to a server is held while <see cref="WorkspaceJobOptions.HoldTransfers"/> says so (a metered connection): it waits
/// before each upload, and the desktop pauses it meanwhile. A send to this PC's archive is never held. A server from before
/// phase 5c files the render without its project and version: the job's log says so. A send queued before the workspace
/// moved works where the workspace is now, and none runs while it is being moved (<see cref="WorkspaceJobRoots"/>).
/// </summary>
internal sealed class WorkspaceSendJobHandler(ChronosDatabase db, ArchiveService archives, ISyncConnector? servers, WorkspaceJobOptions options) : IJobHandler
{
private static readonly TimeSpan LockWait = TimeSpan.FromMinutes(2);
/// <summary>How often a send held on a metered connection looks again.</summary>
internal static TimeSpan HoldPoll { get; set; } = TimeSpan.FromSeconds(2);
public JobKind Kind => JobKind.WorkspaceSend;
public async Task RunAsync(JobContext context)
{
ArgumentNullException.ThrowIfNull(context);
var ct = context.Cancellation;
var parameters = WorkspaceJobs.SendParameters(context.Job) ?? throw new InvalidDataException(Loc.L("Errors.Workspace.JobParameters"));
if (context.Job.TargetIds is not [var sequenceId] || string.IsNullOrWhiteSpace(parameters.WorkspaceRoot))
throw new InvalidDataException(Loc.L("Errors.Workspace.JobParameters"));
var server = parameters.Server is { Length: > 0 } named ? named : null;
// Where the project is now (a workspace moved since the job was queued), never while it is being moved; nothing is
// made where it is not.
var root = WorkspaceJobRoots.Resolve(parameters.WorkspaceRoot, parameters.ProjectId, options.CurrentRoot, "Errors.Workspace.NotCheckedOut");
var project = new WorkspaceStore(root).Find(parameters.ProjectId)
?? throw new InvalidOperationException(Loc.L("Errors.Workspace.NotCheckedOut"));
var workspace = project.ReadBundle().Workspace ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.NotCheckedOut"));
var home = workspace.Home;
if (home.SequenceId != sequenceId || (server is null
? !home.IsArchive
: home.Kind != BundleHome.ServerKind || !string.Equals(home.Server, server, StringComparison.OrdinalIgnoreCase)))
throw new InvalidOperationException(Loc.L("Errors.Workspace.OtherHome"));
var entries = await ToSendAsync(project, parameters.RenderIds, ct).ConfigureAwait(false);
await context.ReportAsync(0, entries.Count).ConfigureAwait(false);
if (entries.Count == 0)
{
context.Log(Loc.L("Archive.Log.WorkspaceSent", workspace.Name, 0, 0));
return;
}
if (server is null)
{
var sequence = await db.Sequences.GetAsync(sequenceId, ct).ConfigureAwait(false)
?? throw new InvalidOperationException(Loc.L("Archive.Log.SequenceMissing", sequenceId));
var archive = await db.Archives.GetAsync(sequence.ArchiveId, ct).ConfigureAwait(false)
?? throw new InvalidOperationException(Loc.L("Errors.Sync.NoArchive"));
var projectHome = new ArchiveProjectHome(db, archives.Projects, sequenceId);
await SendAsync(context, project, workspace, entries, () => projectHome, held: false,
(entry, file, token) => FileAsync(sequence, archive, entry, file, token)).ConfigureAwait(false);
return;
}
using var remote = (servers is null ? null : await servers.ConnectAsync(server, ct).ConfigureAwait(false))
?? throw new InvalidOperationException(Loc.L("Errors.Sync.NotConnected"));
await SendAsync(context, project, workspace, entries,
() => servers!.ProjectHome(home) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.NotConnected")), held: true,
(entry, file, token) => UploadAsync(context, remote, sequenceId, entry, file, token)).ConfigureAwait(false);
}
/// <summary>
/// The entries to send, oldest first: those of <paramref name="renderIds"/> the ledger holds unsent (an id it no longer
/// holds was discarded meanwhile, one sent already is left), every unsent one when none is named.
/// </summary>
private static async Task<IReadOnlyList<WorkspaceRenderEntry>> ToSendAsync(WorkspaceProject project, IReadOnlyList<Guid>? renderIds, CancellationToken ct)
{
WorkspaceRenders ledger;
using (await project.LockAsync(LockWait, ct).ConfigureAwait(false)) ledger = project.ReadRenders();
if (renderIds is not { Count: > 0 }) return ledger.Unsent;
var named = renderIds.ToHashSet();
return [.. ledger.Entries.Where(e => named.Contains(e.RenderId) && e.IsUnsent)];
}
/// <summary>Delivers one checked render home: true when the home took it now, false when it held it already.</summary>
private delegate Task<bool> Deliver(WorkspaceRenderEntry entry, string file, CancellationToken cancellationToken);
private async Task SendAsync(JobContext context, WorkspaceProject project, BundleWorkspace workspace, IReadOnlyList<WorkspaceRenderEntry> entries,
Func<IProjectHome> home, bool held, Deliver deliver)
{
var ct = context.Cancellation;
var (sent, there, refused) = (0, 0, 0);
// The versions this run found at the home, or sent there first, with the id the home holds each by (a replay gives another).
var versions = new Dictionary<Guid, Guid?>();
for (var i = 0; i < entries.Count; i++)
{
ct.ThrowIfCancellationRequested();
if (held) await WhileHeldAsync(context).ConfigureAwait(false);
var entry = entries[i];
var file = project.ContainedExportFile(entry.File);
var refusal = file is null ? Loc.L("Errors.Workspace.SendMissing", Path.GetFileName(entry.File ?? string.Empty)) : await CheckAsync(entry, file, folders: !held, ct).ConfigureAwait(false);
if (refusal is null)
{
if (entry.ProjectVersionId is { } versionId)
{
if (!versions.TryGetValue(versionId, out var atHome))
{
atHome = await EnsureVersionAsync(context, project, workspace, home(), entry, versionId, ct).ConfigureAwait(false);
versions[versionId] = atHome;
}
if (atHome != versionId)
{
// The ledger says from now on what the home holds: later sends agree with this one.
entry = entry with { ProjectVersionId = atHome };
var mapped = atHome;
await UpdateAsync(project, entry.RenderId, e => e with { ProjectVersionId = mapped }).ConfigureAwait(false);
}
}
try
{
if (await deliver(entry, file!, ct).ConfigureAwait(false)) sent++;
else there++;
// The home holds it: recorded whatever stops the job now.
await UpdateAsync(project, entry.RenderId, e => e.AsSent(DateTimeOffset.UtcNow)).ConfigureAwait(false);
}
catch (SendRefusedException ex)
{
refusal = ex.Message;
}
}
if (refusal is not null)
{
refused++;
context.Log(refusal);
await UpdateAsync(project, entry.RenderId, e => e.AsFailed(refusal)).ConfigureAwait(false);
}
await context.ReportAsync(i + 1, entries.Count).ConfigureAwait(false);
}
context.Log(Loc.L("Archive.Log.WorkspaceSent", workspace.Name, sent, there));
if (refused > 0) throw new InvalidOperationException(Loc.L("Errors.Workspace.SendFailed", refused, entries.Count));
}
/// <summary>
/// Why the render cannot be sent as it is (gone from the exports, another file than the one rendered, or an image
/// sequence's folder for a home that takes one file per render); null when it can. A folder (an image sequence, only
/// when <paramref name="folders"/>: this PC's archive files folders as its own renders do) is checked by its total size
/// and the hash of its listing, as the render recorded it (<see cref="WorkspaceRenderJobHandler.HashAsync"/>).
/// </summary>
/// <remarks>A ledger names files inside the exports only (<see cref="WorkspaceProject.ContainedExportFile"/>): one pointing elsewhere is never read.</remarks>
private static async Task<string?> CheckAsync(WorkspaceRenderEntry entry, string file, bool folders, CancellationToken ct)
{
var name = Path.GetFileName(entry.File);
if (Directory.Exists(file))
{
if (!folders) return Loc.L("Errors.Workspace.SendSequence", name);
return WorkspaceCleaner.Measure(file).Bytes != entry.Size
|| !string.Equals(await WorkspaceRenderJobHandler.HashAsync(file).ConfigureAwait(false), entry.Sha256, StringComparison.OrdinalIgnoreCase)
? Loc.L("Errors.Workspace.SendHashMismatch", name)
: null;
}
if (!File.Exists(file)) return Loc.L("Errors.Workspace.SendMissing", name);
if (new FileInfo(file).Length != entry.Size
|| !string.Equals(await ChronosClient.HashFileAsync(file, ct).ConfigureAwait(false), entry.Sha256, StringComparison.OrdinalIgnoreCase))
return Loc.L("Errors.Workspace.SendHashMismatch", name);
return null;
}
/// <summary>How many of the home's newest versions are read for one of the same content, when a replay gave a rendered version another id.</summary>
private const int ReplayedLookBack = 100;
/// <summary>
/// Version <paramref name="versionId"/> of the project at the home, as the id the home holds it by: pushed there first
/// (every local version, in order, as a sync pushes them) when the home lacks it. A push that replays it on a home that
/// moved on gives it another id (and one that changes nothing is not sent at all): the home's version of the content
/// hash that rendered (<see cref="WorkspaceRenderEntry.SettingsHash"/>) stands for it, else none (null: the render goes
/// with its project alone, as a render of a version the home never had). A push that fails fails the job.
/// </summary>
private static async Task<Guid?> EnsureVersionAsync(JobContext context, WorkspaceProject project, BundleWorkspace workspace, IProjectHome home,
WorkspaceRenderEntry entry, Guid versionId, CancellationToken ct)
{
var projectId = entry.ProjectId;
if (await home.GetVersionAsync(projectId, versionId, ct).ConfigureAwait(false) is not null) return versionId;
try
{
await new WorkspaceSync(project.Store, Environment.MachineName) { LockWait = LockWait }.PushAsync(projectId, home, ct).ConfigureAwait(false);
}
catch (Exception ex) when (ex is HomeUnavailableException or WorkspaceSyncException or ProjectException or LeaseUnavailableException
or InvalidOperationException)
{
context.Log(ex.Message);
throw new InvalidOperationException(Loc.L("Errors.Workspace.SendVersion"), ex);
}
if (await home.GetVersionAsync(projectId, versionId, ct).ConfigureAwait(false) is { } pushed)
{
context.Log(Loc.L("Archive.Log.WorkspaceVersionPushed", workspace.Name, pushed.Number));
return versionId;
}
ProjectVersion? same = null;
if (entry.SettingsHash is { Length: > 0 } hash)
{
var history = await home.HistoryAsync(projectId, null, ReplayedLookBack, ct).ConfigureAwait(false);
same = history.FirstOrDefault(v => string.Equals(v.ContentHash, hash, StringComparison.OrdinalIgnoreCase));
}
if (same is not null)
{
Log.Info("workspace", $"Version {versionId} of project {projectId} reached the home as version {same.Id} (a replay): render {entry.RenderId} names that one");
context.Log(Loc.L("Archive.Log.WorkspaceVersionReplayed", workspace.Name, same.Number));
return same.Id;
}
Log.Info("workspace", $"Version {versionId} of project {projectId} is not at the home after the push, nor one of the same content: render {entry.RenderId} goes with its project alone");
context.Log(Loc.L("Archive.Log.WorkspaceVersionNotHome", workspace.Name));
return null;
}
/// <summary>
/// The render filed in this PC's archive, the home's sequence: copied into the archive's staging folder, read back against
/// the ledger's SHA-256, then filed under the sequence's lease with the project and version (once both are checked to be
/// the sequence's). A render id the index holds already for this sequence is the same render, sent before.
/// </summary>
private async Task<bool> FileAsync(Sequence sequence, Archive archive, WorkspaceRenderEntry entry, string file, CancellationToken ct)
{
if (await db.Renders.GetAsync(entry.RenderId, ct).ConfigureAwait(false) is { } known)
return known.SequenceId == sequence.Id ? false : throw new SendRefusedException(Loc.L("Server.Error.RenderExists"));
if (!await ProjectOfAsync(sequence.Id, entry, ct).ConfigureAwait(false)) throw new SendRefusedException(Loc.L("Server.Error.ProjectMismatch"));
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
SyncTransfers.EnsureSpace(paths.Root, entry.Size);
Directory.CreateDirectory(paths.StagingDirectory);
var staged = Path.Combine(paths.StagingDirectory, "workspace-render-" + entry.RenderId.ToString("N") + ".part");
var folder = Directory.Exists(file);
try
{
TryDelete(staged);
if (folder)
{
// An image sequence: every file of its folder, the folder filed whole as the archive's own renders are.
foreach (var source in Directory.EnumerateFiles(file, "*", SearchOption.AllDirectories))
{
var target = Path.Combine(staged, Path.GetRelativePath(file, source));
Directory.CreateDirectory(Path.GetDirectoryName(target)!);
await CopyAsync(source, target, ct).ConfigureAwait(false);
}
Directory.CreateDirectory(staged);
}
else await CopyAsync(file, staged, ct).ConfigureAwait(false);
// Read back from the archive's volume: what is filed is the render as it was written.
var readBack = folder
? await WorkspaceRenderJobHandler.HashAsync(staged).ConfigureAwait(false)
: await ChronosClient.HashFileAsync(staged, ct).ConfigureAwait(false);
if (!string.Equals(readBack, entry.Sha256, StringComparison.OrdinalIgnoreCase))
throw new IOException(Loc.L("Errors.Workspace.CopyMismatch", staged));
var name = SyncTransfers.SafeFileName(Path.GetFileName(entry.File)) ?? entry.RenderId.ToString("N");
var filed = await SyncTransfers.FileRenderAsync(archives, db, sequence, paths, staged, name, new Render
{
Id = entry.RenderId,
SequenceId = sequence.Id,
PresetName = entry.PresetName,
OutputPath = string.Empty,
Codec = entry.Codec,
Width = entry.Width,
Height = entry.Height,
Fps = entry.Fps,
DurationSeconds = entry.DurationSeconds,
Created = entry.Rendered,
ProducedBy = options.Processing.ProducedBy,
ProjectId = entry.ProjectId,
ProjectVersionId = entry.ProjectVersionId,
}, ct).ConfigureAwait(false);
return filed is not null ? true : throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
}
finally
{
TryDelete(staged);
}
}
/// <summary>Copies <paramref name="source"/> to <paramref name="target"/>, flushed to the disk.</summary>
private static async Task CopyAsync(string source, string target, CancellationToken ct)
{
await using var from = new FileStream(source, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 20, FileOptions.Asynchronous | FileOptions.SequentialScan);
await using var to = new FileStream(target, FileMode.Create, FileAccess.Write, FileShare.None, 1 << 20, FileOptions.Asynchronous);
await from.CopyToAsync(to, ct).ConfigureAwait(false);
to.Flush(flushToDisk: true);
}
/// <summary>The project the entry names is one of the sequence's, and its version (when it names one) one of that project's.</summary>
private async Task<bool> ProjectOfAsync(Guid sequenceId, WorkspaceRenderEntry entry, CancellationToken ct)
{
var project = await db.Projects.GetAsync(entry.ProjectId, ct).ConfigureAwait(false);
var version = entry.ProjectVersionId is { } versionId ? await db.Projects.GetVersionAsync(versionId, ct).ConfigureAwait(false) : null;
if (project is null || (entry.ProjectVersionId is not null && version is null))
{
// One the index does not know yet may be in the sequence's sidecars: listing reads them first.
await archives.Projects.ListAsync(sequenceId, ct).ConfigureAwait(false);
project = await db.Projects.GetAsync(entry.ProjectId, ct).ConfigureAwait(false);
if (entry.ProjectVersionId is { } id) version = await db.Projects.GetVersionAsync(id, ct).ConfigureAwait(false);
}
return project is not null && project.SequenceId == sequenceId && (entry.ProjectVersionId is null || version?.ProjectId == project.Id);
}
/// <summary>
/// The render uploaded to its server's sequence, with its project and version, resumable and checked against its SHA-256
/// there; the server answers that it holds the render already when it does. Refused for good (the server takes the
/// project for another sequence's, the id for another sequence's render, the file for too large or not the one hashed):
/// marked failed; anything else (the server out of reach, busy, the sign-in expired) fails the job, the entry left as it was.
/// </summary>
private async Task<bool> UploadAsync(JobContext context, ISyncRemote remote, Guid sequenceId, WorkspaceRenderEntry entry, string file, CancellationToken ct)
{
var name = SyncTransfers.SafeFileName(Path.GetFileName(entry.File)) ?? entry.RenderId.ToString("N");
var request = new RenderUploadRequest(entry.RenderId, name, entry.Size, entry.Sha256.ToLowerInvariant(), entry.PresetName, entry.Codec, entry.Width, entry.Height,
entry.Fps, entry.DurationSeconds, options.Processing.ProducedBy, entry.ProjectId, entry.ProjectVersionId);
try
{
if (await remote.UploadRenderAsync(sequenceId, file, request, ct).ConfigureAwait(false) is not { } filed) return false;
// A server from before phase 5c ignores the project and version: the render is there, without them.
if (filed.ProjectId != entry.ProjectId || filed.ProjectVersionId != entry.ProjectVersionId)
{
Log.Warning("workspace", $"The server filed render {entry.RenderId} without its project {entry.ProjectId} and version {entry.ProjectVersionId} (a server before phase 5c)");
context.Log(Loc.L("Archive.Log.WorkspaceSendNoProvenance", name));
}
return true;
}
catch (ChronosApiException ex) when (ex.Status is HttpStatusCode.BadRequest or HttpStatusCode.Conflict or HttpStatusCode.RequestEntityTooLarge
or HttpStatusCode.UnprocessableEntity)
{
throw new SendRefusedException(string.IsNullOrWhiteSpace(ex.Error.Message) ? ex.Message : ex.Error.Message);
}
}
/// <summary>Waits while transfers with a server are held (a metered connection), saying so once.</summary>
private async Task WhileHeldAsync(JobContext context)
{
if (!options.HoldTransfers()) return;
context.Log(Loc.L("Archive.Log.WorkspaceSendHeld"));
while (options.HoldTransfers()) await Task.Delay(HoldPoll, context.Cancellation).ConfigureAwait(false);
}
/// <summary>
/// The ledger entry of <paramref name="renderId"/> changed by <paramref name="change"/>, under the project's lease, read again
/// first (a render finished meanwhile stays); an entry discarded meanwhile stays discarded. Written whatever stops the job.
/// </summary>
private static async Task UpdateAsync(WorkspaceProject project, Guid renderId, Func<WorkspaceRenderEntry, WorkspaceRenderEntry> change)
{
using (await project.LockAsync(LockWait, CancellationToken.None).ConfigureAwait(false))
{
var ledger = project.ReadRenders();
if (ledger.Find(renderId) is { } entry) project.WriteRenders(ledger.With(change(entry)));
}
}
private static void TryDelete(string path)
{
try
{
if (Directory.Exists(path)) Directory.Delete(path, recursive: true);
else if (File.Exists(path)) File.Delete(path);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Debug("workspace", $"{path} could not be removed: {ex.Message}");
}
}
/// <summary>The home will not take this render as it is: it is marked failed with this message, and the others go on.</summary>
private sealed class SendRefusedException(string message) : Exception(message);
}
+100 -5
View File
@@ -91,17 +91,48 @@ public sealed record WorkspaceConflict(Guid TheirsVersionId, int TheirsNumber, G
/// <summary>
/// Parameters of a <see cref="JobKind.WorkspaceCopy"/> job: target = the home's sequence; the workspace folder, the
/// project whose media to copy and what to copy (<see cref="BundleMedia.Proxies"/>; originals in phase 5c).
/// project whose media to copy and what to copy (<see cref="BundleMedia.Proxies"/>, or <see cref="BundleMedia.Originals"/>:
/// the proxies plus the originals, phase 5c).
/// </summary>
/// <param name="Server">The server the home's sequence is on (<see cref="SyncService.ServerKey"/>, phase 5b); null for a sequence of this PC's archive.</param>
public sealed record WorkspaceCopyParameters(string WorkspaceRoot, Guid ProjectId, string Media, string? Server = null);
/// <summary>
/// Parameters of a <see cref="JobKind.WorkspaceRender"/> job (phase 5c): target = the home's sequence; the workspace folder,
/// the project rendered from the originals its media hold, the version rendered (null: the head version the workspace
/// holds), and the render's export settings as Protocol <c>RenderJobParameters</c> JSON (portable settings, preset,
/// overrides, batch), as <see cref="Processing.ProcessingJobs"/> writes them for a render.
/// </summary>
/// <param name="ProjectVersionId">The version rendered and recorded with each output; null renders the head version the workspace holds.</param>
/// <param name="RenderJson">Protocol <c>RenderJobParameters</c> as JSON.</param>
/// <param name="Server">The server the home's sequence is on (<see cref="SyncService.ServerKey"/>); null for a sequence of this PC's archive.</param>
public sealed record WorkspaceRenderParameters(string WorkspaceRoot, Guid ProjectId, Guid? ProjectVersionId, string RenderJson, string? Server = null);
/// <summary>
/// Parameters of a <see cref="JobKind.WorkspaceSend"/> job (phase 5c): target = the home's sequence; the workspace folder,
/// the project, and the ledger entries (<see cref="WorkspaceRenderEntry.RenderId"/>) whose files go home.
/// </summary>
/// <param name="RenderIds">The renders sent; empty sends every pending entry of the project's ledger.</param>
/// <param name="Server">The server the home's sequence is on (<see cref="SyncService.ServerKey"/>); null for a sequence of this PC's archive.</param>
public sealed record WorkspaceSendParameters(string WorkspaceRoot, Guid ProjectId, IReadOnlyList<Guid> RenderIds, string? Server = null);
/// <summary>
/// Parameters of a <see cref="JobKind.WorkspaceMove"/> job (phase 5c): no target; the workspace folder moved
/// (<paramref name="From"/>), the folder it goes to, and the fingerprint of the plan the user confirmed (the job refuses a
/// workspace that changed since).
/// </summary>
public sealed record WorkspaceMoveParameters(string From, string To, string PlanFingerprint);
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
UseStringEnumConverter = true, WriteIndented = true)]
[JsonSerializable(typeof(WorkspaceManifest))]
[JsonSerializable(typeof(WorkspaceMedia))]
[JsonSerializable(typeof(WorkspaceSyncState))]
[JsonSerializable(typeof(WorkspaceCopyParameters))]
[JsonSerializable(typeof(WorkspaceRenderParameters))]
[JsonSerializable(typeof(WorkspaceSendParameters))]
[JsonSerializable(typeof(WorkspaceMoveParameters))]
[JsonSerializable(typeof(WorkspaceRenders))]
internal sealed partial class WorkspaceJson : JsonSerializerContext;
/// <summary>
@@ -309,9 +340,11 @@ public sealed class WorkspaceStore
/// <summary>
/// One project in the workspace (owner review 2, R8): <c>&lt;root&gt;/&lt;project id&gt;/</c> with
/// <c>project.chronos</c> (the working state: a project file of format 2, its settings placed on the media's frames),
/// <c>media/media.json</c> with <c>media/proxies/&lt;frame id&gt;.jpg</c> (originals later) and <c>media/analysis.json</c>
/// (the home's deep report), <c>versions/&lt;number&gt;-&lt;id&gt;.json</c> (commits not pushed yet, and the versions pulled),
/// <c>sync.json</c>, <c>exports/</c> and the lease <c>.lock.lease</c> that one writer at a time holds.
/// <c>media/media.json</c> with <c>media/proxies/&lt;frame id&gt;.jpg</c>, <c>media/originals/&lt;frame id&gt;-&lt;sha8&gt;&lt;ext&gt;</c>
/// (media of kind originals, phase 5c) and <c>media/analysis.json</c> (the home's deep report),
/// <c>versions/&lt;number&gt;-&lt;id&gt;.json</c> (commits not pushed yet, and the versions pulled), <c>sync.json</c>,
/// <c>exports/</c> with the local renders and their ledger <c>exports/renders.json</c> (phase 5c), and the lease
/// <c>.lock.lease</c> that one writer at a time holds.
/// </summary>
public sealed class WorkspaceProject
{
@@ -324,6 +357,12 @@ public sealed class WorkspaceProject
public const string ExportsFolder = "exports";
public const string SyncFile = "sync.json";
/// <summary>The name of the ledger of the project's local renders (<see cref="WorkspaceRenders"/>) in <see cref="ExportsFolder"/>.</summary>
public const string RendersFileName = "renders.json";
/// <summary>The ledger of the project's local renders (phase 5c), relative to <see cref="Folder"/>.</summary>
public const string RendersFile = ExportsFolder + "/" + RendersFileName;
/// <summary>
/// Old local commits replaced by their replay on top of the home's head, files the home's copy replaced, and the working
/// states a take of the server's values replaced: kept up to a month, the newest 20 at most (<see cref="PruneReplaced"/>).
@@ -365,6 +404,9 @@ public sealed class WorkspaceProject
public string ExportsDirectory => Path.Combine(Folder, ExportsFolder);
public string SyncPath => Path.Combine(Folder, SyncFile);
/// <summary>The ledger of the project's local renders (<see cref="RendersFile"/>, phase 5c).</summary>
public string RendersPath => Path.Combine(ExportsDirectory, RendersFileName);
public bool Exists => File.Exists(BundlePath);
/// <summary>The file a frame's proxy is copied to, relative to <see cref="MediaDirectory"/>.</summary>
@@ -380,9 +422,62 @@ public sealed class WorkspaceProject
return ProxiesFolder + "/" + frameId.ToString("N") + "-" + sha256[..Math.Min(16, sha256.Length)].ToLowerInvariant() + ".jpg";
}
/// <summary>
/// The file a frame's original is copied to, relative to <see cref="MediaDirectory"/> (phase 5c):
/// <c>originals/&lt;frame id&gt;-&lt;sha8&gt;&lt;ext&gt;</c>. The first 8 hex digits of its SHA-256 tell two encodings of one
/// frame apart (a conversion since: the editor keeps the one it reads until the copy commits), and the original's own
/// extension is kept, lower case, since decoders go by it.
/// </summary>
/// <param name="extension">The original's extension, with or without its dot (".CR3", "dng", or the file name's <see cref="Path.GetExtension(string)"/>); empty or null for none.</param>
public static string OriginalName(Guid frameId, string sha256, string? extension)
{
ArgumentException.ThrowIfNullOrEmpty(sha256);
var ext = (extension ?? string.Empty).Trim().TrimStart('.').ToLowerInvariant();
if (ext.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0 || ext.Contains('/') || ext.Contains('\\')) ext = string.Empty;
return OriginalsFolder + "/" + frameId.ToString("N") + "-" + sha256[..Math.Min(8, sha256.Length)].ToLowerInvariant() + (ext.Length > 0 ? "." + ext : string.Empty);
}
/// <summary>The ledger of the project's local renders; empty while it has none (<see cref="WorkspaceRenders.Read"/>).</summary>
/// <exception cref="InvalidDataException">The ledger is damaged or newer.</exception>
/// <exception cref="IOException">The ledger cannot be read.</exception>
public WorkspaceRenders ReadRenders() => WorkspaceRenders.Read(RendersPath);
/// <summary>Writes the ledger of the project's local renders whole and atomically (<see cref="WorkspaceRenders.Write"/>); the caller holds the project's lease.</summary>
public void WriteRenders(WorkspaceRenders renders) => WorkspaceRenders.Write(RendersPath, renders);
/// <summary>The absolute path of a file named relative to <see cref="MediaDirectory"/>.</summary>
public string MediaFile(string relative) => LongPath.FromPortableRelative(MediaDirectory, relative);
/// <summary>The absolute path of a local render named relative to <see cref="ExportsDirectory"/> (<see cref="WorkspaceRenderEntry.File"/>).</summary>
public string ExportFile(string relative) => LongPath.FromPortableRelative(ExportsDirectory, relative);
/// <summary>
/// The absolute path of a local render named relative to <see cref="ExportsDirectory"/>, only when it lies strictly inside
/// that folder and is not its ledger; null for a name that is empty, rooted, climbs out (<c>..</c>), names the folder
/// itself (<c>.</c>) or the ledger. What reads, sends or deletes a ledger's file goes through this, so a damaged or
/// hand-edited ledger never points it at anything else.
/// </summary>
public string? ContainedExportFile(string? relative)
{
if (string.IsNullOrWhiteSpace(relative)) return null;
var portable = relative.Replace('\\', '/');
if (Path.IsPathRooted(relative) || portable.StartsWith('/') || portable.Contains(':', StringComparison.Ordinal)
|| portable.Split('/').Any(part => part is "" or "." or "..")) return null;
string full, folder;
try
{
full = Path.GetFullPath(ExportFile(relative));
folder = Path.TrimEndingDirectorySeparator(Path.GetFullPath(ExportsDirectory)) + Path.DirectorySeparatorChar;
}
catch (Exception ex) when (ex is ArgumentException or NotSupportedException or PathTooLongException)
{
return null;
}
var comparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
if (!full.StartsWith(folder, comparison) || full.Length <= folder.Length) return null;
return string.Equals(full, Path.GetFullPath(RendersPath), comparison) ? null : full;
}
/// <summary>Holds the project for one writer (this program's sync, a copy job) until disposed.</summary>
/// <exception cref="LeaseUnavailableException">Another writer kept it for longer than <paramref name="wait"/>.</exception>
public async Task<IDisposable> LockAsync(TimeSpan wait, CancellationToken cancellationToken = default)
@@ -441,7 +536,7 @@ public sealed class WorkspaceProject
internal void WriteMedia(WorkspaceMedia media)
{
Directory.CreateDirectory(MediaDirectory);
AtomicFile.WriteAllText(MediaIndexPath, JsonSerializer.Serialize(media, WorkspaceJson.Default.WorkspaceMedia));
AtomicFile.WriteAllText(MediaIndexPath, JsonSerializer.Serialize(media with { FormatVersion = media.WrittenFormatVersion }, WorkspaceJson.Default.WorkspaceMedia));
}
/// <summary>
+4 -2
View File
@@ -1200,7 +1200,7 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName)
// A local commit the home stored although its answer was lost reads as behind by the commits it holds: they are ours.
if (head is not null && locals.Any(v => v.Id == head.Id)) behind = 0;
var state = gone ? WorkspaceState.Gone
: media is null ? WorkspaceState.MediaRemoved
: media is null || media.IsCleaned ? WorkspaceState.MediaRemoved
: damaged > 0 ? WorkspaceState.Damaged
: conflict is not null ? WorkspaceState.Conflict
: offline ? WorkspaceState.Offline
@@ -1210,7 +1210,9 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName)
: WorkspaceState.InSync;
// A project the home no longer has holds no conflict to answer (nothing can be merged with it): the one recorded stays
// in sync.json, for the record and for a project the home holds again.
return new WorkspaceStatus(state, modified, locals.Count, behind, offline, media is null, bundle.IsReadOnly, gone ? null : conflict, workspace.Base, head)
// Media the clean-up emptied hold no file to edit: removed, as media never copied, until a copy brings them again.
return new WorkspaceStatus(state, modified, locals.Count, behind, offline, media is null || media.IsCleaned, bundle.IsReadOnly, gone ? null : conflict,
workspace.Base, head)
{
MediaOutdated = outdated,
Damaged = damaged,
@@ -0,0 +1,259 @@
using System.Collections.Concurrent;
using System.Globalization;
using System.Security.Cryptography;
using System.Text;
using Chronos.Core.Diagnostics;
namespace Chronos.Data.Workspaces;
/// <summary>One entry of a workspace's trash: its name in the trash folder, its bytes and files, and when it was put there.</summary>
public sealed record WorkspaceTrashEntry(string Name, long Bytes, int Files, DateTimeOffset Trashed);
/// <summary>
/// What a workspace's trash holds now (<see cref="WorkspaceTrash.List"/>): its entries, oldest first, their bytes, and a
/// fingerprint over the name, size and time of every file in them, which an emptying checks first.
/// </summary>
public sealed record WorkspaceTrashListing(string Folder, IReadOnlyList<WorkspaceTrashEntry> Entries, long Bytes, string Fingerprint)
{
/// <summary>The trash holds nothing.</summary>
public bool IsEmpty => Entries.Count == 0;
}
/// <summary>The workspace's trash changed after it was listed: nothing was erased.</summary>
public sealed class WorkspaceTrashChangedException(string message) : IOException(message);
/// <summary>
/// The trash of a workspace (<c>&lt;workspace&gt;/.trash</c>, phase 5c): what a removed local copy kept of its work
/// (<c>&lt;project id&gt;-&lt;ticks&gt;</c>), and the originals a clean-up found the home no longer holds
/// (<c>&lt;project id&gt;-&lt;ticks&gt;/originals</c>). An entry is filled under a staging name the listing leaves out
/// (<c>.incoming-&lt;project id&gt;-&lt;ticks&gt;</c>, <see cref="Begin"/>) and takes its own name in one rename once it is
/// whole, so nothing writes into an entry a plan lists; one a crash left half filled lists as an entry of its own. Only the
/// user's Empty… erases them: the plan lists them (<see cref="List"/>), and <see cref="Empty"/> erases exactly those, only
/// while the fingerprint of the plan still matches. Schedules and the size limit never empty it.
/// </summary>
public static class WorkspaceTrash
{
/// <summary>The trash's folder name in the workspace (the store lists only folders named by a project id).</summary>
public const string FolderName = ".trash";
/// <summary>The name an entry has while it is filled (<see cref="Begin"/>).</summary>
internal const string IncomingPrefix = ".incoming-";
/// <summary>The staging folders being filled now in this program (by name: a project id and ticks): the listing leaves them out.</summary>
private static readonly ConcurrentDictionary<string, byte> Filling = new(StringComparer.OrdinalIgnoreCase);
/// <summary>The trash folder of the workspace at <paramref name="workspaceRoot"/>.</summary>
public static string FolderOf(string workspaceRoot)
{
ArgumentException.ThrowIfNullOrWhiteSpace(workspaceRoot);
return Path.Combine(workspaceRoot, FolderName);
}
/// <summary>
/// The path of a new entry of the trash for <paramref name="projectId"/> (<c>&lt;id&gt;-&lt;ticks&gt;</c>, the ticks of
/// <paramref name="nowUtc"/> moved on while an entry of that name exists); not made.
/// </summary>
public static string NewEntry(string workspaceRoot, Guid projectId, DateTime nowUtc)
{
var folder = FolderOf(workspaceRoot);
for (var ticks = nowUtc.Ticks; ; ticks++)
{
var path = Path.Combine(folder, projectId.ToString("N") + "-" + ticks.ToString(CultureInfo.InvariantCulture));
if (!Directory.Exists(path) && !File.Exists(path)) return path;
}
}
/// <summary>
/// A new entry of the trash for <paramref name="projectId"/>, filled under a staging name the listing leaves out; it takes
/// its own name (<see cref="NewEntry"/>) when the builder is disposed, or goes when nothing came into it. The staging
/// folder is made when the builder is.
/// </summary>
public static WorkspaceTrashBuilder Begin(string workspaceRoot, Guid projectId, DateTime nowUtc)
{
var folder = FolderOf(workspaceRoot);
for (var ticks = nowUtc.Ticks; ; ticks++)
{
var path = Path.Combine(folder, IncomingPrefix + projectId.ToString("N") + "-" + ticks.ToString(CultureInfo.InvariantCulture));
// Named as being filled before it is made: a listing never sees it half made.
if (Directory.Exists(path) || File.Exists(path) || !Filling.TryAdd(Path.GetFileName(path), 0)) continue;
try
{
Directory.CreateDirectory(path);
}
catch
{
Filling.TryRemove(Path.GetFileName(path), out _);
throw;
}
return new WorkspaceTrashBuilder(workspaceRoot, projectId, path);
}
}
/// <summary>The staging folder <paramref name="staging"/> made an entry of its own (its own name), or removed when empty; no longer left out of the listing.</summary>
internal static string? Finish(string workspaceRoot, Guid projectId, string staging)
{
try
{
if (!Directory.Exists(staging)) return null;
if (!Directory.EnumerateFileSystemEntries(staging).Any())
{
Directory.Delete(staging);
return null;
}
var entry = NewEntry(workspaceRoot, projectId, DateTime.UtcNow);
Directory.Move(staging, entry);
return entry;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// Listed from now on under its staging name: what it holds stays in the trash, and Empty… can erase it.
Log.Warning("workspace", $"The trash entry {staging} could not take its own name; it stays as it is", ex);
return Directory.Exists(staging) ? staging : null;
}
finally
{
Filling.TryRemove(Path.GetFileName(staging), out _);
}
}
/// <summary>The entries of the workspace's trash at <paramref name="workspaceRoot"/>, with their fingerprint; empty when there is no trash.</summary>
/// <exception cref="IOException">The trash cannot be read.</exception>
public static WorkspaceTrashListing List(string workspaceRoot, CancellationToken cancellationToken = default)
{
var folder = FolderOf(workspaceRoot);
var entries = new List<WorkspaceTrashEntry>();
var lines = new List<string>();
if (Directory.Exists(folder))
{
foreach (var entry in new DirectoryInfo(folder).EnumerateFileSystemInfos())
{
cancellationToken.ThrowIfCancellationRequested();
// An entry being filled now is no entry yet: a plan never lists it, so Empty… never erases it.
if (entry is DirectoryInfo && Filling.ContainsKey(entry.Name)) continue;
var (files, bytes) = (0, 0L);
if (entry is DirectoryInfo directory)
{
// A folder counts by its name only: NTFS updates a folder's time in its parent's index lazily, so the
// enumeration may report a stale one; its files (read from their own records) cover what it holds.
lines.Add(entry.Name + "/");
foreach (var file in directory.EnumerateFiles("*", SearchOption.AllDirectories))
{
file.Refresh();
files++;
bytes += file.Length;
lines.Add(Line(Path.GetRelativePath(folder, file.FullName).Replace('\\', '/'), file.Length, file.LastWriteTimeUtc));
}
}
else if (entry is FileInfo single)
{
single.Refresh();
(files, bytes) = (1, single.Length);
lines.Add(Line(entry.Name, single.Length, single.LastWriteTimeUtc));
}
entries.Add(new WorkspaceTrashEntry(entry.Name, bytes, files, TrashedAt(entry)));
}
}
lines.Sort(StringComparer.Ordinal);
var fingerprint = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(string.Join('\n', lines))));
IReadOnlyList<WorkspaceTrashEntry> ordered = [.. entries.OrderBy(e => e.Trashed).ThenBy(e => e.Name, StringComparer.Ordinal)];
return new WorkspaceTrashListing(folder, ordered, ordered.Sum(e => e.Bytes), fingerprint);
static string Line(string name, long size, DateTime written) =>
string.Create(CultureInfo.InvariantCulture, $"{name}|{size}|{written.Ticks}");
}
/// <summary>
/// Erases the entries of <paramref name="plan"/> from the workspace's trash for good, the user having confirmed it; refused,
/// erasing nothing, when the trash is not as the plan listed it (an entry added, removed or changed since).
/// </summary>
/// <returns>The entries and bytes erased.</returns>
/// <exception cref="WorkspaceTrashChangedException">The trash changed since the plan; nothing was erased.</exception>
/// <exception cref="IOException">An entry could not be erased (a file held open); the others were, and it stays.</exception>
public static (int Entries, long Bytes) Empty(string workspaceRoot, WorkspaceTrashListing plan, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(plan);
var now = List(workspaceRoot, cancellationToken);
if (!string.Equals(Path.GetFullPath(now.Folder), Path.GetFullPath(plan.Folder), StringComparison.OrdinalIgnoreCase)
|| !string.Equals(now.Fingerprint, plan.Fingerprint, StringComparison.Ordinal))
throw new WorkspaceTrashChangedException($"The trash {now.Folder} changed after it was listed.");
var (erased, bytes) = (0, 0L);
var failed = new List<string>();
foreach (var entry in plan.Entries)
{
cancellationToken.ThrowIfCancellationRequested();
var path = Path.Combine(now.Folder, entry.Name);
try
{
Erase(path);
erased++;
bytes += entry.Bytes;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("workspace", $"The trash entry {path} could not be erased", ex);
failed.Add(entry.Name);
}
}
Log.Info("workspace", $"The workspace's trash {now.Folder} emptied: {erased} entr(ies), {bytes} bytes erased");
if (failed.Count > 0) throw new IOException($"Trash entries not erased: {string.Join(", ", failed)}.");
return (erased, bytes);
}
/// <summary>When an entry was put in the trash: the ticks of its name (<c>&lt;id&gt;-&lt;ticks&gt;</c>), else when it was written last.</summary>
private static DateTimeOffset TrashedAt(FileSystemInfo entry)
{
var dash = entry.Name.LastIndexOf('-');
if (dash > 0 && long.TryParse(entry.Name.AsSpan(dash + 1), NumberStyles.None, CultureInfo.InvariantCulture, out var ticks)
&& ticks >= DateTime.MinValue.Ticks && ticks <= DateTime.MaxValue.Ticks)
return new DateTimeOffset(ticks, TimeSpan.Zero);
return new DateTimeOffset(entry.LastWriteTimeUtc, TimeSpan.Zero);
}
/// <summary>A trash entry erased, files marked read-only included.</summary>
private static void Erase(string path)
{
if (File.Exists(path))
{
File.SetAttributes(path, FileAttributes.Normal);
File.Delete(path);
return;
}
if (!Directory.Exists(path)) return;
foreach (var file in new DirectoryInfo(path).EnumerateFiles("*", SearchOption.AllDirectories))
{
if ((file.Attributes & FileAttributes.ReadOnly) != 0) file.Attributes &= ~FileAttributes.ReadOnly;
}
Directory.Delete(path, recursive: true);
}
}
/// <summary>
/// A trash entry being filled (<see cref="WorkspaceTrash.Begin"/>): <see cref="Path"/> is its staging folder, left out of every
/// listing; disposing it gives the entry its own name (<see cref="Entry"/>), or removes it when nothing came into it.
/// </summary>
public sealed class WorkspaceTrashBuilder : IDisposable
{
private readonly string _root;
private readonly Guid _projectId;
private int _done;
internal WorkspaceTrashBuilder(string root, Guid projectId, string path)
{
(_root, _projectId, Path) = (root, projectId, path);
}
/// <summary>The staging folder to fill.</summary>
public string Path { get; }
/// <summary>The entry once finished; null before, or when nothing came into it.</summary>
public string? Entry { get; private set; }
/// <summary>Gives the entry its own name (or removes it when empty); later calls do nothing.</summary>
public string? Finish()
{
if (Interlocked.Exchange(ref _done, 1) == 0) Entry = WorkspaceTrash.Finish(_root, _projectId, Path);
return Entry;
}
public void Dispose() => Finish();
}