feat: owner review 2 phase 5c: originals, local render, clean-up and the workspace settings (#18)
CI / Build and test (push) Successful in 9m26s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 8s
Release / Build, test and publish (push) Successful in 14m31s
CI / PostgreSQL 16 integration (push) Successful in 9m53s
CI / PostgreSQL 17 integration (push) Successful in 9m22s
CI / Docker image (push) Successful in 44s
Release / Server image (push) Successful in 1m36s
CI / Build and test (push) Successful in 9m26s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 8s
Release / Build, test and publish (push) Successful in 14m31s
CI / PostgreSQL 16 integration (push) Successful in 9m53s
CI / PostgreSQL 17 integration (push) Successful in 9m22s
CI / Docker image (push) Successful in 44s
Release / Server image (push) Successful in 1m36s
Originals in the workspace, local render and send home with provenance, the workspace clean-up, size limit and trash, moving the workspace folder, and the complete Settings › Workspace page (owner review 2, R8, phase 5c). One adversarial review round (35 findings, 32 fixed), a copy-cancel race fixed, 184 strings in all 38 languages. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
This commit was merged in pull request #18.
This commit is contained in:
1 parent
81c303f484
commit
db389a54f2
364 files changed
+17417
-595
No files matched your search
@@ -0,0 +1,470 @@
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.Diagnostics;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Core.Project;
|
||||
using Chronos.Data.Projects;
|
||||
|
||||
namespace Chronos.Data.Workspaces;
|
||||
|
||||
/// <summary>Why the workspace's clean-up kept the media of a project it would have cleaned (each reason is logged with the project).</summary>
|
||||
public enum WorkspaceKeep
|
||||
{
|
||||
/// <summary>The project is open in Edit.</summary>
|
||||
Open,
|
||||
|
||||
/// <summary>The project is not in step with its home: changes not saved, versions not sent or not taken, a conflict, a damaged version.</summary>
|
||||
NotInSync,
|
||||
|
||||
/// <summary>The project's home cannot be asked now: whether it holds everything is not known.</summary>
|
||||
Offline,
|
||||
|
||||
/// <summary>A copy, render or send of the project, or a move of the workspace, is queued, running or paused.</summary>
|
||||
Jobs,
|
||||
|
||||
/// <summary>The project's ledger holds renders made here that its home does not have yet.</summary>
|
||||
UnsentRenders,
|
||||
|
||||
/// <summary>The project's working state, media index or render ledger cannot be read.</summary>
|
||||
Unreadable,
|
||||
|
||||
/// <summary>The project holds no media files to remove.</summary>
|
||||
NoMedia,
|
||||
|
||||
/// <summary>The project's media were copied just now (the size limit after a copy never takes that one).</summary>
|
||||
JustCopied,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// What a run of the workspace's clean-up applies (phase 5c, Settings › Workspace): Clean after (projects not opened for that
|
||||
/// many days) and Size limit (the projects' folders above it, the least recently used first); 0 turns either off.
|
||||
/// </summary>
|
||||
public sealed record WorkspaceCleanOptions
|
||||
{
|
||||
/// <summary>Days a project may go unopened before its media are removed; 0 never removes them for their age.</summary>
|
||||
public int CleanAfterDays { get; init; }
|
||||
|
||||
/// <summary>Bytes the projects' folders may take (the trash is not counted: only the user empties it); 0 sets no limit.</summary>
|
||||
public long SizeLimitBytes { get; init; }
|
||||
|
||||
/// <summary>When each project was opened last (Settings' <c>WorkspaceOpened</c>); a project not listed counts from when its working state was saved.</summary>
|
||||
public IReadOnlyDictionary<Guid, DateTimeOffset> Opened { get; init; } = new Dictionary<Guid, DateTimeOffset>();
|
||||
|
||||
/// <summary>Whether a project is open in Edit now (asked again under the project's lease, just before anything goes).</summary>
|
||||
public Func<Guid, bool> IsOpen { get; init; } = _ => false;
|
||||
|
||||
/// <summary>The project whose media a copy just brought (the size limit after a copy): never cleaned by this run.</summary>
|
||||
public Guid? JustCopied { get; init; }
|
||||
|
||||
/// <summary>The time the ages count from.</summary>
|
||||
public DateTimeOffset Now { get; init; } = DateTimeOffset.UtcNow;
|
||||
}
|
||||
|
||||
/// <summary>A project whose media the clean-up removed: the bytes freed and the originals set aside in the workspace's trash.</summary>
|
||||
public sealed record WorkspaceCleanedProject(Guid ProjectId, string Name, bool IsMain, string SequenceTitle, long Freed, int Trashed);
|
||||
|
||||
/// <summary>A project the clean-up would have cleaned and kept, with why.</summary>
|
||||
public sealed record WorkspaceKeptProject(Guid ProjectId, string Name, bool IsMain, string SequenceTitle, WorkspaceKeep Reason);
|
||||
|
||||
/// <summary>What a run of the workspace's clean-up did: the projects cleaned, those kept with why, and the projects' bytes before and after.</summary>
|
||||
public sealed record WorkspaceCleanReport(IReadOnlyList<WorkspaceCleanedProject> Cleaned, IReadOnlyList<WorkspaceKeptProject> Kept, long UsedBefore, long UsedAfter)
|
||||
{
|
||||
/// <summary>A run over a workspace with nothing in it.</summary>
|
||||
public static WorkspaceCleanReport Nothing { get; } = new([], [], 0, 0);
|
||||
|
||||
/// <summary>The bytes the run freed.</summary>
|
||||
public long Freed => Cleaned.Sum(c => c.Freed);
|
||||
|
||||
/// <summary>The originals the run set aside in the workspace's trash (their home no longer holds them).</summary>
|
||||
public int Trashed => Cleaned.Sum(c => c.Trashed);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The workspace's clean-up (phase 5c, P4): removes the media of projects whose home holds everything they hold, so they are
|
||||
/// copied again when next opened. A project is cleaned only when it is in step with its home, asked now (never while the
|
||||
/// home is out of reach), not open in Edit, with no copy, render or send of it queued or running (nor a move of the
|
||||
/// workspace), and no local render waiting to be sent. Cleaning removes <c>media/proxies</c> and <c>media/originals</c>
|
||||
/// (the media index stays, its frames naming no file: <see cref="WorkspaceMedia.Cleaned"/>) and the renders of
|
||||
/// <c>exports/</c> the home took; never the working state, the versions, the sync state, the deep report or a render not
|
||||
/// sent. An original is deleted only when the home still holds the same frame with the same SHA-256; otherwise it is moved
|
||||
/// to the workspace's trash (<c>.trash/<id>-<ticks>/originals</c>). The trash itself is never emptied here.
|
||||
/// </summary>
|
||||
public sealed class WorkspaceCleaner
|
||||
{
|
||||
private static readonly JobKind[] ProjectJobs = [JobKind.WorkspaceCopy, JobKind.WorkspaceRender, JobKind.WorkspaceSend];
|
||||
|
||||
private readonly WorkspaceSync _sync;
|
||||
private readonly ChronosDatabase _db;
|
||||
private readonly Func<BundleHome, IProjectHome?> _homes;
|
||||
|
||||
/// <param name="homes">The home of a workspace project; null for one this PC cannot reach now.</param>
|
||||
public WorkspaceCleaner(WorkspaceSync sync, ChronosDatabase db, Func<BundleHome, IProjectHome?> homes)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(sync);
|
||||
ArgumentNullException.ThrowIfNull(db);
|
||||
ArgumentNullException.ThrowIfNull(homes);
|
||||
_sync = sync;
|
||||
_db = db;
|
||||
_homes = homes;
|
||||
}
|
||||
|
||||
/// <summary>How long a project another writer holds is waited for; past it the project is kept (<see cref="WorkspaceKeep.Jobs"/>).</summary>
|
||||
public TimeSpan LockWait { get; init; } = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>
|
||||
/// One run: the projects not opened for <see cref="WorkspaceCleanOptions.CleanAfterDays"/>, oldest first, then, while the
|
||||
/// projects' folders take more than <see cref="WorkspaceCleanOptions.SizeLimitBytes"/>, the least recently used others.
|
||||
/// Each project the policies name is cleaned or kept with its reason.
|
||||
/// </summary>
|
||||
public async Task<WorkspaceCleanReport> CleanAsync(WorkspaceCleanOptions options, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
var candidates = new List<Candidate>();
|
||||
var used = 0L;
|
||||
foreach (var project in _sync.Store.List())
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var candidate = Candidate.Of(project, options);
|
||||
used += candidate.FolderBytes;
|
||||
candidates.Add(candidate);
|
||||
}
|
||||
var before = used;
|
||||
var cleaned = new List<WorkspaceCleanedProject>();
|
||||
var kept = new List<WorkspaceKeptProject>();
|
||||
var named = new HashSet<Guid>();
|
||||
var moving = await _db.Jobs.HasPendingAsync([JobKind.WorkspaceMove], null, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
async Task CleanOneAsync(Candidate candidate)
|
||||
{
|
||||
named.Add(candidate.Project.ProjectId);
|
||||
var (reason, freed, trashed) = await TryCleanAsync(candidate, options, moving, cancellationToken).ConfigureAwait(false);
|
||||
if (reason is { } why)
|
||||
{
|
||||
kept.Add(new WorkspaceKeptProject(candidate.Project.ProjectId, candidate.Name, candidate.IsMain, candidate.Title, why));
|
||||
return;
|
||||
}
|
||||
cleaned.Add(new WorkspaceCleanedProject(candidate.Project.ProjectId, candidate.Name, candidate.IsMain, candidate.Title, freed, trashed));
|
||||
used -= freed;
|
||||
}
|
||||
|
||||
if (options.CleanAfterDays > 0)
|
||||
{
|
||||
var cutoff = options.Now - TimeSpan.FromDays(options.CleanAfterDays);
|
||||
foreach (var candidate in candidates.Where(c => c.Targeted && c.LastUsed < cutoff).OrderBy(c => c.LastUsed).ToList())
|
||||
await CleanOneAsync(candidate).ConfigureAwait(false);
|
||||
}
|
||||
if (options.SizeLimitBytes > 0)
|
||||
{
|
||||
foreach (var candidate in candidates.Where(c => c.Targeted && !named.Contains(c.Project.ProjectId)).OrderBy(c => c.LastUsed).ToList())
|
||||
{
|
||||
if (used <= options.SizeLimitBytes) break;
|
||||
await CleanOneAsync(candidate).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
foreach (var project in cleaned)
|
||||
Log.Info("workspace", $"Media of project {project.ProjectId} ('{project.Name}') removed by the clean-up: {project.Freed} bytes freed, {project.Trashed} original(s) kept in the trash");
|
||||
foreach (var project in kept) Log.Info("workspace", $"Media of project {project.ProjectId} ('{project.Name}') kept by the clean-up: {project.Reason}");
|
||||
return new WorkspaceCleanReport(cleaned, kept, before, Math.Max(0, used));
|
||||
}
|
||||
|
||||
/// <summary>The frames of <paramref name="media"/> whose original the home no longer holds as it is here (all of them when the home's frames are not known).</summary>
|
||||
public static IReadOnlyList<WorkspaceFrame> LastOriginals(WorkspaceMedia? media, HomeFrames? home)
|
||||
{
|
||||
if (media is null) return [];
|
||||
var held = home?.Frames.ToDictionary(f => f.Id);
|
||||
return [.. media.Frames.Where(f => f.HasOriginal && (held is null || !Holds(held, f)))];
|
||||
|
||||
static bool Holds(Dictionary<Guid, Frame> held, WorkspaceFrame frame) =>
|
||||
frame.OriginalSha256 is { Length: > 0 } sha && held.TryGetValue(frame.Id, out var there) && (there.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0
|
||||
&& string.Equals(there.Sha256, sha, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The local renders of the project its home does not have yet (its ledger's unsent entries); a ledger that cannot be
|
||||
/// read counts the files of <c>exports/</c> besides it, at least one: whether they went home is not known.
|
||||
/// </summary>
|
||||
public static int UnsentRenders(WorkspaceProject project)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(project);
|
||||
try
|
||||
{
|
||||
return project.ReadRenders().Unsent.Count;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("workspace", $"The render ledger of project {project.ProjectId} cannot be read: its renders count as not sent", ex);
|
||||
var files = Directory.Exists(project.ExportsDirectory)
|
||||
? Directory.EnumerateFiles(project.ExportsDirectory, "*", SearchOption.AllDirectories)
|
||||
.Count(f => !string.Equals(Path.GetFileName(f), WorkspaceProject.RendersFileName, StringComparison.OrdinalIgnoreCase))
|
||||
: 0;
|
||||
return Math.Max(1, files);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The workspace holds the project's media to edit: a media index whose files the clean-up did not remove (the folders of
|
||||
/// proxies or originals there say so at once; otherwise the index is read).
|
||||
/// </summary>
|
||||
public static bool HoldsMedia(WorkspaceProject project)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(project);
|
||||
if (!File.Exists(project.MediaIndexPath)) return false;
|
||||
if (Directory.Exists(project.ProxiesDirectory) || Directory.Exists(project.OriginalsDirectory)) return true;
|
||||
try
|
||||
{
|
||||
return project.ReadMedia() is { IsCleaned: false };
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
|
||||
{
|
||||
// The index is there: whatever it holds, opening it says so.
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Files and bytes under <paramref name="folder"/>, leases left out; nothing when it is not there.</summary>
|
||||
public static (int Files, long Bytes) Measure(string folder)
|
||||
{
|
||||
var (files, bytes) = (0, 0L);
|
||||
if (!Directory.Exists(folder)) return (files, bytes);
|
||||
foreach (var file in new DirectoryInfo(folder).EnumerateFiles("*", SearchOption.AllDirectories))
|
||||
{
|
||||
if (WorkspaceProject.IsLeaseFile(file.Name)) continue;
|
||||
files++;
|
||||
bytes += file.Length;
|
||||
}
|
||||
return (files, bytes);
|
||||
}
|
||||
|
||||
/// <summary>The project cleaned, or why it is kept.</summary>
|
||||
private async Task<(WorkspaceKeep? Kept, long Freed, int Trashed)> TryCleanAsync(Candidate candidate, WorkspaceCleanOptions options, bool moving,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var project = candidate.Project;
|
||||
var id = project.ProjectId;
|
||||
if (candidate.Unreadable || candidate.Bundle is not { Workspace: { } workspace }) return (WorkspaceKeep.Unreadable, 0, 0);
|
||||
if (options.JustCopied == id) return (WorkspaceKeep.JustCopied, 0, 0);
|
||||
if (options.IsOpen(id)) return (WorkspaceKeep.Open, 0, 0);
|
||||
if (moving || await BusyAsync(id, workspace.Home.SequenceId, cancellationToken).ConfigureAwait(false)) return (WorkspaceKeep.Jobs, 0, 0);
|
||||
if (candidate.CleanableBytes == 0) return (WorkspaceKeep.NoMedia, 0, 0);
|
||||
if (ReadLedger(project) is not { } ledger) return (WorkspaceKeep.Unreadable, 0, 0);
|
||||
if (ledger.Unsent.Count > 0) return (WorkspaceKeep.UnsentRenders, 0, 0);
|
||||
if (_homes(workspace.Home) is not { } home) return (WorkspaceKeep.Offline, 0, 0);
|
||||
WorkspaceStatus status;
|
||||
try
|
||||
{
|
||||
status = await _sync.StatusAsync(id, home, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is ProjectException or DatabaseException or IOException or InvalidDataException or LeaseUnavailableException)
|
||||
{
|
||||
Log.Info("workspace", $"The home of project {id} could not be asked for the clean-up: {ex.Message}");
|
||||
return (WorkspaceKeep.Offline, 0, 0);
|
||||
}
|
||||
if (status.Offline && status.State != WorkspaceState.Gone) return (WorkspaceKeep.Offline, 0, 0);
|
||||
if (status.State != WorkspaceState.InSync) return (WorkspaceKeep.NotInSync, 0, 0);
|
||||
// The home's frames, asked now: an original goes only when the home holds the same bytes. Every file of the folder of
|
||||
// originals counts, named by the media index or not (one a removal could not move aside stays unnamed there).
|
||||
HomeFrames? held = null;
|
||||
if (HoldsOriginalFiles(project))
|
||||
{
|
||||
try
|
||||
{
|
||||
held = await home.FramesAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is HomeUnavailableException or ProjectException or DatabaseException or IOException)
|
||||
{
|
||||
Log.Info("workspace", $"The frames of project {id}'s home could not be read for the clean-up: {ex.Message}");
|
||||
return (WorkspaceKeep.Offline, 0, 0);
|
||||
}
|
||||
}
|
||||
IDisposable lease;
|
||||
try
|
||||
{
|
||||
lease = await project.LockAsync(LockWait, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (LeaseUnavailableException)
|
||||
{
|
||||
return (WorkspaceKeep.Jobs, 0, 0);
|
||||
}
|
||||
using (lease)
|
||||
{
|
||||
// Asked again under the lease: the editor may have opened it, a save or a render may have come meanwhile.
|
||||
if (options.IsOpen(id)) return (WorkspaceKeep.Open, 0, 0);
|
||||
var local = await _sync.StatusAsync(id, null, cancellationToken).ConfigureAwait(false);
|
||||
if (local.State != WorkspaceState.InSync) return (WorkspaceKeep.NotInSync, 0, 0);
|
||||
if (ReadLedger(project) is not { } renders) return (WorkspaceKeep.Unreadable, 0, 0);
|
||||
if (renders.Unsent.Count > 0) return (WorkspaceKeep.UnsentRenders, 0, 0);
|
||||
WorkspaceMedia? media;
|
||||
try
|
||||
{
|
||||
media = project.ReadMedia();
|
||||
}
|
||||
catch (InvalidDataException)
|
||||
{
|
||||
return (WorkspaceKeep.Unreadable, 0, 0);
|
||||
}
|
||||
if (media is null || media.IsCleaned) return (WorkspaceKeep.NoMedia, 0, 0);
|
||||
var (freed, trashed) = Clear(project, media, renders, held, options.Now);
|
||||
return (null, freed, trashed);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes the project's media and its sent renders, under its lease: the media index rewritten with no file first (so
|
||||
/// nothing ever names a file that is gone), then the sent renders (only files inside <c>exports/</c>), then the
|
||||
/// originals, each hashed now and deleted only when the home holds the same bytes, else moved to the trash
|
||||
/// (<see cref="WorkspaceOriginals.ClearOriginals"/>), and last the proxies. A file that cannot be deleted or moved (held
|
||||
/// open) stays, and is logged.
|
||||
/// </summary>
|
||||
private static (long Freed, int Trashed) Clear(WorkspaceProject project, WorkspaceMedia media, WorkspaceRenders renders, HomeFrames? held, DateTimeOffset now)
|
||||
{
|
||||
var before = Cleanable(project, renders);
|
||||
project.WriteMedia(media with
|
||||
{
|
||||
Frames = [.. media.Frames.Select(f => f with { File = null, FileSha256 = null, FileBytes = 0, FileKind = null, Original = null, OriginalSha256 = null, OriginalBytes = 0 })],
|
||||
Cleaned = now,
|
||||
});
|
||||
var ledger = renders;
|
||||
foreach (var entry in renders.Entries.Where(e => e.IsSent))
|
||||
{
|
||||
if (project.ContainedExportFile(entry.File) is not { } path)
|
||||
{
|
||||
Log.Warning("workspace", $"The ledger of project {project.ProjectId} names '{entry.File}' for render {entry.RenderId}, not a file of its exports: nothing is removed for it");
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(path)) Directory.Delete(path, recursive: true);
|
||||
else if (File.Exists(path)) File.Delete(path);
|
||||
ledger = ledger.Without(entry.RenderId);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("workspace", $"The sent render {path} could not be removed; it stays", ex);
|
||||
}
|
||||
}
|
||||
if (!ReferenceEquals(ledger, renders)) project.WriteRenders(ledger);
|
||||
var (trashed, trashedBytes) = (0, 0L);
|
||||
if (Directory.Exists(project.OriginalsDirectory))
|
||||
{
|
||||
using var trash = WorkspaceTrash.Begin(project.Store.Root, project.ProjectId, now.UtcDateTime);
|
||||
var cleared = WorkspaceOriginals.ClearOriginals(project, held?.Frames, trash.Path);
|
||||
(trashed, trashedBytes) = (cleared.Trashed, cleared.TrashedBytes);
|
||||
if (trash.Finish() is { } aside) Log.Info("workspace", $"{trashed} original(s) of project {project.ProjectId} its home does not hold as they are kept in {aside}");
|
||||
if (cleared.Left > 0) Log.Warning("workspace", $"{cleared.Left} original(s) of project {project.ProjectId} could be neither deleted nor set aside; they stay");
|
||||
}
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(project.ProxiesDirectory)) Directory.Delete(project.ProxiesDirectory, recursive: true);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("workspace", $"{project.ProxiesDirectory} could not be removed whole; what is left stays", ex);
|
||||
}
|
||||
var after = Cleanable(project, ledger);
|
||||
return (Math.Max(0, before - after - trashedBytes), trashed);
|
||||
}
|
||||
|
||||
/// <summary>The project's folder of originals holds a file (an original, named by the media index or not, or a part of one).</summary>
|
||||
public static bool HoldsOriginalFiles(WorkspaceProject project)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(project);
|
||||
try
|
||||
{
|
||||
return Directory.Exists(project.OriginalsDirectory) && Directory.EnumerateFiles(project.OriginalsDirectory, "*", SearchOption.AllDirectories).Any();
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
// Not known: counted as holding some, so the home is asked.
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>What a clean-up may remove of the project: its proxies, its originals and the renders its home took.</summary>
|
||||
private static long Cleanable(WorkspaceProject project, WorkspaceRenders renders) =>
|
||||
Measure(project.ProxiesDirectory).Bytes + Measure(project.OriginalsDirectory).Bytes + SentBytes(project, renders);
|
||||
|
||||
/// <summary>The bytes of the renders of <paramref name="renders"/> the home took that are still in <c>exports/</c> (an image sequence's folder counts its files).</summary>
|
||||
private static long SentBytes(WorkspaceProject project, WorkspaceRenders renders) =>
|
||||
renders.Entries.Where(e => e.IsSent).Select(e => project.ContainedExportFile(e.File)).OfType<string>()
|
||||
.Sum(path => File.Exists(path) ? new FileInfo(path).Length : Measure(path).Bytes);
|
||||
|
||||
/// <summary>The project's ledger; null (logged) when it cannot be read.</summary>
|
||||
private static WorkspaceRenders? ReadLedger(WorkspaceProject project)
|
||||
{
|
||||
try
|
||||
{
|
||||
return project.ReadRenders();
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("workspace", $"The render ledger of project {project.ProjectId} cannot be read: the clean-up keeps the project", ex);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A copy, render or send of the project is queued, running or paused (one whose parameters do not read counts too).</summary>
|
||||
private async Task<bool> BusyAsync(Guid projectId, Guid sequenceId, CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (var kind in ProjectJobs)
|
||||
{
|
||||
foreach (var job in await _db.Jobs.ListPendingAsync(kind, sequenceId, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var named = kind switch
|
||||
{
|
||||
JobKind.WorkspaceCopy => WorkspaceJobs.CopyParameters(job)?.ProjectId,
|
||||
JobKind.WorkspaceRender => WorkspaceJobs.RenderParameters(job)?.ProjectId,
|
||||
_ => WorkspaceJobs.SendParameters(job)?.ProjectId,
|
||||
};
|
||||
if (named is null || named == projectId) return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>A project of the workspace as the clean-up weighs it.</summary>
|
||||
private sealed record Candidate(WorkspaceProject Project, ProjectBundle? Bundle, WorkspaceMedia? Media, bool Unreadable, string Name, bool IsMain, string Title,
|
||||
DateTimeOffset LastUsed, long FolderBytes, long CleanableBytes)
|
||||
{
|
||||
/// <summary>The policies may name it: media a copy wrote (cleaned ones are done), or a copy that cannot be read holding media files.</summary>
|
||||
public bool Targeted => Media is { IsCleaned: false } || (Unreadable && CleanableBytes > 0);
|
||||
|
||||
public static Candidate Of(WorkspaceProject project, WorkspaceCleanOptions options)
|
||||
{
|
||||
var folder = Measure(project.Folder).Bytes;
|
||||
var media = Measure(project.ProxiesDirectory).Bytes + Measure(project.OriginalsDirectory).Bytes;
|
||||
var saved = File.Exists(project.BundlePath) ? new DateTimeOffset(File.GetLastWriteTimeUtc(project.BundlePath), TimeSpan.Zero) : DateTimeOffset.MinValue;
|
||||
ProjectBundle bundle;
|
||||
try
|
||||
{
|
||||
bundle = project.ReadBundle();
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
|
||||
{
|
||||
return new Candidate(project, null, null, true, project.ProjectId.ToString("N"), false, string.Empty, Used(saved), folder, media);
|
||||
}
|
||||
var workspace = bundle.Workspace!;
|
||||
var title = bundle.Sequence.Title is { Length: > 0 } t ? t : bundle.Sequence.Folder;
|
||||
WorkspaceMedia? index;
|
||||
var unreadable = false;
|
||||
try
|
||||
{
|
||||
index = project.ReadMedia();
|
||||
}
|
||||
catch (InvalidDataException)
|
||||
{
|
||||
(index, unreadable) = (null, true);
|
||||
}
|
||||
var sent = 0L;
|
||||
try
|
||||
{
|
||||
sent = SentBytes(project, project.ReadRenders());
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
|
||||
{
|
||||
// Weighed by its media alone: cleaning it is refused while its ledger cannot be read.
|
||||
}
|
||||
return new Candidate(project, bundle, index, unreadable, workspace.Name, workspace.IsMain, title, Used(bundle.Saved), folder, media + sent);
|
||||
|
||||
DateTimeOffset Used(DateTimeOffset fallback) => options.Opened.TryGetValue(project.ProjectId, out var opened) ? opened : fallback;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -19,60 +19,6 @@ using SequenceManifest = Chronos.Protocol.SequenceManifest;
|
||||
|
||||
namespace Chronos.Data.Workspaces;
|
||||
|
||||
/// <summary>Builds the jobs of the workspace (owner review 2, R8), and their handlers for the desktop's engine.</summary>
|
||||
public static class WorkspaceJobs
|
||||
{
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.WorkspaceCopy"/>: the media of workspace project <paramref name="projectId"/> (checked out first,
|
||||
/// <see cref="WorkspaceSync.CheckoutAsync"/>) copied from its home, the sequence of this PC's archive.
|
||||
/// </summary>
|
||||
public static ArchiveJobRequest Copy(Sequence sequence, string workspaceRoot, Guid projectId, string media = BundleMedia.Proxies)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(sequence);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(workspaceRoot);
|
||||
return new ArchiveJobRequest(JobKind.WorkspaceCopy, [sequence.Id], sequence.DisplayTitle,
|
||||
JsonSerializer.Serialize(new WorkspaceCopyParameters(workspaceRoot, projectId, media), WorkspaceJson.Default.WorkspaceCopyParameters), JobPriority.User);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.WorkspaceCopy"/> of a project whose home is a server (phase 5b): the media of workspace project
|
||||
/// <paramref name="projectId"/> (checked out first) downloaded from sequence <paramref name="sequenceId"/> of
|
||||
/// <paramref name="server"/> (<see cref="SyncService.ServerKey"/>), into the workspace alone. The job's target is the
|
||||
/// server's sequence, as for a sync download.
|
||||
/// </summary>
|
||||
public static ArchiveJobRequest CopyFromServer(Guid sequenceId, string title, string server, string workspaceRoot, Guid projectId, string media = BundleMedia.Proxies)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(server);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(workspaceRoot);
|
||||
return new ArchiveJobRequest(JobKind.WorkspaceCopy, [sequenceId], title,
|
||||
JsonSerializer.Serialize(new WorkspaceCopyParameters(workspaceRoot, projectId, media, server), WorkspaceJson.Default.WorkspaceCopyParameters), JobPriority.User);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The handlers of the workspace's jobs for a desktop's engine (never a server's: the job writes into a folder its
|
||||
/// parameters name); copies from a server fail without a connection to it.
|
||||
/// </summary>
|
||||
public static IReadOnlyList<IJobHandler> CreateHandlers(ChronosDatabase db, ArchiveService archives) => CreateHandlers(db, archives, null);
|
||||
|
||||
/// <param name="servers">The connection a copy from a server's sequence downloads through (the desktop's signed-in session).</param>
|
||||
public static IReadOnlyList<IJobHandler> CreateHandlers(ChronosDatabase db, ArchiveService archives, ISyncConnector? servers) =>
|
||||
[new WorkspaceCopyJobHandler(db, archives, servers)];
|
||||
|
||||
/// <summary>
|
||||
/// What a first copy of a sequence's proxies writes into a workspace, as the copy reckons it before it starts (the
|
||||
/// archive's proxies' sizes, an estimate for those it makes), and the free bytes on <paramref name="workspaceRoot"/>'s
|
||||
/// drive (null when it cannot be measured): "Open in workspace" shows both. The copy keeps <see cref="CopySpaceReserve"/>
|
||||
/// free besides.
|
||||
/// </summary>
|
||||
/// <exception cref="InvalidOperationException">The index has no such sequence or archive.</exception>
|
||||
public static Task<(long Bytes, long? Free)> CopyFactsAsync(ChronosDatabase db, ArchiveService archives, Guid sequenceId, string workspaceRoot,
|
||||
CancellationToken cancellationToken = default) =>
|
||||
new WorkspaceCopyJobHandler(db, archives).FactsAsync(sequenceId, workspaceRoot, cancellationToken);
|
||||
|
||||
/// <summary>Bytes the copy leaves free on the workspace's drive besides what it copies.</summary>
|
||||
public static long CopySpaceReserve => WorkspaceCopyJobHandler.SpaceReserve;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.WorkspaceCopy"/> for a sequence of this PC's archive (owner review 2, R8): its proxies into the
|
||||
/// workspace project's <c>media/proxies/<frame id>.jpg</c>, the archive's own when current, else made here from the
|
||||
@@ -91,7 +37,14 @@ public static class WorkspaceJobs
|
||||
/// removes the media it wrote itself, once it has stopped writing and under the project's lease, unless another copy of the
|
||||
/// project goes on with them (<see cref="RemovePartialMediaAsync"/>); one cancelled while paused or failed, which no handler
|
||||
/// runs, has them removed the same way (<see cref="CancelledAsync"/>).
|
||||
/// Originals come in phase 5c: asked for, the job fails saying so.
|
||||
/// With <see cref="BundleMedia.Originals"/> (phase 5c) the proxies are copied first as above, then each frame's original into
|
||||
/// <c>media/originals/<frame id>-<sha8><ext></c> (<see cref="WorkspaceProject.OriginalName"/>): streamed into a
|
||||
/// <c>.part</c> file and hashed as it is written, the SHA-256 checked against the home's (recorded as copied when the home has
|
||||
/// none), the file read back after its rename; one already there with the same SHA-256 is kept. A frame whose original the
|
||||
/// home lacks (gone, flagged missing or damaged, refused by a server, not matching its SHA-256 twice) keeps its proxy: the
|
||||
/// copy finishes saying how many. The space reckoned adds the originals not held yet; the checkpoint of that stage is
|
||||
/// <c>originals:<frame>:<frames version>:<folder></c>. Originals the committed media no longer name are deleted
|
||||
/// only when the home holds a frame of the same SHA-256, else kept in the workspace's trash (<see cref="WorkspaceOriginals"/>).
|
||||
/// A project whose home is a server (phase 5b, <see cref="WorkspaceJobs.CopyFromServer"/>) has its media downloaded from
|
||||
/// the server through the desktop's session (<see cref="ISyncConnector"/>), never through this PC's archive: the frames as
|
||||
/// of one frames version with the aliases of the server's conversions (<c>GET …/projects/frames</c>), each frame's proxy as
|
||||
@@ -100,10 +53,19 @@ public static class WorkspaceJobs
|
||||
/// arrives and checked against the file written, four at a time; then the server's deep report. A file the editor's media
|
||||
/// or this run hold of the same photo is hashed again and kept; the editor's file is never written over, as above. The
|
||||
/// space, the passes, the checkpoint and a cancelled first copy go as for a copy from this PC's archive (the server's
|
||||
/// folder of the sequence in the checkpoint). Like the sync's downloads it is not held on a metered connection.
|
||||
/// folder of the sequence in the checkpoint). On a metered connection it waits before it downloads anything while
|
||||
/// <see cref="WorkspaceJobOptions.HoldTransfers"/> says so (the desktop pauses it meanwhile).
|
||||
/// A copy queued before the workspace moved works where the workspace is now, and none runs while the workspace is being
|
||||
/// moved (<see cref="WorkspaceJobRoots"/>). A copy of originals the user cancelled over media a copy finished before
|
||||
/// removes the originals it wrote that the media do not name, each deleted only when the home holds its bytes
|
||||
/// (<see cref="RemoveCancelledOriginalsAsync"/>).
|
||||
/// </summary>
|
||||
internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService archives, ISyncConnector? servers = null) : ICancelledJobHandler
|
||||
internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService archives, ISyncConnector? servers = null, WorkspaceJobOptions? options = null)
|
||||
: ICancelledJobHandler
|
||||
{
|
||||
/// <summary>How often a copy held on a metered connection looks again.</summary>
|
||||
internal static TimeSpan HoldPoll { get; set; } = TimeSpan.FromSeconds(2);
|
||||
|
||||
private const int PageSize = 1000;
|
||||
private const int Passes = 3;
|
||||
|
||||
@@ -135,8 +97,7 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
{
|
||||
var ct = context.Cancellation;
|
||||
var parameters = Parameters(context.Job);
|
||||
if (parameters.Media == BundleMedia.Originals) throw new NotSupportedException(Loc.L("Errors.Workspace.OriginalsLater"));
|
||||
if (parameters.Media != BundleMedia.Proxies) throw new InvalidDataException(Loc.L("Errors.Workspace.Parameters"));
|
||||
if (parameters.Media is not (BundleMedia.Proxies or BundleMedia.Originals)) throw new InvalidDataException(Loc.L("Errors.Workspace.Parameters"));
|
||||
if (parameters.Server is { Length: > 0 } server)
|
||||
{
|
||||
await RunFromServerAsync(context, parameters, server).ConfigureAwait(false);
|
||||
@@ -149,7 +110,13 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
// R8), and a workspace inside one would put its copies among that library's own files.
|
||||
if (await InsideALibraryAsync(parameters.WorkspaceRoot, ct).ConfigureAwait(false))
|
||||
throw new InvalidOperationException(Loc.L("Errors.Workspace.InsideLibrary"));
|
||||
var store = new WorkspaceStore(parameters.WorkspaceRoot);
|
||||
// Where the project is now (a workspace moved since the job was queued), never while it is being moved; nothing is
|
||||
// made where it is not.
|
||||
var root = WorkspaceJobRoots.Resolve(parameters.WorkspaceRoot, parameters.ProjectId, options?.CurrentRoot, "Errors.Workspace.NotCheckedOut");
|
||||
if (!PathsEqual(root, parameters.WorkspaceRoot) && await InsideALibraryAsync(root, ct).ConfigureAwait(false))
|
||||
throw new InvalidOperationException(Loc.L("Errors.Workspace.InsideLibrary"));
|
||||
parameters = parameters with { WorkspaceRoot = root };
|
||||
var store = new WorkspaceStore(root);
|
||||
var project = store.Find(parameters.ProjectId) ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.NotCheckedOut"));
|
||||
if (project.ReadBundle().Workspace is not { Home: { IsArchive: true } home } || home.SequenceId != sequenceId)
|
||||
throw new InvalidOperationException(Loc.L("Errors.Workspace.OtherHome"));
|
||||
@@ -162,6 +129,7 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
{
|
||||
// Stopped writing: what a first copy the user cancelled wrote goes (a pause or a shutdown resumes it).
|
||||
await RemovePartialMediaAsync(context.Job, project).ConfigureAwait(false);
|
||||
await RemoveCancelledOriginalsAsync(context.Job, project, parameters).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -183,13 +151,27 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
// Opening a store makes its folder: one gone stays gone.
|
||||
if (!Directory.Exists(Path.Combine(parameters.WorkspaceRoot, parameters.ProjectId.ToString("N")))) return Task.CompletedTask;
|
||||
string root;
|
||||
try
|
||||
{
|
||||
// Where the project is now; opening a store makes its folder, so one gone stays gone. A workspace being moved is
|
||||
// left as it is: the next copy checks every file it finds.
|
||||
root = WorkspaceJobRoots.Resolve(parameters.WorkspaceRoot, parameters.ProjectId, options?.CurrentRoot, "Errors.Workspace.NotCheckedOut");
|
||||
}
|
||||
catch (InvalidOperationException ex)
|
||||
{
|
||||
Log.Info("workspace", $"The media of cancelled copy {job.Id} are left as they are: {ex.Message}");
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
_ = Task.Run(async () =>
|
||||
{
|
||||
try
|
||||
{
|
||||
if (new WorkspaceStore(parameters.WorkspaceRoot).Find(parameters.ProjectId) is { } project) await RemovePartialMediaAsync(job, project).ConfigureAwait(false);
|
||||
if (new WorkspaceStore(root).Find(parameters.ProjectId) is { } project)
|
||||
{
|
||||
await RemovePartialMediaAsync(job, project).ConfigureAwait(false);
|
||||
await RemoveCancelledOriginalsAsync(job, project, parameters).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is not OutOfMemoryException)
|
||||
{
|
||||
@@ -241,6 +223,83 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
static bool Clearable(WorkspaceProject project) => project.Exists && !File.Exists(project.MediaIndexPath) && Directory.Exists(project.MediaDirectory);
|
||||
}
|
||||
|
||||
/// <summary>Waits while transfers with a server are held (a metered connection), saying so once.</summary>
|
||||
private async Task WhileHeldAsync(JobContext context)
|
||||
{
|
||||
if (options?.HoldTransfers is not { } held || !held()) return;
|
||||
context.Log(Loc.L("Archive.Log.WorkspaceCopyHeld"));
|
||||
while (held()) await Task.Delay(HoldPoll, context.Cancellation).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A copy of originals the user cancelled over media a copy finished before (<c>media.json</c> there, so
|
||||
/// <see cref="RemovePartialMediaAsync"/> keeps them): the originals it wrote that those media do not name go, under the
|
||||
/// project's lease, unless another copy of the project goes on with them; each is hashed and deleted only when the home's
|
||||
/// frames, read now, hold its bytes, else kept in the workspace's trash. What a download left half done goes. A home that
|
||||
/// cannot be asked leaves the whole files where they are (the next copy reuses them, a clean-up hashes them).
|
||||
/// </summary>
|
||||
private async Task RemoveCancelledOriginalsAsync(Job job, WorkspaceProject project, WorkspaceCopyParameters parameters)
|
||||
{
|
||||
if (parameters.Media != BundleMedia.Originals) return;
|
||||
try
|
||||
{
|
||||
if (!project.Exists || !File.Exists(project.MediaIndexPath) || !Directory.Exists(project.OriginalsDirectory)) return;
|
||||
if (!await StillToClearAsync(job, project).ConfigureAwait(false)) return;
|
||||
var home = await HomeFramesOrNullAsync(job, parameters).ConfigureAwait(false);
|
||||
using (await project.LockExistingAsync(LockWait, CancellationToken.None).ConfigureAwait(false))
|
||||
{
|
||||
if (!await StillToClearAsync(job, project).ConfigureAwait(false)) return;
|
||||
lock (WritersGate)
|
||||
{
|
||||
// A copy of this program that started since the jobs were read: its files stay.
|
||||
if (Writers.ContainsKey(project.Folder)) return;
|
||||
}
|
||||
if (TryReadMedia(project) is not { } media || !Directory.Exists(project.OriginalsDirectory)) return;
|
||||
var named = media.Frames.Where(f => f.Original is not null).Select(f => Path.GetFileName(f.Original!)).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var staging = Path.Combine(project.OriginalsDirectory, OriginalsStaging);
|
||||
if (Directory.Exists(staging)) Directory.Delete(staging, recursive: true);
|
||||
var written = Directory.EnumerateFiles(project.OriginalsDirectory).Where(f => !named.Contains(Path.GetFileName(f))).ToList();
|
||||
foreach (var part in written.Where(IsPart)) TryDelete(part);
|
||||
var whole = written.Where(f => !IsPart(f)).ToList();
|
||||
if (home is null)
|
||||
{
|
||||
if (whole.Count > 0) Log.Info("workspace", $"The home of project {project.ProjectId} could not be asked: the {whole.Count} original(s) cancelled copy {job.Id} wrote stay");
|
||||
return;
|
||||
}
|
||||
var trashed = WorkspaceOriginals.DeleteOrTrash(project, whole, home);
|
||||
if (!Directory.EnumerateFileSystemEntries(project.OriginalsDirectory).Any()) Directory.Delete(project.OriginalsDirectory);
|
||||
if (whole.Count > 0) Log.Info("workspace", $"The {whole.Count} original(s) cancelled copy {job.Id} of project {project.ProjectId} wrote were removed ({trashed} kept in the trash)");
|
||||
}
|
||||
}
|
||||
catch (DirectoryNotFoundException)
|
||||
{
|
||||
// The project's folder went meanwhile (its removal): nothing is left to clear.
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or DatabaseException or LeaseUnavailableException)
|
||||
{
|
||||
Log.Warning("workspace", $"The originals of a cancelled copy of project {project.ProjectId} could not be removed", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The frames of the copy's home as they are now (this PC's index, or the server's); null when they cannot be read.</summary>
|
||||
private async Task<IReadOnlyList<Frame>?> HomeFramesOrNullAsync(Job job, WorkspaceCopyParameters parameters)
|
||||
{
|
||||
if (job.TargetIds is not [var sequenceId]) return null;
|
||||
try
|
||||
{
|
||||
if (parameters.Server is not { Length: > 0 } server) return (await FramesAsync(sequenceId, CancellationToken.None).ConfigureAwait(false)).Frames;
|
||||
if (servers is null) return null;
|
||||
using var remote = await servers.ConnectAsync(server, CancellationToken.None).ConfigureAwait(false);
|
||||
if (remote is null) return null;
|
||||
return (await ServerFramesAsync(remote, sequenceId, CancellationToken.None).ConfigureAwait(false)).Frames;
|
||||
}
|
||||
catch (Exception ex) when (ex is not OutOfMemoryException)
|
||||
{
|
||||
Log.Info("workspace", $"The home of cancelled copy {job.Id} could not be asked for its frames: {ex.Message}");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The cancelled copy still reads so, and no other copy of the project into this workspace waits, runs or is paused.</summary>
|
||||
private async Task<bool> StillToClearAsync(Job job, WorkspaceProject project)
|
||||
{
|
||||
@@ -313,6 +372,7 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
// Read only: a read-only archive is copied from like any other.
|
||||
if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(archive.RootPath, ArchiveAvailability.Offline);
|
||||
var (copied, checkpointVersion, checkpointFolder) = Parse(context.Checkpoint);
|
||||
var withOriginals = parameters.Media == BundleMedia.Originals;
|
||||
for (var pass = 1; ; pass++)
|
||||
{
|
||||
var (row, frames) = await FramesAsync(sequenceId, ct).ConfigureAwait(false);
|
||||
@@ -328,15 +388,18 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
// Files kept from an earlier run are this frames version's and this folder's only (a conversion keeps a frame's id,
|
||||
// not its photo; a sequence moved meanwhile may have left its photos' names to another sequence's).
|
||||
if ((checkpointVersion is { } version && version != row.FramesVersion) || (checkpointFolder is { } read && read != row.Folder)) copied = 0;
|
||||
var total = (long)frames.Count;
|
||||
var total = (long)frames.Count * (withOriginals ? 2 : 1);
|
||||
var tally = new Tally();
|
||||
var entries = new WorkspaceFrame[frames.Count];
|
||||
// The media the editor reads until this copy commits: their files stay until then (RemoveStale).
|
||||
var current = TryReadMedia(project)?.Frames.Where(f => f.File is not null).ToDictionary(f => f.Id) ?? [];
|
||||
var held = TryReadMedia(project)?.Frames ?? [];
|
||||
var current = held.Where(f => f.File is not null).ToDictionary(f => f.Id);
|
||||
var heldOriginals = held.Where(f => f.Original is not null).ToDictionary(f => f.Id);
|
||||
Directory.CreateDirectory(project.ProxiesDirectory);
|
||||
// The workspace is on this PC's own drive, with the index, the settings and the logs: the copy never fills it.
|
||||
var remaining = Remaining(project, folder, frames, policy);
|
||||
EnsureSpace(parameters.WorkspaceRoot, remaining[0]);
|
||||
var originalsLeft = withOriginals ? RemainingOriginals(project, frames, heldOriginals, (f, _) => f.Size) : new long[frames.Count + 1];
|
||||
EnsureSpace(parameters.WorkspaceRoot, remaining[0] + originalsLeft[0]);
|
||||
var clock = System.Diagnostics.Stopwatch.StartNew();
|
||||
for (var i = 0; i < frames.Count; i++)
|
||||
{
|
||||
@@ -346,7 +409,7 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
{
|
||||
await context.CheckpointAsync(Checkpoint(Stage.Copy, i + 1, row.FramesVersion, row.Folder), i + 1, total).ConfigureAwait(false);
|
||||
// Another program may take the space meanwhile.
|
||||
EnsureSpace(parameters.WorkspaceRoot, remaining[i + 1]);
|
||||
EnsureSpace(parameters.WorkspaceRoot, remaining[i + 1] + originalsLeft[0]);
|
||||
clock.Restart();
|
||||
}
|
||||
else
|
||||
@@ -354,6 +417,27 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
await context.ReportAsync(i + 1, total).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
var originals = new OriginalsTally();
|
||||
if (withOriginals)
|
||||
{
|
||||
for (var i = 0; i < frames.Count; i++)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
entries[i] = await CopyOriginalAsync(project, paths, frames[i], entries[i], heldOriginals.GetValueOrDefault(frames[i].Id),
|
||||
originals, ct).ConfigureAwait(false);
|
||||
var done = frames.Count + i + 1;
|
||||
if (clock.Elapsed >= CheckpointEvery)
|
||||
{
|
||||
await context.CheckpointAsync(Checkpoint(Stage.Originals, i + 1, row.FramesVersion, row.Folder), done, total).ConfigureAwait(false);
|
||||
EnsureSpace(parameters.WorkspaceRoot, originalsLeft[i + 1]);
|
||||
clock.Restart();
|
||||
}
|
||||
else
|
||||
{
|
||||
await context.ReportAsync(done, total).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
var analysis = CopyAnalysis(folder, project, frames.Count);
|
||||
var (after, now) = await FramesAsync(sequenceId, ct).ConfigureAwait(false);
|
||||
var moved = after.Folder != row.Folder || !SequenceFolders.Holds(paths.Sequence(after.Folder), sequenceId) || Renamed(frames, now, entries);
|
||||
@@ -376,15 +460,19 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
}
|
||||
}
|
||||
var aliases = await HomePhotos.AliasesAsync(db, sequenceId, ct).ConfigureAwait(false);
|
||||
var media = new WorkspaceMedia(WorkspaceMedia.FormatName, WorkspaceMedia.CurrentFormatVersion, BundleMedia.Proxies, sequenceId, row.FramesVersion, entries,
|
||||
var media = new WorkspaceMedia(WorkspaceMedia.FormatName, WorkspaceMedia.CurrentFormatVersion, parameters.Media, sequenceId, row.FramesVersion, entries,
|
||||
DateTimeOffset.UtcNow, analysis, aliases.Count > 0 ? aliases : null);
|
||||
// The working state is the editor's meanwhile: it is held only while the media index and the positions move.
|
||||
using (await project.LockAsync(LockWait, ct).ConfigureAwait(false))
|
||||
{
|
||||
// The commit point: a cancel that came while the copy waited (a query it did not stop, the lease) wins here,
|
||||
// so a cancelled copy never commits and its clean-up removes what it wrote.
|
||||
ct.ThrowIfCancellationRequested();
|
||||
Commit(project, project.ReadBundle(), media, row.Folder, row.DisplayTitle);
|
||||
RemoveStale(project, media);
|
||||
RemoveStale(project, media, frames);
|
||||
}
|
||||
context.Log(Loc.L("Archive.Log.WorkspaceCopied", row.DisplayTitle, tally.Copied, tally.Made, tally.Kept, tally.Missing));
|
||||
if (withOriginals) originals.LogTo(context, "Archive.Log.WorkspaceOriginals", row.DisplayTitle);
|
||||
await context.CheckpointAsync(Checkpoint(Stage.Done, frames.Count, row.FramesVersion, row.Folder), total, total).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
@@ -400,13 +488,19 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
if (context.Job.TargetIds is not [var sequenceId]) throw new InvalidDataException(Loc.L("Errors.Workspace.Parameters"));
|
||||
// Asked before the store is opened, which makes its folder (as for a copy from this PC's archive).
|
||||
if (await InsideALibraryAsync(parameters.WorkspaceRoot, ct).ConfigureAwait(false)) throw new InvalidOperationException(Loc.L("Errors.Workspace.InsideLibrary"));
|
||||
var store = new WorkspaceStore(parameters.WorkspaceRoot);
|
||||
var root = WorkspaceJobRoots.Resolve(parameters.WorkspaceRoot, parameters.ProjectId, options?.CurrentRoot, "Errors.Workspace.NotCheckedOut");
|
||||
if (!PathsEqual(root, parameters.WorkspaceRoot) && await InsideALibraryAsync(root, ct).ConfigureAwait(false))
|
||||
throw new InvalidOperationException(Loc.L("Errors.Workspace.InsideLibrary"));
|
||||
parameters = parameters with { WorkspaceRoot = root };
|
||||
var store = new WorkspaceStore(root);
|
||||
var project = store.Find(parameters.ProjectId) ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.NotCheckedOut"));
|
||||
if (project.ReadBundle().Workspace is not { Home: { Kind: BundleHome.ServerKind } home } || home.SequenceId != sequenceId
|
||||
|| !string.Equals(home.Server, server, StringComparison.OrdinalIgnoreCase))
|
||||
throw new InvalidOperationException(Loc.L("Errors.Workspace.OtherHome"));
|
||||
try
|
||||
{
|
||||
// Nothing is downloaded while transfers with a server are held (a metered connection).
|
||||
await WhileHeldAsync(context).ConfigureAwait(false);
|
||||
using (WritingMedia(project))
|
||||
{
|
||||
using var remote = (servers is null ? null : await servers.ConnectAsync(server, ct).ConfigureAwait(false))
|
||||
@@ -418,6 +512,7 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
{
|
||||
// Stopped writing: what a first copy the user cancelled wrote goes (a pause or a shutdown resumes it).
|
||||
await RemovePartialMediaAsync(context.Job, project).ConfigureAwait(false);
|
||||
await RemoveCancelledOriginalsAsync(context.Job, project, parameters).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -426,6 +521,7 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
CancellationToken ct)
|
||||
{
|
||||
var (copied, checkpointVersion, checkpointFolder) = Parse(context.Checkpoint);
|
||||
var withOriginals = parameters.Media == BundleMedia.Originals;
|
||||
// What this run downloaded, pass after pass: kept where a frame still shows the same photo.
|
||||
var fetched = new Dictionary<Guid, WorkspaceFrame>();
|
||||
for (var pass = 1; ; pass++)
|
||||
@@ -433,15 +529,20 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
var (sequence, frames, assets, aliases, framesVersion, spec) = await ServerFramesAsync(remote, sequenceId, ct).ConfigureAwait(false);
|
||||
var folder = sequence.Folder ?? string.Empty;
|
||||
if ((checkpointVersion is { } version && version != framesVersion) || (checkpointFolder is { } read && read != folder)) copied = 0;
|
||||
var total = (long)frames.Count;
|
||||
var total = (long)frames.Count * (withOriginals ? 2 : 1);
|
||||
var tally = new Tally();
|
||||
var entries = new WorkspaceFrame[frames.Count];
|
||||
// The media the editor reads until this copy commits: their files stay until then (RemoveStale).
|
||||
var current = TryReadMedia(project)?.Frames.Where(f => f.File is not null).ToDictionary(f => f.Id) ?? [];
|
||||
var held = TryReadMedia(project)?.Frames ?? [];
|
||||
var current = held.Where(f => f.File is not null).ToDictionary(f => f.Id);
|
||||
var heldOriginals = held.Where(f => f.Original is not null).ToDictionary(f => f.Id);
|
||||
Directory.CreateDirectory(project.ProxiesDirectory);
|
||||
// The workspace is on this PC's own drive: the copy never fills it (the server's files estimated at its proxy size).
|
||||
var remaining = RemainingDownloads(project, frames, assets, spec);
|
||||
EnsureSpace(parameters.WorkspaceRoot, remaining[0]);
|
||||
var originalsLeft = withOriginals
|
||||
? RemainingOriginals(project, frames, heldOriginals, (f, _) => assets.GetValueOrDefault(f.Id) is { } asset ? asset.Size : 0)
|
||||
: new long[frames.Count + 1];
|
||||
EnsureSpace(parameters.WorkspaceRoot, remaining[0] + originalsLeft[0]);
|
||||
var clock = System.Diagnostics.Stopwatch.StartNew();
|
||||
var options = new ParallelOptions { MaxDegreeOfParallelism = Downloads, CancellationToken = ct };
|
||||
for (var start = 0; start < frames.Count; start += DownloadBatch)
|
||||
@@ -455,7 +556,7 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
{
|
||||
await context.CheckpointAsync(Checkpoint(Stage.Copy, end, framesVersion, folder), end, total).ConfigureAwait(false);
|
||||
// Another program may take the space meanwhile.
|
||||
EnsureSpace(parameters.WorkspaceRoot, remaining[end]);
|
||||
EnsureSpace(parameters.WorkspaceRoot, remaining[end] + originalsLeft[0]);
|
||||
clock.Restart();
|
||||
}
|
||||
else
|
||||
@@ -464,6 +565,27 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
}
|
||||
}
|
||||
foreach (var entry in entries.Where(e => e.File is not null)) fetched[entry.Id] = entry;
|
||||
var originals = new OriginalsTally();
|
||||
if (withOriginals)
|
||||
{
|
||||
for (var start = 0; start < frames.Count; start += DownloadBatch)
|
||||
{
|
||||
var end = Math.Min(frames.Count, start + DownloadBatch);
|
||||
await Parallel.ForEachAsync(Enumerable.Range(start, end - start), options, async (i, token) =>
|
||||
entries[i] = await DownloadOriginalAsync(remote, project, frames[i], assets.GetValueOrDefault(frames[i].Id), entries[i],
|
||||
heldOriginals.GetValueOrDefault(frames[i].Id), originals, token).ConfigureAwait(false)).ConfigureAwait(false);
|
||||
if (clock.Elapsed >= CheckpointEvery || end == frames.Count)
|
||||
{
|
||||
await context.CheckpointAsync(Checkpoint(Stage.Originals, end, framesVersion, folder), frames.Count + end, total).ConfigureAwait(false);
|
||||
EnsureSpace(parameters.WorkspaceRoot, originalsLeft[end]);
|
||||
clock.Restart();
|
||||
}
|
||||
else
|
||||
{
|
||||
await context.ReportAsync(frames.Count + end, total).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
var analysis = await CopyServerAnalysisAsync(remote, sequenceId, project, frames.Count, ct).ConfigureAwait(false);
|
||||
var after = await remote.GetSequenceAsync(sequenceId, ct).ConfigureAwait(false);
|
||||
if (after.FramesVersion != framesVersion && pass < Passes)
|
||||
@@ -476,15 +598,19 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
continue;
|
||||
}
|
||||
// Frames still arriving are copied as they were read (Update media takes the rest).
|
||||
var media = new WorkspaceMedia(WorkspaceMedia.FormatName, WorkspaceMedia.CurrentFormatVersion, BundleMedia.Proxies, sequenceId, framesVersion, entries,
|
||||
var media = new WorkspaceMedia(WorkspaceMedia.FormatName, WorkspaceMedia.CurrentFormatVersion, parameters.Media, sequenceId, framesVersion, entries,
|
||||
DateTimeOffset.UtcNow, analysis, aliases.Count > 0 ? aliases : null);
|
||||
// The working state is the editor's meanwhile: it is held only while the media index and the positions move.
|
||||
using (await project.LockAsync(LockWait, ct).ConfigureAwait(false))
|
||||
{
|
||||
// The commit point: a cancel that came while the copy waited (a query it did not stop, the lease) wins here,
|
||||
// so a cancelled copy never commits and its clean-up removes what it wrote.
|
||||
ct.ThrowIfCancellationRequested();
|
||||
Commit(project, project.ReadBundle(), media, folder, sequence.DisplayTitle);
|
||||
RemoveStale(project, media);
|
||||
RemoveStale(project, media, frames);
|
||||
}
|
||||
context.Log(Loc.L("Archive.Log.WorkspaceDownloaded", sequence.DisplayTitle, tally.Copied, tally.Made, tally.Kept, tally.Missing));
|
||||
if (withOriginals) originals.LogTo(context, "Archive.Log.WorkspaceOriginalsDownloaded", sequence.DisplayTitle);
|
||||
await context.CheckpointAsync(Checkpoint(Stage.Done, frames.Count, framesVersion, folder), total, total).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
@@ -626,6 +752,166 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
entry is { FileSha256: { } held } && string.Equals(held, sha, StringComparison.OrdinalIgnoreCase) ? entry.FileKind ?? ProxyCache.Proxy : null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A frame's original from this PC's archive into <c>media/originals/</c> (phase 5c), beside its proxy in
|
||||
/// <paramref name="entry"/>: kept when the workspace holds it already with the same SHA-256, else streamed into a
|
||||
/// <c>.part</c> file hashed as it is written (never held in memory), checked against the home's SHA-256, renamed and read
|
||||
/// back. A frame flagged missing or damaged at the home, or whose file is gone, keeps its proxy alone; so does one whose
|
||||
/// original twice does not match the SHA-256 the home has of it (the part is removed, the copy goes on).
|
||||
/// </summary>
|
||||
private static async Task<WorkspaceFrame> CopyOriginalAsync(WorkspaceProject project, ArchivePaths paths, Frame frame, WorkspaceFrame entry,
|
||||
WorkspaceFrame? held, OriginalsTally tally, CancellationToken cancellationToken)
|
||||
{
|
||||
var extension = OriginalExtension(frame);
|
||||
var expected = string.IsNullOrEmpty(frame.Sha256) ? null : frame.Sha256.ToLowerInvariant();
|
||||
if ((frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0) return tally.Missed(entry, frame.Index);
|
||||
if (await HeldOriginalAsync(project, frame, expected, extension, held, cancellationToken).ConfigureAwait(false) is { } kept)
|
||||
{
|
||||
tally.Keep();
|
||||
return entry with { Original = kept.Name, OriginalSha256 = kept.Sha256, OriginalBytes = kept.Bytes };
|
||||
}
|
||||
var source = paths.FromRelative(frame.RelativePath);
|
||||
if (!File.Exists(source)) return tally.Missed(entry, frame.Index);
|
||||
Directory.CreateDirectory(project.OriginalsDirectory);
|
||||
var part = Path.Combine(project.OriginalsDirectory, frame.Id.ToString("N") + ".part");
|
||||
for (var attempt = 1; ; attempt++)
|
||||
{
|
||||
string sha;
|
||||
long bytes;
|
||||
try
|
||||
{
|
||||
await using var input = new FileStream(source, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete, StreamBuffer,
|
||||
FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
await using var output = new FileStream(part, FileMode.Create, FileAccess.Write, FileShare.None, StreamBuffer, FileOptions.Asynchronous);
|
||||
using var hashing = new HashingStream(output);
|
||||
await input.CopyToAsync(hashing, StreamBuffer, cancellationToken).ConfigureAwait(false);
|
||||
output.Flush(flushToDisk: true);
|
||||
(sha, bytes) = (hashing.Sha256(), hashing.Written);
|
||||
}
|
||||
catch (FileNotFoundException)
|
||||
{
|
||||
// Gone while it was read (a Reorganize): the frame keeps its proxy, the pass's check of the folder decides the rest.
|
||||
TryDelete(part);
|
||||
return tally.Missed(entry, frame.Index);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// A part is never resumed from (a source is read whole): it goes, so a cancel leaves nothing half written.
|
||||
TryDelete(part);
|
||||
throw;
|
||||
}
|
||||
if (expected is not null && sha != expected)
|
||||
{
|
||||
TryDelete(part);
|
||||
if (attempt == 1) continue;
|
||||
Log.Warning("workspace", Loc.L("Errors.Workspace.OriginalMismatch", source));
|
||||
return tally.Missed(entry, frame.Index);
|
||||
}
|
||||
var name = WorkspaceProject.OriginalName(frame.Id, sha, extension);
|
||||
var target = project.MediaFile(name);
|
||||
File.Move(part, target, overwrite: true);
|
||||
if (string.Equals(await ContentHash.Sha256Async(target, cancellationToken: cancellationToken).ConfigureAwait(false), sha, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
tally.Copy();
|
||||
return entry with { Original = name, OriginalSha256 = sha, OriginalBytes = bytes };
|
||||
}
|
||||
TryDelete(target);
|
||||
if (attempt == 2) throw new IOException(Loc.L("Errors.Workspace.CopyMismatch", target));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A frame's original from a server into <c>media/originals/</c> (phase 5c): kept when held already, else downloaded with
|
||||
/// <see cref="SyncTransfers.FetchOriginalAsync"/> (resumable, checked against the server's SHA-256, two attempts) into a
|
||||
/// staging folder beside the originals, then renamed into place. A frame the server has no original of (flagged missing
|
||||
/// or damaged, 404), one this account may not read (403), or one that does not match twice keeps its proxy alone.
|
||||
/// </summary>
|
||||
private static async Task<WorkspaceFrame> DownloadOriginalAsync(ISyncRemote remote, WorkspaceProject project, Frame frame, ManifestFrame? asset, WorkspaceFrame entry,
|
||||
WorkspaceFrame? held, OriginalsTally tally, CancellationToken cancellationToken)
|
||||
{
|
||||
if (asset is null || (frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0 || (((FrameFlags)asset.Flags) & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0)
|
||||
return tally.Missed(entry, frame.Index);
|
||||
var extension = Path.GetExtension(asset.FileName);
|
||||
var expected = string.IsNullOrEmpty(asset.Sha256) ? null : asset.Sha256.ToLowerInvariant();
|
||||
if (await HeldOriginalAsync(project, frame, expected, extension, held, cancellationToken).ConfigureAwait(false) is { } kept)
|
||||
{
|
||||
tally.Keep();
|
||||
return entry with { Original = kept.Name, OriginalSha256 = kept.Sha256, OriginalBytes = kept.Bytes };
|
||||
}
|
||||
var staging = Path.Combine(project.OriginalsDirectory, OriginalsStaging);
|
||||
Directory.CreateDirectory(staging);
|
||||
string path, sha;
|
||||
try
|
||||
{
|
||||
(path, sha) = await SyncTransfers.FetchOriginalAsync(remote, asset, staging, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (ChronosApiException ex) when (ex.Status is HttpStatusCode.NotFound or HttpStatusCode.Forbidden)
|
||||
{
|
||||
return tally.Missed(entry, frame.Index);
|
||||
}
|
||||
catch (InvalidDataException ex)
|
||||
{
|
||||
Log.Warning("workspace", $"The original of frame {frame.Id} did not match the server's SHA-256", ex);
|
||||
return tally.Missed(entry, frame.Index);
|
||||
}
|
||||
var name = WorkspaceProject.OriginalName(frame.Id, sha, extension);
|
||||
var target = project.MediaFile(name);
|
||||
var bytes = new FileInfo(path).Length;
|
||||
File.Move(path, target, overwrite: true);
|
||||
tally.Copy();
|
||||
return entry with { Original = name, OriginalSha256 = sha, OriginalBytes = bytes };
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The original the workspace holds of <paramref name="frame"/>, hashed again: the one the media name for it (the same
|
||||
/// photo, of the SHA-256 the home has when it has one), else the file a copy of <paramref name="expected"/> takes; null when
|
||||
/// none holds it (a file that no longer matches is copied anew).
|
||||
/// </summary>
|
||||
private static async Task<(string Name, string Sha256, long Bytes)?> HeldOriginalAsync(WorkspaceProject project, Frame frame, string? expected, string extension,
|
||||
WorkspaceFrame? held, CancellationToken cancellationToken)
|
||||
{
|
||||
var candidates = new List<(string Name, string? Sha)>();
|
||||
if (held is { Original: { } heldName, OriginalSha256: { } heldSha } && SamePhoto(held, frame) && (expected is null || string.Equals(heldSha, expected, StringComparison.OrdinalIgnoreCase)))
|
||||
candidates.Add((heldName, heldSha.ToLowerInvariant()));
|
||||
if (expected is not null) candidates.Add((WorkspaceProject.OriginalName(frame.Id, expected, extension), expected));
|
||||
foreach (var (name, sha) in candidates)
|
||||
{
|
||||
var path = project.MediaFile(name);
|
||||
if (!File.Exists(path)) continue;
|
||||
var found = await ContentHash.Sha256Async(path, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
if (string.Equals(found, sha, StringComparison.OrdinalIgnoreCase)) return (name, found.ToLowerInvariant(), new FileInfo(path).Length);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The bytes a copy of the originals still writes from each frame on (index <c>i</c>: frames <c>i</c> to the last; one
|
||||
/// more entry, 0): <paramref name="size"/> of each frame whose original the workspace does not hold yet under the name a
|
||||
/// copy gives it, nothing for one the media name already, or flagged missing.
|
||||
/// </summary>
|
||||
private static long[] RemainingOriginals(WorkspaceProject project, List<Frame> frames, Dictionary<Guid, WorkspaceFrame> held, Func<Frame, int, long> size)
|
||||
{
|
||||
var remaining = new long[frames.Count + 1];
|
||||
for (var i = frames.Count - 1; i >= 0; i--)
|
||||
{
|
||||
var frame = frames[i];
|
||||
var there = (held.GetValueOrDefault(frame.Id) is { Original: { } name } && File.Exists(project.MediaFile(name)))
|
||||
|| (!string.IsNullOrEmpty(frame.Sha256) && File.Exists(project.MediaFile(WorkspaceProject.OriginalName(frame.Id, frame.Sha256, OriginalExtension(frame)))));
|
||||
var bytes = there || (frame.Flags & FrameFlags.Missing) != 0 ? 0 : Math.Max(0, size(frame, i));
|
||||
remaining[i] = remaining[i + 1] + bytes;
|
||||
}
|
||||
return remaining;
|
||||
}
|
||||
|
||||
/// <summary>The extension of a frame's original as its path at the home has it (else its file name), which decoders go by.</summary>
|
||||
private static string OriginalExtension(Frame frame) => Path.GetExtension(frame.RelativePath) is { Length: > 0 } extension ? extension : Path.GetExtension(frame.FileName);
|
||||
|
||||
/// <summary>The folder beside the originals a download from a server stages them in (resumable parts), emptied by the commit.</summary>
|
||||
private const string OriginalsStaging = ".staging";
|
||||
|
||||
/// <summary>The buffer an original is streamed through.</summary>
|
||||
private const int StreamBuffer = 1 << 20;
|
||||
|
||||
/// <summary>
|
||||
/// The bytes a copy from a server still writes from each frame on (index <c>i</c>: frames <c>i</c> to the last; one more
|
||||
/// entry, 0): nothing for a frame the workspace holds a file of already or the server has neither proxy nor thumbnail of,
|
||||
@@ -678,6 +964,9 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
}
|
||||
|
||||
/// <summary>Whether <paramref name="folder"/> is an archive's root of this index or lies inside one.</summary>
|
||||
private static bool PathsEqual(string a, string b) =>
|
||||
string.Equals(Path.TrimEndingDirectorySeparator(Path.GetFullPath(a)), Path.TrimEndingDirectorySeparator(Path.GetFullPath(b)), OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal);
|
||||
|
||||
private async Task<bool> InsideALibraryAsync(string folder, CancellationToken cancellationToken)
|
||||
{
|
||||
var path = Plain(Path.GetFullPath(folder));
|
||||
@@ -839,7 +1128,7 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
private static void Commit(WorkspaceProject project, ProjectBundle bundle, WorkspaceMedia media, string folder, string title)
|
||||
{
|
||||
var old = TryReadMedia(project);
|
||||
var workspace = bundle.Workspace! with { Media = new BundleMedia(BundleMedia.Proxies, media.FramesVersion) };
|
||||
var workspace = bundle.Workspace! with { Media = new BundleMedia(media.Kind, media.FramesVersion) };
|
||||
var sequence = bundle.Sequence with { Folder = folder, Title = title, Frames = media.Frames.Count };
|
||||
var working = bundle.Project;
|
||||
// Without the media index the positions were made against, they are taken as naming these frames.
|
||||
@@ -868,8 +1157,36 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
|
||||
/// <summary>
|
||||
/// Proxies of frames the sequence no longer has, and downloads a stopped copy from a server left half done: regenerable
|
||||
/// copies, removed so the media folder holds the frames listed.
|
||||
/// copies, removed so the media folder holds the frames listed. Media of kind originals (phase 5c) clear their folder
|
||||
/// alike, but an original is never erased on a guess: one the media no longer name is deleted only when the home's
|
||||
/// frames (<paramref name="home"/>, as this copy read them) hold a photo of its SHA-256, else it goes to the workspace's
|
||||
/// trash (<see cref="WorkspaceOriginals.DeleteOrTrash"/>). A copy of the proxies alone leaves the originals as they are.
|
||||
/// </summary>
|
||||
private static void RemoveStale(WorkspaceProject project, WorkspaceMedia media, IReadOnlyList<Frame> home)
|
||||
{
|
||||
RemoveStale(project, media);
|
||||
if (!media.HoldsOriginals || !Directory.Exists(project.OriginalsDirectory)) return;
|
||||
var staging = Path.Combine(project.OriginalsDirectory, OriginalsStaging);
|
||||
if (Directory.Exists(staging))
|
||||
{
|
||||
try
|
||||
{
|
||||
Directory.Delete(staging, recursive: true);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("workspace", $"{staging} could not be removed", ex);
|
||||
}
|
||||
}
|
||||
var originals = media.Frames.Where(f => f.Original is not null).Select(f => Path.GetFileName(f.Original!)).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var stale = Directory.EnumerateFiles(project.OriginalsDirectory).Where(f => !originals.Contains(Path.GetFileName(f))).ToList();
|
||||
foreach (var part in stale.Where(IsPart)) TryDelete(part);
|
||||
WorkspaceOriginals.DeleteOrTrash(project, [.. stale.Where(f => !IsPart(f))], home);
|
||||
}
|
||||
|
||||
private static bool IsPart(string file) => file.EndsWith(".part", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>The proxies' folder cleared as <see cref="RemoveStale(WorkspaceProject, WorkspaceMedia, IReadOnlyList{Frame})"/> says.</summary>
|
||||
private static void RemoveStale(WorkspaceProject project, WorkspaceMedia media)
|
||||
{
|
||||
var listed = media.Frames.Where(f => f.File is not null).Select(f => Path.GetFileName(f.File!)).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
@@ -943,6 +1260,18 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
return (Remaining([], paths.Sequence(row.Folder), frames, policy)[0], FreeBytes(workspaceRoot));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The bytes a copy of the sequence's originals writes (phase 5c): every frame's original the archive has, or, for a
|
||||
/// workspace project that holds some already (<paramref name="project"/>), only those it lacks.
|
||||
/// </summary>
|
||||
internal async Task<long> OriginalsBytesAsync(Guid sequenceId, WorkspaceProject? project, CancellationToken cancellationToken)
|
||||
{
|
||||
var (_, frames) = await FramesAsync(sequenceId, cancellationToken).ConfigureAwait(false);
|
||||
if (project is null || !File.Exists(project.MediaIndexPath)) return frames.Where(f => (f.Flags & FrameFlags.Missing) == 0).Sum(f => Math.Max(0, f.Size));
|
||||
var held = (TryReadMedia(project)?.Frames ?? []).Where(f => f.Original is not null).ToDictionary(f => f.Id);
|
||||
return RemainingOriginals(project, frames, held, (f, _) => f.Size)[0];
|
||||
}
|
||||
|
||||
/// <summary>A proxy's size when the archive has none to measure: some 1.5 MB at 1920 px, scaled by the area at the copy's long edge.</summary>
|
||||
private static long Estimate(Frame frame, ProxyPolicy policy)
|
||||
{
|
||||
@@ -1034,6 +1363,9 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
{
|
||||
Copy,
|
||||
Done,
|
||||
|
||||
/// <summary>The proxies are copied; the originals are (phase 5c).</summary>
|
||||
Originals,
|
||||
}
|
||||
|
||||
private static string Checkpoint(Stage stage, int frame, int framesVersion, string folder) =>
|
||||
@@ -1048,11 +1380,46 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
private static (int Copied, int? FramesVersion, string? Folder) Parse(string? checkpoint)
|
||||
{
|
||||
var parts = checkpoint?.Split(':', 4);
|
||||
if (parts is not ([_, _] or [_, _, _] or [_, _, _, _]) || !Enum.TryParse<Stage>(parts[0], ignoreCase: true, out var stage) || stage != Stage.Copy
|
||||
if (parts is not ([_, _] or [_, _, _] or [_, _, _, _]) || !Enum.TryParse<Stage>(parts[0], ignoreCase: true, out var stage) || stage is not (Stage.Copy or Stage.Originals)
|
||||
|| !int.TryParse(parts[1], NumberStyles.None, CultureInfo.InvariantCulture, out var next))
|
||||
return (0, null, null);
|
||||
int? version = parts.Length >= 3 && int.TryParse(parts[2], NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture, out var v) ? v : null;
|
||||
return (next, version, parts.Length == 4 ? parts[3] : null);
|
||||
return (stage == Stage.Originals ? int.MaxValue : next, version, parts.Length == 4 ? parts[3] : null);
|
||||
}
|
||||
|
||||
/// <summary>What a copy of the originals did with each frame (phase 5c), counted from the workers that download them at once.</summary>
|
||||
private sealed class OriginalsTally
|
||||
{
|
||||
/// <summary>Frames named one by one in the job's log when they have no original; the count says the rest.</summary>
|
||||
private const int Named = 20;
|
||||
|
||||
private readonly System.Collections.Concurrent.ConcurrentQueue<(int Index, string Name)> _missed = new();
|
||||
private int _copied;
|
||||
private int _kept;
|
||||
|
||||
public int Copied => Volatile.Read(ref _copied);
|
||||
|
||||
public int Kept => Volatile.Read(ref _kept);
|
||||
|
||||
public int Missing => _missed.Count;
|
||||
|
||||
public void Copy() => Interlocked.Increment(ref _copied);
|
||||
|
||||
public void Keep() => Interlocked.Increment(ref _kept);
|
||||
|
||||
/// <summary>The frame keeps its proxy alone: <paramref name="entry"/> with no original.</summary>
|
||||
public WorkspaceFrame Missed(WorkspaceFrame entry, int index)
|
||||
{
|
||||
_missed.Enqueue((index, entry.FileName));
|
||||
return entry with { Original = null, OriginalSha256 = null, OriginalBytes = 0 };
|
||||
}
|
||||
|
||||
/// <summary>Says in the job's log what was copied (<paramref name="key"/>), and names the first frames without an original.</summary>
|
||||
public void LogTo(JobContext context, string key, string title)
|
||||
{
|
||||
foreach (var (_, name) in _missed.OrderBy(m => m.Index).Take(Named)) context.Log(Loc.L("Archive.Log.WorkspaceOriginalMissing", title, name));
|
||||
context.Log(Loc.L(key, title, Copied, Kept, Missing));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>What a copy did with each frame, counted from the workers that download them at once.</summary>
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
using System.Text.Json;
|
||||
using Chronos.Core.Diagnostics;
|
||||
using Chronos.Core.Localization;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Core.Project;
|
||||
using Chronos.Data.Archiving;
|
||||
using Chronos.Data.Archiving.Jobs;
|
||||
using Chronos.Data.Jobs;
|
||||
using Chronos.Data.Processing;
|
||||
using Chronos.Data.Sync;
|
||||
|
||||
namespace Chronos.Data.Workspaces;
|
||||
|
||||
/// <summary>
|
||||
/// What the workspace's job handlers take from the desktop besides the index and the archive engine (phase 5c).
|
||||
/// </summary>
|
||||
/// <param name="Processing">
|
||||
/// Decoders, encoders and tuning of a local render (<see cref="JobKind.WorkspaceRender"/>): the desktop's
|
||||
/// <see cref="ProcessingService.Options"/>.
|
||||
/// </param>
|
||||
/// <param name="HoldTransfers">
|
||||
/// True while transfers with a server are held (Settings › Server › Pause metered transfers on a metered connection): a
|
||||
/// <see cref="JobKind.WorkspaceCopy"/> or <see cref="JobKind.WorkspaceSend"/> naming a server waits before it downloads or
|
||||
/// uploads anything then. Never holds a copy from or a send to this PC's archive.
|
||||
/// </param>
|
||||
/// <param name="Moved">
|
||||
/// Told once a <see cref="JobKind.WorkspaceMove"/> verified every file at its target (from, to): the desktop names the new
|
||||
/// folder in Settings › Workspace and refreshes the workspace; null where nothing follows the folder.
|
||||
/// </param>
|
||||
/// <param name="CurrentRoot">
|
||||
/// The workspace folder in use now (the desktop's Settings › Workspace): a copy, render or send queued before the workspace
|
||||
/// moved finds its project there when the folder its parameters name no longer holds it; null where only that one counts.
|
||||
/// </param>
|
||||
public sealed record WorkspaceJobOptions(ProcessingOptions Processing, Func<bool> HoldTransfers, Func<string, string, Task>? Moved = null,
|
||||
Func<string?>? CurrentRoot = null)
|
||||
{
|
||||
/// <summary>The options of an engine without a processing service of its own: the archive engine's decoders, nothing held, no move followed.</summary>
|
||||
public static WorkspaceJobOptions For(ArchiveService archives)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(archives);
|
||||
return new WorkspaceJobOptions(new ProcessingOptions { Archive = new ArchiveServiceOptions { Decoder = archives.Decoder } }, () => false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Builds the jobs of the workspace (owner review 2, R8), and their handlers for the desktop's engine.</summary>
|
||||
public static class WorkspaceJobs
|
||||
{
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.WorkspaceCopy"/>: the media of workspace project <paramref name="projectId"/> (checked out first,
|
||||
/// <see cref="WorkspaceSync.CheckoutAsync"/>) copied from its home, the sequence of this PC's archive.
|
||||
/// </summary>
|
||||
public static ArchiveJobRequest Copy(Sequence sequence, string workspaceRoot, Guid projectId, string media = BundleMedia.Proxies)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(sequence);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(workspaceRoot);
|
||||
return new ArchiveJobRequest(JobKind.WorkspaceCopy, [sequence.Id], sequence.DisplayTitle,
|
||||
JsonSerializer.Serialize(new WorkspaceCopyParameters(workspaceRoot, projectId, media), WorkspaceJson.Default.WorkspaceCopyParameters), JobPriority.User);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.WorkspaceCopy"/> of a project whose home is a server (phase 5b): the media of workspace project
|
||||
/// <paramref name="projectId"/> (checked out first) downloaded from sequence <paramref name="sequenceId"/> of
|
||||
/// <paramref name="server"/> (<see cref="SyncService.ServerKey"/>), into the workspace alone. The job's target is the
|
||||
/// server's sequence, as for a sync download.
|
||||
/// </summary>
|
||||
public static ArchiveJobRequest CopyFromServer(Guid sequenceId, string title, string server, string workspaceRoot, Guid projectId, string media = BundleMedia.Proxies)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(server);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(workspaceRoot);
|
||||
return new ArchiveJobRequest(JobKind.WorkspaceCopy, [sequenceId], title,
|
||||
JsonSerializer.Serialize(new WorkspaceCopyParameters(workspaceRoot, projectId, media, server), WorkspaceJson.Default.WorkspaceCopyParameters), JobPriority.User);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.WorkspaceRender"/> (phase 5c): workspace project <paramref name="projectId"/> rendered on this PC from
|
||||
/// the originals its media hold, into its <c>exports/</c>, as <paramref name="renderJson"/> says (Protocol
|
||||
/// <c>RenderJobParameters</c> with the portable settings: what <see cref="ProcessingJobs"/> writes for a render), each
|
||||
/// output recorded with <paramref name="projectVersionId"/> in the project's ledger and sent home after. The target is the
|
||||
/// home's sequence (this PC's archive, or <paramref name="server"/>'s, <see cref="SyncService.ServerKey"/>), so the
|
||||
/// editor follows it as it follows a render of that sequence.
|
||||
/// </summary>
|
||||
public static ArchiveJobRequest Render(Guid sequenceId, string title, string workspaceRoot, Guid projectId, Guid? projectVersionId, string renderJson,
|
||||
string? server = null)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(workspaceRoot);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(renderJson);
|
||||
return new ArchiveJobRequest(JobKind.WorkspaceRender, [sequenceId], title,
|
||||
JsonSerializer.Serialize(new WorkspaceRenderParameters(workspaceRoot, projectId, projectVersionId, renderJson, server),
|
||||
WorkspaceJson.Default.WorkspaceRenderParameters), JobPriority.User);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.WorkspaceSend"/> (phase 5c): the local renders <paramref name="renderIds"/> of workspace project
|
||||
/// <paramref name="projectId"/> (every unsent one of its ledger when empty) sent to its home, the sequence of this PC's
|
||||
/// archive or of <paramref name="server"/> (<see cref="SyncService.ServerKey"/>), with their project and version.
|
||||
/// </summary>
|
||||
public static ArchiveJobRequest Send(Guid sequenceId, string title, string workspaceRoot, Guid projectId, IReadOnlyList<Guid> renderIds, string? server = null,
|
||||
JobPriority priority = JobPriority.User)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(workspaceRoot);
|
||||
ArgumentNullException.ThrowIfNull(renderIds);
|
||||
return new ArchiveJobRequest(JobKind.WorkspaceSend, [sequenceId], title,
|
||||
JsonSerializer.Serialize(new WorkspaceSendParameters(workspaceRoot, projectId, [.. renderIds], server), WorkspaceJson.Default.WorkspaceSendParameters),
|
||||
priority);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.WorkspaceMove"/> (phase 5c): the workspace folder <paramref name="from"/> moved with its projects to
|
||||
/// <paramref name="to"/>, as the plan of fingerprint <paramref name="planFingerprint"/> the user confirmed showed it. The
|
||||
/// job has no target; its label is the folder it moves to.
|
||||
/// </summary>
|
||||
public static ArchiveJobRequest Move(string from, string to, string planFingerprint)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(from);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(to);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(planFingerprint);
|
||||
return new ArchiveJobRequest(JobKind.WorkspaceMove, [], to,
|
||||
JsonSerializer.Serialize(new WorkspaceMoveParameters(from, to, planFingerprint), WorkspaceJson.Default.WorkspaceMoveParameters), JobPriority.User);
|
||||
}
|
||||
|
||||
/// <summary>The parameters of a <see cref="JobKind.WorkspaceRender"/> job; null when it has none that read.</summary>
|
||||
public static WorkspaceRenderParameters? RenderParameters(Job job) => Read(job, JobKind.WorkspaceRender, WorkspaceJson.Default.WorkspaceRenderParameters);
|
||||
|
||||
/// <summary>The parameters of a <see cref="JobKind.WorkspaceSend"/> job; null when it has none that read.</summary>
|
||||
public static WorkspaceSendParameters? SendParameters(Job job) => Read(job, JobKind.WorkspaceSend, WorkspaceJson.Default.WorkspaceSendParameters);
|
||||
|
||||
/// <summary>The parameters of a <see cref="JobKind.WorkspaceMove"/> job; null when it has none that read.</summary>
|
||||
public static WorkspaceMoveParameters? MoveParameters(Job job) => Read(job, JobKind.WorkspaceMove, WorkspaceJson.Default.WorkspaceMoveParameters);
|
||||
|
||||
/// <summary>The parameters of a <see cref="JobKind.WorkspaceCopy"/> job; null when it has none that read.</summary>
|
||||
public static WorkspaceCopyParameters? CopyParameters(Job job) => Read(job, JobKind.WorkspaceCopy, WorkspaceJson.Default.WorkspaceCopyParameters);
|
||||
|
||||
/// <summary>
|
||||
/// The handlers of the workspace's jobs for a desktop's engine (never a server's: the job writes into a folder its
|
||||
/// parameters name); copies from a server fail without a connection to it.
|
||||
/// </summary>
|
||||
public static IReadOnlyList<IJobHandler> CreateHandlers(ChronosDatabase db, ArchiveService archives) => CreateHandlers(db, archives, null);
|
||||
|
||||
/// <param name="servers">The connection a copy from a server's sequence downloads through (the desktop's signed-in session).</param>
|
||||
public static IReadOnlyList<IJobHandler> CreateHandlers(ChronosDatabase db, ArchiveService archives, ISyncConnector? servers) =>
|
||||
CreateHandlers(db, archives, servers, WorkspaceJobOptions.For(archives));
|
||||
|
||||
/// <summary>
|
||||
/// The handlers of every workspace job (phase 5c): the copy of a project's media, its local render, the sending of its
|
||||
/// renders home and the move of the workspace folder.
|
||||
/// </summary>
|
||||
/// <param name="servers">The connection a copy from, or a send to, a server's sequence goes through (the desktop's signed-in session).</param>
|
||||
/// <param name="options">The desktop's processing options, its hold on transfers and what follows a moved folder.</param>
|
||||
public static IReadOnlyList<IJobHandler> CreateHandlers(ChronosDatabase db, ArchiveService archives, ISyncConnector? servers, WorkspaceJobOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(db);
|
||||
ArgumentNullException.ThrowIfNull(archives);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
return
|
||||
[
|
||||
new WorkspaceCopyJobHandler(db, archives, servers, options),
|
||||
new WorkspaceRenderJobHandler(db, archives, options),
|
||||
new WorkspaceSendJobHandler(db, archives, servers, options),
|
||||
new WorkspaceMoveJobHandler(db, options),
|
||||
];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// What a first copy of a sequence's proxies writes into a workspace, as the copy reckons it before it starts (the
|
||||
/// archive's proxies' sizes, an estimate for those it makes), and the free bytes on <paramref name="workspaceRoot"/>'s
|
||||
/// drive (null when it cannot be measured): "Open in workspace" shows both. The copy keeps <see cref="CopySpaceReserve"/>
|
||||
/// free besides.
|
||||
/// </summary>
|
||||
/// <exception cref="InvalidOperationException">The index has no such sequence or archive.</exception>
|
||||
public static Task<(long Bytes, long? Free)> CopyFactsAsync(ChronosDatabase db, ArchiveService archives, Guid sequenceId, string workspaceRoot,
|
||||
CancellationToken cancellationToken = default) =>
|
||||
new WorkspaceCopyJobHandler(db, archives).FactsAsync(sequenceId, workspaceRoot, cancellationToken);
|
||||
|
||||
/// <summary>Bytes the copy leaves free on the workspace's drive besides what it copies.</summary>
|
||||
public static long CopySpaceReserve => WorkspaceCopyJobHandler.SpaceReserve;
|
||||
|
||||
private static T? Read<T>(Job job, JobKind kind, System.Text.Json.Serialization.Metadata.JsonTypeInfo<T> info) where T : class
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(job);
|
||||
if (job.Kind != kind || job.ParametersJson is not { Length: > 0 } json) return null;
|
||||
try
|
||||
{
|
||||
return JsonSerializer.Deserialize(json, info);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Where a copy, render or send of a workspace project works (phase 5c): the folder its parameters name, or the one the
|
||||
/// workspace moved to since. Nothing of the workspace is written while it is being moved, and nothing is made where the
|
||||
/// project is not (opening a store makes its folder).
|
||||
/// </summary>
|
||||
internal static class WorkspaceJobRoots
|
||||
{
|
||||
/// <summary>
|
||||
/// The workspace folder that holds <paramref name="projectId"/> for a job queued with <paramref name="named"/>: that folder
|
||||
/// when it holds the project; else the target of a move of it past its switch, or the folder in use now
|
||||
/// (<paramref name="current"/>), when either holds it. Refused while a move of either folder has not switched yet
|
||||
/// (<c>Errors.Workspace.Moving</c>: the job fails and can be retried once the move ends); <paramref name="missing"/> (a
|
||||
/// resource key) when no folder holds the project.
|
||||
/// </summary>
|
||||
/// <exception cref="InvalidOperationException">The workspace is being moved, or no folder holds the project.</exception>
|
||||
public static string Resolve(string named, Guid projectId, Func<string?>? current, string missing)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(named);
|
||||
var candidates = new List<string>();
|
||||
string? currentRoot = null;
|
||||
try
|
||||
{
|
||||
currentRoot = current?.Invoke() is { Length: > 0 } root ? WorkspaceStore.FullRoot(root) : null;
|
||||
}
|
||||
catch (Exception ex) when (ex is ArgumentException or IOException or NotSupportedException)
|
||||
{
|
||||
Log.Debug("workspace", $"The workspace folder in use cannot be read: {ex.Message}");
|
||||
}
|
||||
foreach (var root in new[] { WorkspaceStore.FullRoot(named), currentRoot })
|
||||
{
|
||||
if (root is null) continue;
|
||||
if (WorkspaceMover.HasJournal(root))
|
||||
{
|
||||
// Past its switch the move's target is the workspace, and what is left here is a source still to clear.
|
||||
if (WorkspaceMover.ReadJournal(root) is not { Stage: WorkspaceMoveStage.Switched } journal) throw new InvalidOperationException(Loc.L("Errors.Workspace.Moving"));
|
||||
candidates.Add(WorkspaceStore.FullRoot(journal.To));
|
||||
continue;
|
||||
}
|
||||
candidates.Add(root);
|
||||
}
|
||||
foreach (var root in candidates.Distinct(StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
if (!File.Exists(Path.Combine(root, projectId.ToString("N"), WorkspaceProject.BundleFile))) continue;
|
||||
if (WorkspaceMover.HasJournal(root)) throw new InvalidOperationException(Loc.L("Errors.Workspace.Moving"));
|
||||
if (!string.Equals(root, WorkspaceStore.FullRoot(named), StringComparison.OrdinalIgnoreCase))
|
||||
Log.Info("workspace", $"Project {projectId} is no longer in {named}: the job works in {root}, where the workspace is now");
|
||||
return root;
|
||||
}
|
||||
throw new InvalidOperationException(Loc.L(missing));
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,13 @@ namespace Chronos.Data.Workspaces;
|
||||
/// What the file is when a server sent it (phase 5b): <c>proxy</c>, or <c>thumb</c>, a thumbnail standing in while the server
|
||||
/// had no proxy of the frame, which a later copy replaces once it has one; null for a proxy (copies from this PC's archive).
|
||||
/// </param>
|
||||
/// <param name="Original">
|
||||
/// The frame's original the workspace holds, relative to the media folder (<see cref="WorkspaceProject.OriginalName"/>:
|
||||
/// "originals/<frame id>-<sha8><ext>", phase 5c), in media of kind <see cref="BundleMedia.Originals"/>; null
|
||||
/// when it holds none (proxies only, or the home had no original of the frame: the proxy stands in for editing).
|
||||
/// </param>
|
||||
/// <param name="OriginalSha256">The original's SHA-256 as copied and checked (lower case); null with <paramref name="Original"/>.</param>
|
||||
/// <param name="OriginalBytes">The original's size as copied; 0 with no original.</param>
|
||||
public sealed record WorkspaceFrame(
|
||||
Guid Id,
|
||||
int Index,
|
||||
@@ -46,7 +53,10 @@ public sealed record WorkspaceFrame(
|
||||
string? File,
|
||||
string? FileSha256,
|
||||
long FileBytes,
|
||||
string? FileKind = null)
|
||||
string? FileKind = null,
|
||||
string? Original = null,
|
||||
string? OriginalSha256 = null,
|
||||
long OriginalBytes = 0)
|
||||
{
|
||||
/// <summary>The record of a home's frame, with the file the workspace holds for it (<paramref name="fileKind"/>: what a server sent).</summary>
|
||||
public static WorkspaceFrame Of(Frame frame, string? file, string? fileSha256, long fileBytes, string? fileKind = null)
|
||||
@@ -62,6 +72,24 @@ public sealed record WorkspaceFrame(
|
||||
[JsonIgnore]
|
||||
public bool IsStandIn => FileKind == Sync.ProxyCache.Thumb;
|
||||
|
||||
/// <summary>The workspace holds the frame's original (media of kind <see cref="BundleMedia.Originals"/>).</summary>
|
||||
[JsonIgnore]
|
||||
public bool HasOriginal => Original is not null;
|
||||
|
||||
/// <summary>
|
||||
/// The frame as a render from the workspace's originals reads it (phase 5c): its file is <see cref="Original"/>, and
|
||||
/// it is flagged missing while the workspace holds no original for it (a render refuses such media first).
|
||||
/// </summary>
|
||||
public Frame ToOriginalFrame(Guid sequenceId)
|
||||
{
|
||||
var frame = ToFrame(sequenceId);
|
||||
return frame with
|
||||
{
|
||||
RelativePath = Original ?? FileName,
|
||||
Flags = Original is null ? Flags | FrameFlags.Missing : Flags,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>The frame as the processing engine reads it; flagged missing while the workspace holds no file for it.</summary>
|
||||
public Frame ToFrame(Guid sequenceId) => new()
|
||||
{
|
||||
@@ -97,16 +125,34 @@ public sealed record WorkspaceFrame(
|
||||
|
||||
/// <summary>
|
||||
/// What a workspace project holds of its sequence's photos (<c>media/media.json</c>, owner review 2, R8): the kind
|
||||
/// (<see cref="BundleMedia.Proxies"/>; originals in phase 5c), the sequence and its frames version at the home when they
|
||||
/// were copied, and every frame in index order. Written last by a copy, so a copy cut short leaves the previous index.
|
||||
/// (<see cref="BundleMedia.Proxies"/>, or from phase 5c <see cref="BundleMedia.Originals"/>: the proxies plus each frame's
|
||||
/// original, <see cref="WorkspaceFrame.Original"/>), the sequence and its frames version at the home when they were
|
||||
/// copied, and every frame in index order. Written last by a copy, so a copy cut short leaves the previous index. An index
|
||||
/// of phase 5b has no original fields: it reads as proxies only. One that holds originals or was cleaned is written as
|
||||
/// format 2 (<see cref="WrittenFormatVersion"/>), which a Chronos before phase 5c refuses as newer and reads as media
|
||||
/// removed (copied again) rather than as proxies whose files are all missing; proxies alone stay format 1, which it reads.
|
||||
/// </summary>
|
||||
/// <param name="Aliases">The encodings of the sequence's photos a conversion joined at the home (R7, its tombstones of reason
|
||||
/// "replaced") when the media were copied: a version parked on a photo before its file was converted still finds its frame.</param>
|
||||
/// <param name="Cleaned">
|
||||
/// When the workspace's clean-up removed the files of these media (phase 5c, <see cref="WorkspaceCleaner"/>): the frames stay,
|
||||
/// none names a file, and the project reads as <see cref="WorkspaceState.MediaRemoved"/> until a copy brings them again (a
|
||||
/// copy writes a new index, without it). Null for media a copy wrote.
|
||||
/// </param>
|
||||
public sealed record WorkspaceMedia(string Format, int FormatVersion, string Kind, Guid SequenceId, int FramesVersion, IReadOnlyList<WorkspaceFrame> Frames,
|
||||
DateTimeOffset Copied, string? AnalysisSha256 = null, IReadOnlyList<PhotoAlias>? Aliases = null)
|
||||
DateTimeOffset Copied, string? AnalysisSha256 = null, IReadOnlyList<PhotoAlias>? Aliases = null, DateTimeOffset? Cleaned = null)
|
||||
{
|
||||
public const string FormatName = "chronos-workspace-media";
|
||||
public const int CurrentFormatVersion = 1;
|
||||
|
||||
/// <summary>The newest format this Chronos reads (2: media with originals, or cleaned).</summary>
|
||||
public const int CurrentFormatVersion = 2;
|
||||
|
||||
/// <summary>The format of media of proxies only that a copy wrote, which a Chronos before phase 5c reads too.</summary>
|
||||
public const int ProxiesFormatVersion = 1;
|
||||
|
||||
/// <summary>The format these media are written in: 2 when they hold originals (or name one) or were cleaned, else 1.</summary>
|
||||
[JsonIgnore]
|
||||
public int WrittenFormatVersion => HoldsOriginals || IsCleaned || Frames.Any(f => f.HasOriginal) ? CurrentFormatVersion : ProxiesFormatVersion;
|
||||
|
||||
/// <summary>The frames' photos in index order, as project positions name them, with the encodings conversions joined.</summary>
|
||||
[JsonIgnore]
|
||||
@@ -115,6 +161,17 @@ public sealed record WorkspaceMedia(string Format, int FormatVersion, string Kin
|
||||
/// <summary>The frames held, as the processing engine reads them.</summary>
|
||||
public IReadOnlyList<Frame> ToFrames() => [.. Frames.Select(f => f.ToFrame(SequenceId))];
|
||||
|
||||
/// <summary>The frames over the originals the workspace holds (<see cref="WorkspaceFrame.ToOriginalFrame"/>), as a local render reads them.</summary>
|
||||
public IReadOnlyList<Frame> ToOriginalFrames() => [.. Frames.Select(f => f.ToOriginalFrame(SequenceId))];
|
||||
|
||||
/// <summary>The clean-up removed the files of these media (<see cref="Cleaned"/>): nothing to edit until they are copied again.</summary>
|
||||
[JsonIgnore]
|
||||
public bool IsCleaned => Cleaned is not null;
|
||||
|
||||
/// <summary>Media of kind <see cref="BundleMedia.Originals"/>: the proxies plus the originals the frames name.</summary>
|
||||
[JsonIgnore]
|
||||
public bool HoldsOriginals => Kind == BundleMedia.Originals;
|
||||
|
||||
/// <summary>
|
||||
/// These media with their photos named as <paramref name="home"/> names them now (owner review 2, R8): a frame copied
|
||||
/// before the home hashed its photo (a folder adopted, verified since) takes the SHA-256 the home's frame of that id has
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
using System.Globalization;
|
||||
using Chronos.Core.Diagnostics;
|
||||
using Chronos.Core.Localization;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Data.Jobs;
|
||||
|
||||
namespace Chronos.Data.Workspaces;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.WorkspaceMove"/> (phase 5c, <see cref="WorkspaceJobs.Move"/>): the workspace folder moved with its
|
||||
/// projects, after the plan the user confirmed (its fingerprint checked again): journaled in <c><from>/.move.json</c>,
|
||||
/// moved per project on one volume, copied and hash-verified on both sides across volumes, a project's source deleted only
|
||||
/// once every file of it is verified, resumable from the journal; at the end <see cref="WorkspaceJobOptions.Moved"/> is told.
|
||||
/// On failure the source stays the workspace and unverified partial copies are removed (<see cref="WorkspaceMover"/>).
|
||||
/// A move that starts afresh is refused while another job of the workspace waits, runs or is paused (they write into the
|
||||
/// folder that moves). Cancelled by the user, while it runs or while it is paused or failed, what it did is undone as
|
||||
/// after a failure, unless the target already became the workspace; a pause or the program closing keeps the journal,
|
||||
/// from which the next run goes on. The checkpoint carries the progress only: the journal and the files' names say what
|
||||
/// is done (a file reaches its final name in the target only once verified).
|
||||
/// </summary>
|
||||
internal sealed class WorkspaceMoveJobHandler(ChronosDatabase db, WorkspaceJobOptions options) : ICancelledJobHandler
|
||||
{
|
||||
/// <summary>The other jobs of the workspace, which write into the folder a move takes away.</summary>
|
||||
private static readonly JobKind[] WorkspaceKinds = [JobKind.WorkspaceCopy, JobKind.WorkspaceRender, JobKind.WorkspaceSend];
|
||||
|
||||
public JobKind Kind => JobKind.WorkspaceMove;
|
||||
|
||||
/// <summary>Tests: the mover the job runs (one volume or not, a crash, a damaged copy); null makes the real one.</summary>
|
||||
internal Func<WorkspaceMoveParameters, WorkspaceMover>? Movers { get; init; }
|
||||
|
||||
public async Task RunAsync(JobContext context)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
var ct = context.Cancellation;
|
||||
var parameters = WorkspaceJobs.MoveParameters(context.Job) is { From.Length: > 0, To.Length: > 0, PlanFingerprint.Length: > 0 } read
|
||||
? read
|
||||
: throw new InvalidDataException(Loc.L("Errors.Workspace.JobParameters"));
|
||||
var mover = Mover(parameters);
|
||||
if (WorkspaceMover.ReadJournal(mover.From) is null && await db.Jobs.HasPendingAsync(WorkspaceKinds, cancellationToken: ct).ConfigureAwait(false))
|
||||
throw new InvalidOperationException(Loc.L("Errors.Workspace.JobsRunning"));
|
||||
try
|
||||
{
|
||||
await mover.MoveAsync(parameters.PlanFingerprint, options.Moved, (done, total, durable) => durable
|
||||
? context.CheckpointAsync(string.Create(CultureInfo.InvariantCulture, $"move:{done}"), done, total)
|
||||
: context.ReportAsync(done, total),
|
||||
context.Log, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
// Stopped: the user's cancel undoes what was done; a pause, a yield or the program closing goes on from the journal.
|
||||
if (await db.Jobs.GetAsync(context.Job.Id, CancellationToken.None).ConfigureAwait(false) is { State: JobState.Cancelled }) Undo(mover, context.Job);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A move cancelled while no handler ran it (paused, failed): what it did is undone, unless the target is the workspace already.</summary>
|
||||
public Task CancelledAsync(Job job, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(job);
|
||||
if (WorkspaceJobs.MoveParameters(job) is not { From.Length: > 0, To.Length: > 0 } parameters) return Task.CompletedTask;
|
||||
_ = Task.Run(() =>
|
||||
{
|
||||
try
|
||||
{
|
||||
Undo(Mover(parameters), job);
|
||||
}
|
||||
catch (Exception ex) when (ex is not OutOfMemoryException)
|
||||
{
|
||||
// Nobody waits for this task: the journal stays, and the move can be taken up.
|
||||
Log.Warning("workspace", $"Cancelled move {job.Id} could not be undone", ex);
|
||||
}
|
||||
}, CancellationToken.None);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private WorkspaceMover Mover(WorkspaceMoveParameters parameters) => Movers?.Invoke(parameters) ?? new WorkspaceMover(parameters.From, parameters.To);
|
||||
|
||||
private static void Undo(WorkspaceMover mover, Job job)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!mover.Abandon()) Log.Info("workspace", $"Move {job.Id} was cancelled after {mover.To} became the workspace; it is not undone");
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("workspace", $"Cancelled move {job.Id} could not be undone; its journal stays to take it up", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,713 @@
|
||||
using System.Globalization;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Chronos.Core.Diagnostics;
|
||||
using Chronos.Core.IO;
|
||||
using Chronos.Core.Localization;
|
||||
|
||||
namespace Chronos.Data.Workspaces;
|
||||
|
||||
/// <summary>Why the workspace folder cannot move to a folder (<see cref="WorkspaceMoveSurvey.Refusal"/>).</summary>
|
||||
public enum WorkspaceMoveRefusal
|
||||
{
|
||||
/// <summary>The target is the workspace folder itself or lies inside it (<c>Errors.Workspace.MoveSameFolder</c>).</summary>
|
||||
SameFolder,
|
||||
|
||||
/// <summary>The target holds a workspace (its manifest, its trash, a move's journal) or a folder of the same name as one moved (<c>Errors.Workspace.MoveTargetNotEmpty</c>).</summary>
|
||||
TargetNotEmpty,
|
||||
|
||||
/// <summary>The target's drive lacks the room for a copy across drives (<c>Errors.Workspace.MoveNoRoom</c>).</summary>
|
||||
NoRoom,
|
||||
}
|
||||
|
||||
/// <summary>A folder of a project the workspace holds, as a move surveys it.</summary>
|
||||
/// <param name="IsProject">It holds a working state (<c>project.chronos</c>); a folder that does not (a leftover) moves all the same.</param>
|
||||
public sealed record WorkspaceMoveFolder(Guid ProjectId, bool IsProject, int Files, long Bytes);
|
||||
|
||||
/// <summary>
|
||||
/// What moving a workspace folder <see cref="From"/> to <see cref="To"/> would take, read without writing anything: the
|
||||
/// entries that move (every folder named by a project id, the manifest, the trash), their files and bytes (leases left
|
||||
/// out), the target drive's free bytes, whether both lie on one volume, why it is refused, and a fingerprint of the files
|
||||
/// read (their relative names, sizes and times), which the move checks again before it starts.
|
||||
/// </summary>
|
||||
/// <param name="Free">The target drive's free bytes; null when they cannot be measured (a share), which is not refused.</param>
|
||||
/// <param name="Missing">With <see cref="WorkspaceMoveRefusal.NoRoom"/>, the bytes the target's drive lacks; 0 otherwise.</param>
|
||||
public sealed record WorkspaceMoveSurvey(string From, string To, IReadOnlyList<WorkspaceMoveFolder> Projects, IReadOnlyList<string> Entries, int Files, long Bytes,
|
||||
long? Free, bool SameVolume, WorkspaceMoveRefusal? Refusal, long Missing, string Fingerprint)
|
||||
{
|
||||
public bool CanMove => Refusal is null;
|
||||
|
||||
/// <summary>The refusal as one localized sentence; null when the move can go.</summary>
|
||||
public string? RefusalText => Refusal switch
|
||||
{
|
||||
WorkspaceMoveRefusal.SameFolder => Loc.L("Errors.Workspace.MoveSameFolder", To),
|
||||
WorkspaceMoveRefusal.TargetNotEmpty => Loc.L("Errors.Workspace.MoveTargetNotEmpty", To),
|
||||
WorkspaceMoveRefusal.NoRoom => Loc.L("Errors.Workspace.MoveNoRoom", (Missing / 1e9).ToString("0.0", Loc.Culture)),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>How far a move of the workspace folder went, as its journal records it.</summary>
|
||||
public enum WorkspaceMoveStage
|
||||
{
|
||||
/// <summary>The entries are being moved (one volume) or copied and verified (across volumes); the source is the workspace.</summary>
|
||||
Moving,
|
||||
|
||||
/// <summary>Every entry is verified at the target and the desktop was told: the target is the workspace; what is left of the source goes.</summary>
|
||||
Switched,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The journal of a move of the workspace folder (<c><from>/.move.json</c>, <see cref="WorkspaceMover.JournalFile"/>):
|
||||
/// where it goes, the plan's fingerprint, whether it renames folders on one volume, the entries it moves in their order,
|
||||
/// those copied and verified at the target already (across volumes), and how far it went. Written whole and atomically
|
||||
/// before every step that depends on it, so a move stopped at any moment is taken up, or undone, from it.
|
||||
/// </summary>
|
||||
public sealed record WorkspaceMoveJournal(string Format, int FormatVersion, string From, string To, string Fingerprint, bool SameVolume,
|
||||
IReadOnlyList<string> Entries, IReadOnlyList<string> Verified, WorkspaceMoveStage Stage, DateTimeOffset Started)
|
||||
{
|
||||
public const string FormatName = "chronos-workspace-move";
|
||||
public const int CurrentFormatVersion = 1;
|
||||
}
|
||||
|
||||
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
UseStringEnumConverter = true, WriteIndented = true)]
|
||||
[JsonSerializable(typeof(WorkspaceMoveJournal))]
|
||||
internal sealed partial class WorkspaceMoveJson : JsonSerializerContext;
|
||||
|
||||
/// <summary>
|
||||
/// Moves a workspace folder with its projects (phase 5c, <see cref="Core.Model.JobKind.WorkspaceMove"/>). What moves: every
|
||||
/// folder named by a project id, then <c>workspace.json</c>, then <c>.trash</c>, in that order; anything else in the folder
|
||||
/// stays where it is. On one volume each entry is renamed into the target (<see cref="Directory.Move(string, string)"/>);
|
||||
/// across volumes each file is copied into a <c>.part</c> beside its place, hashed as it is read and read back to hash the
|
||||
/// copy, and only once both SHA-256 agree renamed into place (with the source's time), so a file at its final name in the
|
||||
/// target is always one verified. The source is the workspace until every entry is at the target: then the desktop is told
|
||||
/// (the switch), and only after it does the source of a copy go. The journal (<see cref="WorkspaceMoveJournal"/>) is
|
||||
/// written in the source before the first step and at every entry: a move stopped (paused, the program closed, the PC
|
||||
/// down) goes on from it, trusting the files at their final names and the entries it records as verified; a move that
|
||||
/// fails before the switch is undone (<see cref="Abandon"/>): renamed entries go back, copies at the target are removed,
|
||||
/// the journal goes. After the switch a source entry goes only when each of its files still matches its copy at the target
|
||||
/// (name, size and time, as verified); one that changed since is kept and logged, and one that cannot be removed (a file held
|
||||
/// open) stays with the journal, so the move taken up later clears it: neither fails a move whose target is the workspace.
|
||||
/// A rename refused because the folders lie on two volumes after all (a volume mounted in a folder) turns the move into a
|
||||
/// copy before anything was renamed.
|
||||
/// </summary>
|
||||
public sealed class WorkspaceMover
|
||||
{
|
||||
/// <summary>The journal of a move under way, in the workspace folder it leaves.</summary>
|
||||
public const string JournalFile = ".move.json";
|
||||
|
||||
/// <summary>Where a file copied across volumes is written until its hash is checked.</summary>
|
||||
internal const string PartSuffix = ".part";
|
||||
|
||||
private const int BufferSize = 1 << 20;
|
||||
private const int DeleteAttempts = 5;
|
||||
|
||||
/// <param name="from">The workspace folder in use.</param>
|
||||
/// <param name="to">The folder it moves to (made when it is not there).</param>
|
||||
public WorkspaceMover(string from, string to)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(from);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(to);
|
||||
From = WorkspaceStore.FullRoot(from);
|
||||
To = WorkspaceStore.FullRoot(to);
|
||||
}
|
||||
|
||||
public string From { get; }
|
||||
|
||||
public string To { get; }
|
||||
|
||||
/// <summary>Free bytes on the volume of a folder (the host's measure); null takes the drive's (<see cref="DriveInfo"/>).</summary>
|
||||
public Func<string, long?>? FreeSpace { get; init; }
|
||||
|
||||
/// <summary>Whether both folders lie on one volume; null decides from their drives (tests set it to try the copy on one disk).</summary>
|
||||
public bool? SameVolume { get; init; }
|
||||
|
||||
/// <summary>Bytes a copy across volumes leaves free on the target's drive besides what it copies (as a copy into the workspace keeps).</summary>
|
||||
public long Reserve { get; init; } = WorkspaceCopyJobHandler.SpaceReserve;
|
||||
|
||||
/// <summary>
|
||||
/// Called at named steps with the path concerned (tests: a crash, a file damaged on its way): <c>verify</c> (a copy's
|
||||
/// <c>.part</c>, written and about to be read back), <c>copied</c> (a file at its final name), <c>moved</c> (an entry
|
||||
/// renamed into the target), <c>rename</c> (an entry about to be renamed into the target), <c>switched</c> (the desktop told).
|
||||
/// </summary>
|
||||
internal Action<string, string>? Cut { get; init; }
|
||||
|
||||
/// <summary>The journal of a move under way out of <paramref name="root"/>; null when there is none, or it cannot be read (logged).</summary>
|
||||
public static WorkspaceMoveJournal? ReadJournal(string root)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(root);
|
||||
var path = Path.Combine(WorkspaceStore.FullRoot(root), JournalFile);
|
||||
if (!File.Exists(path)) return null;
|
||||
try
|
||||
{
|
||||
if (JsonSerializer.Deserialize(AtomicFile.ReadAllText(path), WorkspaceMoveJson.Default.WorkspaceMoveJournal) is
|
||||
{ Format: WorkspaceMoveJournal.FormatName, From.Length: > 0, To.Length: > 0, Entries: not null, Verified: not null } journal
|
||||
&& journal.FormatVersion <= WorkspaceMoveJournal.CurrentFormatVersion)
|
||||
return journal;
|
||||
Log.Warning("workspace", $"{path} is not a journal of a move this Chronos reads");
|
||||
}
|
||||
catch (Exception ex) when (ex is JsonException or IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("workspace", $"{path} cannot be read", ex);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True when <paramref name="root"/> holds a move's journal, readable or not: the folder is being moved, or a move of it
|
||||
/// stopped before it ended, and nothing may write there until it is taken up or undone.
|
||||
/// </summary>
|
||||
public static bool HasJournal(string root) => File.Exists(Path.Combine(WorkspaceStore.FullRoot(root), JournalFile));
|
||||
|
||||
private string JournalPath => Path.Combine(From, JournalFile);
|
||||
|
||||
/// <summary>What the move would take and whether it can go (<see cref="WorkspaceMoveSurvey"/>); nothing is written.</summary>
|
||||
/// <exception cref="IOException">The workspace folder cannot be read.</exception>
|
||||
public WorkspaceMoveSurvey Survey()
|
||||
{
|
||||
var comparison = PathComparison;
|
||||
var projects = new List<WorkspaceMoveFolder>();
|
||||
var entries = new List<string>();
|
||||
var (files, bytes) = (0, 0L);
|
||||
var fingerprint = new StringBuilder();
|
||||
fingerprint.Append(From).Append('\n').Append(To).Append('\n');
|
||||
if (Directory.Exists(From))
|
||||
{
|
||||
foreach (var directory in Directory.EnumerateDirectories(From).Order(StringComparer.Ordinal))
|
||||
{
|
||||
var name = Path.GetFileName(directory);
|
||||
if (!Guid.TryParseExact(name, "N", out var id)) continue;
|
||||
var (count, size) = Read(directory, name, fingerprint);
|
||||
projects.Add(new WorkspaceMoveFolder(id, File.Exists(Path.Combine(directory, WorkspaceProject.BundleFile)), count, size));
|
||||
entries.Add(name);
|
||||
(files, bytes) = (files + count, bytes + size);
|
||||
}
|
||||
var manifest = Path.Combine(From, WorkspaceStore.ManifestFile);
|
||||
if (File.Exists(manifest))
|
||||
{
|
||||
var info = new FileInfo(manifest);
|
||||
entries.Add(WorkspaceStore.ManifestFile);
|
||||
Line(fingerprint, WorkspaceStore.ManifestFile, info);
|
||||
(files, bytes) = (files + 1, bytes + info.Length);
|
||||
}
|
||||
var trash = Path.Combine(From, TrashFolder);
|
||||
if (Directory.Exists(trash))
|
||||
{
|
||||
var (count, size) = Read(trash, TrashFolder, fingerprint);
|
||||
entries.Add(TrashFolder);
|
||||
(files, bytes) = (files + count, bytes + size);
|
||||
}
|
||||
}
|
||||
var same = SameVolume ?? OnOneVolume(From, To);
|
||||
var free = FreeBytes(To);
|
||||
WorkspaceMoveRefusal? refusal = null;
|
||||
long missing = 0;
|
||||
if (string.Equals(From, To, comparison) || To.StartsWith(From + Path.DirectorySeparatorChar, comparison))
|
||||
{
|
||||
refusal = WorkspaceMoveRefusal.SameFolder;
|
||||
}
|
||||
else if (File.Exists(To) || TargetHolds(entries))
|
||||
{
|
||||
refusal = WorkspaceMoveRefusal.TargetNotEmpty;
|
||||
}
|
||||
else if (!same && free is { } room && room < bytes + Reserve)
|
||||
{
|
||||
refusal = WorkspaceMoveRefusal.NoRoom;
|
||||
missing = bytes + Reserve - room;
|
||||
}
|
||||
return new WorkspaceMoveSurvey(From, To, projects, entries, files, bytes, free, same, refusal, missing,
|
||||
Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(fingerprint.ToString()))));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Moves the workspace, or takes up the move its journal records: the plan of <paramref name="fingerprint"/> is checked
|
||||
/// again first (a move taken up has checked it), every entry is moved or copied and verified, <paramref name="switched"/>
|
||||
/// is told (from, to) once all are at the target, then what is left of the source goes and the journal with it.
|
||||
/// <paramref name="progress"/> hears the files done and their total, with true where the job should keep it as its
|
||||
/// checkpoint; <paramref name="log"/> the lines of the job's log. A failure before the switch undoes what was done
|
||||
/// (<see cref="Abandon"/>) and is thrown; a cancel (a pause, the program closing) leaves the journal for the next run.
|
||||
/// </summary>
|
||||
/// <returns>The survey of the move as it began, and whether it was taken up from a journal.</returns>
|
||||
/// <exception cref="InvalidOperationException">
|
||||
/// The workspace changed since its plan (<c>Errors.Workspace.MoveChanged</c>), the move is refused now (the target filled,
|
||||
/// the room went), another move's journal is there (<c>Errors.Workspace.Moving</c>), or a copy did not match its source
|
||||
/// (<c>Errors.Workspace.MoveVerify</c>).
|
||||
/// </exception>
|
||||
/// <exception cref="IOException">A file or folder could not be read, written or moved.</exception>
|
||||
public async Task<(WorkspaceMoveSurvey Survey, bool Resumed)> MoveAsync(string fingerprint, Func<string, string, Task>? switched = null,
|
||||
Func<long, long, bool, ValueTask>? progress = null, Action<string>? log = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(fingerprint);
|
||||
var survey = Survey();
|
||||
var journal = ReadJournal(From);
|
||||
var resumed = journal is not null;
|
||||
if (journal is null)
|
||||
{
|
||||
if (HasJournal(From)) throw new InvalidOperationException(Loc.L("Errors.Workspace.Moving"));
|
||||
if (!string.Equals(survey.Fingerprint, fingerprint, StringComparison.Ordinal)) throw new InvalidOperationException(Loc.L("Errors.Workspace.MoveChanged"));
|
||||
if (survey.RefusalText is { } refused) throw new InvalidOperationException(refused);
|
||||
journal = new WorkspaceMoveJournal(WorkspaceMoveJournal.FormatName, WorkspaceMoveJournal.CurrentFormatVersion, From, To, fingerprint, survey.SameVolume,
|
||||
survey.Entries, [], WorkspaceMoveStage.Moving, DateTimeOffset.UtcNow);
|
||||
Directory.CreateDirectory(To);
|
||||
WriteJournal(journal);
|
||||
Log.Info("workspace", $"Moving the workspace {From} to {To} ({survey.Entries.Count} entries, {survey.Files} files, {survey.Bytes} bytes, "
|
||||
+ (survey.SameVolume ? "renamed" : "copied") + ")");
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!string.Equals(journal.To, To, PathComparison) || !string.Equals(journal.From, From, PathComparison))
|
||||
throw new InvalidOperationException(Loc.L("Errors.Workspace.Moving"));
|
||||
log?.Invoke(Loc.L("Archive.Log.WorkspaceMoveResumed", JournalPath));
|
||||
Log.Info("workspace", $"The move of the workspace {From} to {To} is taken up from its journal ({journal.Stage})");
|
||||
}
|
||||
|
||||
if (journal.Stage == WorkspaceMoveStage.Moving)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (journal.SameVolume)
|
||||
{
|
||||
try
|
||||
{
|
||||
journal = await RenameAsync(journal, progress, log, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (IOException ex) when (IsCrossVolume(ex) && NothingRenamed(journal))
|
||||
{
|
||||
// Two volumes under one drive letter (a volume mounted in a folder): the files are copied and verified.
|
||||
Log.Info("workspace", $"{From} and {To} lie on two volumes after all: the move copies and verifies the files ({ex.Message})");
|
||||
if (FreeBytes(To) is { } room && room < survey.Bytes + Reserve)
|
||||
throw new InvalidOperationException(Loc.L("Errors.Workspace.MoveNoRoom", ((survey.Bytes + Reserve - room) / 1e9).ToString("0.0", Loc.Culture)), ex);
|
||||
journal = journal with { SameVolume = false };
|
||||
WriteJournal(journal);
|
||||
journal = await CopyAsync(journal, survey, progress, log, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
journal = await CopyAsync(journal, survey, progress, log, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException || !cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
Log.Warning("workspace", $"The move of the workspace {From} to {To} failed; it is undone", ex);
|
||||
TryAbandon();
|
||||
throw;
|
||||
}
|
||||
journal = journal with { Stage = WorkspaceMoveStage.Switched };
|
||||
WriteJournal(journal);
|
||||
}
|
||||
// From here on the target is the workspace: told again when a run taken up after the switch did not see its answer.
|
||||
if (switched is not null) await switched(From, To).ConfigureAwait(false);
|
||||
Cut?.Invoke("switched", To);
|
||||
var left = Clean(journal);
|
||||
log?.Invoke(Loc.L("Archive.Log.WorkspaceMoved", To, journal.Entries.Count(e => Guid.TryParseExact(e, "N", out _)), survey.Files));
|
||||
if (left > 0) log?.Invoke(Loc.L("Archive.Log.WorkspaceMoveLeft", From, left));
|
||||
Log.Info("workspace", $"The workspace moved from {From} to {To}" + (left > 0 ? $"; {left} entr(ies) of the old folder stay there" : string.Empty));
|
||||
return (survey, resumed);
|
||||
}
|
||||
|
||||
/// <summary>A move or rename refused because source and target lie on two volumes (Windows <c>ERROR_NOT_SAME_DEVICE</c>, Unix <c>EXDEV</c>).</summary>
|
||||
private static bool IsCrossVolume(IOException ex) => (ex.HResult & 0xFFFF) == (OperatingSystem.IsWindows() ? 0x11 : 18);
|
||||
|
||||
/// <summary>No entry of the journal was renamed into the target yet.</summary>
|
||||
private bool NothingRenamed(WorkspaceMoveJournal journal) => journal.Entries.All(e => !Exists(Path.Combine(To, e)));
|
||||
|
||||
/// <summary>
|
||||
/// Undoes a move its journal records as not switched yet, so the source stays the workspace: on one volume the entries
|
||||
/// renamed into the target go back; across volumes the copies it made at the target are removed (what cannot be removed
|
||||
/// is logged: the source is whole). The journal then goes. A move that switched is not undone (false): the target is the
|
||||
/// workspace, and the move is taken up to finish clearing the source. True when there was nothing left to undo, or it was.
|
||||
/// </summary>
|
||||
/// <exception cref="IOException">An entry renamed into the target could not go back; the journal stays, and the move can be taken up.</exception>
|
||||
public bool Abandon()
|
||||
{
|
||||
var journal = ReadJournal(From);
|
||||
if (journal is null) return true;
|
||||
if (journal.Stage != WorkspaceMoveStage.Moving) return false;
|
||||
foreach (var entry in journal.Entries.Reverse())
|
||||
{
|
||||
var source = Path.Combine(From, entry);
|
||||
var target = Path.Combine(To, entry);
|
||||
if (journal.SameVolume)
|
||||
{
|
||||
if (Exists(target) && !Exists(source)) Rename(target, source);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
// Refused when the target held the entry: whatever is there is this move's copy.
|
||||
if (Directory.Exists(target)) Directory.Delete(target, recursive: true);
|
||||
else if (File.Exists(target)) File.Delete(target);
|
||||
var part = target + PartSuffix;
|
||||
if (File.Exists(part)) File.Delete(part);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("workspace", $"The copy {target} of an undone move could not be removed", ex);
|
||||
}
|
||||
}
|
||||
File.Delete(JournalPath);
|
||||
TryDeleteEmpty(To);
|
||||
Log.Info("workspace", $"The move of the workspace {From} to {To} was undone; the workspace stays in {From}");
|
||||
return true;
|
||||
}
|
||||
|
||||
private void TryAbandon()
|
||||
{
|
||||
try
|
||||
{
|
||||
Abandon();
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("workspace", $"The move of the workspace {From} to {To} could not be undone; its journal stays to take it up", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>One volume: each entry renamed into the target, an entry already there (a run that stopped after it) taken as moved.</summary>
|
||||
private async Task<WorkspaceMoveJournal> RenameAsync(WorkspaceMoveJournal journal, Func<long, long, bool, ValueTask>? progress, Action<string>? log,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var moved = 0;
|
||||
foreach (var entry in journal.Entries)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var source = Path.Combine(From, entry);
|
||||
var target = Path.Combine(To, entry);
|
||||
if (Exists(source))
|
||||
{
|
||||
if (Exists(target)) throw new IOException(Loc.L("Errors.Workspace.MoveTargetNotEmpty", To));
|
||||
Cut?.Invoke("rename", source);
|
||||
Rename(source, target);
|
||||
Cut?.Invoke("moved", target);
|
||||
if (Guid.TryParseExact(entry, "N", out _)) log?.Invoke(Loc.L("Archive.Log.WorkspaceMoveProject", Name(target)));
|
||||
}
|
||||
else if (!Exists(target))
|
||||
{
|
||||
// Gone from both sides: nothing to move (the trash emptied meanwhile cannot happen while the move holds the workspace).
|
||||
Log.Warning("workspace", $"{entry} of the workspace {From} is neither there nor at {To}");
|
||||
}
|
||||
moved++;
|
||||
if (progress is not null) await progress(moved, journal.Entries.Count, true).ConfigureAwait(false);
|
||||
}
|
||||
return journal;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Across volumes: each entry's files copied and verified (<see cref="CopyFileAsync"/>), the entry recorded as verified in
|
||||
/// the journal once all of its files are; nothing of the source is touched.
|
||||
/// </summary>
|
||||
private async Task<WorkspaceMoveJournal> CopyAsync(WorkspaceMoveJournal journal, WorkspaceMoveSurvey survey, Func<long, long, bool, ValueTask>? progress,
|
||||
Action<string>? log, CancellationToken cancellationToken)
|
||||
{
|
||||
var total = (long)survey.Files;
|
||||
long done = 0;
|
||||
var clock = System.Diagnostics.Stopwatch.StartNew();
|
||||
foreach (var entry in journal.Entries)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var source = Path.Combine(From, entry);
|
||||
var target = Path.Combine(To, entry);
|
||||
if (journal.Verified.Contains(entry, StringComparer.Ordinal))
|
||||
{
|
||||
done += Directory.Exists(source) ? Listed(source).Count : File.Exists(source) ? 1 : 0;
|
||||
continue;
|
||||
}
|
||||
if (File.Exists(source))
|
||||
{
|
||||
await CopyFileAsync(source, target, cancellationToken).ConfigureAwait(false);
|
||||
done++;
|
||||
}
|
||||
else if (Directory.Exists(source))
|
||||
{
|
||||
Directory.CreateDirectory(target);
|
||||
foreach (var folder in new DirectoryInfo(source).EnumerateDirectories("*", SearchOption.AllDirectories))
|
||||
Directory.CreateDirectory(Path.Combine(target, Path.GetRelativePath(source, folder.FullName)));
|
||||
foreach (var relative in Listed(source))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
await CopyFileAsync(Path.Combine(source, relative), Path.Combine(target, relative), cancellationToken).ConfigureAwait(false);
|
||||
done++;
|
||||
var durable = clock.Elapsed >= CheckpointEvery;
|
||||
if (durable) clock.Restart();
|
||||
if (progress is not null) await progress(done, total, durable).ConfigureAwait(false);
|
||||
}
|
||||
if (Guid.TryParseExact(entry, "N", out _)) log?.Invoke(Loc.L("Archive.Log.WorkspaceMoveProject", Name(source)));
|
||||
}
|
||||
journal = journal with { Verified = [.. journal.Verified, entry] };
|
||||
WriteJournal(journal);
|
||||
if (progress is not null) await progress(done, total, true).ConfigureAwait(false);
|
||||
}
|
||||
return journal;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One file across volumes: a file at its final name with the source's size was verified by an earlier run and stays;
|
||||
/// otherwise the source is streamed into <c><target>.part</c> and hashed as it is read, the copy is read back and
|
||||
/// hashed, and only when both agree is it renamed into place, with the source's time.
|
||||
/// </summary>
|
||||
/// <exception cref="InvalidOperationException">The copy does not match its source (<c>Errors.Workspace.MoveVerify</c>); the <c>.part</c> is removed.</exception>
|
||||
private async Task CopyFileAsync(string source, string target, CancellationToken cancellationToken)
|
||||
{
|
||||
var info = new FileInfo(source);
|
||||
if (File.Exists(target) && new FileInfo(target).Length == info.Length) return;
|
||||
var part = target + PartSuffix;
|
||||
try
|
||||
{
|
||||
string sha;
|
||||
using (var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256))
|
||||
{
|
||||
var buffer = new byte[BufferSize];
|
||||
await using (var input = new FileStream(source, FileMode.Open, FileAccess.Read, FileShare.Read, BufferSize, FileOptions.Asynchronous | FileOptions.SequentialScan))
|
||||
await using (var output = new FileStream(part, FileMode.Create, FileAccess.Write, FileShare.None, BufferSize, FileOptions.Asynchronous))
|
||||
{
|
||||
int read;
|
||||
while ((read = await input.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) > 0)
|
||||
{
|
||||
hash.AppendData(buffer, 0, read);
|
||||
await output.WriteAsync(buffer.AsMemory(0, read), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
await output.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
output.Flush(flushToDisk: true);
|
||||
}
|
||||
sha = Convert.ToHexStringLower(hash.GetHashAndReset());
|
||||
}
|
||||
Cut?.Invoke("verify", part);
|
||||
if (!string.Equals(await Sha256Async(part, cancellationToken).ConfigureAwait(false), sha, StringComparison.Ordinal))
|
||||
throw new InvalidOperationException(Loc.L("Errors.Workspace.MoveVerify", LongPath.ToPortableRelative(From, source)));
|
||||
File.Move(part, target, overwrite: true);
|
||||
File.SetLastWriteTimeUtc(target, info.LastWriteTimeUtc);
|
||||
Cut?.Invoke("copied", target);
|
||||
}
|
||||
catch
|
||||
{
|
||||
TryDelete(part);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// After the switch: across volumes every entry of the source goes, folders first, the manifest last, each only while
|
||||
/// every file of it still matches its copy at the target (<see cref="StillAsCopied"/>): one that changed since it was
|
||||
/// verified is kept, and logged, never deleted. Then the journal, and the source folder when nothing else is left in it.
|
||||
/// An entry that cannot be removed (a file held open past a few attempts) stays, and the journal with it naming only
|
||||
/// such entries, so the move taken up clears them.
|
||||
/// </summary>
|
||||
/// <returns>The entries of the source left where they are.</returns>
|
||||
private int Clean(WorkspaceMoveJournal journal)
|
||||
{
|
||||
var stuck = new List<string>();
|
||||
var kept = 0;
|
||||
if (!journal.SameVolume)
|
||||
{
|
||||
foreach (var entry in journal.Entries.OrderBy(e => e == WorkspaceStore.ManifestFile ? 1 : 0))
|
||||
{
|
||||
var source = Path.Combine(From, entry);
|
||||
if (!Exists(source)) continue;
|
||||
if (!StillAsCopied(source, Path.Combine(To, entry)))
|
||||
{
|
||||
kept++;
|
||||
Log.Warning("workspace", $"{source} changed after it was copied to {To}: it is kept where it is, never deleted");
|
||||
continue;
|
||||
}
|
||||
for (var attempt = 1; ; attempt++)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(source)) Directory.Delete(source, recursive: true);
|
||||
else if (File.Exists(source)) File.Delete(source);
|
||||
break;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
if (attempt >= DeleteAttempts)
|
||||
{
|
||||
Log.Warning("workspace", $"{source} could not be removed after the move: it stays, and the move taken up removes it", ex);
|
||||
stuck.Add(entry);
|
||||
break;
|
||||
}
|
||||
// An indexer or an antivirus reading a file a moment: tried again shortly.
|
||||
Thread.Sleep(100 * attempt);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (stuck.Count > 0)
|
||||
{
|
||||
WriteJournal(journal with { Entries = stuck, Verified = [.. journal.Verified.Where(stuck.Contains)] });
|
||||
return stuck.Count + kept;
|
||||
}
|
||||
File.Delete(JournalPath);
|
||||
TryDeleteEmpty(From);
|
||||
return kept;
|
||||
}
|
||||
|
||||
/// <summary>Clock steps a target's file system may round a file's time to (FAT keeps two seconds).</summary>
|
||||
private static readonly TimeSpan TimeSlack = TimeSpan.FromSeconds(2);
|
||||
|
||||
/// <summary>
|
||||
/// Every file of the source entry (a file, or a folder's files, leases left out) is at the target with the same relative
|
||||
/// name and size and the time the copy gave it (the source's, within the target's rounding): nothing came or changed in
|
||||
/// the source since it was verified.
|
||||
/// </summary>
|
||||
private static bool StillAsCopied(string source, string target)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (File.Exists(source)) return Matches(new FileInfo(source), new FileInfo(target));
|
||||
return Listed(source).All(relative => Matches(new FileInfo(Path.Combine(source, relative)), new FileInfo(Path.Combine(target, relative))));
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Debug("workspace", $"{source} could not be compared with {target}: {ex.Message}");
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool Matches(FileInfo from, FileInfo to) =>
|
||||
to.Exists && from.Length == to.Length && (from.LastWriteTimeUtc - to.LastWriteTimeUtc).Duration() <= TimeSlack;
|
||||
}
|
||||
|
||||
/// <summary>A folder's files relative to it, in ordinal order, the leases left out (they belong to whoever holds the project there).</summary>
|
||||
private static List<string> Listed(string folder) =>
|
||||
[.. new DirectoryInfo(folder).EnumerateFiles("*", SearchOption.AllDirectories).Where(f => !WorkspaceProject.IsLeaseFile(f.Name))
|
||||
.Select(f => Path.GetRelativePath(folder, f.FullName)).Order(StringComparer.Ordinal)];
|
||||
|
||||
/// <summary>Files and bytes of an entry, each file's name, size and time added to the fingerprint.</summary>
|
||||
private static (int Files, long Bytes) Read(string folder, string name, StringBuilder fingerprint)
|
||||
{
|
||||
var (files, bytes) = (0, 0L);
|
||||
foreach (var relative in Listed(folder))
|
||||
{
|
||||
var info = new FileInfo(Path.Combine(folder, relative));
|
||||
Line(fingerprint, name + "/" + relative.Replace('\\', '/'), info);
|
||||
files++;
|
||||
bytes += info.Length;
|
||||
}
|
||||
return (files, bytes);
|
||||
}
|
||||
|
||||
private static void Line(StringBuilder fingerprint, string name, FileInfo info) =>
|
||||
fingerprint.Append(name).Append('|').Append(info.Length.ToString(CultureInfo.InvariantCulture)).Append('|')
|
||||
.Append(info.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture)).Append('\n');
|
||||
|
||||
/// <summary>The target holds a workspace's own files, or a folder or file named as one of the entries that move.</summary>
|
||||
private bool TargetHolds(IEnumerable<string> entries) =>
|
||||
Directory.Exists(To) && (Exists(Path.Combine(To, WorkspaceStore.ManifestFile)) || Exists(Path.Combine(To, TrashFolder)) || Exists(Path.Combine(To, JournalFile))
|
||||
|| entries.Any(e => Exists(Path.Combine(To, e))));
|
||||
|
||||
private const string TrashFolder = ".trash";
|
||||
|
||||
private static readonly TimeSpan CheckpointEvery = TimeSpan.FromSeconds(5);
|
||||
|
||||
private static StringComparison PathComparison => OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
|
||||
|
||||
private static bool Exists(string path) => Directory.Exists(path) || File.Exists(path);
|
||||
|
||||
private static void Rename(string source, string target)
|
||||
{
|
||||
if (Directory.Exists(source)) Directory.Move(source, target);
|
||||
else File.Move(source, target);
|
||||
}
|
||||
|
||||
/// <summary>The name a project's working state gives it, for the log; its folder's name when it cannot be read.</summary>
|
||||
private static string Name(string folder)
|
||||
{
|
||||
var bundle = Path.Combine(folder, WorkspaceProject.BundleFile);
|
||||
try
|
||||
{
|
||||
if (File.Exists(bundle) && Core.Project.ProjectBundle.Load(bundle).Workspace?.Name is { Length: > 0 } name) return name;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException or JsonException)
|
||||
{
|
||||
Log.Debug("workspace", $"{bundle} names no project: {ex.Message}");
|
||||
}
|
||||
return Path.GetFileName(folder);
|
||||
}
|
||||
|
||||
private void WriteJournal(WorkspaceMoveJournal journal) =>
|
||||
AtomicFile.WriteAllText(JournalPath, JsonSerializer.Serialize(journal, WorkspaceMoveJson.Default.WorkspaceMoveJournal));
|
||||
|
||||
/// <summary>Free bytes on the volume of <paramref name="folder"/> (its nearest folder that exists); null when they cannot be measured.</summary>
|
||||
private long? FreeBytes(string folder)
|
||||
{
|
||||
var existing = Nearest(folder);
|
||||
if (existing is null) return null;
|
||||
if (FreeSpace is { } measure) return measure(existing);
|
||||
try
|
||||
{
|
||||
// Windows wants a drive root; Unix accepts any directory and reports the volume it is mounted on.
|
||||
return new DriveInfo(OperatingSystem.IsWindows() ? Path.GetPathRoot(existing)! : existing).AvailableFreeSpace;
|
||||
}
|
||||
catch (Exception ex) when (ex is ArgumentException or IOException or UnauthorizedAccessException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Both folders on one volume: on Windows the same drive; on Unix the same mount point. A doubt reads as two volumes,
|
||||
/// whose copy is the safe way (a rename across volumes would fail).
|
||||
/// </summary>
|
||||
private static bool OnOneVolume(string from, string to)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (Nearest(to) is not { } target) return false;
|
||||
if (OperatingSystem.IsWindows())
|
||||
return string.Equals(Path.GetPathRoot(from), Path.GetPathRoot(target), StringComparison.OrdinalIgnoreCase)
|
||||
&& !Path.GetPathRoot(from)!.StartsWith(@"\\", StringComparison.Ordinal);
|
||||
var mounts = DriveInfo.GetDrives().Select(d => Path.TrimEndingDirectorySeparator(d.RootDirectory.FullName)).ToList();
|
||||
return string.Equals(Mount(from, mounts), Mount(target, mounts), StringComparison.Ordinal);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static string? Mount(string path, List<string> mounts) =>
|
||||
mounts.Where(m => path == m || path.StartsWith(m.EndsWith('/') ? m : m + "/", StringComparison.Ordinal)).MaxBy(m => m.Length);
|
||||
}
|
||||
|
||||
/// <summary><paramref name="folder"/> or its nearest parent that exists; null when none does.</summary>
|
||||
private static string? Nearest(string folder)
|
||||
{
|
||||
for (var current = folder; !string.IsNullOrEmpty(current); current = Path.GetDirectoryName(current))
|
||||
{
|
||||
if (Directory.Exists(current)) return current;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static async Task<string> Sha256Async(string path, CancellationToken cancellationToken)
|
||||
{
|
||||
await using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, BufferSize, FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
return Convert.ToHexStringLower(await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false));
|
||||
}
|
||||
|
||||
private static void TryDelete(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
File.Delete(path);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Debug("workspace", $"{path} stays: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private static void TryDeleteEmpty(string folder)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(folder) && !Directory.EnumerateFileSystemEntries(folder).Any()) Directory.Delete(folder);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Debug("workspace", $"{folder} stays: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
using System.Globalization;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using Chronos.Core.Diagnostics;
|
||||
using Chronos.Core.Localization;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Core.Project;
|
||||
using Chronos.Data.Archiving;
|
||||
using Chronos.Data.Projects;
|
||||
|
||||
namespace Chronos.Data.Workspaces;
|
||||
|
||||
/// <summary>
|
||||
/// What "Proxies only" (phase 5c) does to a workspace project that holds its originals: <paramref name="Originals"/> files
|
||||
/// of <paramref name="Bytes"/> the home holds with the same SHA-256, deleted; <paramref name="ToTrash"/> files of
|
||||
/// <paramref name="TrashBytes"/> it does not (gone there, damaged, or another photo now), moved to the workspace's trash.
|
||||
/// <paramref name="Fingerprint"/> names the files and the home's answer the plan read: an apply refuses a plan the
|
||||
/// project or the home changed since.
|
||||
/// </summary>
|
||||
public sealed record WorkspaceProxiesOnlyPlan(Guid ProjectId, int Originals, long Bytes, int ToTrash, long TrashBytes, string Fingerprint)
|
||||
{
|
||||
/// <summary>Nothing to remove: the project holds no originals.</summary>
|
||||
public bool IsEmpty => Originals == 0 && ToTrash == 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The originals of a workspace project (phase 5c): what a copy of them writes, and their removal. An original is never
|
||||
/// erased on a guess: a file is deleted only when the home holds a photo of its SHA-256, read from the file at that moment,
|
||||
/// that the home has not flagged missing or damaged; any other goes to the workspace's trash
|
||||
/// (<c><root>/.trash/<project id>-<ticks>/originals/</c>, which the clean-up empties).
|
||||
/// </summary>
|
||||
public static class WorkspaceOriginals
|
||||
{
|
||||
/// <summary>The workspace's trash folder, under its root (the App's <c>WorkspaceHost.TrashFolder</c>).</summary>
|
||||
public const string TrashFolder = ".trash";
|
||||
|
||||
/// <summary>
|
||||
/// The bytes a copy of the sequence's originals writes from this PC's archive: all the archive has, or only those
|
||||
/// <paramref name="project"/> lacks when it holds some already.
|
||||
/// </summary>
|
||||
public static Task<long> ArchiveOriginalsBytesAsync(ChronosDatabase db, ArchiveService archives, Guid sequenceId, WorkspaceProject? project = null,
|
||||
CancellationToken cancellationToken = default) =>
|
||||
new WorkspaceCopyJobHandler(db, archives).OriginalsBytesAsync(sequenceId, project, cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// What "Proxies only" removes from <paramref name="project"/>, each original hashed now and weighed against
|
||||
/// <paramref name="home"/> (the home's frames as read just before): nothing when it holds none.
|
||||
/// </summary>
|
||||
/// <exception cref="InvalidDataException">The media index is damaged or newer.</exception>
|
||||
public static WorkspaceProxiesOnlyPlan PlanProxiesOnly(WorkspaceProject project, HomeFrames home)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(project);
|
||||
ArgumentNullException.ThrowIfNull(home);
|
||||
var files = Files(project);
|
||||
var held = HeldAtHome(home.Frames);
|
||||
var fingerprint = new StringBuilder(home.FramesVersion.ToString(CultureInfo.InvariantCulture));
|
||||
int deleted = 0, trashed = 0;
|
||||
long deletedBytes = 0, trashedBytes = 0;
|
||||
foreach (var file in files)
|
||||
{
|
||||
var (sha, bytes) = Hash(file);
|
||||
var atHome = sha is not null && held.Contains(sha);
|
||||
if (atHome) (deleted, deletedBytes) = (deleted + 1, deletedBytes + bytes);
|
||||
else (trashed, trashedBytes) = (trashed + 1, trashedBytes + bytes);
|
||||
fingerprint.Append('|').Append(Path.GetFileName(file)).Append(':').Append(sha).Append(':').Append(atHome ? 'd' : 't');
|
||||
}
|
||||
var digest = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(fingerprint.ToString())));
|
||||
return new WorkspaceProxiesOnlyPlan(project.ProjectId, deleted, deletedBytes, trashed, trashedBytes, digest);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Applies a plan of <see cref="PlanProxiesOnly"/> under the project's lease: planned again, refused when anything changed
|
||||
/// (<c>Errors.Workspace.MediaChanged</c>); then the media index and the project file say proxies first (so a stop
|
||||
/// half way leaves media that name no original that is gone), and last each original is deleted or trashed as the plan
|
||||
/// says, hashed again as it goes.
|
||||
/// </summary>
|
||||
/// <returns>The files moved to the trash.</returns>
|
||||
/// <exception cref="InvalidOperationException">The project or the home changed since the plan.</exception>
|
||||
/// <exception cref="Chronos.Core.Archiving.LeaseUnavailableException">Another writer holds the project.</exception>
|
||||
public static async Task<int> ApplyProxiesOnlyAsync(WorkspaceProject project, WorkspaceProxiesOnlyPlan plan, HomeFrames home, TimeSpan lockWait,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(project);
|
||||
ArgumentNullException.ThrowIfNull(plan);
|
||||
ArgumentNullException.ThrowIfNull(home);
|
||||
using (await project.LockAsync(lockWait, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var now = PlanProxiesOnly(project, home);
|
||||
if (now.ProjectId != plan.ProjectId || now.Fingerprint != plan.Fingerprint) throw new InvalidOperationException(Loc.L("Errors.Workspace.MediaChanged"));
|
||||
var media = project.ReadMedia();
|
||||
var files = Files(project);
|
||||
if (media is not null)
|
||||
{
|
||||
project.WriteMedia(media with
|
||||
{
|
||||
Kind = BundleMedia.Proxies,
|
||||
Frames = [.. media.Frames.Select(f => f with { Original = null, OriginalSha256 = null, OriginalBytes = 0 })],
|
||||
});
|
||||
}
|
||||
var bundle = project.ReadBundle();
|
||||
if (bundle.Workspace is { } workspace && workspace.Media.Kind != BundleMedia.Proxies)
|
||||
project.WriteBundle(bundle with { Workspace = workspace with { Media = workspace.Media with { Kind = BundleMedia.Proxies } } });
|
||||
var trashed = DeleteOrTrash(project, files, home.Frames);
|
||||
var staging = Path.Combine(project.OriginalsDirectory, StagingFolder);
|
||||
if (Directory.Exists(staging)) TryDeleteFolder(staging);
|
||||
if (Directory.Exists(project.OriginalsDirectory) && !Directory.EnumerateFileSystemEntries(project.OriginalsDirectory).Any()) TryDeleteFolder(project.OriginalsDirectory);
|
||||
Log.Info("workspace", $"Originals of project {project.ProjectId} removed: {plan.Originals} deleted, {trashed} kept in {TrashFolder}");
|
||||
return trashed;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes each of <paramref name="files"/> (originals of <paramref name="project"/>) whose SHA-256, read now, is that of a
|
||||
/// photo <paramref name="home"/> holds, not flagged missing or damaged; moves the others to the workspace's trash (one
|
||||
/// entry, filled under its staging name and named once whole). A file that can be neither deleted nor moved (held open)
|
||||
/// stays where it is, and is logged.
|
||||
/// </summary>
|
||||
/// <returns>The files moved to the trash.</returns>
|
||||
internal static int DeleteOrTrash(WorkspaceProject project, IReadOnlyList<string> files, IReadOnlyList<Frame> home)
|
||||
{
|
||||
if (files.Count == 0) return 0;
|
||||
using var trash = WorkspaceTrash.Begin(project.Store.Root, project.ProjectId, DateTime.UtcNow);
|
||||
return DeleteOrTrash(project, files, home, Path.Combine(trash.Path, WorkspaceProject.OriginalsFolder)).Trashed;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Clears the project's folder of originals for good: every file in it, whether the media index names it or not, hashed
|
||||
/// now and deleted only when <paramref name="home"/> (the home's frames as just read; null when they are not known) holds
|
||||
/// a photo of that SHA-256 not flagged missing or damaged, else moved into <paramref name="trashEntry"/>'s
|
||||
/// <c>originals</c> folder (made when needed). The parts a copy left (<c>.part</c>, <c>.staging</c>: downloads not
|
||||
/// finished) are deleted, and the folder goes once nothing is left in it. The caller holds the project's lease.
|
||||
/// </summary>
|
||||
/// <returns>
|
||||
/// The files deleted, those moved to the trash with their bytes, and those that stay where they are (neither deleted nor
|
||||
/// moved: held open), which the folder keeps.
|
||||
/// </returns>
|
||||
public static (int Deleted, int Trashed, long TrashedBytes, int Left) ClearOriginals(WorkspaceProject project, IReadOnlyList<Frame>? home, string trashEntry)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(project);
|
||||
ArgumentException.ThrowIfNullOrEmpty(trashEntry);
|
||||
var folder = project.OriginalsDirectory;
|
||||
if (!Directory.Exists(folder)) return (0, 0, 0, 0);
|
||||
var staging = Path.Combine(folder, StagingFolder);
|
||||
var files = new List<string>();
|
||||
var parts = new List<string>();
|
||||
foreach (var file in Directory.EnumerateFiles(folder, "*", SearchOption.AllDirectories))
|
||||
{
|
||||
if (Inside(staging, file)) continue;
|
||||
(file.EndsWith(".part", StringComparison.OrdinalIgnoreCase) ? parts : files).Add(file);
|
||||
}
|
||||
var (deleted, trashed, trashedBytes, left) = DeleteOrTrash(project, files, home ?? [], Path.Combine(trashEntry, WorkspaceProject.OriginalsFolder));
|
||||
foreach (var part in parts)
|
||||
{
|
||||
try
|
||||
{
|
||||
File.Delete(part);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("workspace", $"{part} could not be removed", ex);
|
||||
}
|
||||
}
|
||||
if (Directory.Exists(staging)) TryDeleteFolder(staging);
|
||||
RemoveEmptyFolders(folder);
|
||||
return (deleted, trashed, trashedBytes, left);
|
||||
}
|
||||
|
||||
/// <summary>The folder of originals a copy from a server downloads into until a file is whole.</summary>
|
||||
internal const string StagingFolder = ".staging";
|
||||
|
||||
private static (int Deleted, int Trashed, long TrashedBytes, int Left) DeleteOrTrash(WorkspaceProject project, IReadOnlyList<string> files, IReadOnlyList<Frame> home,
|
||||
string trash)
|
||||
{
|
||||
var held = HeldAtHome(home);
|
||||
int deleted = 0, trashed = 0, left = 0;
|
||||
long trashedBytes = 0;
|
||||
foreach (var file in files)
|
||||
{
|
||||
try
|
||||
{
|
||||
var (sha, bytes) = Hash(file);
|
||||
if (sha is not null && held.Contains(sha))
|
||||
{
|
||||
File.Delete(file);
|
||||
deleted++;
|
||||
continue;
|
||||
}
|
||||
var target = Path.Combine(trash, Path.GetRelativePath(project.OriginalsDirectory, file));
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(target)!);
|
||||
// Never over another file of the entry: one that cannot go there stays where it is.
|
||||
File.Move(file, target, overwrite: false);
|
||||
trashed++;
|
||||
trashedBytes += bytes;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
left++;
|
||||
Log.Warning("workspace", $"{file} could not be removed; it stays where it is", ex);
|
||||
}
|
||||
}
|
||||
return (deleted, trashed, trashedBytes, left);
|
||||
}
|
||||
|
||||
private static bool Inside(string folder, string path) =>
|
||||
path.StartsWith(Path.TrimEndingDirectorySeparator(folder) + Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>Removes the empty folders under <paramref name="folder"/>, and <paramref name="folder"/> itself when it is left empty.</summary>
|
||||
private static void RemoveEmptyFolders(string folder)
|
||||
{
|
||||
try
|
||||
{
|
||||
foreach (var sub in Directory.EnumerateDirectories(folder, "*", SearchOption.AllDirectories).OrderByDescending(d => d.Length).ToList())
|
||||
{
|
||||
if (!Directory.EnumerateFileSystemEntries(sub).Any()) Directory.Delete(sub);
|
||||
}
|
||||
if (!Directory.EnumerateFileSystemEntries(folder).Any()) Directory.Delete(folder);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Debug("workspace", $"{folder} stays: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The original files in the project's folder of originals (never a part a copy writes, nor its staging folder).</summary>
|
||||
private static List<string> Files(WorkspaceProject project)
|
||||
{
|
||||
if (!Directory.Exists(project.OriginalsDirectory)) return [];
|
||||
return [.. Directory.EnumerateFiles(project.OriginalsDirectory)
|
||||
.Where(f => !f.EndsWith(".part", StringComparison.OrdinalIgnoreCase))
|
||||
.Order(StringComparer.OrdinalIgnoreCase)];
|
||||
}
|
||||
|
||||
/// <summary>The SHA-256 (lower case) of the photos the home holds, flagged neither missing nor damaged.</summary>
|
||||
private static HashSet<string> HeldAtHome(IReadOnlyList<Frame> home) =>
|
||||
home.Where(f => !string.IsNullOrEmpty(f.Sha256) && (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0)
|
||||
.Select(f => f.Sha256!.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
|
||||
|
||||
private static (string? Sha256, long Bytes) Hash(string file)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var stream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 20, FileOptions.SequentialScan);
|
||||
return (Convert.ToHexStringLower(SHA256.HashData(stream)), stream.Length);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("workspace", $"{file} could not be read", ex);
|
||||
return (null, 0);
|
||||
}
|
||||
}
|
||||
|
||||
private static void TryDeleteFolder(string folder)
|
||||
{
|
||||
try
|
||||
{
|
||||
Directory.Delete(folder, recursive: true);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("workspace", $"{folder} could not be removed", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,302 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Chronos.Core.Analysis.Deep;
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.Export;
|
||||
using Chronos.Core.Localization;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Core.Project;
|
||||
using Chronos.Core.Render;
|
||||
using Chronos.Data.Archiving;
|
||||
using Chronos.Data.Jobs;
|
||||
using Chronos.Data.Processing;
|
||||
using Chronos.Data.Processing.Jobs;
|
||||
using SequenceFrames = Chronos.Data.Processing.SequenceFrames;
|
||||
|
||||
namespace Chronos.Data.Workspaces;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.WorkspaceRender"/> (phase 5c, <see cref="WorkspaceJobs.Render"/>): a workspace project rendered on this
|
||||
/// PC from the originals its media hold, as <see cref="Processing.Jobs.RenderJobHandler"/> renders an archive's sequence,
|
||||
/// into the project's <c>exports/</c> (each output written by the encoder under a hidden temporary name, then renamed), each
|
||||
/// output hashed and recorded in the project's ledger (<see cref="WorkspaceRenders"/>, its id
|
||||
/// <c>RenderJobHandler.RenderId(jobId, index)</c>), then a <see cref="JobKind.WorkspaceSend"/> queued of every render of the
|
||||
/// project not sent yet (this run's, an earlier run's of this job, and those whose send failed before), whenever this job
|
||||
/// ever recorded one that is not sent: a resumed or retried run queues it too. A render queued before the workspace moved
|
||||
/// works where the workspace is now, and none runs while it is being moved (<see cref="WorkspaceJobRoots"/>).
|
||||
/// Media without an original for every placed frame are refused. The project's lease is held while the inputs are read
|
||||
/// and while each ledger entry is written, not for the whole render: the editor keeps saving its working state meanwhile.
|
||||
/// The settings rendered are the ones the job embeds (the editor's portable settings), else the version it names from
|
||||
/// <c>versions/</c>; the version is recorded only when <c>versions/</c> holds it with the content hash of what rendered.
|
||||
/// Checkpoint: the entries recorded and the output being written; a resumed job removes that output's leftovers (never one
|
||||
/// the ledger names) and renders only what is not recorded yet.
|
||||
/// </summary>
|
||||
internal sealed class WorkspaceRenderJobHandler(ChronosDatabase db, ArchiveService archives, WorkspaceJobOptions options) : IJobHandler
|
||||
{
|
||||
private static readonly TimeSpan LockWait = TimeSpan.FromMinutes(2);
|
||||
|
||||
public JobKind Kind => JobKind.WorkspaceRender;
|
||||
|
||||
public async Task RunAsync(JobContext context)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
// The index and the archive engine are not read: everything a local render needs is in the workspace project.
|
||||
_ = (db, archives);
|
||||
var cancellation = context.Cancellation;
|
||||
var parameters = WorkspaceJobs.RenderParameters(context.Job) ?? throw new InvalidDataException(Loc.L("Errors.Workspace.JobParameters"));
|
||||
var (render, export) = ProcessingJobs.ReadRender(parameters.RenderJson);
|
||||
// Where the project is now (a workspace moved since the job was queued), never while it is being moved; nothing is
|
||||
// made where it is not.
|
||||
var root = WorkspaceJobRoots.Resolve(parameters.WorkspaceRoot, parameters.ProjectId, options.CurrentRoot, "Errors.Workspace.RenderProject");
|
||||
parameters = parameters with { WorkspaceRoot = root };
|
||||
var store = new WorkspaceStore(root);
|
||||
var project = store.Find(parameters.ProjectId) ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.RenderProject"));
|
||||
var inputs = await ReadInputsAsync(project, parameters, render, cancellation).ConfigureAwait(false);
|
||||
var (sequence, frames, settings, versionId) = (inputs.Sequence, inputs.Frames, inputs.Settings, inputs.VersionId);
|
||||
// A local render's outputs stay in the project's exports, from where they are sent home.
|
||||
var projectSettings = ProjectStore.Parse(settings) with { Export = export with { Folder = null } };
|
||||
var settingsJson = projectSettings.ToJson();
|
||||
var settingsHash = ProjectContent.Hash(inputs.Portable);
|
||||
|
||||
var exports = project.ExportsDirectory;
|
||||
Directory.CreateDirectory(exports);
|
||||
using var names = new OutputNames();
|
||||
var all = RenderJobHandler.Targets(projectSettings.Export, render.PresetName);
|
||||
var resume = RenderCheckpoint.Parse(context.Checkpoint);
|
||||
var done = resume.Done.ToList();
|
||||
var ledger = project.ReadRenders();
|
||||
for (var i = 0; i < all.Count; i++)
|
||||
{
|
||||
if (!done.Contains(i) && ledger.Find(RenderJobHandler.RenderId(context.Job.Id, i)) is not null) done.Add(i);
|
||||
}
|
||||
if (resume is { Pending: { } pending, PendingOutput: { } chosen } && !done.Contains(pending) && Inside(exports, chosen) && names.TryReserve(chosen))
|
||||
{
|
||||
Discard(chosen, keepOutput: ledger.Entries.Any(e => SamePath(project.ExportFile(e.File), chosen)));
|
||||
names.Release(chosen);
|
||||
}
|
||||
var targets = all.Select((t, i) => (i, t)).Where(t => !done.Contains(t.i)).ToList();
|
||||
// What an earlier run of this job recorded and did not send (it stopped before queueing the send, or that failed).
|
||||
var recorded = ledger.Entries.Where(e => e.JobId == context.Job.Id && e.IsUnsent).Select(e => e.RenderId).ToList();
|
||||
if (targets.Count == 0)
|
||||
{
|
||||
await QueueSendAsync(context, parameters, sequence, recorded).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
if (targets.Count > 0)
|
||||
{
|
||||
string? writing = null;
|
||||
IReadOnlyList<TargetOutcome> outcomes;
|
||||
try
|
||||
{
|
||||
await using var renderer = new SequenceRender(frames, projectSettings, inputs.Analysis, options.Processing, originals: true, companions: null);
|
||||
outcomes = await renderer.RenderAllAsync(
|
||||
targets,
|
||||
(target, setup) => names.Reserve(taken => ExportNaming.OutputPath(projectSettings.Export, exports, SequenceRender.NameContext(sequence, target, setup),
|
||||
name => taken(name) || ledger.Entries.Any(e => SamePath(project.ExportFile(e.File), name)))),
|
||||
(index, path) =>
|
||||
{
|
||||
writing = path;
|
||||
return context.CheckpointAsync(new RenderCheckpoint(done, index, path).ToJson(), context.Done, context.Total);
|
||||
},
|
||||
async (index, target, result) =>
|
||||
{
|
||||
// The output is complete: recorded even if the job is being cancelled, so it is never orphaned.
|
||||
var id = RenderJobHandler.RenderId(context.Job.Id, index);
|
||||
var entry = await EntryAsync(project, context.Job.Id, id, target, result, versionId, settingsHash).ConfigureAwait(false);
|
||||
using (await project.LockAsync(LockWait, CancellationToken.None).ConfigureAwait(false))
|
||||
{
|
||||
ledger = project.ReadRenders().With(entry);
|
||||
project.WriteRenders(ledger);
|
||||
}
|
||||
done.Add(index);
|
||||
if (!recorded.Contains(id)) recorded.Add(id);
|
||||
writing = null;
|
||||
await context.CheckpointAsync(new RenderCheckpoint(done, null, null).ToJson(), context.Done, context.Total).ConfigureAwait(false);
|
||||
context.Log(Loc.L("Export.Log.Rendered", target.PresetName, Path.GetFileName(result.OutputPath)));
|
||||
foreach (var remark in SequenceRender.Remarks(result)) context.Log(remark);
|
||||
},
|
||||
context.ReportAsync,
|
||||
cancellation).ConfigureAwait(false);
|
||||
if (renderer.StandIns.Count > 0)
|
||||
context.Log(Loc.L("Export.Log.StandIns", string.Join(", ", renderer.StandIns.Select(i => (i + 1).ToString(Loc.Culture)))));
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellation.IsCancellationRequested)
|
||||
{
|
||||
// What the encoder was writing goes; every output recorded stays, and a resumed job renders the rest.
|
||||
if (writing is { } partial) Discard(partial, keepOutput: true);
|
||||
await QueueSendAsync(context, parameters, sequence, recorded).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
var failed = outcomes.Where(o => o.Error is not null).ToList();
|
||||
foreach (var outcome in failed) context.Log(Loc.L("Export.Log.Failed", outcome.Target.PresetName, outcome.Error));
|
||||
await QueueSendAsync(context, parameters, sequence, recorded).ConfigureAwait(false);
|
||||
if (failed.Count > 0)
|
||||
throw new InvalidOperationException(failed.Count == 1 ? failed[0].Error : Loc.L("Errors.Render.Failed", failed.Count, outcomes.Count));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>What the render reads, taken under the project's lease so the media, the versions and the project file agree.</summary>
|
||||
private sealed record Inputs(Sequence Sequence, SequenceFrames Frames, string Settings, string Portable, Guid? VersionId, DeepAnalysisReport? Analysis);
|
||||
|
||||
/// <summary>
|
||||
/// The sequence over the workspace's originals, the settings to render (placed on its frames) and the version they are,
|
||||
/// read under the project's lease.
|
||||
/// </summary>
|
||||
/// <exception cref="InvalidOperationException">
|
||||
/// The project is gone (<c>Errors.Workspace.RenderProject</c>), holds no originals (<c>Errors.Workspace.OriginalsMissing</c>),
|
||||
/// lacks the original of a frame the render places (<c>Errors.Workspace.RenderNoOriginals</c>, with their number), or the
|
||||
/// version the job names is not in <c>versions/</c> (<c>Errors.Workspace.RenderVersion</c>).
|
||||
/// </exception>
|
||||
private static async Task<Inputs> ReadInputsAsync(WorkspaceProject project, WorkspaceRenderParameters parameters, Protocol.RenderJobParameters render,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
using (await project.LockAsync(LockWait, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (!File.Exists(project.BundlePath)) throw new InvalidOperationException(Loc.L("Errors.Workspace.RenderProject"));
|
||||
var bundle = project.ReadBundle();
|
||||
var workspace = bundle.Workspace!;
|
||||
var media = project.ReadMedia();
|
||||
if (media is not { HoldsOriginals: true })
|
||||
throw new InvalidOperationException(Loc.L(workspace.Home.IsArchive ? "Errors.Workspace.OriginalsMissing" : "Errors.Workspace.OriginalsMissing.Server"));
|
||||
var versions = project.Versions();
|
||||
var named = parameters.ProjectVersionId ?? render.ProjectVersionId;
|
||||
var version = named is { } id ? versions.FirstOrDefault(v => v.Id == id) : null;
|
||||
string portable;
|
||||
if (render.SettingsJson is { Length: > 0 } embedded) portable = embedded;
|
||||
else if (named is not null) portable = version?.SettingsJson ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.RenderVersion"));
|
||||
else portable = versions.Count > 0 ? versions[^1].SettingsJson : ProjectContent.ToPortable(bundle.Project, media.Photos);
|
||||
// Recorded only for the settings it holds (a version is never named for settings it did not render).
|
||||
Guid? versionId = version is not null && ProjectContent.Verifies(portable, version.ContentHash) ? version.Id : null;
|
||||
var settings = portable.Contains("pendingFrames", StringComparison.OrdinalIgnoreCase) ? Place(portable, media.Photos) : portable;
|
||||
var placed = ProjectStore.Parse(settings);
|
||||
var excluded = placed.ExcludedFrames.ToHashSet();
|
||||
var lacking = media.Frames.Count(f => f.Original is null && !excluded.Contains(f.Index) && (f.Flags & SequenceFrames.NotRendered) == 0);
|
||||
if (lacking > 0) throw new InvalidOperationException(Loc.L("Errors.Workspace.RenderNoOriginals", lacking));
|
||||
var frames = media.ToOriginalFrames();
|
||||
var files = media.Frames.Select(f => f.Original is { } original ? project.MediaFile(original) : null).ToList();
|
||||
var sequence = SequenceOf(bundle, media, frames);
|
||||
return new Inputs(sequence, new SequenceFrames(sequence, frames, files), settings, portable, versionId, Analysis(project, frames.Count));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The home's sequence as the project file and the media name it (the workspace keeps no index row of it).</summary>
|
||||
private static Sequence SequenceOf(ProjectBundle bundle, WorkspaceMedia media, IReadOnlyList<Frame> frames)
|
||||
{
|
||||
var times = frames.Where(f => f.CapturedAt is not null).Select(f => f.CapturedAt!.Value).Order().ToList();
|
||||
var home = bundle.Workspace!.Home;
|
||||
return new Sequence
|
||||
{
|
||||
Id = home.SequenceId,
|
||||
ArchiveId = bundle.Archive.Id,
|
||||
Title = bundle.Sequence.Title,
|
||||
Folder = bundle.Sequence.Folder,
|
||||
FrameCount = frames.Count,
|
||||
FirstCapture = times.Count > 0 ? times[0] : null,
|
||||
LastCapture = times.Count > 0 ? times[^1] : null,
|
||||
FramesVersion = media.FramesVersion,
|
||||
Origin = home.IsArchive ? SequenceOrigin.Local : SequenceOrigin.Server,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>The deep report the copy brought from the home, when it describes these frames; null otherwise.</summary>
|
||||
private static DeepAnalysisReport? Analysis(WorkspaceProject project, int frameCount)
|
||||
{
|
||||
if (!File.Exists(project.AnalysisPath)) return null;
|
||||
try
|
||||
{
|
||||
var report = DeepAnalysisReport.Load(project.AnalysisPath);
|
||||
return report.FrameCount == frameCount ? report : null;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or JsonException or UnauthorizedAccessException)
|
||||
{
|
||||
Core.Diagnostics.Log.Warning("workspace", $"{project.AnalysisPath} cannot be read", ex);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Portable settings placed on the media's frames; others as they are.</summary>
|
||||
private static string Place(string portable, FramePhotos photos)
|
||||
{
|
||||
try
|
||||
{
|
||||
return ProjectContent.Place(portable, photos);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
// Not a project: reading it says so.
|
||||
return portable;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The ledger entry of a finished output: hashed as it is on the disk now.</summary>
|
||||
private static async Task<WorkspaceRenderEntry> EntryAsync(WorkspaceProject project, Guid jobId, Guid renderId, ExportTarget target, RenderResult result,
|
||||
Guid? versionId, string settingsHash)
|
||||
{
|
||||
var sha = await HashAsync(result.OutputPath).ConfigureAwait(false);
|
||||
var file = Path.GetRelativePath(project.ExportsDirectory, result.OutputPath).Replace('\\', '/');
|
||||
return new WorkspaceRenderEntry(renderId, jobId, file, sha, result.Bytes, target.PresetName, result.Codec.ToString(), result.Width, result.Height,
|
||||
result.FrameRate.Value, result.DurationSeconds, project.ProjectId, versionId, settingsHash, DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The SHA-256 of an output: of the file, or, for an image sequence's folder, of its files' relative paths and SHA-256 in
|
||||
/// order (a send checks the folder the same way).
|
||||
/// </summary>
|
||||
internal static async Task<string> HashAsync(string output)
|
||||
{
|
||||
if (File.Exists(output)) return await ContentHash.Sha256Async(output, cancellationToken: CancellationToken.None).ConfigureAwait(false);
|
||||
var listing = new StringBuilder();
|
||||
foreach (var file in Directory.EnumerateFiles(output, "*", SearchOption.AllDirectories).Order(StringComparer.Ordinal))
|
||||
{
|
||||
var sha = await ContentHash.Sha256Async(file, cancellationToken: CancellationToken.None).ConfigureAwait(false);
|
||||
listing.Append(Path.GetRelativePath(output, file).Replace('\\', '/')).Append(' ').Append(sha).Append('\n');
|
||||
}
|
||||
return Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(listing.ToString())));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queues the send home of every render of the project not sent yet (an empty list of ids: the send reads the ledger when
|
||||
/// it runs), whatever stops the job now, once this job recorded one that is not sent (they are done).
|
||||
/// </summary>
|
||||
private static async Task QueueSendAsync(JobContext context, WorkspaceRenderParameters parameters, Sequence sequence, List<Guid> recorded)
|
||||
{
|
||||
if (recorded.Count == 0) return;
|
||||
var send = WorkspaceJobs.Send(sequence.Id, sequence.DisplayTitle, parameters.WorkspaceRoot, parameters.ProjectId, [], parameters.Server,
|
||||
context.Job.Priority);
|
||||
await context.EnqueueFollowUpAsync(send.Kind, send.Targets, send.Label, send.ParametersJson, send.Priority, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static bool Inside(string folder, string path)
|
||||
{
|
||||
var relative = Path.GetRelativePath(folder, path);
|
||||
return !relative.StartsWith("..", StringComparison.Ordinal) && !Path.IsPathRooted(relative);
|
||||
}
|
||||
|
||||
private static bool SamePath(string a, string b) =>
|
||||
string.Equals(Path.GetFullPath(a), Path.GetFullPath(b), OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal);
|
||||
|
||||
/// <summary>
|
||||
/// Removes what an interrupted attempt left of <paramref name="output"/>: the encoder's hidden temporary file or folder,
|
||||
/// and an output completed but never recorded (it renders again under the same name) unless <paramref name="keepOutput"/>.
|
||||
/// </summary>
|
||||
private static void Discard(string output, bool keepOutput)
|
||||
{
|
||||
var parent = Path.GetDirectoryName(output);
|
||||
if (parent is null || !Directory.Exists(parent)) return;
|
||||
var leftovers = Directory.EnumerateFileSystemEntries(parent, "." + Path.GetFileName(output) + ".*.tmp").ToList();
|
||||
if (!keepOutput) leftovers.Add(output);
|
||||
foreach (var leftover in leftovers)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(leftover)) Directory.Delete(leftover, recursive: true);
|
||||
else if (File.Exists(leftover)) File.Delete(leftover);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Core.Diagnostics.Log.Warning("workspace", $"{leftover} could not be removed", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Chronos.Core.IO;
|
||||
using Chronos.Core.Model;
|
||||
|
||||
namespace Chronos.Data.Workspaces;
|
||||
|
||||
/// <summary>
|
||||
/// One local render of a workspace project (phase 5c), as its ledger (<see cref="WorkspaceRenders"/>) records it: the output
|
||||
/// a <see cref="JobKind.WorkspaceRender"/> wrote into the project's <c>exports/</c>, its SHA-256 and size as written, what it
|
||||
/// is (preset, codec, size, rate, length), the project and version it was rendered from, and whether it went home
|
||||
/// (<see cref="JobKind.WorkspaceSend"/>). The file stays in <c>exports/</c> after it is sent, until a clean-up removes it.
|
||||
/// </summary>
|
||||
/// <param name="RenderId">
|
||||
/// The render's id, the same at the home once sent (<c>RenderJobHandler.RenderId(jobId, index)</c>): a send repeated after a
|
||||
/// cut finds the home holding it already, and nothing is filed twice.
|
||||
/// </param>
|
||||
/// <param name="JobId">The <see cref="JobKind.WorkspaceRender"/> job that wrote it.</param>
|
||||
/// <param name="File">The output, relative to the project's <c>exports/</c> folder (forward slashes; <see cref="WorkspaceProject.ExportFile"/>).</param>
|
||||
/// <param name="Sha256">The output's SHA-256 (lower case) as it was written: a send hashes the file again and refuses another.</param>
|
||||
/// <param name="Size">The output's size in bytes.</param>
|
||||
/// <param name="ProjectVersionId">The project version rendered; null for a working state no version names.</param>
|
||||
/// <param name="SettingsHash">The content hash of the portable settings rendered (<c>ProjectContent</c>); null when not known.</param>
|
||||
/// <param name="Rendered">When the output was finished.</param>
|
||||
/// <param name="State"><see cref="Pending"/>, <see cref="SentState"/> or <see cref="Failed"/> (a later Chronos may write others: read as pending).</param>
|
||||
/// <param name="Sent">When the home took it; null until then.</param>
|
||||
/// <param name="Error">Why the last send failed (one localized sentence); null otherwise.</param>
|
||||
public sealed record WorkspaceRenderEntry(
|
||||
Guid RenderId,
|
||||
Guid JobId,
|
||||
string File,
|
||||
string Sha256,
|
||||
long Size,
|
||||
string PresetName,
|
||||
string? Codec,
|
||||
int Width,
|
||||
int Height,
|
||||
double Fps,
|
||||
double DurationSeconds,
|
||||
Guid ProjectId,
|
||||
Guid? ProjectVersionId,
|
||||
string? SettingsHash,
|
||||
DateTimeOffset Rendered,
|
||||
string State = WorkspaceRenderEntry.Pending,
|
||||
DateTimeOffset? Sent = null,
|
||||
string? Error = null)
|
||||
{
|
||||
/// <summary>Written, not at the home yet.</summary>
|
||||
public const string Pending = "pending";
|
||||
|
||||
/// <summary>The home holds it (a render row of the home's sequence with this id).</summary>
|
||||
public const string SentState = "sent";
|
||||
|
||||
/// <summary>The last send failed (<see cref="Error"/> says why); sent again by the next send.</summary>
|
||||
public const string Failed = "failed";
|
||||
|
||||
/// <summary>The home holds this render.</summary>
|
||||
[JsonIgnore]
|
||||
public bool IsSent => State == SentState;
|
||||
|
||||
/// <summary>Not at the home yet: pending, failed, or a state this Chronos does not know. A clean-up never removes its file.</summary>
|
||||
[JsonIgnore]
|
||||
public bool IsUnsent => !IsSent;
|
||||
|
||||
/// <summary>This entry as taken by the home at <paramref name="at"/>.</summary>
|
||||
public WorkspaceRenderEntry AsSent(DateTimeOffset at) => this with { State = SentState, Sent = at, Error = null };
|
||||
|
||||
/// <summary>This entry after a send that failed for <paramref name="error"/>; it stays to be sent again.</summary>
|
||||
public WorkspaceRenderEntry AsFailed(string error) => this with { State = Failed, Error = error };
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The ledger of a workspace project's local renders (<c>exports/renders.json</c>, phase 5c): every output a
|
||||
/// <see cref="JobKind.WorkspaceRender"/> finished, oldest first, one entry per render id. Read whole and written whole and
|
||||
/// atomically (temporary file, flush, rename), always under the project's lease; a missing ledger is an empty one.
|
||||
/// </summary>
|
||||
public sealed record WorkspaceRenders(string Format, int FormatVersion, IReadOnlyList<WorkspaceRenderEntry> Entries)
|
||||
{
|
||||
public const string FormatName = "chronos-workspace-renders";
|
||||
public const int CurrentFormatVersion = 1;
|
||||
|
||||
/// <summary>A ledger without entries.</summary>
|
||||
public static WorkspaceRenders Empty { get; } = new(FormatName, CurrentFormatVersion, []);
|
||||
|
||||
/// <summary>The entries not at the home yet (<see cref="WorkspaceRenderEntry.IsUnsent"/>), oldest first.</summary>
|
||||
[JsonIgnore]
|
||||
public IReadOnlyList<WorkspaceRenderEntry> Unsent => [.. Entries.Where(e => e.IsUnsent)];
|
||||
|
||||
/// <summary>The entry of <paramref name="renderId"/>; null when the ledger has none.</summary>
|
||||
public WorkspaceRenderEntry? Find(Guid renderId) => Entries.FirstOrDefault(e => e.RenderId == renderId);
|
||||
|
||||
/// <summary>This ledger with <paramref name="entry"/> in place of the entry of its render id, or appended when it has none.</summary>
|
||||
public WorkspaceRenders With(WorkspaceRenderEntry entry)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(entry);
|
||||
var entries = Entries.ToList();
|
||||
var at = entries.FindIndex(e => e.RenderId == entry.RenderId);
|
||||
if (at >= 0) entries[at] = entry;
|
||||
else entries.Add(entry);
|
||||
return this with { Entries = entries };
|
||||
}
|
||||
|
||||
/// <summary>This ledger without the entry of <paramref name="renderId"/> (a sent file a clean-up removed, a discarded render).</summary>
|
||||
public WorkspaceRenders Without(Guid renderId) => Entries.Any(e => e.RenderId == renderId) ? this with { Entries = [.. Entries.Where(e => e.RenderId != renderId)] } : this;
|
||||
|
||||
/// <summary>The ledger at <paramref name="path"/>; <see cref="Empty"/> when there is none.</summary>
|
||||
/// <exception cref="InvalidDataException">The file is no ledger, is damaged, or a newer Chronos wrote it.</exception>
|
||||
/// <exception cref="IOException">The file cannot be read.</exception>
|
||||
public static WorkspaceRenders Read(string path)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrEmpty(path);
|
||||
if (!System.IO.File.Exists(path)) return Empty;
|
||||
WorkspaceRenders? renders;
|
||||
try
|
||||
{
|
||||
renders = JsonSerializer.Deserialize(AtomicFile.ReadAllText(path), WorkspaceJson.Default.WorkspaceRenders);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
throw new InvalidDataException($"{path} cannot be read.", ex);
|
||||
}
|
||||
if (renders is null || renders.Format != FormatName || renders.Entries is null) throw new InvalidDataException($"{path} is not a renders ledger.");
|
||||
if (renders.FormatVersion > CurrentFormatVersion) throw new InvalidDataException($"{path} is newer than this Chronos.");
|
||||
return renders;
|
||||
}
|
||||
|
||||
/// <summary>Writes <paramref name="renders"/> to <paramref name="path"/> whole and atomically, making its folder when needed.</summary>
|
||||
/// <exception cref="IOException">The file cannot be written; the ledger there stays as it was.</exception>
|
||||
public static void Write(string path, WorkspaceRenders renders)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrEmpty(path);
|
||||
ArgumentNullException.ThrowIfNull(renders);
|
||||
if (Path.GetDirectoryName(Path.GetFullPath(path)) is { Length: > 0 } folder) Directory.CreateDirectory(folder);
|
||||
AtomicFile.WriteAllText(path, JsonSerializer.Serialize(renders with { Format = FormatName, FormatVersion = CurrentFormatVersion }, WorkspaceJson.Default.WorkspaceRenders));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
using System.Net;
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.Diagnostics;
|
||||
using Chronos.Core.Localization;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Core.Project;
|
||||
using Chronos.Data.Archiving;
|
||||
using Chronos.Data.Jobs;
|
||||
using Chronos.Data.Projects;
|
||||
using Chronos.Data.Sync;
|
||||
using Chronos.Protocol;
|
||||
using Chronos.Protocol.Client;
|
||||
|
||||
namespace Chronos.Data.Workspaces;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.WorkspaceSend"/> (phase 5c, <see cref="WorkspaceJobs.Send"/>): the local renders of a workspace project
|
||||
/// sent home with their project and version, per unsent ledger entry (<see cref="WorkspaceRenders"/>; the ids the job names,
|
||||
/// or every unsent one): the file hashed again (one missing or changed since it was rendered is refused, marked failed in the
|
||||
/// ledger and left where it is), the version it names pushed first when the home lacks it (a push that fails fails the job,
|
||||
/// the entry left as it was; a version the push replayed under a new id, after the home moved on, is found again by the
|
||||
/// content hash of what rendered, else the render goes with its project alone, and the entry says so from then on), then
|
||||
/// filed in this PC's archive (copied into the archive's staging folder, read back against
|
||||
/// its SHA-256, and filed under the sequence's lease as the sync files a server's exports,
|
||||
/// <see cref="SyncTransfers.FileRenderAsync"/>, after checking the project and version are the sequence's) or uploaded to its
|
||||
/// server (<see cref="RenderUploadRequest"/> with <c>ProjectId</c> and <c>ProjectVersionId</c>, which the server checks the
|
||||
/// same way), and the entry marked sent. The file stays in <c>exports/</c> until a clean-up removes it.
|
||||
/// Idempotent by render id: a send repeated after a cut finds the home holding the render already and only marks it sent; a
|
||||
/// render id the home files under another sequence is refused. The ledger is read and written under the project's lease,
|
||||
/// each entry read again before it is written, so a render finished meanwhile is kept.
|
||||
/// A send to a server is held while <see cref="WorkspaceJobOptions.HoldTransfers"/> says so (a metered connection): it waits
|
||||
/// before each upload, and the desktop pauses it meanwhile. A send to this PC's archive is never held. A server from before
|
||||
/// phase 5c files the render without its project and version: the job's log says so. A send queued before the workspace
|
||||
/// moved works where the workspace is now, and none runs while it is being moved (<see cref="WorkspaceJobRoots"/>).
|
||||
/// </summary>
|
||||
internal sealed class WorkspaceSendJobHandler(ChronosDatabase db, ArchiveService archives, ISyncConnector? servers, WorkspaceJobOptions options) : IJobHandler
|
||||
{
|
||||
private static readonly TimeSpan LockWait = TimeSpan.FromMinutes(2);
|
||||
|
||||
/// <summary>How often a send held on a metered connection looks again.</summary>
|
||||
internal static TimeSpan HoldPoll { get; set; } = TimeSpan.FromSeconds(2);
|
||||
|
||||
public JobKind Kind => JobKind.WorkspaceSend;
|
||||
|
||||
public async Task RunAsync(JobContext context)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
var ct = context.Cancellation;
|
||||
var parameters = WorkspaceJobs.SendParameters(context.Job) ?? throw new InvalidDataException(Loc.L("Errors.Workspace.JobParameters"));
|
||||
if (context.Job.TargetIds is not [var sequenceId] || string.IsNullOrWhiteSpace(parameters.WorkspaceRoot))
|
||||
throw new InvalidDataException(Loc.L("Errors.Workspace.JobParameters"));
|
||||
var server = parameters.Server is { Length: > 0 } named ? named : null;
|
||||
// Where the project is now (a workspace moved since the job was queued), never while it is being moved; nothing is
|
||||
// made where it is not.
|
||||
var root = WorkspaceJobRoots.Resolve(parameters.WorkspaceRoot, parameters.ProjectId, options.CurrentRoot, "Errors.Workspace.NotCheckedOut");
|
||||
var project = new WorkspaceStore(root).Find(parameters.ProjectId)
|
||||
?? throw new InvalidOperationException(Loc.L("Errors.Workspace.NotCheckedOut"));
|
||||
var workspace = project.ReadBundle().Workspace ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.NotCheckedOut"));
|
||||
var home = workspace.Home;
|
||||
if (home.SequenceId != sequenceId || (server is null
|
||||
? !home.IsArchive
|
||||
: home.Kind != BundleHome.ServerKind || !string.Equals(home.Server, server, StringComparison.OrdinalIgnoreCase)))
|
||||
throw new InvalidOperationException(Loc.L("Errors.Workspace.OtherHome"));
|
||||
|
||||
var entries = await ToSendAsync(project, parameters.RenderIds, ct).ConfigureAwait(false);
|
||||
await context.ReportAsync(0, entries.Count).ConfigureAwait(false);
|
||||
if (entries.Count == 0)
|
||||
{
|
||||
context.Log(Loc.L("Archive.Log.WorkspaceSent", workspace.Name, 0, 0));
|
||||
return;
|
||||
}
|
||||
if (server is null)
|
||||
{
|
||||
var sequence = await db.Sequences.GetAsync(sequenceId, ct).ConfigureAwait(false)
|
||||
?? throw new InvalidOperationException(Loc.L("Archive.Log.SequenceMissing", sequenceId));
|
||||
var archive = await db.Archives.GetAsync(sequence.ArchiveId, ct).ConfigureAwait(false)
|
||||
?? throw new InvalidOperationException(Loc.L("Errors.Sync.NoArchive"));
|
||||
var projectHome = new ArchiveProjectHome(db, archives.Projects, sequenceId);
|
||||
await SendAsync(context, project, workspace, entries, () => projectHome, held: false,
|
||||
(entry, file, token) => FileAsync(sequence, archive, entry, file, token)).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
using var remote = (servers is null ? null : await servers.ConnectAsync(server, ct).ConfigureAwait(false))
|
||||
?? throw new InvalidOperationException(Loc.L("Errors.Sync.NotConnected"));
|
||||
await SendAsync(context, project, workspace, entries,
|
||||
() => servers!.ProjectHome(home) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.NotConnected")), held: true,
|
||||
(entry, file, token) => UploadAsync(context, remote, sequenceId, entry, file, token)).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The entries to send, oldest first: those of <paramref name="renderIds"/> the ledger holds unsent (an id it no longer
|
||||
/// holds was discarded meanwhile, one sent already is left), every unsent one when none is named.
|
||||
/// </summary>
|
||||
private static async Task<IReadOnlyList<WorkspaceRenderEntry>> ToSendAsync(WorkspaceProject project, IReadOnlyList<Guid>? renderIds, CancellationToken ct)
|
||||
{
|
||||
WorkspaceRenders ledger;
|
||||
using (await project.LockAsync(LockWait, ct).ConfigureAwait(false)) ledger = project.ReadRenders();
|
||||
if (renderIds is not { Count: > 0 }) return ledger.Unsent;
|
||||
var named = renderIds.ToHashSet();
|
||||
return [.. ledger.Entries.Where(e => named.Contains(e.RenderId) && e.IsUnsent)];
|
||||
}
|
||||
|
||||
/// <summary>Delivers one checked render home: true when the home took it now, false when it held it already.</summary>
|
||||
private delegate Task<bool> Deliver(WorkspaceRenderEntry entry, string file, CancellationToken cancellationToken);
|
||||
|
||||
private async Task SendAsync(JobContext context, WorkspaceProject project, BundleWorkspace workspace, IReadOnlyList<WorkspaceRenderEntry> entries,
|
||||
Func<IProjectHome> home, bool held, Deliver deliver)
|
||||
{
|
||||
var ct = context.Cancellation;
|
||||
var (sent, there, refused) = (0, 0, 0);
|
||||
// The versions this run found at the home, or sent there first, with the id the home holds each by (a replay gives another).
|
||||
var versions = new Dictionary<Guid, Guid?>();
|
||||
for (var i = 0; i < entries.Count; i++)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
if (held) await WhileHeldAsync(context).ConfigureAwait(false);
|
||||
var entry = entries[i];
|
||||
var file = project.ContainedExportFile(entry.File);
|
||||
var refusal = file is null ? Loc.L("Errors.Workspace.SendMissing", Path.GetFileName(entry.File ?? string.Empty)) : await CheckAsync(entry, file, folders: !held, ct).ConfigureAwait(false);
|
||||
if (refusal is null)
|
||||
{
|
||||
if (entry.ProjectVersionId is { } versionId)
|
||||
{
|
||||
if (!versions.TryGetValue(versionId, out var atHome))
|
||||
{
|
||||
atHome = await EnsureVersionAsync(context, project, workspace, home(), entry, versionId, ct).ConfigureAwait(false);
|
||||
versions[versionId] = atHome;
|
||||
}
|
||||
if (atHome != versionId)
|
||||
{
|
||||
// The ledger says from now on what the home holds: later sends agree with this one.
|
||||
entry = entry with { ProjectVersionId = atHome };
|
||||
var mapped = atHome;
|
||||
await UpdateAsync(project, entry.RenderId, e => e with { ProjectVersionId = mapped }).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
try
|
||||
{
|
||||
if (await deliver(entry, file!, ct).ConfigureAwait(false)) sent++;
|
||||
else there++;
|
||||
// The home holds it: recorded whatever stops the job now.
|
||||
await UpdateAsync(project, entry.RenderId, e => e.AsSent(DateTimeOffset.UtcNow)).ConfigureAwait(false);
|
||||
}
|
||||
catch (SendRefusedException ex)
|
||||
{
|
||||
refusal = ex.Message;
|
||||
}
|
||||
}
|
||||
if (refusal is not null)
|
||||
{
|
||||
refused++;
|
||||
context.Log(refusal);
|
||||
await UpdateAsync(project, entry.RenderId, e => e.AsFailed(refusal)).ConfigureAwait(false);
|
||||
}
|
||||
await context.ReportAsync(i + 1, entries.Count).ConfigureAwait(false);
|
||||
}
|
||||
context.Log(Loc.L("Archive.Log.WorkspaceSent", workspace.Name, sent, there));
|
||||
if (refused > 0) throw new InvalidOperationException(Loc.L("Errors.Workspace.SendFailed", refused, entries.Count));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Why the render cannot be sent as it is (gone from the exports, another file than the one rendered, or an image
|
||||
/// sequence's folder for a home that takes one file per render); null when it can. A folder (an image sequence, only
|
||||
/// when <paramref name="folders"/>: this PC's archive files folders as its own renders do) is checked by its total size
|
||||
/// and the hash of its listing, as the render recorded it (<see cref="WorkspaceRenderJobHandler.HashAsync"/>).
|
||||
/// </summary>
|
||||
/// <remarks>A ledger names files inside the exports only (<see cref="WorkspaceProject.ContainedExportFile"/>): one pointing elsewhere is never read.</remarks>
|
||||
private static async Task<string?> CheckAsync(WorkspaceRenderEntry entry, string file, bool folders, CancellationToken ct)
|
||||
{
|
||||
var name = Path.GetFileName(entry.File);
|
||||
if (Directory.Exists(file))
|
||||
{
|
||||
if (!folders) return Loc.L("Errors.Workspace.SendSequence", name);
|
||||
return WorkspaceCleaner.Measure(file).Bytes != entry.Size
|
||||
|| !string.Equals(await WorkspaceRenderJobHandler.HashAsync(file).ConfigureAwait(false), entry.Sha256, StringComparison.OrdinalIgnoreCase)
|
||||
? Loc.L("Errors.Workspace.SendHashMismatch", name)
|
||||
: null;
|
||||
}
|
||||
if (!File.Exists(file)) return Loc.L("Errors.Workspace.SendMissing", name);
|
||||
if (new FileInfo(file).Length != entry.Size
|
||||
|| !string.Equals(await ChronosClient.HashFileAsync(file, ct).ConfigureAwait(false), entry.Sha256, StringComparison.OrdinalIgnoreCase))
|
||||
return Loc.L("Errors.Workspace.SendHashMismatch", name);
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>How many of the home's newest versions are read for one of the same content, when a replay gave a rendered version another id.</summary>
|
||||
private const int ReplayedLookBack = 100;
|
||||
|
||||
/// <summary>
|
||||
/// Version <paramref name="versionId"/> of the project at the home, as the id the home holds it by: pushed there first
|
||||
/// (every local version, in order, as a sync pushes them) when the home lacks it. A push that replays it on a home that
|
||||
/// moved on gives it another id (and one that changes nothing is not sent at all): the home's version of the content
|
||||
/// hash that rendered (<see cref="WorkspaceRenderEntry.SettingsHash"/>) stands for it, else none (null: the render goes
|
||||
/// with its project alone, as a render of a version the home never had). A push that fails fails the job.
|
||||
/// </summary>
|
||||
private static async Task<Guid?> EnsureVersionAsync(JobContext context, WorkspaceProject project, BundleWorkspace workspace, IProjectHome home,
|
||||
WorkspaceRenderEntry entry, Guid versionId, CancellationToken ct)
|
||||
{
|
||||
var projectId = entry.ProjectId;
|
||||
if (await home.GetVersionAsync(projectId, versionId, ct).ConfigureAwait(false) is not null) return versionId;
|
||||
try
|
||||
{
|
||||
await new WorkspaceSync(project.Store, Environment.MachineName) { LockWait = LockWait }.PushAsync(projectId, home, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is HomeUnavailableException or WorkspaceSyncException or ProjectException or LeaseUnavailableException
|
||||
or InvalidOperationException)
|
||||
{
|
||||
context.Log(ex.Message);
|
||||
throw new InvalidOperationException(Loc.L("Errors.Workspace.SendVersion"), ex);
|
||||
}
|
||||
if (await home.GetVersionAsync(projectId, versionId, ct).ConfigureAwait(false) is { } pushed)
|
||||
{
|
||||
context.Log(Loc.L("Archive.Log.WorkspaceVersionPushed", workspace.Name, pushed.Number));
|
||||
return versionId;
|
||||
}
|
||||
ProjectVersion? same = null;
|
||||
if (entry.SettingsHash is { Length: > 0 } hash)
|
||||
{
|
||||
var history = await home.HistoryAsync(projectId, null, ReplayedLookBack, ct).ConfigureAwait(false);
|
||||
same = history.FirstOrDefault(v => string.Equals(v.ContentHash, hash, StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
if (same is not null)
|
||||
{
|
||||
Log.Info("workspace", $"Version {versionId} of project {projectId} reached the home as version {same.Id} (a replay): render {entry.RenderId} names that one");
|
||||
context.Log(Loc.L("Archive.Log.WorkspaceVersionReplayed", workspace.Name, same.Number));
|
||||
return same.Id;
|
||||
}
|
||||
Log.Info("workspace", $"Version {versionId} of project {projectId} is not at the home after the push, nor one of the same content: render {entry.RenderId} goes with its project alone");
|
||||
context.Log(Loc.L("Archive.Log.WorkspaceVersionNotHome", workspace.Name));
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The render filed in this PC's archive, the home's sequence: copied into the archive's staging folder, read back against
|
||||
/// the ledger's SHA-256, then filed under the sequence's lease with the project and version (once both are checked to be
|
||||
/// the sequence's). A render id the index holds already for this sequence is the same render, sent before.
|
||||
/// </summary>
|
||||
private async Task<bool> FileAsync(Sequence sequence, Archive archive, WorkspaceRenderEntry entry, string file, CancellationToken ct)
|
||||
{
|
||||
if (await db.Renders.GetAsync(entry.RenderId, ct).ConfigureAwait(false) is { } known)
|
||||
return known.SequenceId == sequence.Id ? false : throw new SendRefusedException(Loc.L("Server.Error.RenderExists"));
|
||||
if (!await ProjectOfAsync(sequence.Id, entry, ct).ConfigureAwait(false)) throw new SendRefusedException(Loc.L("Server.Error.ProjectMismatch"));
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
SyncTransfers.EnsureSpace(paths.Root, entry.Size);
|
||||
Directory.CreateDirectory(paths.StagingDirectory);
|
||||
var staged = Path.Combine(paths.StagingDirectory, "workspace-render-" + entry.RenderId.ToString("N") + ".part");
|
||||
var folder = Directory.Exists(file);
|
||||
try
|
||||
{
|
||||
TryDelete(staged);
|
||||
if (folder)
|
||||
{
|
||||
// An image sequence: every file of its folder, the folder filed whole as the archive's own renders are.
|
||||
foreach (var source in Directory.EnumerateFiles(file, "*", SearchOption.AllDirectories))
|
||||
{
|
||||
var target = Path.Combine(staged, Path.GetRelativePath(file, source));
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(target)!);
|
||||
await CopyAsync(source, target, ct).ConfigureAwait(false);
|
||||
}
|
||||
Directory.CreateDirectory(staged);
|
||||
}
|
||||
else await CopyAsync(file, staged, ct).ConfigureAwait(false);
|
||||
// Read back from the archive's volume: what is filed is the render as it was written.
|
||||
var readBack = folder
|
||||
? await WorkspaceRenderJobHandler.HashAsync(staged).ConfigureAwait(false)
|
||||
: await ChronosClient.HashFileAsync(staged, ct).ConfigureAwait(false);
|
||||
if (!string.Equals(readBack, entry.Sha256, StringComparison.OrdinalIgnoreCase))
|
||||
throw new IOException(Loc.L("Errors.Workspace.CopyMismatch", staged));
|
||||
var name = SyncTransfers.SafeFileName(Path.GetFileName(entry.File)) ?? entry.RenderId.ToString("N");
|
||||
var filed = await SyncTransfers.FileRenderAsync(archives, db, sequence, paths, staged, name, new Render
|
||||
{
|
||||
Id = entry.RenderId,
|
||||
SequenceId = sequence.Id,
|
||||
PresetName = entry.PresetName,
|
||||
OutputPath = string.Empty,
|
||||
Codec = entry.Codec,
|
||||
Width = entry.Width,
|
||||
Height = entry.Height,
|
||||
Fps = entry.Fps,
|
||||
DurationSeconds = entry.DurationSeconds,
|
||||
Created = entry.Rendered,
|
||||
ProducedBy = options.Processing.ProducedBy,
|
||||
ProjectId = entry.ProjectId,
|
||||
ProjectVersionId = entry.ProjectVersionId,
|
||||
}, ct).ConfigureAwait(false);
|
||||
return filed is not null ? true : throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
|
||||
}
|
||||
finally
|
||||
{
|
||||
TryDelete(staged);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Copies <paramref name="source"/> to <paramref name="target"/>, flushed to the disk.</summary>
|
||||
private static async Task CopyAsync(string source, string target, CancellationToken ct)
|
||||
{
|
||||
await using var from = new FileStream(source, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 20, FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
await using var to = new FileStream(target, FileMode.Create, FileAccess.Write, FileShare.None, 1 << 20, FileOptions.Asynchronous);
|
||||
await from.CopyToAsync(to, ct).ConfigureAwait(false);
|
||||
to.Flush(flushToDisk: true);
|
||||
}
|
||||
|
||||
/// <summary>The project the entry names is one of the sequence's, and its version (when it names one) one of that project's.</summary>
|
||||
private async Task<bool> ProjectOfAsync(Guid sequenceId, WorkspaceRenderEntry entry, CancellationToken ct)
|
||||
{
|
||||
var project = await db.Projects.GetAsync(entry.ProjectId, ct).ConfigureAwait(false);
|
||||
var version = entry.ProjectVersionId is { } versionId ? await db.Projects.GetVersionAsync(versionId, ct).ConfigureAwait(false) : null;
|
||||
if (project is null || (entry.ProjectVersionId is not null && version is null))
|
||||
{
|
||||
// One the index does not know yet may be in the sequence's sidecars: listing reads them first.
|
||||
await archives.Projects.ListAsync(sequenceId, ct).ConfigureAwait(false);
|
||||
project = await db.Projects.GetAsync(entry.ProjectId, ct).ConfigureAwait(false);
|
||||
if (entry.ProjectVersionId is { } id) version = await db.Projects.GetVersionAsync(id, ct).ConfigureAwait(false);
|
||||
}
|
||||
return project is not null && project.SequenceId == sequenceId && (entry.ProjectVersionId is null || version?.ProjectId == project.Id);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The render uploaded to its server's sequence, with its project and version, resumable and checked against its SHA-256
|
||||
/// there; the server answers that it holds the render already when it does. Refused for good (the server takes the
|
||||
/// project for another sequence's, the id for another sequence's render, the file for too large or not the one hashed):
|
||||
/// marked failed; anything else (the server out of reach, busy, the sign-in expired) fails the job, the entry left as it was.
|
||||
/// </summary>
|
||||
private async Task<bool> UploadAsync(JobContext context, ISyncRemote remote, Guid sequenceId, WorkspaceRenderEntry entry, string file, CancellationToken ct)
|
||||
{
|
||||
var name = SyncTransfers.SafeFileName(Path.GetFileName(entry.File)) ?? entry.RenderId.ToString("N");
|
||||
var request = new RenderUploadRequest(entry.RenderId, name, entry.Size, entry.Sha256.ToLowerInvariant(), entry.PresetName, entry.Codec, entry.Width, entry.Height,
|
||||
entry.Fps, entry.DurationSeconds, options.Processing.ProducedBy, entry.ProjectId, entry.ProjectVersionId);
|
||||
try
|
||||
{
|
||||
if (await remote.UploadRenderAsync(sequenceId, file, request, ct).ConfigureAwait(false) is not { } filed) return false;
|
||||
// A server from before phase 5c ignores the project and version: the render is there, without them.
|
||||
if (filed.ProjectId != entry.ProjectId || filed.ProjectVersionId != entry.ProjectVersionId)
|
||||
{
|
||||
Log.Warning("workspace", $"The server filed render {entry.RenderId} without its project {entry.ProjectId} and version {entry.ProjectVersionId} (a server before phase 5c)");
|
||||
context.Log(Loc.L("Archive.Log.WorkspaceSendNoProvenance", name));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
catch (ChronosApiException ex) when (ex.Status is HttpStatusCode.BadRequest or HttpStatusCode.Conflict or HttpStatusCode.RequestEntityTooLarge
|
||||
or HttpStatusCode.UnprocessableEntity)
|
||||
{
|
||||
throw new SendRefusedException(string.IsNullOrWhiteSpace(ex.Error.Message) ? ex.Message : ex.Error.Message);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Waits while transfers with a server are held (a metered connection), saying so once.</summary>
|
||||
private async Task WhileHeldAsync(JobContext context)
|
||||
{
|
||||
if (!options.HoldTransfers()) return;
|
||||
context.Log(Loc.L("Archive.Log.WorkspaceSendHeld"));
|
||||
while (options.HoldTransfers()) await Task.Delay(HoldPoll, context.Cancellation).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The ledger entry of <paramref name="renderId"/> changed by <paramref name="change"/>, under the project's lease, read again
|
||||
/// first (a render finished meanwhile stays); an entry discarded meanwhile stays discarded. Written whatever stops the job.
|
||||
/// </summary>
|
||||
private static async Task UpdateAsync(WorkspaceProject project, Guid renderId, Func<WorkspaceRenderEntry, WorkspaceRenderEntry> change)
|
||||
{
|
||||
using (await project.LockAsync(LockWait, CancellationToken.None).ConfigureAwait(false))
|
||||
{
|
||||
var ledger = project.ReadRenders();
|
||||
if (ledger.Find(renderId) is { } entry) project.WriteRenders(ledger.With(change(entry)));
|
||||
}
|
||||
}
|
||||
|
||||
private static void TryDelete(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(path)) Directory.Delete(path, recursive: true);
|
||||
else if (File.Exists(path)) File.Delete(path);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Debug("workspace", $"{path} could not be removed: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The home will not take this render as it is: it is marked failed with this message, and the others go on.</summary>
|
||||
private sealed class SendRefusedException(string message) : Exception(message);
|
||||
}
|
||||
@@ -91,17 +91,48 @@ public sealed record WorkspaceConflict(Guid TheirsVersionId, int TheirsNumber, G
|
||||
|
||||
/// <summary>
|
||||
/// Parameters of a <see cref="JobKind.WorkspaceCopy"/> job: target = the home's sequence; the workspace folder, the
|
||||
/// project whose media to copy and what to copy (<see cref="BundleMedia.Proxies"/>; originals in phase 5c).
|
||||
/// project whose media to copy and what to copy (<see cref="BundleMedia.Proxies"/>, or <see cref="BundleMedia.Originals"/>:
|
||||
/// the proxies plus the originals, phase 5c).
|
||||
/// </summary>
|
||||
/// <param name="Server">The server the home's sequence is on (<see cref="SyncService.ServerKey"/>, phase 5b); null for a sequence of this PC's archive.</param>
|
||||
public sealed record WorkspaceCopyParameters(string WorkspaceRoot, Guid ProjectId, string Media, string? Server = null);
|
||||
|
||||
/// <summary>
|
||||
/// Parameters of a <see cref="JobKind.WorkspaceRender"/> job (phase 5c): target = the home's sequence; the workspace folder,
|
||||
/// the project rendered from the originals its media hold, the version rendered (null: the head version the workspace
|
||||
/// holds), and the render's export settings as Protocol <c>RenderJobParameters</c> JSON (portable settings, preset,
|
||||
/// overrides, batch), as <see cref="Processing.ProcessingJobs"/> writes them for a render.
|
||||
/// </summary>
|
||||
/// <param name="ProjectVersionId">The version rendered and recorded with each output; null renders the head version the workspace holds.</param>
|
||||
/// <param name="RenderJson">Protocol <c>RenderJobParameters</c> as JSON.</param>
|
||||
/// <param name="Server">The server the home's sequence is on (<see cref="SyncService.ServerKey"/>); null for a sequence of this PC's archive.</param>
|
||||
public sealed record WorkspaceRenderParameters(string WorkspaceRoot, Guid ProjectId, Guid? ProjectVersionId, string RenderJson, string? Server = null);
|
||||
|
||||
/// <summary>
|
||||
/// Parameters of a <see cref="JobKind.WorkspaceSend"/> job (phase 5c): target = the home's sequence; the workspace folder,
|
||||
/// the project, and the ledger entries (<see cref="WorkspaceRenderEntry.RenderId"/>) whose files go home.
|
||||
/// </summary>
|
||||
/// <param name="RenderIds">The renders sent; empty sends every pending entry of the project's ledger.</param>
|
||||
/// <param name="Server">The server the home's sequence is on (<see cref="SyncService.ServerKey"/>); null for a sequence of this PC's archive.</param>
|
||||
public sealed record WorkspaceSendParameters(string WorkspaceRoot, Guid ProjectId, IReadOnlyList<Guid> RenderIds, string? Server = null);
|
||||
|
||||
/// <summary>
|
||||
/// Parameters of a <see cref="JobKind.WorkspaceMove"/> job (phase 5c): no target; the workspace folder moved
|
||||
/// (<paramref name="From"/>), the folder it goes to, and the fingerprint of the plan the user confirmed (the job refuses a
|
||||
/// workspace that changed since).
|
||||
/// </summary>
|
||||
public sealed record WorkspaceMoveParameters(string From, string To, string PlanFingerprint);
|
||||
|
||||
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
UseStringEnumConverter = true, WriteIndented = true)]
|
||||
[JsonSerializable(typeof(WorkspaceManifest))]
|
||||
[JsonSerializable(typeof(WorkspaceMedia))]
|
||||
[JsonSerializable(typeof(WorkspaceSyncState))]
|
||||
[JsonSerializable(typeof(WorkspaceCopyParameters))]
|
||||
[JsonSerializable(typeof(WorkspaceRenderParameters))]
|
||||
[JsonSerializable(typeof(WorkspaceSendParameters))]
|
||||
[JsonSerializable(typeof(WorkspaceMoveParameters))]
|
||||
[JsonSerializable(typeof(WorkspaceRenders))]
|
||||
internal sealed partial class WorkspaceJson : JsonSerializerContext;
|
||||
|
||||
/// <summary>
|
||||
@@ -309,9 +340,11 @@ public sealed class WorkspaceStore
|
||||
/// <summary>
|
||||
/// One project in the workspace (owner review 2, R8): <c><root>/<project id>/</c> with
|
||||
/// <c>project.chronos</c> (the working state: a project file of format 2, its settings placed on the media's frames),
|
||||
/// <c>media/media.json</c> with <c>media/proxies/<frame id>.jpg</c> (originals later) and <c>media/analysis.json</c>
|
||||
/// (the home's deep report), <c>versions/<number>-<id>.json</c> (commits not pushed yet, and the versions pulled),
|
||||
/// <c>sync.json</c>, <c>exports/</c> and the lease <c>.lock.lease</c> that one writer at a time holds.
|
||||
/// <c>media/media.json</c> with <c>media/proxies/<frame id>.jpg</c>, <c>media/originals/<frame id>-<sha8><ext></c>
|
||||
/// (media of kind originals, phase 5c) and <c>media/analysis.json</c> (the home's deep report),
|
||||
/// <c>versions/<number>-<id>.json</c> (commits not pushed yet, and the versions pulled), <c>sync.json</c>,
|
||||
/// <c>exports/</c> with the local renders and their ledger <c>exports/renders.json</c> (phase 5c), and the lease
|
||||
/// <c>.lock.lease</c> that one writer at a time holds.
|
||||
/// </summary>
|
||||
public sealed class WorkspaceProject
|
||||
{
|
||||
@@ -324,6 +357,12 @@ public sealed class WorkspaceProject
|
||||
public const string ExportsFolder = "exports";
|
||||
public const string SyncFile = "sync.json";
|
||||
|
||||
/// <summary>The name of the ledger of the project's local renders (<see cref="WorkspaceRenders"/>) in <see cref="ExportsFolder"/>.</summary>
|
||||
public const string RendersFileName = "renders.json";
|
||||
|
||||
/// <summary>The ledger of the project's local renders (phase 5c), relative to <see cref="Folder"/>.</summary>
|
||||
public const string RendersFile = ExportsFolder + "/" + RendersFileName;
|
||||
|
||||
/// <summary>
|
||||
/// Old local commits replaced by their replay on top of the home's head, files the home's copy replaced, and the working
|
||||
/// states a take of the server's values replaced: kept up to a month, the newest 20 at most (<see cref="PruneReplaced"/>).
|
||||
@@ -365,6 +404,9 @@ public sealed class WorkspaceProject
|
||||
public string ExportsDirectory => Path.Combine(Folder, ExportsFolder);
|
||||
public string SyncPath => Path.Combine(Folder, SyncFile);
|
||||
|
||||
/// <summary>The ledger of the project's local renders (<see cref="RendersFile"/>, phase 5c).</summary>
|
||||
public string RendersPath => Path.Combine(ExportsDirectory, RendersFileName);
|
||||
|
||||
public bool Exists => File.Exists(BundlePath);
|
||||
|
||||
/// <summary>The file a frame's proxy is copied to, relative to <see cref="MediaDirectory"/>.</summary>
|
||||
@@ -380,9 +422,62 @@ public sealed class WorkspaceProject
|
||||
return ProxiesFolder + "/" + frameId.ToString("N") + "-" + sha256[..Math.Min(16, sha256.Length)].ToLowerInvariant() + ".jpg";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The file a frame's original is copied to, relative to <see cref="MediaDirectory"/> (phase 5c):
|
||||
/// <c>originals/<frame id>-<sha8><ext></c>. The first 8 hex digits of its SHA-256 tell two encodings of one
|
||||
/// frame apart (a conversion since: the editor keeps the one it reads until the copy commits), and the original's own
|
||||
/// extension is kept, lower case, since decoders go by it.
|
||||
/// </summary>
|
||||
/// <param name="extension">The original's extension, with or without its dot (".CR3", "dng", or the file name's <see cref="Path.GetExtension(string)"/>); empty or null for none.</param>
|
||||
public static string OriginalName(Guid frameId, string sha256, string? extension)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrEmpty(sha256);
|
||||
var ext = (extension ?? string.Empty).Trim().TrimStart('.').ToLowerInvariant();
|
||||
if (ext.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0 || ext.Contains('/') || ext.Contains('\\')) ext = string.Empty;
|
||||
return OriginalsFolder + "/" + frameId.ToString("N") + "-" + sha256[..Math.Min(8, sha256.Length)].ToLowerInvariant() + (ext.Length > 0 ? "." + ext : string.Empty);
|
||||
}
|
||||
|
||||
/// <summary>The ledger of the project's local renders; empty while it has none (<see cref="WorkspaceRenders.Read"/>).</summary>
|
||||
/// <exception cref="InvalidDataException">The ledger is damaged or newer.</exception>
|
||||
/// <exception cref="IOException">The ledger cannot be read.</exception>
|
||||
public WorkspaceRenders ReadRenders() => WorkspaceRenders.Read(RendersPath);
|
||||
|
||||
/// <summary>Writes the ledger of the project's local renders whole and atomically (<see cref="WorkspaceRenders.Write"/>); the caller holds the project's lease.</summary>
|
||||
public void WriteRenders(WorkspaceRenders renders) => WorkspaceRenders.Write(RendersPath, renders);
|
||||
|
||||
/// <summary>The absolute path of a file named relative to <see cref="MediaDirectory"/>.</summary>
|
||||
public string MediaFile(string relative) => LongPath.FromPortableRelative(MediaDirectory, relative);
|
||||
|
||||
/// <summary>The absolute path of a local render named relative to <see cref="ExportsDirectory"/> (<see cref="WorkspaceRenderEntry.File"/>).</summary>
|
||||
public string ExportFile(string relative) => LongPath.FromPortableRelative(ExportsDirectory, relative);
|
||||
|
||||
/// <summary>
|
||||
/// The absolute path of a local render named relative to <see cref="ExportsDirectory"/>, only when it lies strictly inside
|
||||
/// that folder and is not its ledger; null for a name that is empty, rooted, climbs out (<c>..</c>), names the folder
|
||||
/// itself (<c>.</c>) or the ledger. What reads, sends or deletes a ledger's file goes through this, so a damaged or
|
||||
/// hand-edited ledger never points it at anything else.
|
||||
/// </summary>
|
||||
public string? ContainedExportFile(string? relative)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(relative)) return null;
|
||||
var portable = relative.Replace('\\', '/');
|
||||
if (Path.IsPathRooted(relative) || portable.StartsWith('/') || portable.Contains(':', StringComparison.Ordinal)
|
||||
|| portable.Split('/').Any(part => part is "" or "." or "..")) return null;
|
||||
string full, folder;
|
||||
try
|
||||
{
|
||||
full = Path.GetFullPath(ExportFile(relative));
|
||||
folder = Path.TrimEndingDirectorySeparator(Path.GetFullPath(ExportsDirectory)) + Path.DirectorySeparatorChar;
|
||||
}
|
||||
catch (Exception ex) when (ex is ArgumentException or NotSupportedException or PathTooLongException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
var comparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
|
||||
if (!full.StartsWith(folder, comparison) || full.Length <= folder.Length) return null;
|
||||
return string.Equals(full, Path.GetFullPath(RendersPath), comparison) ? null : full;
|
||||
}
|
||||
|
||||
/// <summary>Holds the project for one writer (this program's sync, a copy job) until disposed.</summary>
|
||||
/// <exception cref="LeaseUnavailableException">Another writer kept it for longer than <paramref name="wait"/>.</exception>
|
||||
public async Task<IDisposable> LockAsync(TimeSpan wait, CancellationToken cancellationToken = default)
|
||||
@@ -441,7 +536,7 @@ public sealed class WorkspaceProject
|
||||
internal void WriteMedia(WorkspaceMedia media)
|
||||
{
|
||||
Directory.CreateDirectory(MediaDirectory);
|
||||
AtomicFile.WriteAllText(MediaIndexPath, JsonSerializer.Serialize(media, WorkspaceJson.Default.WorkspaceMedia));
|
||||
AtomicFile.WriteAllText(MediaIndexPath, JsonSerializer.Serialize(media with { FormatVersion = media.WrittenFormatVersion }, WorkspaceJson.Default.WorkspaceMedia));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -1200,7 +1200,7 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName)
|
||||
// A local commit the home stored although its answer was lost reads as behind by the commits it holds: they are ours.
|
||||
if (head is not null && locals.Any(v => v.Id == head.Id)) behind = 0;
|
||||
var state = gone ? WorkspaceState.Gone
|
||||
: media is null ? WorkspaceState.MediaRemoved
|
||||
: media is null || media.IsCleaned ? WorkspaceState.MediaRemoved
|
||||
: damaged > 0 ? WorkspaceState.Damaged
|
||||
: conflict is not null ? WorkspaceState.Conflict
|
||||
: offline ? WorkspaceState.Offline
|
||||
@@ -1210,7 +1210,9 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName)
|
||||
: WorkspaceState.InSync;
|
||||
// A project the home no longer has holds no conflict to answer (nothing can be merged with it): the one recorded stays
|
||||
// in sync.json, for the record and for a project the home holds again.
|
||||
return new WorkspaceStatus(state, modified, locals.Count, behind, offline, media is null, bundle.IsReadOnly, gone ? null : conflict, workspace.Base, head)
|
||||
// Media the clean-up emptied hold no file to edit: removed, as media never copied, until a copy brings them again.
|
||||
return new WorkspaceStatus(state, modified, locals.Count, behind, offline, media is null || media.IsCleaned, bundle.IsReadOnly, gone ? null : conflict,
|
||||
workspace.Base, head)
|
||||
{
|
||||
MediaOutdated = outdated,
|
||||
Damaged = damaged,
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Globalization;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using Chronos.Core.Diagnostics;
|
||||
|
||||
namespace Chronos.Data.Workspaces;
|
||||
|
||||
/// <summary>One entry of a workspace's trash: its name in the trash folder, its bytes and files, and when it was put there.</summary>
|
||||
public sealed record WorkspaceTrashEntry(string Name, long Bytes, int Files, DateTimeOffset Trashed);
|
||||
|
||||
/// <summary>
|
||||
/// What a workspace's trash holds now (<see cref="WorkspaceTrash.List"/>): its entries, oldest first, their bytes, and a
|
||||
/// fingerprint over the name, size and time of every file in them, which an emptying checks first.
|
||||
/// </summary>
|
||||
public sealed record WorkspaceTrashListing(string Folder, IReadOnlyList<WorkspaceTrashEntry> Entries, long Bytes, string Fingerprint)
|
||||
{
|
||||
/// <summary>The trash holds nothing.</summary>
|
||||
public bool IsEmpty => Entries.Count == 0;
|
||||
}
|
||||
|
||||
/// <summary>The workspace's trash changed after it was listed: nothing was erased.</summary>
|
||||
public sealed class WorkspaceTrashChangedException(string message) : IOException(message);
|
||||
|
||||
/// <summary>
|
||||
/// The trash of a workspace (<c><workspace>/.trash</c>, phase 5c): what a removed local copy kept of its work
|
||||
/// (<c><project id>-<ticks></c>), and the originals a clean-up found the home no longer holds
|
||||
/// (<c><project id>-<ticks>/originals</c>). An entry is filled under a staging name the listing leaves out
|
||||
/// (<c>.incoming-<project id>-<ticks></c>, <see cref="Begin"/>) and takes its own name in one rename once it is
|
||||
/// whole, so nothing writes into an entry a plan lists; one a crash left half filled lists as an entry of its own. Only the
|
||||
/// user's Empty… erases them: the plan lists them (<see cref="List"/>), and <see cref="Empty"/> erases exactly those, only
|
||||
/// while the fingerprint of the plan still matches. Schedules and the size limit never empty it.
|
||||
/// </summary>
|
||||
public static class WorkspaceTrash
|
||||
{
|
||||
/// <summary>The trash's folder name in the workspace (the store lists only folders named by a project id).</summary>
|
||||
public const string FolderName = ".trash";
|
||||
|
||||
/// <summary>The name an entry has while it is filled (<see cref="Begin"/>).</summary>
|
||||
internal const string IncomingPrefix = ".incoming-";
|
||||
|
||||
/// <summary>The staging folders being filled now in this program (by name: a project id and ticks): the listing leaves them out.</summary>
|
||||
private static readonly ConcurrentDictionary<string, byte> Filling = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>The trash folder of the workspace at <paramref name="workspaceRoot"/>.</summary>
|
||||
public static string FolderOf(string workspaceRoot)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(workspaceRoot);
|
||||
return Path.Combine(workspaceRoot, FolderName);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The path of a new entry of the trash for <paramref name="projectId"/> (<c><id>-<ticks></c>, the ticks of
|
||||
/// <paramref name="nowUtc"/> moved on while an entry of that name exists); not made.
|
||||
/// </summary>
|
||||
public static string NewEntry(string workspaceRoot, Guid projectId, DateTime nowUtc)
|
||||
{
|
||||
var folder = FolderOf(workspaceRoot);
|
||||
for (var ticks = nowUtc.Ticks; ; ticks++)
|
||||
{
|
||||
var path = Path.Combine(folder, projectId.ToString("N") + "-" + ticks.ToString(CultureInfo.InvariantCulture));
|
||||
if (!Directory.Exists(path) && !File.Exists(path)) return path;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A new entry of the trash for <paramref name="projectId"/>, filled under a staging name the listing leaves out; it takes
|
||||
/// its own name (<see cref="NewEntry"/>) when the builder is disposed, or goes when nothing came into it. The staging
|
||||
/// folder is made when the builder is.
|
||||
/// </summary>
|
||||
public static WorkspaceTrashBuilder Begin(string workspaceRoot, Guid projectId, DateTime nowUtc)
|
||||
{
|
||||
var folder = FolderOf(workspaceRoot);
|
||||
for (var ticks = nowUtc.Ticks; ; ticks++)
|
||||
{
|
||||
var path = Path.Combine(folder, IncomingPrefix + projectId.ToString("N") + "-" + ticks.ToString(CultureInfo.InvariantCulture));
|
||||
// Named as being filled before it is made: a listing never sees it half made.
|
||||
if (Directory.Exists(path) || File.Exists(path) || !Filling.TryAdd(Path.GetFileName(path), 0)) continue;
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(path);
|
||||
}
|
||||
catch
|
||||
{
|
||||
Filling.TryRemove(Path.GetFileName(path), out _);
|
||||
throw;
|
||||
}
|
||||
return new WorkspaceTrashBuilder(workspaceRoot, projectId, path);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The staging folder <paramref name="staging"/> made an entry of its own (its own name), or removed when empty; no longer left out of the listing.</summary>
|
||||
internal static string? Finish(string workspaceRoot, Guid projectId, string staging)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!Directory.Exists(staging)) return null;
|
||||
if (!Directory.EnumerateFileSystemEntries(staging).Any())
|
||||
{
|
||||
Directory.Delete(staging);
|
||||
return null;
|
||||
}
|
||||
var entry = NewEntry(workspaceRoot, projectId, DateTime.UtcNow);
|
||||
Directory.Move(staging, entry);
|
||||
return entry;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
// Listed from now on under its staging name: what it holds stays in the trash, and Empty… can erase it.
|
||||
Log.Warning("workspace", $"The trash entry {staging} could not take its own name; it stays as it is", ex);
|
||||
return Directory.Exists(staging) ? staging : null;
|
||||
}
|
||||
finally
|
||||
{
|
||||
Filling.TryRemove(Path.GetFileName(staging), out _);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The entries of the workspace's trash at <paramref name="workspaceRoot"/>, with their fingerprint; empty when there is no trash.</summary>
|
||||
/// <exception cref="IOException">The trash cannot be read.</exception>
|
||||
public static WorkspaceTrashListing List(string workspaceRoot, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var folder = FolderOf(workspaceRoot);
|
||||
var entries = new List<WorkspaceTrashEntry>();
|
||||
var lines = new List<string>();
|
||||
if (Directory.Exists(folder))
|
||||
{
|
||||
foreach (var entry in new DirectoryInfo(folder).EnumerateFileSystemInfos())
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
// An entry being filled now is no entry yet: a plan never lists it, so Empty… never erases it.
|
||||
if (entry is DirectoryInfo && Filling.ContainsKey(entry.Name)) continue;
|
||||
var (files, bytes) = (0, 0L);
|
||||
if (entry is DirectoryInfo directory)
|
||||
{
|
||||
// A folder counts by its name only: NTFS updates a folder's time in its parent's index lazily, so the
|
||||
// enumeration may report a stale one; its files (read from their own records) cover what it holds.
|
||||
lines.Add(entry.Name + "/");
|
||||
foreach (var file in directory.EnumerateFiles("*", SearchOption.AllDirectories))
|
||||
{
|
||||
file.Refresh();
|
||||
files++;
|
||||
bytes += file.Length;
|
||||
lines.Add(Line(Path.GetRelativePath(folder, file.FullName).Replace('\\', '/'), file.Length, file.LastWriteTimeUtc));
|
||||
}
|
||||
}
|
||||
else if (entry is FileInfo single)
|
||||
{
|
||||
single.Refresh();
|
||||
(files, bytes) = (1, single.Length);
|
||||
lines.Add(Line(entry.Name, single.Length, single.LastWriteTimeUtc));
|
||||
}
|
||||
entries.Add(new WorkspaceTrashEntry(entry.Name, bytes, files, TrashedAt(entry)));
|
||||
}
|
||||
}
|
||||
lines.Sort(StringComparer.Ordinal);
|
||||
var fingerprint = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(string.Join('\n', lines))));
|
||||
IReadOnlyList<WorkspaceTrashEntry> ordered = [.. entries.OrderBy(e => e.Trashed).ThenBy(e => e.Name, StringComparer.Ordinal)];
|
||||
return new WorkspaceTrashListing(folder, ordered, ordered.Sum(e => e.Bytes), fingerprint);
|
||||
|
||||
static string Line(string name, long size, DateTime written) =>
|
||||
string.Create(CultureInfo.InvariantCulture, $"{name}|{size}|{written.Ticks}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Erases the entries of <paramref name="plan"/> from the workspace's trash for good, the user having confirmed it; refused,
|
||||
/// erasing nothing, when the trash is not as the plan listed it (an entry added, removed or changed since).
|
||||
/// </summary>
|
||||
/// <returns>The entries and bytes erased.</returns>
|
||||
/// <exception cref="WorkspaceTrashChangedException">The trash changed since the plan; nothing was erased.</exception>
|
||||
/// <exception cref="IOException">An entry could not be erased (a file held open); the others were, and it stays.</exception>
|
||||
public static (int Entries, long Bytes) Empty(string workspaceRoot, WorkspaceTrashListing plan, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(plan);
|
||||
var now = List(workspaceRoot, cancellationToken);
|
||||
if (!string.Equals(Path.GetFullPath(now.Folder), Path.GetFullPath(plan.Folder), StringComparison.OrdinalIgnoreCase)
|
||||
|| !string.Equals(now.Fingerprint, plan.Fingerprint, StringComparison.Ordinal))
|
||||
throw new WorkspaceTrashChangedException($"The trash {now.Folder} changed after it was listed.");
|
||||
var (erased, bytes) = (0, 0L);
|
||||
var failed = new List<string>();
|
||||
foreach (var entry in plan.Entries)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var path = Path.Combine(now.Folder, entry.Name);
|
||||
try
|
||||
{
|
||||
Erase(path);
|
||||
erased++;
|
||||
bytes += entry.Bytes;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("workspace", $"The trash entry {path} could not be erased", ex);
|
||||
failed.Add(entry.Name);
|
||||
}
|
||||
}
|
||||
Log.Info("workspace", $"The workspace's trash {now.Folder} emptied: {erased} entr(ies), {bytes} bytes erased");
|
||||
if (failed.Count > 0) throw new IOException($"Trash entries not erased: {string.Join(", ", failed)}.");
|
||||
return (erased, bytes);
|
||||
}
|
||||
|
||||
/// <summary>When an entry was put in the trash: the ticks of its name (<c><id>-<ticks></c>), else when it was written last.</summary>
|
||||
private static DateTimeOffset TrashedAt(FileSystemInfo entry)
|
||||
{
|
||||
var dash = entry.Name.LastIndexOf('-');
|
||||
if (dash > 0 && long.TryParse(entry.Name.AsSpan(dash + 1), NumberStyles.None, CultureInfo.InvariantCulture, out var ticks)
|
||||
&& ticks >= DateTime.MinValue.Ticks && ticks <= DateTime.MaxValue.Ticks)
|
||||
return new DateTimeOffset(ticks, TimeSpan.Zero);
|
||||
return new DateTimeOffset(entry.LastWriteTimeUtc, TimeSpan.Zero);
|
||||
}
|
||||
|
||||
/// <summary>A trash entry erased, files marked read-only included.</summary>
|
||||
private static void Erase(string path)
|
||||
{
|
||||
if (File.Exists(path))
|
||||
{
|
||||
File.SetAttributes(path, FileAttributes.Normal);
|
||||
File.Delete(path);
|
||||
return;
|
||||
}
|
||||
if (!Directory.Exists(path)) return;
|
||||
foreach (var file in new DirectoryInfo(path).EnumerateFiles("*", SearchOption.AllDirectories))
|
||||
{
|
||||
if ((file.Attributes & FileAttributes.ReadOnly) != 0) file.Attributes &= ~FileAttributes.ReadOnly;
|
||||
}
|
||||
Directory.Delete(path, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A trash entry being filled (<see cref="WorkspaceTrash.Begin"/>): <see cref="Path"/> is its staging folder, left out of every
|
||||
/// listing; disposing it gives the entry its own name (<see cref="Entry"/>), or removes it when nothing came into it.
|
||||
/// </summary>
|
||||
public sealed class WorkspaceTrashBuilder : IDisposable
|
||||
{
|
||||
private readonly string _root;
|
||||
private readonly Guid _projectId;
|
||||
private int _done;
|
||||
|
||||
internal WorkspaceTrashBuilder(string root, Guid projectId, string path)
|
||||
{
|
||||
(_root, _projectId, Path) = (root, projectId, path);
|
||||
}
|
||||
|
||||
/// <summary>The staging folder to fill.</summary>
|
||||
public string Path { get; }
|
||||
|
||||
/// <summary>The entry once finished; null before, or when nothing came into it.</summary>
|
||||
public string? Entry { get; private set; }
|
||||
|
||||
/// <summary>Gives the entry its own name (or removes it when empty); later calls do nothing.</summary>
|
||||
public string? Finish()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _done, 1) == 0) Entry = WorkspaceTrash.Finish(_root, _projectId, Path);
|
||||
return Entry;
|
||||
}
|
||||
|
||||
public void Dispose() => Finish();
|
||||
}
|
||||
Reference in new issue
Block a user