using System.Globalization;
using System.Security.Cryptography;
using System.Text.Json;
using Chronos.Core.Archiving;
using Chronos.Core.Diagnostics;
using Chronos.Core.IO;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Data.Archiving;
using Chronos.Data.Archiving.Jobs;
using Chronos.Data.Jobs;
using Chronos.Data.Projects;
using Chronos.Data.Sync;
namespace Chronos.Data.Workspaces;
/// Builds the jobs of the workspace (owner review 2, R8), and their handlers for the desktop's engine.
public static class WorkspaceJobs
{
///
/// : the media of workspace project (checked out first,
/// ) copied from its home, the sequence of this PC's archive.
///
public static ArchiveJobRequest Copy(Sequence sequence, string workspaceRoot, Guid projectId, string media = BundleMedia.Proxies)
{
ArgumentNullException.ThrowIfNull(sequence);
ArgumentException.ThrowIfNullOrWhiteSpace(workspaceRoot);
return new ArchiveJobRequest(JobKind.WorkspaceCopy, [sequence.Id], sequence.DisplayTitle,
JsonSerializer.Serialize(new WorkspaceCopyParameters(workspaceRoot, projectId, media), WorkspaceJson.Default.WorkspaceCopyParameters), JobPriority.User);
}
///
/// The handlers of the workspace's jobs for a desktop's engine (never a server's: the job writes into a folder its
/// parameters name).
///
public static IReadOnlyList CreateHandlers(ChronosDatabase db, ArchiveService archives) => [new WorkspaceCopyJobHandler(db, archives)];
///
/// What a first copy of a sequence's proxies writes into a workspace, as the copy reckons it before it starts (the
/// archive's proxies' sizes, an estimate for those it makes), and the free bytes on 's
/// drive (null when it cannot be measured): "Open in workspace" shows both. The copy keeps
/// free besides.
///
/// The index has no such sequence or archive.
public static Task<(long Bytes, long? Free)> CopyFactsAsync(ChronosDatabase db, ArchiveService archives, Guid sequenceId, string workspaceRoot,
CancellationToken cancellationToken = default) =>
new WorkspaceCopyJobHandler(db, archives).FactsAsync(sequenceId, workspaceRoot, cancellationToken);
/// Bytes the copy leaves free on the workspace's drive besides what it copies.
public static long CopySpaceReserve => WorkspaceCopyJobHandler.SpaceReserve;
}
///
/// for a sequence of this PC's archive (owner review 2, R8): its proxies into the
/// workspace project's media/proxies/<frame id>.jpg, the archive's own when current, else made here from the
/// original (at the archive's proxy size, or the default one when the archive makes none), each written atomically and read
/// back to check its SHA-256; then the deep report (media/analysis.json) and, last, media/media.json with the
/// encodings conversions joined at the home. The job only reads the archive: it never writes there, not even proxies (the
/// archive's own pass makes those), refuses a workspace folder inside any library of the index before it writes anything,
/// and reads only a folder that holds the sequence ().
/// Frames changed at the home during the copy, or the folder moved (a Reorganize), make it start again (three times at
/// most; a folder still moving fails it, to be retried). The working state's positions follow the new frames: by frame
/// where a frame stays (one whose file a conversion replaced keeps its positions), by photo where frames came or went.
/// Before each pass and at each checkpoint the workspace's drive must hold what is still to copy plus a reserve, else the
/// job stops saying how much is missing (the workspace shares this PC's drive with the index and the logs).
/// Checkpoint: copy:<frame>:<frames version>:<folder>; a run resumed on other frames or in another folder
/// checks every file again, and a pass started again stores that before it reads anything. A first copy the user cancels
/// removes the media it wrote itself, once it has stopped writing and under the project's lease, unless another copy of the
/// project goes on with them (); one cancelled while paused or failed, which no handler
/// runs, has them removed the same way ().
/// Originals come in phase 5c: asked for, the job fails saying so.
///
internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService archives) : ICancelledJobHandler
{
private const int PageSize = 1000;
private const int Passes = 3;
private static readonly TimeSpan LockWait = TimeSpan.FromMinutes(2);
private static readonly TimeSpan CheckpointEvery = TimeSpan.FromSeconds(5);
/// Bytes left free on the workspace's drive besides the copy, as a download keeps ().
internal const long SpaceReserve = 512L << 20;
/// A proxy at px, when the archive has none to measure: some 1.5 MB.
private const long ProxyEstimate = 3L << 19;
private const int ProxyEstimateEdge = 1920;
/// The long edge taken for a full-size proxy of a frame whose size is not known.
private const int UnknownFullSize = 6000;
public JobKind Kind => JobKind.WorkspaceCopy;
public async Task RunAsync(JobContext context)
{
var ct = context.Cancellation;
var parameters = Parameters(context.Job);
if (parameters.Media == BundleMedia.Originals) throw new NotSupportedException(Loc.L("Errors.Workspace.OriginalsLater"));
if (parameters.Media != BundleMedia.Proxies) throw new InvalidDataException(Loc.L("Errors.Workspace.Parameters"));
if (context.Job.TargetIds is not [var sequenceId] || await db.Sequences.GetAsync(sequenceId, ct).ConfigureAwait(false) is not { } sequence
|| await db.Archives.GetAsync(sequence.ArchiveId, ct).ConfigureAwait(false) is not { } archive)
throw new InvalidOperationException(Loc.L("Archive.Log.SequenceMissing", context.Job.TargetIds.FirstOrDefault()));
// Asked before the store is opened, which makes its folder: editing happens outside every library (owner review 2,
// R8), and a workspace inside one would put its copies among that library's own files.
if (await InsideALibraryAsync(parameters.WorkspaceRoot, ct).ConfigureAwait(false))
throw new InvalidOperationException(Loc.L("Errors.Workspace.InsideLibrary"));
var store = new WorkspaceStore(parameters.WorkspaceRoot);
var project = store.Find(parameters.ProjectId) ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.NotCheckedOut"));
if (project.ReadBundle().Workspace is not { Home: { IsArchive: true } home } || home.SequenceId != sequenceId)
throw new InvalidOperationException(Loc.L("Errors.Workspace.OtherHome"));
try
{
using (WritingMedia(project)) await CopyAsync(context, project, sequenceId, archive, parameters, ct).ConfigureAwait(false);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
// Stopped writing: what a first copy the user cancelled wrote goes (a pause or a shutdown resumes it).
await RemovePartialMediaAsync(context.Job, project).ConfigureAwait(false);
throw;
}
}
///
/// A copy cancelled while no handler ran it (paused, failed): what a first copy wrote goes as after a cancel while it ran,
/// under the project's lease, in the background (the lease may be held for a while: the cancel does not wait for it).
/// A workspace folder or a project no longer there has nothing to clear, and nothing is made for it.
///
public Task CancelledAsync(Job job, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(job);
WorkspaceCopyParameters parameters;
try
{
parameters = Parameters(job);
}
catch (InvalidDataException)
{
return Task.CompletedTask;
}
// Opening a store makes its folder: one gone stays gone.
if (!Directory.Exists(Path.Combine(parameters.WorkspaceRoot, parameters.ProjectId.ToString("N")))) return Task.CompletedTask;
_ = Task.Run(async () =>
{
try
{
if (new WorkspaceStore(parameters.WorkspaceRoot).Find(parameters.ProjectId) is { } project) await RemovePartialMediaAsync(job, project).ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OutOfMemoryException)
{
// Nobody waits for this task: whatever stopped it is logged; the next copy checks every file it finds.
Log.Warning("workspace", $"The media of cancelled copy {job.Id} could not be cleared", ex);
}
}, CancellationToken.None);
return Task.CompletedTask;
}
///
/// A first copy of the project's media the user cancelled (the job reads ), once it has
/// stopped writing: the photos it wrote (copies the home can make again) are removed under the project's lease, so the
/// workspace holds no half media. Media that a copy ever finished (media.json) stay, as do those of a copy paused,
/// yielded or stopped with the program, which goes on from its checkpoint, and those another copy of the project into
/// this workspace still writes or goes on with (queued, running or paused: ). The last
/// look is taken under the lease, and no copy of this program starts writing while the media go
/// (). A folder that is no project any more (its removal under way) is left as it is, and the
/// lease never makes it again. A failure is logged: the next copy checks every file it finds.
///
private async Task RemovePartialMediaAsync(Job job, WorkspaceProject project)
{
try
{
// Nothing half written (or the copy removed meanwhile): the lease is not taken.
if (!Clearable(project) || !await StillToClearAsync(job, project).ConfigureAwait(false)) return;
using (await project.LockExistingAsync(LockWait, CancellationToken.None).ConfigureAwait(false))
{
if (!Clearable(project) || !await StillToClearAsync(job, project).ConfigureAwait(false)) return;
lock (WritersGate)
{
// A copy of this program that started since the jobs were read: its files stay.
if (Writers.ContainsKey(project.Folder) || !Clearable(project)) return;
Directory.Delete(project.MediaDirectory, recursive: true);
}
}
Log.Info("workspace", $"The media a cancelled first copy of project {project.ProjectId} wrote were removed");
}
catch (DirectoryNotFoundException)
{
// The project's folder went meanwhile (its removal): nothing is left to clear.
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or DatabaseException)
{
Log.Warning("workspace", $"The media of a cancelled copy of project {project.ProjectId} could not be removed", ex);
}
// A project (its working state there) with media no copy ever finished.
static bool Clearable(WorkspaceProject project) => project.Exists && !File.Exists(project.MediaIndexPath) && Directory.Exists(project.MediaDirectory);
}
/// The cancelled copy still reads so, and no other copy of the project into this workspace waits, runs or is paused.
private async Task StillToClearAsync(Job job, WorkspaceProject project)
{
if (await db.Jobs.GetAsync(job.Id, CancellationToken.None).ConfigureAwait(false) is not { State: JobState.Cancelled }) return false;
if (await OtherCopyAsync(job, project).ConfigureAwait(false) is not { } other) return true;
Log.Info("workspace", $"The media of cancelled copy {job.Id} stay: copy {other.Id} of project {project.ProjectId} is {other.State}");
return false;
}
///
/// Another copy of the project into the same workspace that is queued, running or paused: it writes the media folder, or
/// goes on with what is there. A job whose parameters cannot be read counts, so nothing is removed on a guess.
///
private async Task OtherCopyAsync(Job job, WorkspaceProject project)
{
var sequenceId = job.TargetIds.FirstOrDefault();
foreach (var other in await db.Jobs.ListPendingAsync(JobKind.WorkspaceCopy, sequenceId, CancellationToken.None).ConfigureAwait(false))
{
if (other.Id == job.Id) continue;
try
{
var parameters = Parameters(other);
if (parameters.ProjectId != project.ProjectId) continue;
// Case aside: a match on a case-sensitive drive only keeps media that could have gone.
if (!string.Equals(WorkspaceStore.FullRoot(parameters.WorkspaceRoot), project.Store.Root, StringComparison.OrdinalIgnoreCase)) continue;
}
catch (Exception ex) when (ex is InvalidDataException or ArgumentException or IOException or NotSupportedException or System.Security.SecurityException)
{
Log.Debug("workspace", $"Copy {other.Id} cannot be read: it counts as one of project {project.ProjectId}: {ex.Message}");
}
return other;
}
return null;
}
/// The project folders copies of this program write media into now, with how many copies write each.
private static readonly Dictionary Writers = new(StringComparer.OrdinalIgnoreCase);
private static readonly Lock WritersGate = new();
///
/// Marks the copy into as writing its media until disposed: a clean-up after a cancelled copy
/// leaves them (), and one under way finishes before the copy writes anything.
///
private static IDisposable WritingMedia(WorkspaceProject project)
{
lock (WritersGate) Writers[project.Folder] = Writers.GetValueOrDefault(project.Folder) + 1;
return new Writing(project.Folder);
}
private sealed class Writing(string folder) : IDisposable
{
private int _done;
public void Dispose()
{
if (Interlocked.Exchange(ref _done, 1) != 0) return;
lock (WritersGate)
{
if (Writers.GetValueOrDefault(folder) <= 1) Writers.Remove(folder);
else Writers[folder]--;
}
}
}
private async Task CopyAsync(JobContext context, WorkspaceProject project, Guid sequenceId, Archive archive, WorkspaceCopyParameters parameters,
CancellationToken ct)
{
var paths = new ArchivePaths(archive.RootPath);
// Read only: a read-only archive is copied from like any other.
if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(archive.RootPath, ArchiveAvailability.Offline);
var (copied, checkpointVersion, checkpointFolder) = Parse(context.Checkpoint);
for (var pass = 1; ; pass++)
{
var (row, frames) = await FramesAsync(sequenceId, ct).ConfigureAwait(false);
var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy;
var folder = paths.Sequence(row.Folder);
// Another index moved the sequence away, maybe another one in: its files are not this sequence's photos. A retry
// checks every file again (the files of this run may be of that folder).
if (!SequenceFolders.Holds(folder, sequenceId))
{
await context.CheckpointAsync(Checkpoint(Stage.Copy, 0, row.FramesVersion, row.Folder), 0, frames.Count).ConfigureAwait(false);
throw new SequenceFolderMovedException();
}
// Files kept from an earlier run are this frames version's and this folder's only (a conversion keeps a frame's id,
// not its photo; a sequence moved meanwhile may have left its photos' names to another sequence's).
if ((checkpointVersion is { } version && version != row.FramesVersion) || (checkpointFolder is { } read && read != row.Folder)) copied = 0;
var total = (long)frames.Count;
var tally = new Tally();
var entries = new WorkspaceFrame[frames.Count];
// The media the editor reads until this copy commits: their files stay until then (RemoveStale).
var current = TryReadMedia(project)?.Frames.Where(f => f.File is not null).ToDictionary(f => f.Id) ?? [];
Directory.CreateDirectory(project.ProxiesDirectory);
// The workspace is on this PC's own drive, with the index, the settings and the logs: the copy never fills it.
var remaining = Remaining(project, folder, frames, policy);
EnsureSpace(parameters.WorkspaceRoot, remaining[0]);
var clock = System.Diagnostics.Stopwatch.StartNew();
for (var i = 0; i < frames.Count; i++)
{
ct.ThrowIfCancellationRequested();
entries[i] = Copy(project, folder, paths, frames[i], policy, done: i < copied, current.GetValueOrDefault(frames[i].Id), tally, ct);
if (clock.Elapsed >= CheckpointEvery)
{
await context.CheckpointAsync(Checkpoint(Stage.Copy, i + 1, row.FramesVersion, row.Folder), i + 1, total).ConfigureAwait(false);
// Another program may take the space meanwhile.
EnsureSpace(parameters.WorkspaceRoot, remaining[i + 1]);
clock.Restart();
}
else
{
await context.ReportAsync(i + 1, total).ConfigureAwait(false);
}
}
var analysis = CopyAnalysis(folder, project, frames.Count);
var (after, now) = await FramesAsync(sequenceId, ct).ConfigureAwait(false);
var moved = after.Folder != row.Folder || !SequenceFolders.Holds(paths.Sequence(after.Folder), sequenceId) || Renamed(frames, now, entries);
if (after.FramesVersion != row.FramesVersion || moved)
{
// The files read may be of other frames, or missing where a Reorganize took them: read them again, and so
// does a run resumed after a crash in the next pass (the checkpoint stored says so).
if (pass < Passes)
{
context.Log(Loc.L("Archive.Log.WorkspaceFramesChanged", row.DisplayTitle));
await context.CheckpointAsync(Checkpoint(Stage.Copy, 0, after.FramesVersion, after.Folder), 0, total).ConfigureAwait(false);
(copied, checkpointVersion, checkpointFolder) = (0, null, null);
continue;
}
// Frames still arriving are copied as they were read (Update media takes the rest); a folder still moving is not.
if (moved)
{
await context.CheckpointAsync(Checkpoint(Stage.Copy, 0, after.FramesVersion, after.Folder), 0, total).ConfigureAwait(false);
throw new SequenceFolderMovedException();
}
}
var aliases = await HomePhotos.AliasesAsync(db, sequenceId, ct).ConfigureAwait(false);
var media = new WorkspaceMedia(WorkspaceMedia.FormatName, WorkspaceMedia.CurrentFormatVersion, BundleMedia.Proxies, sequenceId, row.FramesVersion, entries,
DateTimeOffset.UtcNow, analysis, aliases.Count > 0 ? aliases : null);
// The working state is the editor's meanwhile: it is held only while the media index and the positions move.
using (await project.LockAsync(LockWait, ct).ConfigureAwait(false))
{
Commit(project, project.ReadBundle(), media, row);
RemoveStale(project, media);
}
context.Log(Loc.L("Archive.Log.WorkspaceCopied", row.DisplayTitle, tally.Copied, tally.Made, tally.Kept, tally.Missing));
await context.CheckpointAsync(Checkpoint(Stage.Done, frames.Count, row.FramesVersion, row.Folder), total, total).ConfigureAwait(false);
return;
}
}
/// Whether is an archive's root of this index or lies inside one.
private async Task InsideALibraryAsync(string folder, CancellationToken cancellationToken)
{
var path = Plain(Path.GetFullPath(folder));
foreach (var archive in await db.Archives.ListAsync(cancellationToken).ConfigureAwait(false))
{
if (DataLocation.IsUnder(path, Plain(Path.GetFullPath(archive.RootPath)))) return true;
}
return false;
// Without the long-path prefix, which only one of the two may carry.
static string Plain(string path) =>
path.StartsWith(@"\\?\UNC\", StringComparison.Ordinal) ? @"\\" + path[8..]
: path.StartsWith(@"\\?\", StringComparison.Ordinal) ? path[4..]
: path;
}
///
/// A frame the copy found no file for whose original the index has under another name now (a Reorganize renamed the
/// files during the copy, keeping the frames version): the copy runs again.
///
private static bool Renamed(List read, List now, WorkspaceFrame[] entries)
{
var paths = now.ToDictionary(f => f.Id, f => f.RelativePath);
for (var i = 0; i < read.Count; i++)
{
if (entries[i].File is null && paths.TryGetValue(read[i].Id, out var path) && path != read[i].RelativePath) return true;
}
return false;
}
///
/// The frame's file in the workspace: kept when an earlier run copied it (before the checkpoint, or the same bytes as the
/// archive's proxy), else the archive's current proxy, else one made from the original; none for a frame without either.
/// The file the media the editor reads now name for the frame () is never removed nor written
/// over here: the editor keeps it while the copy runs and if it fails, other bytes for it go under a name of their own
/// (), and a copy that commits without it removes it then
/// (). Nor is it taken as this run's copy of a frame whose photo changed since.
///
private WorkspaceFrame Copy(WorkspaceProject project, SequencePaths folder, ArchivePaths paths, Frame frame, ProxyPolicy policy, bool done,
WorkspaceFrame? current, Tally tally, CancellationToken cancellationToken)
{
var name = WorkspaceProject.ProxyName(frame.Id);
var target = project.MediaFile(name);
var listed = current is not null && string.Equals(current.File, name, StringComparison.OrdinalIgnoreCase);
if (done && File.Exists(target))
{
var kept = AtomicFile.ReadAllBytes(target);
var keptSha = Sha(kept);
// The editor's file of another photo of the frame (a conversion since) was left by an earlier copy, not this run.
if (!listed || keptSha != current!.FileSha256 || SamePhoto(current, frame))
{
tally.Kept++;
return WorkspaceFrame.Of(frame, name, keptSha, kept.Length);
}
}
var source = folder.Proxy(frame.Index);
byte[]? bytes = null;
if (File.Exists(source) && ArchiveImages.IsProxyOf(source, frame.CapturedAt, frame.Width, frame.Height, null)) bytes = AtomicFile.ReadAllBytes(source);
var made = false;
if (bytes is null && (frame.Flags & FrameFlags.Missing) == 0)
{
var original = paths.FromRelative(frame.RelativePath);
try
{
var edge = policy.LongEdge == ProxyPolicy.Disabled ? new ProxyPolicy().LongEdge : policy.LongEdge;
bytes = File.Exists(original) ? ArchiveImages.MakeProxy(archives.Decoder, original, frame, null, edge, policy.Quality, cancellationToken) : null;
made = bytes is not null;
}
catch (IOException ex)
{
Log.Warning("workspace", $"No proxy of {original} could be made", ex);
}
}
if (bytes is null)
{
// A file an earlier copy left for the frame (of other frames, or another folder's) is never kept as its photo by a
// resumed run: a regenerable copy, it goes, unless the editor's media name it (it goes once this copy commits).
if (!listed && File.Exists(target)) TryDelete(target);
tally.Missing++;
return WorkspaceFrame.Of(frame, null, null, 0);
}
var sha = Sha(bytes);
// The file the editor's media name for the frame holds these bytes already (under its own name, maybe): kept.
if (current?.File is { } held && !string.Equals(held, name, StringComparison.OrdinalIgnoreCase)
&& string.Equals(current.FileSha256, sha, StringComparison.OrdinalIgnoreCase) && Holds(project.MediaFile(held), sha))
{
tally.Kept++;
return WorkspaceFrame.Of(frame, held, sha, bytes.Length);
}
if (Holds(target, sha))
{
tally.Kept++;
return WorkspaceFrame.Of(frame, name, sha, bytes.Length);
}
// Other bytes for the file the editor's media name (a conversion since): written under a name of their own, never over
// the editor's file, which the commit switches the media from (RemoveStale then removes it). A copy that fails before
// its commit leaves the editor reading the bytes its entry names. A file that no longer holds them (damaged, gone)
// is mended where it is.
if (listed && current!.FileSha256 is { } editors && Holds(target, editors.ToLowerInvariant()))
{
name = WorkspaceProject.ProxyName(frame.Id, sha);
target = project.MediaFile(name);
if (Holds(target, sha))
{
tally.Kept++;
return WorkspaceFrame.Of(frame, name, sha, bytes.Length);
}
}
WriteVerified(target, bytes, sha);
if (made) tally.Made++;
else tally.Copied++;
return WorkspaceFrame.Of(frame, name, sha, bytes.Length);
}
/// Whether the file at is there and hashes as .
private static bool Holds(string path, string sha) => File.Exists(path) && Sha(AtomicFile.ReadAllBytes(path)) == sha;
/// Writes atomically and reads back: a file that does not hash as its source is written again once, then refused.
private static void WriteVerified(string path, byte[] bytes, string sha)
{
for (var attempt = 1; ; attempt++)
{
AtomicFile.WriteAllBytes(path, bytes);
if (Sha(AtomicFile.ReadAllBytes(path)) == sha) return;
if (attempt == 2) throw new IOException(Loc.L("Errors.Workspace.CopyMismatch", path));
}
}
/// The home's deep report into the media folder when it describes these frames; its SHA-256, or null without one.
private static string? CopyAnalysis(SequencePaths folder, WorkspaceProject project, int frameCount)
{
if (!File.Exists(folder.AnalysisPath))
{
if (File.Exists(project.AnalysisPath)) File.Delete(project.AnalysisPath);
return null;
}
var bytes = AtomicFile.ReadAllBytes(folder.AnalysisPath);
try
{
if (Core.Analysis.Deep.DeepAnalysisReport.FromJson(System.Text.Encoding.UTF8.GetString(bytes)).FrameCount != frameCount) return null;
}
catch (Exception ex) when (ex is InvalidDataException or JsonException or IOException)
{
Log.Warning("workspace", $"{folder.AnalysisPath} cannot be read; the workspace takes no deep report", ex);
return null;
}
var sha = Sha(bytes);
WriteVerified(project.AnalysisPath, bytes, sha);
return sha;
}
///
/// The media index and the working state moved onto it: first the working state with every position parked by photo
/// (valid whatever the frames), then the index, then the working state placed on the new frames. A crash in between
/// leaves positions that wait for their photos, which the next open places. A frame the new media still hold is parked
/// as the photo they hold for it, so one whose file a conversion replaced (the same frame, another SHA-256) keeps its
/// positions, as the archive's own settings do ().
///
private static void Commit(WorkspaceProject project, ProjectBundle bundle, WorkspaceMedia media, Sequence row)
{
var old = TryReadMedia(project);
var workspace = bundle.Workspace! with { Media = new BundleMedia(BundleMedia.Proxies, media.FramesVersion) };
var sequence = bundle.Sequence with { Folder = row.Folder, Title = row.DisplayTitle, Frames = media.Frames.Count };
var working = bundle.Project;
// Without the media index the positions were made against, they are taken as naming these frames.
if (!working.IsNewerFormat && old is not null)
{
var held = media.Frames.ToDictionary(f => f.Id);
var before = new FramePhotos(old.Frames.Select(f => (f.Index, Photo(held.TryGetValue(f.Id, out var now) ? now : f))), old.Aliases);
var portable = ProjectSettings.FromJson(ProjectContent.ToPortable(working, before), allowNewer: true);
project.WriteBundle(bundle with { Project = portable, Workspace = workspace, Sequence = sequence });
project.WriteMedia(media);
working = ProjectSettings.FromJson(ProjectContent.Place(portable.ToJson(), media.Photos), allowNewer: true);
}
else
{
project.WriteMedia(media);
}
project.WriteBundle(bundle with { Project = working, Workspace = workspace, Sequence = sequence, Saved = DateTimeOffset.UtcNow });
}
private static FramePhoto Photo(WorkspaceFrame frame) => new(frame.Sha256, frame.Size, frame.CapturedAt);
/// Whether the media's entry of a frame is of the photo the frame holds now (a conversion gives it another file).
private static bool SamePhoto(WorkspaceFrame held, Frame frame) =>
held.Size == frame.Size && held.CapturedAt == frame.CapturedAt
&& (string.IsNullOrEmpty(held.Sha256) || string.IsNullOrEmpty(frame.Sha256) || string.Equals(held.Sha256, frame.Sha256, StringComparison.OrdinalIgnoreCase));
/// Proxies of frames the sequence no longer has: regenerable copies, removed so the media folder holds the frames listed.
private static void RemoveStale(WorkspaceProject project, WorkspaceMedia media)
{
var listed = media.Frames.Where(f => f.File is not null).Select(f => Path.GetFileName(f.File!)).ToHashSet(StringComparer.OrdinalIgnoreCase);
foreach (var file in Directory.EnumerateFiles(project.ProxiesDirectory, "*.jpg"))
{
if (!listed.Contains(Path.GetFileName(file))) TryDelete(file);
}
}
private static void TryDelete(string file)
{
try
{
File.Delete(file);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("workspace", $"{file} could not be removed", ex);
}
}
///
/// The bytes the copy still writes from each frame on (index i: frames i to the last; one more entry, 0):
/// nothing for a frame the workspace holds a file of already (kept, or replaced by one of about its size; under its own
/// name or the one other bytes of it take, ), the archive's
/// proxy's size where it has one, else an estimate at the proxy size the copy makes ().
///
private static long[] Remaining(WorkspaceProject project, SequencePaths folder, List frames, ProxyPolicy policy) =>
Remaining(Directory.Exists(project.ProxiesDirectory) ? HeldFrames(Directory.EnumerateFiles(project.ProxiesDirectory, "*.jpg")) : [], folder, frames, policy);
/// The frames the files at are proxies of: "<frame id>.jpg" and "<frame id>-<sha16>.jpg".
internal static HashSet HeldFrames(IEnumerable paths)
{
var held = new HashSet();
foreach (var path in paths)
{
var name = Path.GetFileNameWithoutExtension(path);
if (name.Length >= 32 && (name.Length == 32 || name[32] == '-') && Guid.TryParseExact(name[..32], "N", out var id)) held.Add(id);
}
return held;
}
private static long[] Remaining(HashSet held, SequencePaths folder, List frames, ProxyPolicy policy)
{
var proxies = Directory.Exists(folder.Proxies)
? new DirectoryInfo(folder.Proxies).EnumerateFiles().ToDictionary(f => f.Name, f => f.Length, StringComparer.OrdinalIgnoreCase)
: [];
var remaining = new long[frames.Count + 1];
for (var i = frames.Count - 1; i >= 0; i--)
{
var frame = frames[i];
var bytes = held.Contains(frame.Id) ? 0
: proxies.TryGetValue(ArchiveLayout.ProxyFileName(frame.Index), out var length) ? length
: (frame.Flags & FrameFlags.Missing) != 0 ? 0
: Estimate(frame, policy);
remaining[i] = remaining[i + 1] + bytes;
}
return remaining;
}
/// What a first copy of the sequence writes (nothing held yet), and the free bytes of the workspace's drive.
internal async Task<(long Bytes, long? Free)> FactsAsync(Guid sequenceId, string workspaceRoot, CancellationToken cancellationToken)
{
var (row, frames) = await FramesAsync(sequenceId, cancellationToken).ConfigureAwait(false);
var archive = await db.Archives.GetAsync(row.ArchiveId, cancellationToken).ConfigureAwait(false)
?? throw new InvalidOperationException(Loc.L("Archive.Log.SequenceMissing", sequenceId));
var paths = new ArchivePaths(archive.RootPath);
var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy;
return (Remaining([], paths.Sequence(row.Folder), frames, policy)[0], FreeBytes(workspaceRoot));
}
/// A proxy's size when the archive has none to measure: some 1.5 MB at 1920 px, scaled by the area at the copy's long edge.
private static long Estimate(Frame frame, ProxyPolicy policy)
{
var edge = policy.LongEdge == ProxyPolicy.Disabled ? new ProxyPolicy().LongEdge : policy.LongEdge;
var longest = Math.Max(frame.Width, frame.Height);
var side = edge == ProxyPolicy.FullSize ? (longest > 0 ? longest : UnknownFullSize) : longest > 0 ? Math.Min(edge, longest) : edge;
return (long)(ProxyEstimate * Math.Pow(side / (double)ProxyEstimateEdge, 2));
}
///
/// Refuses to go on while the workspace's drive lacks besides , with the
/// gigabytes missing; a drive that cannot be measured (a network share) is not refused, the copy itself then says so.
///
/// The drive lacks the space (Errors.Workspace.DiskFull).
private void EnsureSpace(string root, long bytes)
{
if (bytes <= 0 || FreeBytes(root) is not { } free || free >= bytes + SpaceReserve) return;
throw new IOException(Loc.L("Errors.Workspace.DiskFull", ((bytes + SpaceReserve - free) / 1e9).ToString("0.0", Loc.Culture)));
}
/// Free bytes on the volume of , as the host measures them (), else its drive's.
private long? FreeBytes(string folder)
{
if (archives.FreeSpace is { } measure) return measure(folder);
try
{
var full = Path.GetFullPath(folder);
// Windows wants a drive root; Unix accepts any directory and reports the volume it is mounted on.
return new DriveInfo(OperatingSystem.IsWindows() ? Path.GetPathRoot(full)! : full).AvailableFreeSpace;
}
catch (Exception ex) when (ex is ArgumentException or IOException or UnauthorizedAccessException)
{
return null;
}
}
private static WorkspaceMedia? TryReadMedia(WorkspaceProject project)
{
try
{
return project.ReadMedia();
}
catch (InvalidDataException ex)
{
Log.Warning("workspace", $"The media index of {project.Folder} cannot be read; positions are taken as they are", ex);
return null;
}
}
/// The sequence row with its frames as one state (listed between two reads of the row at the same frames version).
private async Task<(Sequence Row, List Frames)> FramesAsync(Guid sequenceId, CancellationToken cancellationToken)
{
for (var attempt = 0; ; attempt++)
{
var before = await db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
?? throw new InvalidOperationException(Loc.L("Archive.Log.SequenceMissing", sequenceId));
var frames = new List();
for (var after = -1; ;)
{
var page = await db.Frames.ListBySequenceAsync(sequenceId, after, PageSize, cancellationToken).ConfigureAwait(false);
frames.AddRange(page);
if (page.Count < PageSize) break;
after = page[^1].Index;
}
var now = await db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? before;
if (now.FramesVersion == before.FramesVersion || attempt == 2) return (now, frames);
}
}
private static WorkspaceCopyParameters Parameters(Job job)
{
try
{
if (job.ParametersJson is { Length: > 0 } json
&& JsonSerializer.Deserialize(json, WorkspaceJson.Default.WorkspaceCopyParameters) is { WorkspaceRoot.Length: > 0 } parameters
&& parameters.ProjectId != Guid.Empty)
return parameters with { Media = parameters.Media ?? BundleMedia.Proxies };
}
catch (JsonException ex)
{
throw new InvalidDataException(Loc.L("Errors.Workspace.Parameters"), ex);
}
throw new InvalidDataException(Loc.L("Errors.Workspace.Parameters"));
}
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes));
private enum Stage
{
Copy,
Done,
}
private static string Checkpoint(Stage stage, int frame, int framesVersion, string folder) =>
string.Create(CultureInfo.InvariantCulture, $"{stage.ToString().ToLowerInvariant()}:{frame}:{framesVersion}:{folder}");
///
/// The frames copied by an earlier run (copy:<frame>[:<frames version>[:<folder>]], the folder
/// last as it may hold a colon), the frames version and the sequence's folder they were of (null when the checkpoint does
/// not say); nothing for a finished run, or another stage (an earlier build's archive proxy pass), whose files the copy
/// checks again.
///
private static (int Copied, int? FramesVersion, string? Folder) Parse(string? checkpoint)
{
var parts = checkpoint?.Split(':', 4);
if (parts is not ([_, _] or [_, _, _] or [_, _, _, _]) || !Enum.TryParse(parts[0], ignoreCase: true, out var stage) || stage != Stage.Copy
|| !int.TryParse(parts[1], NumberStyles.None, CultureInfo.InvariantCulture, out var next))
return (0, null, null);
int? version = parts.Length >= 3 && int.TryParse(parts[2], NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture, out var v) ? v : null;
return (next, version, parts.Length == 4 ? parts[3] : null);
}
private sealed class Tally
{
public int Copied { get; set; }
public int Made { get; set; }
public int Kept { get; set; }
public int Missing { get; set; }
}
}