using System.Globalization; using System.Security.Cryptography; using System.Text.Json; using Chronos.Core.Archiving; using Chronos.Core.Diagnostics; using Chronos.Core.IO; using Chronos.Core.Localization; using Chronos.Core.Model; using Chronos.Core.Project; using Chronos.Data.Archiving; using Chronos.Data.Archiving.Jobs; using Chronos.Data.Jobs; using Chronos.Data.Projects; using Chronos.Data.Sync; namespace Chronos.Data.Workspaces; /// Builds the jobs of the workspace (owner review 2, R8), and their handlers for the desktop's engine. public static class WorkspaceJobs { /// /// : the media of workspace project (checked out first, /// ) copied from its home, the sequence of this PC's archive. /// public static ArchiveJobRequest Copy(Sequence sequence, string workspaceRoot, Guid projectId, string media = BundleMedia.Proxies) { ArgumentNullException.ThrowIfNull(sequence); ArgumentException.ThrowIfNullOrWhiteSpace(workspaceRoot); return new ArchiveJobRequest(JobKind.WorkspaceCopy, [sequence.Id], sequence.DisplayTitle, JsonSerializer.Serialize(new WorkspaceCopyParameters(workspaceRoot, projectId, media), WorkspaceJson.Default.WorkspaceCopyParameters), JobPriority.User); } /// /// The handlers of the workspace's jobs for a desktop's engine (never a server's: the job writes into a folder its /// parameters name). /// public static IReadOnlyList CreateHandlers(ChronosDatabase db, ArchiveService archives) => [new WorkspaceCopyJobHandler(db, archives)]; /// /// What a first copy of a sequence's proxies writes into a workspace, as the copy reckons it before it starts (the /// archive's proxies' sizes, an estimate for those it makes), and the free bytes on 's /// drive (null when it cannot be measured): "Open in workspace" shows both. The copy keeps /// free besides. /// /// The index has no such sequence or archive. public static Task<(long Bytes, long? Free)> CopyFactsAsync(ChronosDatabase db, ArchiveService archives, Guid sequenceId, string workspaceRoot, CancellationToken cancellationToken = default) => new WorkspaceCopyJobHandler(db, archives).FactsAsync(sequenceId, workspaceRoot, cancellationToken); /// Bytes the copy leaves free on the workspace's drive besides what it copies. public static long CopySpaceReserve => WorkspaceCopyJobHandler.SpaceReserve; } /// /// for a sequence of this PC's archive (owner review 2, R8): its proxies into the /// workspace project's media/proxies/<frame id>.jpg, the archive's own when current, else made here from the /// original (at the archive's proxy size, or the default one when the archive makes none), each written atomically and read /// back to check its SHA-256; then the deep report (media/analysis.json) and, last, media/media.json with the /// encodings conversions joined at the home. The job only reads the archive: it never writes there, not even proxies (the /// archive's own pass makes those), refuses a workspace folder inside any library of the index before it writes anything, /// and reads only a folder that holds the sequence (). /// Frames changed at the home during the copy, or the folder moved (a Reorganize), make it start again (three times at /// most; a folder still moving fails it, to be retried). The working state's positions follow the new frames: by frame /// where a frame stays (one whose file a conversion replaced keeps its positions), by photo where frames came or went. /// Before each pass and at each checkpoint the workspace's drive must hold what is still to copy plus a reserve, else the /// job stops saying how much is missing (the workspace shares this PC's drive with the index and the logs). /// Checkpoint: copy:<frame>:<frames version>:<folder>; a run resumed on other frames or in another folder /// checks every file again, and a pass started again stores that before it reads anything. A first copy the user cancels /// removes the media it wrote itself, once it has stopped writing and under the project's lease, unless another copy of the /// project goes on with them (); one cancelled while paused or failed, which no handler /// runs, has them removed the same way (). /// Originals come in phase 5c: asked for, the job fails saying so. /// internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService archives) : ICancelledJobHandler { private const int PageSize = 1000; private const int Passes = 3; private static readonly TimeSpan LockWait = TimeSpan.FromMinutes(2); private static readonly TimeSpan CheckpointEvery = TimeSpan.FromSeconds(5); /// Bytes left free on the workspace's drive besides the copy, as a download keeps (). internal const long SpaceReserve = 512L << 20; /// A proxy at px, when the archive has none to measure: some 1.5 MB. private const long ProxyEstimate = 3L << 19; private const int ProxyEstimateEdge = 1920; /// The long edge taken for a full-size proxy of a frame whose size is not known. private const int UnknownFullSize = 6000; public JobKind Kind => JobKind.WorkspaceCopy; public async Task RunAsync(JobContext context) { var ct = context.Cancellation; var parameters = Parameters(context.Job); if (parameters.Media == BundleMedia.Originals) throw new NotSupportedException(Loc.L("Errors.Workspace.OriginalsLater")); if (parameters.Media != BundleMedia.Proxies) throw new InvalidDataException(Loc.L("Errors.Workspace.Parameters")); if (context.Job.TargetIds is not [var sequenceId] || await db.Sequences.GetAsync(sequenceId, ct).ConfigureAwait(false) is not { } sequence || await db.Archives.GetAsync(sequence.ArchiveId, ct).ConfigureAwait(false) is not { } archive) throw new InvalidOperationException(Loc.L("Archive.Log.SequenceMissing", context.Job.TargetIds.FirstOrDefault())); // Asked before the store is opened, which makes its folder: editing happens outside every library (owner review 2, // R8), and a workspace inside one would put its copies among that library's own files. if (await InsideALibraryAsync(parameters.WorkspaceRoot, ct).ConfigureAwait(false)) throw new InvalidOperationException(Loc.L("Errors.Workspace.InsideLibrary")); var store = new WorkspaceStore(parameters.WorkspaceRoot); var project = store.Find(parameters.ProjectId) ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.NotCheckedOut")); if (project.ReadBundle().Workspace is not { Home: { IsArchive: true } home } || home.SequenceId != sequenceId) throw new InvalidOperationException(Loc.L("Errors.Workspace.OtherHome")); try { using (WritingMedia(project)) await CopyAsync(context, project, sequenceId, archive, parameters, ct).ConfigureAwait(false); } catch (OperationCanceledException) when (ct.IsCancellationRequested) { // Stopped writing: what a first copy the user cancelled wrote goes (a pause or a shutdown resumes it). await RemovePartialMediaAsync(context.Job, project).ConfigureAwait(false); throw; } } /// /// A copy cancelled while no handler ran it (paused, failed): what a first copy wrote goes as after a cancel while it ran, /// under the project's lease, in the background (the lease may be held for a while: the cancel does not wait for it). /// A workspace folder or a project no longer there has nothing to clear, and nothing is made for it. /// public Task CancelledAsync(Job job, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(job); WorkspaceCopyParameters parameters; try { parameters = Parameters(job); } catch (InvalidDataException) { return Task.CompletedTask; } // Opening a store makes its folder: one gone stays gone. if (!Directory.Exists(Path.Combine(parameters.WorkspaceRoot, parameters.ProjectId.ToString("N")))) return Task.CompletedTask; _ = Task.Run(async () => { try { if (new WorkspaceStore(parameters.WorkspaceRoot).Find(parameters.ProjectId) is { } project) await RemovePartialMediaAsync(job, project).ConfigureAwait(false); } catch (Exception ex) when (ex is not OutOfMemoryException) { // Nobody waits for this task: whatever stopped it is logged; the next copy checks every file it finds. Log.Warning("workspace", $"The media of cancelled copy {job.Id} could not be cleared", ex); } }, CancellationToken.None); return Task.CompletedTask; } /// /// A first copy of the project's media the user cancelled (the job reads ), once it has /// stopped writing: the photos it wrote (copies the home can make again) are removed under the project's lease, so the /// workspace holds no half media. Media that a copy ever finished (media.json) stay, as do those of a copy paused, /// yielded or stopped with the program, which goes on from its checkpoint, and those another copy of the project into /// this workspace still writes or goes on with (queued, running or paused: ). The last /// look is taken under the lease, and no copy of this program starts writing while the media go /// (). A folder that is no project any more (its removal under way) is left as it is, and the /// lease never makes it again. A failure is logged: the next copy checks every file it finds. /// private async Task RemovePartialMediaAsync(Job job, WorkspaceProject project) { try { // Nothing half written (or the copy removed meanwhile): the lease is not taken. if (!Clearable(project) || !await StillToClearAsync(job, project).ConfigureAwait(false)) return; using (await project.LockExistingAsync(LockWait, CancellationToken.None).ConfigureAwait(false)) { if (!Clearable(project) || !await StillToClearAsync(job, project).ConfigureAwait(false)) return; lock (WritersGate) { // A copy of this program that started since the jobs were read: its files stay. if (Writers.ContainsKey(project.Folder) || !Clearable(project)) return; Directory.Delete(project.MediaDirectory, recursive: true); } } Log.Info("workspace", $"The media a cancelled first copy of project {project.ProjectId} wrote were removed"); } catch (DirectoryNotFoundException) { // The project's folder went meanwhile (its removal): nothing is left to clear. } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or DatabaseException) { Log.Warning("workspace", $"The media of a cancelled copy of project {project.ProjectId} could not be removed", ex); } // A project (its working state there) with media no copy ever finished. static bool Clearable(WorkspaceProject project) => project.Exists && !File.Exists(project.MediaIndexPath) && Directory.Exists(project.MediaDirectory); } /// The cancelled copy still reads so, and no other copy of the project into this workspace waits, runs or is paused. private async Task StillToClearAsync(Job job, WorkspaceProject project) { if (await db.Jobs.GetAsync(job.Id, CancellationToken.None).ConfigureAwait(false) is not { State: JobState.Cancelled }) return false; if (await OtherCopyAsync(job, project).ConfigureAwait(false) is not { } other) return true; Log.Info("workspace", $"The media of cancelled copy {job.Id} stay: copy {other.Id} of project {project.ProjectId} is {other.State}"); return false; } /// /// Another copy of the project into the same workspace that is queued, running or paused: it writes the media folder, or /// goes on with what is there. A job whose parameters cannot be read counts, so nothing is removed on a guess. /// private async Task OtherCopyAsync(Job job, WorkspaceProject project) { var sequenceId = job.TargetIds.FirstOrDefault(); foreach (var other in await db.Jobs.ListPendingAsync(JobKind.WorkspaceCopy, sequenceId, CancellationToken.None).ConfigureAwait(false)) { if (other.Id == job.Id) continue; try { var parameters = Parameters(other); if (parameters.ProjectId != project.ProjectId) continue; // Case aside: a match on a case-sensitive drive only keeps media that could have gone. if (!string.Equals(WorkspaceStore.FullRoot(parameters.WorkspaceRoot), project.Store.Root, StringComparison.OrdinalIgnoreCase)) continue; } catch (Exception ex) when (ex is InvalidDataException or ArgumentException or IOException or NotSupportedException or System.Security.SecurityException) { Log.Debug("workspace", $"Copy {other.Id} cannot be read: it counts as one of project {project.ProjectId}: {ex.Message}"); } return other; } return null; } /// The project folders copies of this program write media into now, with how many copies write each. private static readonly Dictionary Writers = new(StringComparer.OrdinalIgnoreCase); private static readonly Lock WritersGate = new(); /// /// Marks the copy into as writing its media until disposed: a clean-up after a cancelled copy /// leaves them (), and one under way finishes before the copy writes anything. /// private static IDisposable WritingMedia(WorkspaceProject project) { lock (WritersGate) Writers[project.Folder] = Writers.GetValueOrDefault(project.Folder) + 1; return new Writing(project.Folder); } private sealed class Writing(string folder) : IDisposable { private int _done; public void Dispose() { if (Interlocked.Exchange(ref _done, 1) != 0) return; lock (WritersGate) { if (Writers.GetValueOrDefault(folder) <= 1) Writers.Remove(folder); else Writers[folder]--; } } } private async Task CopyAsync(JobContext context, WorkspaceProject project, Guid sequenceId, Archive archive, WorkspaceCopyParameters parameters, CancellationToken ct) { var paths = new ArchivePaths(archive.RootPath); // Read only: a read-only archive is copied from like any other. if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(archive.RootPath, ArchiveAvailability.Offline); var (copied, checkpointVersion, checkpointFolder) = Parse(context.Checkpoint); for (var pass = 1; ; pass++) { var (row, frames) = await FramesAsync(sequenceId, ct).ConfigureAwait(false); var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy; var folder = paths.Sequence(row.Folder); // Another index moved the sequence away, maybe another one in: its files are not this sequence's photos. A retry // checks every file again (the files of this run may be of that folder). if (!SequenceFolders.Holds(folder, sequenceId)) { await context.CheckpointAsync(Checkpoint(Stage.Copy, 0, row.FramesVersion, row.Folder), 0, frames.Count).ConfigureAwait(false); throw new SequenceFolderMovedException(); } // Files kept from an earlier run are this frames version's and this folder's only (a conversion keeps a frame's id, // not its photo; a sequence moved meanwhile may have left its photos' names to another sequence's). if ((checkpointVersion is { } version && version != row.FramesVersion) || (checkpointFolder is { } read && read != row.Folder)) copied = 0; var total = (long)frames.Count; var tally = new Tally(); var entries = new WorkspaceFrame[frames.Count]; // The media the editor reads until this copy commits: their files stay until then (RemoveStale). var current = TryReadMedia(project)?.Frames.Where(f => f.File is not null).ToDictionary(f => f.Id) ?? []; Directory.CreateDirectory(project.ProxiesDirectory); // The workspace is on this PC's own drive, with the index, the settings and the logs: the copy never fills it. var remaining = Remaining(project, folder, frames, policy); EnsureSpace(parameters.WorkspaceRoot, remaining[0]); var clock = System.Diagnostics.Stopwatch.StartNew(); for (var i = 0; i < frames.Count; i++) { ct.ThrowIfCancellationRequested(); entries[i] = Copy(project, folder, paths, frames[i], policy, done: i < copied, current.GetValueOrDefault(frames[i].Id), tally, ct); if (clock.Elapsed >= CheckpointEvery) { await context.CheckpointAsync(Checkpoint(Stage.Copy, i + 1, row.FramesVersion, row.Folder), i + 1, total).ConfigureAwait(false); // Another program may take the space meanwhile. EnsureSpace(parameters.WorkspaceRoot, remaining[i + 1]); clock.Restart(); } else { await context.ReportAsync(i + 1, total).ConfigureAwait(false); } } var analysis = CopyAnalysis(folder, project, frames.Count); var (after, now) = await FramesAsync(sequenceId, ct).ConfigureAwait(false); var moved = after.Folder != row.Folder || !SequenceFolders.Holds(paths.Sequence(after.Folder), sequenceId) || Renamed(frames, now, entries); if (after.FramesVersion != row.FramesVersion || moved) { // The files read may be of other frames, or missing where a Reorganize took them: read them again, and so // does a run resumed after a crash in the next pass (the checkpoint stored says so). if (pass < Passes) { context.Log(Loc.L("Archive.Log.WorkspaceFramesChanged", row.DisplayTitle)); await context.CheckpointAsync(Checkpoint(Stage.Copy, 0, after.FramesVersion, after.Folder), 0, total).ConfigureAwait(false); (copied, checkpointVersion, checkpointFolder) = (0, null, null); continue; } // Frames still arriving are copied as they were read (Update media takes the rest); a folder still moving is not. if (moved) { await context.CheckpointAsync(Checkpoint(Stage.Copy, 0, after.FramesVersion, after.Folder), 0, total).ConfigureAwait(false); throw new SequenceFolderMovedException(); } } var aliases = await HomePhotos.AliasesAsync(db, sequenceId, ct).ConfigureAwait(false); var media = new WorkspaceMedia(WorkspaceMedia.FormatName, WorkspaceMedia.CurrentFormatVersion, BundleMedia.Proxies, sequenceId, row.FramesVersion, entries, DateTimeOffset.UtcNow, analysis, aliases.Count > 0 ? aliases : null); // The working state is the editor's meanwhile: it is held only while the media index and the positions move. using (await project.LockAsync(LockWait, ct).ConfigureAwait(false)) { Commit(project, project.ReadBundle(), media, row); RemoveStale(project, media); } context.Log(Loc.L("Archive.Log.WorkspaceCopied", row.DisplayTitle, tally.Copied, tally.Made, tally.Kept, tally.Missing)); await context.CheckpointAsync(Checkpoint(Stage.Done, frames.Count, row.FramesVersion, row.Folder), total, total).ConfigureAwait(false); return; } } /// Whether is an archive's root of this index or lies inside one. private async Task InsideALibraryAsync(string folder, CancellationToken cancellationToken) { var path = Plain(Path.GetFullPath(folder)); foreach (var archive in await db.Archives.ListAsync(cancellationToken).ConfigureAwait(false)) { if (DataLocation.IsUnder(path, Plain(Path.GetFullPath(archive.RootPath)))) return true; } return false; // Without the long-path prefix, which only one of the two may carry. static string Plain(string path) => path.StartsWith(@"\\?\UNC\", StringComparison.Ordinal) ? @"\\" + path[8..] : path.StartsWith(@"\\?\", StringComparison.Ordinal) ? path[4..] : path; } /// /// A frame the copy found no file for whose original the index has under another name now (a Reorganize renamed the /// files during the copy, keeping the frames version): the copy runs again. /// private static bool Renamed(List read, List now, WorkspaceFrame[] entries) { var paths = now.ToDictionary(f => f.Id, f => f.RelativePath); for (var i = 0; i < read.Count; i++) { if (entries[i].File is null && paths.TryGetValue(read[i].Id, out var path) && path != read[i].RelativePath) return true; } return false; } /// /// The frame's file in the workspace: kept when an earlier run copied it (before the checkpoint, or the same bytes as the /// archive's proxy), else the archive's current proxy, else one made from the original; none for a frame without either. /// The file the media the editor reads now name for the frame () is never removed nor written /// over here: the editor keeps it while the copy runs and if it fails, other bytes for it go under a name of their own /// (), and a copy that commits without it removes it then /// (). Nor is it taken as this run's copy of a frame whose photo changed since. /// private WorkspaceFrame Copy(WorkspaceProject project, SequencePaths folder, ArchivePaths paths, Frame frame, ProxyPolicy policy, bool done, WorkspaceFrame? current, Tally tally, CancellationToken cancellationToken) { var name = WorkspaceProject.ProxyName(frame.Id); var target = project.MediaFile(name); var listed = current is not null && string.Equals(current.File, name, StringComparison.OrdinalIgnoreCase); if (done && File.Exists(target)) { var kept = AtomicFile.ReadAllBytes(target); var keptSha = Sha(kept); // The editor's file of another photo of the frame (a conversion since) was left by an earlier copy, not this run. if (!listed || keptSha != current!.FileSha256 || SamePhoto(current, frame)) { tally.Kept++; return WorkspaceFrame.Of(frame, name, keptSha, kept.Length); } } var source = folder.Proxy(frame.Index); byte[]? bytes = null; if (File.Exists(source) && ArchiveImages.IsProxyOf(source, frame.CapturedAt, frame.Width, frame.Height, null)) bytes = AtomicFile.ReadAllBytes(source); var made = false; if (bytes is null && (frame.Flags & FrameFlags.Missing) == 0) { var original = paths.FromRelative(frame.RelativePath); try { var edge = policy.LongEdge == ProxyPolicy.Disabled ? new ProxyPolicy().LongEdge : policy.LongEdge; bytes = File.Exists(original) ? ArchiveImages.MakeProxy(archives.Decoder, original, frame, null, edge, policy.Quality, cancellationToken) : null; made = bytes is not null; } catch (IOException ex) { Log.Warning("workspace", $"No proxy of {original} could be made", ex); } } if (bytes is null) { // A file an earlier copy left for the frame (of other frames, or another folder's) is never kept as its photo by a // resumed run: a regenerable copy, it goes, unless the editor's media name it (it goes once this copy commits). if (!listed && File.Exists(target)) TryDelete(target); tally.Missing++; return WorkspaceFrame.Of(frame, null, null, 0); } var sha = Sha(bytes); // The file the editor's media name for the frame holds these bytes already (under its own name, maybe): kept. if (current?.File is { } held && !string.Equals(held, name, StringComparison.OrdinalIgnoreCase) && string.Equals(current.FileSha256, sha, StringComparison.OrdinalIgnoreCase) && Holds(project.MediaFile(held), sha)) { tally.Kept++; return WorkspaceFrame.Of(frame, held, sha, bytes.Length); } if (Holds(target, sha)) { tally.Kept++; return WorkspaceFrame.Of(frame, name, sha, bytes.Length); } // Other bytes for the file the editor's media name (a conversion since): written under a name of their own, never over // the editor's file, which the commit switches the media from (RemoveStale then removes it). A copy that fails before // its commit leaves the editor reading the bytes its entry names. A file that no longer holds them (damaged, gone) // is mended where it is. if (listed && current!.FileSha256 is { } editors && Holds(target, editors.ToLowerInvariant())) { name = WorkspaceProject.ProxyName(frame.Id, sha); target = project.MediaFile(name); if (Holds(target, sha)) { tally.Kept++; return WorkspaceFrame.Of(frame, name, sha, bytes.Length); } } WriteVerified(target, bytes, sha); if (made) tally.Made++; else tally.Copied++; return WorkspaceFrame.Of(frame, name, sha, bytes.Length); } /// Whether the file at is there and hashes as . private static bool Holds(string path, string sha) => File.Exists(path) && Sha(AtomicFile.ReadAllBytes(path)) == sha; /// Writes atomically and reads back: a file that does not hash as its source is written again once, then refused. private static void WriteVerified(string path, byte[] bytes, string sha) { for (var attempt = 1; ; attempt++) { AtomicFile.WriteAllBytes(path, bytes); if (Sha(AtomicFile.ReadAllBytes(path)) == sha) return; if (attempt == 2) throw new IOException(Loc.L("Errors.Workspace.CopyMismatch", path)); } } /// The home's deep report into the media folder when it describes these frames; its SHA-256, or null without one. private static string? CopyAnalysis(SequencePaths folder, WorkspaceProject project, int frameCount) { if (!File.Exists(folder.AnalysisPath)) { if (File.Exists(project.AnalysisPath)) File.Delete(project.AnalysisPath); return null; } var bytes = AtomicFile.ReadAllBytes(folder.AnalysisPath); try { if (Core.Analysis.Deep.DeepAnalysisReport.FromJson(System.Text.Encoding.UTF8.GetString(bytes)).FrameCount != frameCount) return null; } catch (Exception ex) when (ex is InvalidDataException or JsonException or IOException) { Log.Warning("workspace", $"{folder.AnalysisPath} cannot be read; the workspace takes no deep report", ex); return null; } var sha = Sha(bytes); WriteVerified(project.AnalysisPath, bytes, sha); return sha; } /// /// The media index and the working state moved onto it: first the working state with every position parked by photo /// (valid whatever the frames), then the index, then the working state placed on the new frames. A crash in between /// leaves positions that wait for their photos, which the next open places. A frame the new media still hold is parked /// as the photo they hold for it, so one whose file a conversion replaced (the same frame, another SHA-256) keeps its /// positions, as the archive's own settings do (). /// private static void Commit(WorkspaceProject project, ProjectBundle bundle, WorkspaceMedia media, Sequence row) { var old = TryReadMedia(project); var workspace = bundle.Workspace! with { Media = new BundleMedia(BundleMedia.Proxies, media.FramesVersion) }; var sequence = bundle.Sequence with { Folder = row.Folder, Title = row.DisplayTitle, Frames = media.Frames.Count }; var working = bundle.Project; // Without the media index the positions were made against, they are taken as naming these frames. if (!working.IsNewerFormat && old is not null) { var held = media.Frames.ToDictionary(f => f.Id); var before = new FramePhotos(old.Frames.Select(f => (f.Index, Photo(held.TryGetValue(f.Id, out var now) ? now : f))), old.Aliases); var portable = ProjectSettings.FromJson(ProjectContent.ToPortable(working, before), allowNewer: true); project.WriteBundle(bundle with { Project = portable, Workspace = workspace, Sequence = sequence }); project.WriteMedia(media); working = ProjectSettings.FromJson(ProjectContent.Place(portable.ToJson(), media.Photos), allowNewer: true); } else { project.WriteMedia(media); } project.WriteBundle(bundle with { Project = working, Workspace = workspace, Sequence = sequence, Saved = DateTimeOffset.UtcNow }); } private static FramePhoto Photo(WorkspaceFrame frame) => new(frame.Sha256, frame.Size, frame.CapturedAt); /// Whether the media's entry of a frame is of the photo the frame holds now (a conversion gives it another file). private static bool SamePhoto(WorkspaceFrame held, Frame frame) => held.Size == frame.Size && held.CapturedAt == frame.CapturedAt && (string.IsNullOrEmpty(held.Sha256) || string.IsNullOrEmpty(frame.Sha256) || string.Equals(held.Sha256, frame.Sha256, StringComparison.OrdinalIgnoreCase)); /// Proxies of frames the sequence no longer has: regenerable copies, removed so the media folder holds the frames listed. private static void RemoveStale(WorkspaceProject project, WorkspaceMedia media) { var listed = media.Frames.Where(f => f.File is not null).Select(f => Path.GetFileName(f.File!)).ToHashSet(StringComparer.OrdinalIgnoreCase); foreach (var file in Directory.EnumerateFiles(project.ProxiesDirectory, "*.jpg")) { if (!listed.Contains(Path.GetFileName(file))) TryDelete(file); } } private static void TryDelete(string file) { try { File.Delete(file); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { Log.Warning("workspace", $"{file} could not be removed", ex); } } /// /// The bytes the copy still writes from each frame on (index i: frames i to the last; one more entry, 0): /// nothing for a frame the workspace holds a file of already (kept, or replaced by one of about its size; under its own /// name or the one other bytes of it take, ), the archive's /// proxy's size where it has one, else an estimate at the proxy size the copy makes (). /// private static long[] Remaining(WorkspaceProject project, SequencePaths folder, List frames, ProxyPolicy policy) => Remaining(Directory.Exists(project.ProxiesDirectory) ? HeldFrames(Directory.EnumerateFiles(project.ProxiesDirectory, "*.jpg")) : [], folder, frames, policy); /// The frames the files at are proxies of: "<frame id>.jpg" and "<frame id>-<sha16>.jpg". internal static HashSet HeldFrames(IEnumerable paths) { var held = new HashSet(); foreach (var path in paths) { var name = Path.GetFileNameWithoutExtension(path); if (name.Length >= 32 && (name.Length == 32 || name[32] == '-') && Guid.TryParseExact(name[..32], "N", out var id)) held.Add(id); } return held; } private static long[] Remaining(HashSet held, SequencePaths folder, List frames, ProxyPolicy policy) { var proxies = Directory.Exists(folder.Proxies) ? new DirectoryInfo(folder.Proxies).EnumerateFiles().ToDictionary(f => f.Name, f => f.Length, StringComparer.OrdinalIgnoreCase) : []; var remaining = new long[frames.Count + 1]; for (var i = frames.Count - 1; i >= 0; i--) { var frame = frames[i]; var bytes = held.Contains(frame.Id) ? 0 : proxies.TryGetValue(ArchiveLayout.ProxyFileName(frame.Index), out var length) ? length : (frame.Flags & FrameFlags.Missing) != 0 ? 0 : Estimate(frame, policy); remaining[i] = remaining[i + 1] + bytes; } return remaining; } /// What a first copy of the sequence writes (nothing held yet), and the free bytes of the workspace's drive. internal async Task<(long Bytes, long? Free)> FactsAsync(Guid sequenceId, string workspaceRoot, CancellationToken cancellationToken) { var (row, frames) = await FramesAsync(sequenceId, cancellationToken).ConfigureAwait(false); var archive = await db.Archives.GetAsync(row.ArchiveId, cancellationToken).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Archive.Log.SequenceMissing", sequenceId)); var paths = new ArchivePaths(archive.RootPath); var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy; return (Remaining([], paths.Sequence(row.Folder), frames, policy)[0], FreeBytes(workspaceRoot)); } /// A proxy's size when the archive has none to measure: some 1.5 MB at 1920 px, scaled by the area at the copy's long edge. private static long Estimate(Frame frame, ProxyPolicy policy) { var edge = policy.LongEdge == ProxyPolicy.Disabled ? new ProxyPolicy().LongEdge : policy.LongEdge; var longest = Math.Max(frame.Width, frame.Height); var side = edge == ProxyPolicy.FullSize ? (longest > 0 ? longest : UnknownFullSize) : longest > 0 ? Math.Min(edge, longest) : edge; return (long)(ProxyEstimate * Math.Pow(side / (double)ProxyEstimateEdge, 2)); } /// /// Refuses to go on while the workspace's drive lacks besides , with the /// gigabytes missing; a drive that cannot be measured (a network share) is not refused, the copy itself then says so. /// /// The drive lacks the space (Errors.Workspace.DiskFull). private void EnsureSpace(string root, long bytes) { if (bytes <= 0 || FreeBytes(root) is not { } free || free >= bytes + SpaceReserve) return; throw new IOException(Loc.L("Errors.Workspace.DiskFull", ((bytes + SpaceReserve - free) / 1e9).ToString("0.0", Loc.Culture))); } /// Free bytes on the volume of , as the host measures them (), else its drive's. private long? FreeBytes(string folder) { if (archives.FreeSpace is { } measure) return measure(folder); try { var full = Path.GetFullPath(folder); // Windows wants a drive root; Unix accepts any directory and reports the volume it is mounted on. return new DriveInfo(OperatingSystem.IsWindows() ? Path.GetPathRoot(full)! : full).AvailableFreeSpace; } catch (Exception ex) when (ex is ArgumentException or IOException or UnauthorizedAccessException) { return null; } } private static WorkspaceMedia? TryReadMedia(WorkspaceProject project) { try { return project.ReadMedia(); } catch (InvalidDataException ex) { Log.Warning("workspace", $"The media index of {project.Folder} cannot be read; positions are taken as they are", ex); return null; } } /// The sequence row with its frames as one state (listed between two reads of the row at the same frames version). private async Task<(Sequence Row, List Frames)> FramesAsync(Guid sequenceId, CancellationToken cancellationToken) { for (var attempt = 0; ; attempt++) { var before = await db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Archive.Log.SequenceMissing", sequenceId)); var frames = new List(); for (var after = -1; ;) { var page = await db.Frames.ListBySequenceAsync(sequenceId, after, PageSize, cancellationToken).ConfigureAwait(false); frames.AddRange(page); if (page.Count < PageSize) break; after = page[^1].Index; } var now = await db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? before; if (now.FramesVersion == before.FramesVersion || attempt == 2) return (now, frames); } } private static WorkspaceCopyParameters Parameters(Job job) { try { if (job.ParametersJson is { Length: > 0 } json && JsonSerializer.Deserialize(json, WorkspaceJson.Default.WorkspaceCopyParameters) is { WorkspaceRoot.Length: > 0 } parameters && parameters.ProjectId != Guid.Empty) return parameters with { Media = parameters.Media ?? BundleMedia.Proxies }; } catch (JsonException ex) { throw new InvalidDataException(Loc.L("Errors.Workspace.Parameters"), ex); } throw new InvalidDataException(Loc.L("Errors.Workspace.Parameters")); } private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes)); private enum Stage { Copy, Done, } private static string Checkpoint(Stage stage, int frame, int framesVersion, string folder) => string.Create(CultureInfo.InvariantCulture, $"{stage.ToString().ToLowerInvariant()}:{frame}:{framesVersion}:{folder}"); /// /// The frames copied by an earlier run (copy:<frame>[:<frames version>[:<folder>]], the folder /// last as it may hold a colon), the frames version and the sequence's folder they were of (null when the checkpoint does /// not say); nothing for a finished run, or another stage (an earlier build's archive proxy pass), whose files the copy /// checks again. /// private static (int Copied, int? FramesVersion, string? Folder) Parse(string? checkpoint) { var parts = checkpoint?.Split(':', 4); if (parts is not ([_, _] or [_, _, _] or [_, _, _, _]) || !Enum.TryParse(parts[0], ignoreCase: true, out var stage) || stage != Stage.Copy || !int.TryParse(parts[1], NumberStyles.None, CultureInfo.InvariantCulture, out var next)) return (0, null, null); int? version = parts.Length >= 3 && int.TryParse(parts[2], NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture, out var v) ? v : null; return (next, version, parts.Length == 4 ? parts[3] : null); } private sealed class Tally { public int Copied { get; set; } public int Made { get; set; } public int Kept { get; set; } public int Missing { get; set; } } }