feat: owner review 2 phase 5a: workspace and projects with a local home (#14)
CI / Build and test (push) Successful in 9m43s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 27s
CI / PostgreSQL 16 integration (push) Failing after 8m41s
CI / PostgreSQL 17 integration (push) Successful in 12m37s
CI / Docker image (push) Successful in 56s
Release / Build, test and publish (push) Successful in 14m59s
Release / Server image (push) Successful in 1m51s
CI / Build and test (push) Successful in 9m43s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 27s
CI / PostgreSQL 16 integration (push) Failing after 8m41s
CI / PostgreSQL 17 integration (push) Successful in 12m37s
CI / Docker image (push) Successful in 56s
Release / Build, test and publish (push) Successful in 14m59s
Release / Server image (push) Successful in 1m51s
Edit opens a sequence as a project of a workspace folder outside the library. Projects have linear version histories kept as sidecars in .chronos/projects/, Main mirrors the sequence's settings, and sync merges setting by setting. Migration 0009, JobKind.WorkspaceCopy = 20, Settings › Workspace, and 241 new strings in 38 languages. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
This commit was merged in pull request #14.
This commit is contained in:
1 parent
efa87f68ce
commit
9b24cf74fc
370 files changed
+29237
-287
No files matched your search
@@ -0,0 +1,728 @@
|
||||
using System.Globalization;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text.Json;
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.Diagnostics;
|
||||
using Chronos.Core.IO;
|
||||
using Chronos.Core.Localization;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Core.Project;
|
||||
using Chronos.Data.Archiving;
|
||||
using Chronos.Data.Archiving.Jobs;
|
||||
using Chronos.Data.Jobs;
|
||||
using Chronos.Data.Projects;
|
||||
using Chronos.Data.Sync;
|
||||
|
||||
namespace Chronos.Data.Workspaces;
|
||||
|
||||
/// <summary>Builds the jobs of the workspace (owner review 2, R8), and their handlers for the desktop's engine.</summary>
|
||||
public static class WorkspaceJobs
|
||||
{
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.WorkspaceCopy"/>: the media of workspace project <paramref name="projectId"/> (checked out first,
|
||||
/// <see cref="WorkspaceSync.CheckoutAsync"/>) copied from its home, the sequence of this PC's archive.
|
||||
/// </summary>
|
||||
public static ArchiveJobRequest Copy(Sequence sequence, string workspaceRoot, Guid projectId, string media = BundleMedia.Proxies)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(sequence);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(workspaceRoot);
|
||||
return new ArchiveJobRequest(JobKind.WorkspaceCopy, [sequence.Id], sequence.DisplayTitle,
|
||||
JsonSerializer.Serialize(new WorkspaceCopyParameters(workspaceRoot, projectId, media), WorkspaceJson.Default.WorkspaceCopyParameters), JobPriority.User);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The handlers of the workspace's jobs for a desktop's engine (never a server's: the job writes into a folder its
|
||||
/// parameters name).
|
||||
/// </summary>
|
||||
public static IReadOnlyList<IJobHandler> CreateHandlers(ChronosDatabase db, ArchiveService archives) => [new WorkspaceCopyJobHandler(db, archives)];
|
||||
|
||||
/// <summary>
|
||||
/// What a first copy of a sequence's proxies writes into a workspace, as the copy reckons it before it starts (the
|
||||
/// archive's proxies' sizes, an estimate for those it makes), and the free bytes on <paramref name="workspaceRoot"/>'s
|
||||
/// drive (null when it cannot be measured): "Open in workspace" shows both. The copy keeps <see cref="CopySpaceReserve"/>
|
||||
/// free besides.
|
||||
/// </summary>
|
||||
/// <exception cref="InvalidOperationException">The index has no such sequence or archive.</exception>
|
||||
public static Task<(long Bytes, long? Free)> CopyFactsAsync(ChronosDatabase db, ArchiveService archives, Guid sequenceId, string workspaceRoot,
|
||||
CancellationToken cancellationToken = default) =>
|
||||
new WorkspaceCopyJobHandler(db, archives).FactsAsync(sequenceId, workspaceRoot, cancellationToken);
|
||||
|
||||
/// <summary>Bytes the copy leaves free on the workspace's drive besides what it copies.</summary>
|
||||
public static long CopySpaceReserve => WorkspaceCopyJobHandler.SpaceReserve;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.WorkspaceCopy"/> for a sequence of this PC's archive (owner review 2, R8): its proxies into the
|
||||
/// workspace project's <c>media/proxies/<frame id>.jpg</c>, the archive's own when current, else made here from the
|
||||
/// original (at the archive's proxy size, or the default one when the archive makes none), each written atomically and read
|
||||
/// back to check its SHA-256; then the deep report (<c>media/analysis.json</c>) and, last, <c>media/media.json</c> with the
|
||||
/// encodings conversions joined at the home. The job only reads the archive: it never writes there, not even proxies (the
|
||||
/// archive's own pass makes those), refuses a workspace folder inside any library of the index before it writes anything,
|
||||
/// and reads only a folder that holds the sequence (<see cref="SequenceFolders.Holds"/>).
|
||||
/// Frames changed at the home during the copy, or the folder moved (a Reorganize), make it start again (three times at
|
||||
/// most; a folder still moving fails it, to be retried). The working state's positions follow the new frames: by frame
|
||||
/// where a frame stays (one whose file a conversion replaced keeps its positions), by photo where frames came or went.
|
||||
/// Before each pass and at each checkpoint the workspace's drive must hold what is still to copy plus a reserve, else the
|
||||
/// job stops saying how much is missing (the workspace shares this PC's drive with the index and the logs).
|
||||
/// Checkpoint: <c>copy:<frame>:<frames version>:<folder></c>; a run resumed on other frames or in another folder
|
||||
/// checks every file again, and a pass started again stores that before it reads anything. A first copy the user cancels
|
||||
/// removes the media it wrote itself, once it has stopped writing and under the project's lease, unless another copy of the
|
||||
/// project goes on with them (<see cref="RemovePartialMediaAsync"/>); one cancelled while paused or failed, which no handler
|
||||
/// runs, has them removed the same way (<see cref="CancelledAsync"/>).
|
||||
/// Originals come in phase 5c: asked for, the job fails saying so.
|
||||
/// </summary>
|
||||
internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService archives) : ICancelledJobHandler
|
||||
{
|
||||
private const int PageSize = 1000;
|
||||
private const int Passes = 3;
|
||||
private static readonly TimeSpan LockWait = TimeSpan.FromMinutes(2);
|
||||
private static readonly TimeSpan CheckpointEvery = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>Bytes left free on the workspace's drive besides the copy, as a download keeps (<see cref="SyncTransfers"/>).</summary>
|
||||
internal const long SpaceReserve = 512L << 20;
|
||||
|
||||
/// <summary>A proxy at <see cref="ProxyEstimateEdge"/> px, when the archive has none to measure: some 1.5 MB.</summary>
|
||||
private const long ProxyEstimate = 3L << 19;
|
||||
|
||||
private const int ProxyEstimateEdge = 1920;
|
||||
|
||||
/// <summary>The long edge taken for a full-size proxy of a frame whose size is not known.</summary>
|
||||
private const int UnknownFullSize = 6000;
|
||||
|
||||
public JobKind Kind => JobKind.WorkspaceCopy;
|
||||
|
||||
public async Task RunAsync(JobContext context)
|
||||
{
|
||||
var ct = context.Cancellation;
|
||||
var parameters = Parameters(context.Job);
|
||||
if (parameters.Media == BundleMedia.Originals) throw new NotSupportedException(Loc.L("Errors.Workspace.OriginalsLater"));
|
||||
if (parameters.Media != BundleMedia.Proxies) throw new InvalidDataException(Loc.L("Errors.Workspace.Parameters"));
|
||||
if (context.Job.TargetIds is not [var sequenceId] || await db.Sequences.GetAsync(sequenceId, ct).ConfigureAwait(false) is not { } sequence
|
||||
|| await db.Archives.GetAsync(sequence.ArchiveId, ct).ConfigureAwait(false) is not { } archive)
|
||||
throw new InvalidOperationException(Loc.L("Archive.Log.SequenceMissing", context.Job.TargetIds.FirstOrDefault()));
|
||||
// Asked before the store is opened, which makes its folder: editing happens outside every library (owner review 2,
|
||||
// R8), and a workspace inside one would put its copies among that library's own files.
|
||||
if (await InsideALibraryAsync(parameters.WorkspaceRoot, ct).ConfigureAwait(false))
|
||||
throw new InvalidOperationException(Loc.L("Errors.Workspace.InsideLibrary"));
|
||||
var store = new WorkspaceStore(parameters.WorkspaceRoot);
|
||||
var project = store.Find(parameters.ProjectId) ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.NotCheckedOut"));
|
||||
if (project.ReadBundle().Workspace is not { Home: { IsArchive: true } home } || home.SequenceId != sequenceId)
|
||||
throw new InvalidOperationException(Loc.L("Errors.Workspace.OtherHome"));
|
||||
|
||||
try
|
||||
{
|
||||
using (WritingMedia(project)) await CopyAsync(context, project, sequenceId, archive, parameters, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
// Stopped writing: what a first copy the user cancelled wrote goes (a pause or a shutdown resumes it).
|
||||
await RemovePartialMediaAsync(context.Job, project).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A copy cancelled while no handler ran it (paused, failed): what a first copy wrote goes as after a cancel while it ran,
|
||||
/// under the project's lease, in the background (the lease may be held for a while: the cancel does not wait for it).
|
||||
/// A workspace folder or a project no longer there has nothing to clear, and nothing is made for it.
|
||||
/// </summary>
|
||||
public Task CancelledAsync(Job job, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(job);
|
||||
WorkspaceCopyParameters parameters;
|
||||
try
|
||||
{
|
||||
parameters = Parameters(job);
|
||||
}
|
||||
catch (InvalidDataException)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
// Opening a store makes its folder: one gone stays gone.
|
||||
if (!Directory.Exists(Path.Combine(parameters.WorkspaceRoot, parameters.ProjectId.ToString("N")))) return Task.CompletedTask;
|
||||
_ = Task.Run(async () =>
|
||||
{
|
||||
try
|
||||
{
|
||||
if (new WorkspaceStore(parameters.WorkspaceRoot).Find(parameters.ProjectId) is { } project) await RemovePartialMediaAsync(job, project).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is not OutOfMemoryException)
|
||||
{
|
||||
// Nobody waits for this task: whatever stopped it is logged; the next copy checks every file it finds.
|
||||
Log.Warning("workspace", $"The media of cancelled copy {job.Id} could not be cleared", ex);
|
||||
}
|
||||
}, CancellationToken.None);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A first copy of the project's media the user cancelled (the job reads <see cref="JobState.Cancelled"/>), once it has
|
||||
/// stopped writing: the photos it wrote (copies the home can make again) are removed under the project's lease, so the
|
||||
/// workspace holds no half media. Media that a copy ever finished (<c>media.json</c>) stay, as do those of a copy paused,
|
||||
/// yielded or stopped with the program, which goes on from its checkpoint, and those another copy of the project into
|
||||
/// this workspace still writes or goes on with (queued, running or paused: <see cref="OtherCopyAsync"/>). The last
|
||||
/// look is taken under the lease, and no copy of this program starts writing while the media go
|
||||
/// (<see cref="WritingMedia"/>). A folder that is no project any more (its removal under way) is left as it is, and the
|
||||
/// lease never makes it again. A failure is logged: the next copy checks every file it finds.
|
||||
/// </summary>
|
||||
private async Task RemovePartialMediaAsync(Job job, WorkspaceProject project)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Nothing half written (or the copy removed meanwhile): the lease is not taken.
|
||||
if (!Clearable(project) || !await StillToClearAsync(job, project).ConfigureAwait(false)) return;
|
||||
using (await project.LockExistingAsync(LockWait, CancellationToken.None).ConfigureAwait(false))
|
||||
{
|
||||
if (!Clearable(project) || !await StillToClearAsync(job, project).ConfigureAwait(false)) return;
|
||||
lock (WritersGate)
|
||||
{
|
||||
// A copy of this program that started since the jobs were read: its files stay.
|
||||
if (Writers.ContainsKey(project.Folder) || !Clearable(project)) return;
|
||||
Directory.Delete(project.MediaDirectory, recursive: true);
|
||||
}
|
||||
}
|
||||
Log.Info("workspace", $"The media a cancelled first copy of project {project.ProjectId} wrote were removed");
|
||||
}
|
||||
catch (DirectoryNotFoundException)
|
||||
{
|
||||
// The project's folder went meanwhile (its removal): nothing is left to clear.
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or DatabaseException)
|
||||
{
|
||||
Log.Warning("workspace", $"The media of a cancelled copy of project {project.ProjectId} could not be removed", ex);
|
||||
}
|
||||
|
||||
// A project (its working state there) with media no copy ever finished.
|
||||
static bool Clearable(WorkspaceProject project) => project.Exists && !File.Exists(project.MediaIndexPath) && Directory.Exists(project.MediaDirectory);
|
||||
}
|
||||
|
||||
/// <summary>The cancelled copy still reads so, and no other copy of the project into this workspace waits, runs or is paused.</summary>
|
||||
private async Task<bool> StillToClearAsync(Job job, WorkspaceProject project)
|
||||
{
|
||||
if (await db.Jobs.GetAsync(job.Id, CancellationToken.None).ConfigureAwait(false) is not { State: JobState.Cancelled }) return false;
|
||||
if (await OtherCopyAsync(job, project).ConfigureAwait(false) is not { } other) return true;
|
||||
Log.Info("workspace", $"The media of cancelled copy {job.Id} stay: copy {other.Id} of project {project.ProjectId} is {other.State}");
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Another copy of the project into the same workspace that is queued, running or paused: it writes the media folder, or
|
||||
/// goes on with what is there. A job whose parameters cannot be read counts, so nothing is removed on a guess.
|
||||
/// </summary>
|
||||
private async Task<Job?> OtherCopyAsync(Job job, WorkspaceProject project)
|
||||
{
|
||||
var sequenceId = job.TargetIds.FirstOrDefault();
|
||||
foreach (var other in await db.Jobs.ListPendingAsync(JobKind.WorkspaceCopy, sequenceId, CancellationToken.None).ConfigureAwait(false))
|
||||
{
|
||||
if (other.Id == job.Id) continue;
|
||||
try
|
||||
{
|
||||
var parameters = Parameters(other);
|
||||
if (parameters.ProjectId != project.ProjectId) continue;
|
||||
// Case aside: a match on a case-sensitive drive only keeps media that could have gone.
|
||||
if (!string.Equals(WorkspaceStore.FullRoot(parameters.WorkspaceRoot), project.Store.Root, StringComparison.OrdinalIgnoreCase)) continue;
|
||||
}
|
||||
catch (Exception ex) when (ex is InvalidDataException or ArgumentException or IOException or NotSupportedException or System.Security.SecurityException)
|
||||
{
|
||||
Log.Debug("workspace", $"Copy {other.Id} cannot be read: it counts as one of project {project.ProjectId}: {ex.Message}");
|
||||
}
|
||||
return other;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>The project folders copies of this program write media into now, with how many copies write each.</summary>
|
||||
private static readonly Dictionary<string, int> Writers = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
private static readonly Lock WritersGate = new();
|
||||
|
||||
/// <summary>
|
||||
/// Marks the copy into <paramref name="project"/> as writing its media until disposed: a clean-up after a cancelled copy
|
||||
/// leaves them (<see cref="RemovePartialMediaAsync"/>), and one under way finishes before the copy writes anything.
|
||||
/// </summary>
|
||||
private static IDisposable WritingMedia(WorkspaceProject project)
|
||||
{
|
||||
lock (WritersGate) Writers[project.Folder] = Writers.GetValueOrDefault(project.Folder) + 1;
|
||||
return new Writing(project.Folder);
|
||||
}
|
||||
|
||||
private sealed class Writing(string folder) : IDisposable
|
||||
{
|
||||
private int _done;
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _done, 1) != 0) return;
|
||||
lock (WritersGate)
|
||||
{
|
||||
if (Writers.GetValueOrDefault(folder) <= 1) Writers.Remove(folder);
|
||||
else Writers[folder]--;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task CopyAsync(JobContext context, WorkspaceProject project, Guid sequenceId, Archive archive, WorkspaceCopyParameters parameters,
|
||||
CancellationToken ct)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
// Read only: a read-only archive is copied from like any other.
|
||||
if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(archive.RootPath, ArchiveAvailability.Offline);
|
||||
var (copied, checkpointVersion, checkpointFolder) = Parse(context.Checkpoint);
|
||||
for (var pass = 1; ; pass++)
|
||||
{
|
||||
var (row, frames) = await FramesAsync(sequenceId, ct).ConfigureAwait(false);
|
||||
var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy;
|
||||
var folder = paths.Sequence(row.Folder);
|
||||
// Another index moved the sequence away, maybe another one in: its files are not this sequence's photos. A retry
|
||||
// checks every file again (the files of this run may be of that folder).
|
||||
if (!SequenceFolders.Holds(folder, sequenceId))
|
||||
{
|
||||
await context.CheckpointAsync(Checkpoint(Stage.Copy, 0, row.FramesVersion, row.Folder), 0, frames.Count).ConfigureAwait(false);
|
||||
throw new SequenceFolderMovedException();
|
||||
}
|
||||
// Files kept from an earlier run are this frames version's and this folder's only (a conversion keeps a frame's id,
|
||||
// not its photo; a sequence moved meanwhile may have left its photos' names to another sequence's).
|
||||
if ((checkpointVersion is { } version && version != row.FramesVersion) || (checkpointFolder is { } read && read != row.Folder)) copied = 0;
|
||||
var total = (long)frames.Count;
|
||||
var tally = new Tally();
|
||||
var entries = new WorkspaceFrame[frames.Count];
|
||||
// The media the editor reads until this copy commits: their files stay until then (RemoveStale).
|
||||
var current = TryReadMedia(project)?.Frames.Where(f => f.File is not null).ToDictionary(f => f.Id) ?? [];
|
||||
Directory.CreateDirectory(project.ProxiesDirectory);
|
||||
// The workspace is on this PC's own drive, with the index, the settings and the logs: the copy never fills it.
|
||||
var remaining = Remaining(project, folder, frames, policy);
|
||||
EnsureSpace(parameters.WorkspaceRoot, remaining[0]);
|
||||
var clock = System.Diagnostics.Stopwatch.StartNew();
|
||||
for (var i = 0; i < frames.Count; i++)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
entries[i] = Copy(project, folder, paths, frames[i], policy, done: i < copied, current.GetValueOrDefault(frames[i].Id), tally, ct);
|
||||
if (clock.Elapsed >= CheckpointEvery)
|
||||
{
|
||||
await context.CheckpointAsync(Checkpoint(Stage.Copy, i + 1, row.FramesVersion, row.Folder), i + 1, total).ConfigureAwait(false);
|
||||
// Another program may take the space meanwhile.
|
||||
EnsureSpace(parameters.WorkspaceRoot, remaining[i + 1]);
|
||||
clock.Restart();
|
||||
}
|
||||
else
|
||||
{
|
||||
await context.ReportAsync(i + 1, total).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
var analysis = CopyAnalysis(folder, project, frames.Count);
|
||||
var (after, now) = await FramesAsync(sequenceId, ct).ConfigureAwait(false);
|
||||
var moved = after.Folder != row.Folder || !SequenceFolders.Holds(paths.Sequence(after.Folder), sequenceId) || Renamed(frames, now, entries);
|
||||
if (after.FramesVersion != row.FramesVersion || moved)
|
||||
{
|
||||
// The files read may be of other frames, or missing where a Reorganize took them: read them again, and so
|
||||
// does a run resumed after a crash in the next pass (the checkpoint stored says so).
|
||||
if (pass < Passes)
|
||||
{
|
||||
context.Log(Loc.L("Archive.Log.WorkspaceFramesChanged", row.DisplayTitle));
|
||||
await context.CheckpointAsync(Checkpoint(Stage.Copy, 0, after.FramesVersion, after.Folder), 0, total).ConfigureAwait(false);
|
||||
(copied, checkpointVersion, checkpointFolder) = (0, null, null);
|
||||
continue;
|
||||
}
|
||||
// Frames still arriving are copied as they were read (Update media takes the rest); a folder still moving is not.
|
||||
if (moved)
|
||||
{
|
||||
await context.CheckpointAsync(Checkpoint(Stage.Copy, 0, after.FramesVersion, after.Folder), 0, total).ConfigureAwait(false);
|
||||
throw new SequenceFolderMovedException();
|
||||
}
|
||||
}
|
||||
var aliases = await HomePhotos.AliasesAsync(db, sequenceId, ct).ConfigureAwait(false);
|
||||
var media = new WorkspaceMedia(WorkspaceMedia.FormatName, WorkspaceMedia.CurrentFormatVersion, BundleMedia.Proxies, sequenceId, row.FramesVersion, entries,
|
||||
DateTimeOffset.UtcNow, analysis, aliases.Count > 0 ? aliases : null);
|
||||
// The working state is the editor's meanwhile: it is held only while the media index and the positions move.
|
||||
using (await project.LockAsync(LockWait, ct).ConfigureAwait(false))
|
||||
{
|
||||
Commit(project, project.ReadBundle(), media, row);
|
||||
RemoveStale(project, media);
|
||||
}
|
||||
context.Log(Loc.L("Archive.Log.WorkspaceCopied", row.DisplayTitle, tally.Copied, tally.Made, tally.Kept, tally.Missing));
|
||||
await context.CheckpointAsync(Checkpoint(Stage.Done, frames.Count, row.FramesVersion, row.Folder), total, total).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Whether <paramref name="folder"/> is an archive's root of this index or lies inside one.</summary>
|
||||
private async Task<bool> InsideALibraryAsync(string folder, CancellationToken cancellationToken)
|
||||
{
|
||||
var path = Plain(Path.GetFullPath(folder));
|
||||
foreach (var archive in await db.Archives.ListAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (DataLocation.IsUnder(path, Plain(Path.GetFullPath(archive.RootPath)))) return true;
|
||||
}
|
||||
return false;
|
||||
|
||||
// Without the long-path prefix, which only one of the two may carry.
|
||||
static string Plain(string path) =>
|
||||
path.StartsWith(@"\\?\UNC\", StringComparison.Ordinal) ? @"\\" + path[8..]
|
||||
: path.StartsWith(@"\\?\", StringComparison.Ordinal) ? path[4..]
|
||||
: path;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A frame the copy found no file for whose original the index has under another name now (a Reorganize renamed the
|
||||
/// files during the copy, keeping the frames version): the copy runs again.
|
||||
/// </summary>
|
||||
private static bool Renamed(List<Frame> read, List<Frame> now, WorkspaceFrame[] entries)
|
||||
{
|
||||
var paths = now.ToDictionary(f => f.Id, f => f.RelativePath);
|
||||
for (var i = 0; i < read.Count; i++)
|
||||
{
|
||||
if (entries[i].File is null && paths.TryGetValue(read[i].Id, out var path) && path != read[i].RelativePath) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The frame's file in the workspace: kept when an earlier run copied it (before the checkpoint, or the same bytes as the
|
||||
/// archive's proxy), else the archive's current proxy, else one made from the original; none for a frame without either.
|
||||
/// The file the media the editor reads now name for the frame (<paramref name="current"/>) is never removed nor written
|
||||
/// over here: the editor keeps it while the copy runs and if it fails, other bytes for it go under a name of their own
|
||||
/// (<see cref="WorkspaceProject.ProxyName(Guid, string)"/>), and a copy that commits without it removes it then
|
||||
/// (<see cref="RemoveStale"/>). Nor is it taken as this run's copy of a frame whose photo changed since.
|
||||
/// </summary>
|
||||
private WorkspaceFrame Copy(WorkspaceProject project, SequencePaths folder, ArchivePaths paths, Frame frame, ProxyPolicy policy, bool done,
|
||||
WorkspaceFrame? current, Tally tally, CancellationToken cancellationToken)
|
||||
{
|
||||
var name = WorkspaceProject.ProxyName(frame.Id);
|
||||
var target = project.MediaFile(name);
|
||||
var listed = current is not null && string.Equals(current.File, name, StringComparison.OrdinalIgnoreCase);
|
||||
if (done && File.Exists(target))
|
||||
{
|
||||
var kept = AtomicFile.ReadAllBytes(target);
|
||||
var keptSha = Sha(kept);
|
||||
// The editor's file of another photo of the frame (a conversion since) was left by an earlier copy, not this run.
|
||||
if (!listed || keptSha != current!.FileSha256 || SamePhoto(current, frame))
|
||||
{
|
||||
tally.Kept++;
|
||||
return WorkspaceFrame.Of(frame, name, keptSha, kept.Length);
|
||||
}
|
||||
}
|
||||
var source = folder.Proxy(frame.Index);
|
||||
byte[]? bytes = null;
|
||||
if (File.Exists(source) && ArchiveImages.IsProxyOf(source, frame.CapturedAt, frame.Width, frame.Height, null)) bytes = AtomicFile.ReadAllBytes(source);
|
||||
var made = false;
|
||||
if (bytes is null && (frame.Flags & FrameFlags.Missing) == 0)
|
||||
{
|
||||
var original = paths.FromRelative(frame.RelativePath);
|
||||
try
|
||||
{
|
||||
var edge = policy.LongEdge == ProxyPolicy.Disabled ? new ProxyPolicy().LongEdge : policy.LongEdge;
|
||||
bytes = File.Exists(original) ? ArchiveImages.MakeProxy(archives.Decoder, original, frame, null, edge, policy.Quality, cancellationToken) : null;
|
||||
made = bytes is not null;
|
||||
}
|
||||
catch (IOException ex)
|
||||
{
|
||||
Log.Warning("workspace", $"No proxy of {original} could be made", ex);
|
||||
}
|
||||
}
|
||||
if (bytes is null)
|
||||
{
|
||||
// A file an earlier copy left for the frame (of other frames, or another folder's) is never kept as its photo by a
|
||||
// resumed run: a regenerable copy, it goes, unless the editor's media name it (it goes once this copy commits).
|
||||
if (!listed && File.Exists(target)) TryDelete(target);
|
||||
tally.Missing++;
|
||||
return WorkspaceFrame.Of(frame, null, null, 0);
|
||||
}
|
||||
var sha = Sha(bytes);
|
||||
// The file the editor's media name for the frame holds these bytes already (under its own name, maybe): kept.
|
||||
if (current?.File is { } held && !string.Equals(held, name, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(current.FileSha256, sha, StringComparison.OrdinalIgnoreCase) && Holds(project.MediaFile(held), sha))
|
||||
{
|
||||
tally.Kept++;
|
||||
return WorkspaceFrame.Of(frame, held, sha, bytes.Length);
|
||||
}
|
||||
if (Holds(target, sha))
|
||||
{
|
||||
tally.Kept++;
|
||||
return WorkspaceFrame.Of(frame, name, sha, bytes.Length);
|
||||
}
|
||||
// Other bytes for the file the editor's media name (a conversion since): written under a name of their own, never over
|
||||
// the editor's file, which the commit switches the media from (RemoveStale then removes it). A copy that fails before
|
||||
// its commit leaves the editor reading the bytes its entry names. A file that no longer holds them (damaged, gone)
|
||||
// is mended where it is.
|
||||
if (listed && current!.FileSha256 is { } editors && Holds(target, editors.ToLowerInvariant()))
|
||||
{
|
||||
name = WorkspaceProject.ProxyName(frame.Id, sha);
|
||||
target = project.MediaFile(name);
|
||||
if (Holds(target, sha))
|
||||
{
|
||||
tally.Kept++;
|
||||
return WorkspaceFrame.Of(frame, name, sha, bytes.Length);
|
||||
}
|
||||
}
|
||||
WriteVerified(target, bytes, sha);
|
||||
if (made) tally.Made++;
|
||||
else tally.Copied++;
|
||||
return WorkspaceFrame.Of(frame, name, sha, bytes.Length);
|
||||
}
|
||||
|
||||
/// <summary>Whether the file at <paramref name="path"/> is there and hashes as <paramref name="sha"/>.</summary>
|
||||
private static bool Holds(string path, string sha) => File.Exists(path) && Sha(AtomicFile.ReadAllBytes(path)) == sha;
|
||||
|
||||
/// <summary>Writes atomically and reads back: a file that does not hash as its source is written again once, then refused.</summary>
|
||||
private static void WriteVerified(string path, byte[] bytes, string sha)
|
||||
{
|
||||
for (var attempt = 1; ; attempt++)
|
||||
{
|
||||
AtomicFile.WriteAllBytes(path, bytes);
|
||||
if (Sha(AtomicFile.ReadAllBytes(path)) == sha) return;
|
||||
if (attempt == 2) throw new IOException(Loc.L("Errors.Workspace.CopyMismatch", path));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The home's deep report into the media folder when it describes these frames; its SHA-256, or null without one.</summary>
|
||||
private static string? CopyAnalysis(SequencePaths folder, WorkspaceProject project, int frameCount)
|
||||
{
|
||||
if (!File.Exists(folder.AnalysisPath))
|
||||
{
|
||||
if (File.Exists(project.AnalysisPath)) File.Delete(project.AnalysisPath);
|
||||
return null;
|
||||
}
|
||||
var bytes = AtomicFile.ReadAllBytes(folder.AnalysisPath);
|
||||
try
|
||||
{
|
||||
if (Core.Analysis.Deep.DeepAnalysisReport.FromJson(System.Text.Encoding.UTF8.GetString(bytes)).FrameCount != frameCount) return null;
|
||||
}
|
||||
catch (Exception ex) when (ex is InvalidDataException or JsonException or IOException)
|
||||
{
|
||||
Log.Warning("workspace", $"{folder.AnalysisPath} cannot be read; the workspace takes no deep report", ex);
|
||||
return null;
|
||||
}
|
||||
var sha = Sha(bytes);
|
||||
WriteVerified(project.AnalysisPath, bytes, sha);
|
||||
return sha;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The media index and the working state moved onto it: first the working state with every position parked by photo
|
||||
/// (valid whatever the frames), then the index, then the working state placed on the new frames. A crash in between
|
||||
/// leaves positions that wait for their photos, which the next open places. A frame the new media still hold is parked
|
||||
/// as the photo they hold for it, so one whose file a conversion replaced (the same frame, another SHA-256) keeps its
|
||||
/// positions, as the archive's own settings do (<see cref="ProjectPositions"/>).
|
||||
/// </summary>
|
||||
private static void Commit(WorkspaceProject project, ProjectBundle bundle, WorkspaceMedia media, Sequence row)
|
||||
{
|
||||
var old = TryReadMedia(project);
|
||||
var workspace = bundle.Workspace! with { Media = new BundleMedia(BundleMedia.Proxies, media.FramesVersion) };
|
||||
var sequence = bundle.Sequence with { Folder = row.Folder, Title = row.DisplayTitle, Frames = media.Frames.Count };
|
||||
var working = bundle.Project;
|
||||
// Without the media index the positions were made against, they are taken as naming these frames.
|
||||
if (!working.IsNewerFormat && old is not null)
|
||||
{
|
||||
var held = media.Frames.ToDictionary(f => f.Id);
|
||||
var before = new FramePhotos(old.Frames.Select(f => (f.Index, Photo(held.TryGetValue(f.Id, out var now) ? now : f))), old.Aliases);
|
||||
var portable = ProjectSettings.FromJson(ProjectContent.ToPortable(working, before), allowNewer: true);
|
||||
project.WriteBundle(bundle with { Project = portable, Workspace = workspace, Sequence = sequence });
|
||||
project.WriteMedia(media);
|
||||
working = ProjectSettings.FromJson(ProjectContent.Place(portable.ToJson(), media.Photos), allowNewer: true);
|
||||
}
|
||||
else
|
||||
{
|
||||
project.WriteMedia(media);
|
||||
}
|
||||
project.WriteBundle(bundle with { Project = working, Workspace = workspace, Sequence = sequence, Saved = DateTimeOffset.UtcNow });
|
||||
}
|
||||
|
||||
private static FramePhoto Photo(WorkspaceFrame frame) => new(frame.Sha256, frame.Size, frame.CapturedAt);
|
||||
|
||||
/// <summary>Whether the media's entry of a frame is of the photo the frame holds now (a conversion gives it another file).</summary>
|
||||
private static bool SamePhoto(WorkspaceFrame held, Frame frame) =>
|
||||
held.Size == frame.Size && held.CapturedAt == frame.CapturedAt
|
||||
&& (string.IsNullOrEmpty(held.Sha256) || string.IsNullOrEmpty(frame.Sha256) || string.Equals(held.Sha256, frame.Sha256, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
/// <summary>Proxies of frames the sequence no longer has: regenerable copies, removed so the media folder holds the frames listed.</summary>
|
||||
private static void RemoveStale(WorkspaceProject project, WorkspaceMedia media)
|
||||
{
|
||||
var listed = media.Frames.Where(f => f.File is not null).Select(f => Path.GetFileName(f.File!)).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var file in Directory.EnumerateFiles(project.ProxiesDirectory, "*.jpg"))
|
||||
{
|
||||
if (!listed.Contains(Path.GetFileName(file))) TryDelete(file);
|
||||
}
|
||||
}
|
||||
|
||||
private static void TryDelete(string file)
|
||||
{
|
||||
try
|
||||
{
|
||||
File.Delete(file);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("workspace", $"{file} could not be removed", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The bytes the copy still writes from each frame on (index <c>i</c>: frames <c>i</c> to the last; one more entry, 0):
|
||||
/// nothing for a frame the workspace holds a file of already (kept, or replaced by one of about its size; under its own
|
||||
/// name or the one other bytes of it take, <see cref="WorkspaceProject.ProxyName(Guid, string)"/>), the archive's
|
||||
/// proxy's size where it has one, else an estimate at the proxy size the copy makes (<see cref="Estimate"/>).
|
||||
/// </summary>
|
||||
private static long[] Remaining(WorkspaceProject project, SequencePaths folder, List<Frame> frames, ProxyPolicy policy) =>
|
||||
Remaining(Directory.Exists(project.ProxiesDirectory) ? HeldFrames(Directory.EnumerateFiles(project.ProxiesDirectory, "*.jpg")) : [], folder, frames, policy);
|
||||
|
||||
/// <summary>The frames the files at <paramref name="paths"/> are proxies of: "<frame id>.jpg" and "<frame id>-<sha16>.jpg".</summary>
|
||||
internal static HashSet<Guid> HeldFrames(IEnumerable<string> paths)
|
||||
{
|
||||
var held = new HashSet<Guid>();
|
||||
foreach (var path in paths)
|
||||
{
|
||||
var name = Path.GetFileNameWithoutExtension(path);
|
||||
if (name.Length >= 32 && (name.Length == 32 || name[32] == '-') && Guid.TryParseExact(name[..32], "N", out var id)) held.Add(id);
|
||||
}
|
||||
return held;
|
||||
}
|
||||
|
||||
private static long[] Remaining(HashSet<Guid> held, SequencePaths folder, List<Frame> frames, ProxyPolicy policy)
|
||||
{
|
||||
var proxies = Directory.Exists(folder.Proxies)
|
||||
? new DirectoryInfo(folder.Proxies).EnumerateFiles().ToDictionary(f => f.Name, f => f.Length, StringComparer.OrdinalIgnoreCase)
|
||||
: [];
|
||||
var remaining = new long[frames.Count + 1];
|
||||
for (var i = frames.Count - 1; i >= 0; i--)
|
||||
{
|
||||
var frame = frames[i];
|
||||
var bytes = held.Contains(frame.Id) ? 0
|
||||
: proxies.TryGetValue(ArchiveLayout.ProxyFileName(frame.Index), out var length) ? length
|
||||
: (frame.Flags & FrameFlags.Missing) != 0 ? 0
|
||||
: Estimate(frame, policy);
|
||||
remaining[i] = remaining[i + 1] + bytes;
|
||||
}
|
||||
return remaining;
|
||||
}
|
||||
|
||||
/// <summary>What a first copy of the sequence writes (nothing held yet), and the free bytes of the workspace's drive.</summary>
|
||||
internal async Task<(long Bytes, long? Free)> FactsAsync(Guid sequenceId, string workspaceRoot, CancellationToken cancellationToken)
|
||||
{
|
||||
var (row, frames) = await FramesAsync(sequenceId, cancellationToken).ConfigureAwait(false);
|
||||
var archive = await db.Archives.GetAsync(row.ArchiveId, cancellationToken).ConfigureAwait(false)
|
||||
?? throw new InvalidOperationException(Loc.L("Archive.Log.SequenceMissing", sequenceId));
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy;
|
||||
return (Remaining([], paths.Sequence(row.Folder), frames, policy)[0], FreeBytes(workspaceRoot));
|
||||
}
|
||||
|
||||
/// <summary>A proxy's size when the archive has none to measure: some 1.5 MB at 1920 px, scaled by the area at the copy's long edge.</summary>
|
||||
private static long Estimate(Frame frame, ProxyPolicy policy)
|
||||
{
|
||||
var edge = policy.LongEdge == ProxyPolicy.Disabled ? new ProxyPolicy().LongEdge : policy.LongEdge;
|
||||
var longest = Math.Max(frame.Width, frame.Height);
|
||||
var side = edge == ProxyPolicy.FullSize ? (longest > 0 ? longest : UnknownFullSize) : longest > 0 ? Math.Min(edge, longest) : edge;
|
||||
return (long)(ProxyEstimate * Math.Pow(side / (double)ProxyEstimateEdge, 2));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Refuses to go on while the workspace's drive lacks <paramref name="bytes"/> besides <see cref="SpaceReserve"/>, with the
|
||||
/// gigabytes missing; a drive that cannot be measured (a network share) is not refused, the copy itself then says so.
|
||||
/// </summary>
|
||||
/// <exception cref="IOException">The drive lacks the space (<c>Errors.Workspace.DiskFull</c>).</exception>
|
||||
private void EnsureSpace(string root, long bytes)
|
||||
{
|
||||
if (bytes <= 0 || FreeBytes(root) is not { } free || free >= bytes + SpaceReserve) return;
|
||||
throw new IOException(Loc.L("Errors.Workspace.DiskFull", ((bytes + SpaceReserve - free) / 1e9).ToString("0.0", Loc.Culture)));
|
||||
}
|
||||
|
||||
/// <summary>Free bytes on the volume of <paramref name="folder"/>, as the host measures them (<see cref="ArchiveService.FreeSpace"/>), else its drive's.</summary>
|
||||
private long? FreeBytes(string folder)
|
||||
{
|
||||
if (archives.FreeSpace is { } measure) return measure(folder);
|
||||
try
|
||||
{
|
||||
var full = Path.GetFullPath(folder);
|
||||
// Windows wants a drive root; Unix accepts any directory and reports the volume it is mounted on.
|
||||
return new DriveInfo(OperatingSystem.IsWindows() ? Path.GetPathRoot(full)! : full).AvailableFreeSpace;
|
||||
}
|
||||
catch (Exception ex) when (ex is ArgumentException or IOException or UnauthorizedAccessException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static WorkspaceMedia? TryReadMedia(WorkspaceProject project)
|
||||
{
|
||||
try
|
||||
{
|
||||
return project.ReadMedia();
|
||||
}
|
||||
catch (InvalidDataException ex)
|
||||
{
|
||||
Log.Warning("workspace", $"The media index of {project.Folder} cannot be read; positions are taken as they are", ex);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The sequence row with its frames as one state (listed between two reads of the row at the same frames version).</summary>
|
||||
private async Task<(Sequence Row, List<Frame> Frames)> FramesAsync(Guid sequenceId, CancellationToken cancellationToken)
|
||||
{
|
||||
for (var attempt = 0; ; attempt++)
|
||||
{
|
||||
var before = await db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
|
||||
?? throw new InvalidOperationException(Loc.L("Archive.Log.SequenceMissing", sequenceId));
|
||||
var frames = new List<Frame>();
|
||||
for (var after = -1; ;)
|
||||
{
|
||||
var page = await db.Frames.ListBySequenceAsync(sequenceId, after, PageSize, cancellationToken).ConfigureAwait(false);
|
||||
frames.AddRange(page);
|
||||
if (page.Count < PageSize) break;
|
||||
after = page[^1].Index;
|
||||
}
|
||||
var now = await db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? before;
|
||||
if (now.FramesVersion == before.FramesVersion || attempt == 2) return (now, frames);
|
||||
}
|
||||
}
|
||||
|
||||
private static WorkspaceCopyParameters Parameters(Job job)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (job.ParametersJson is { Length: > 0 } json
|
||||
&& JsonSerializer.Deserialize(json, WorkspaceJson.Default.WorkspaceCopyParameters) is { WorkspaceRoot.Length: > 0 } parameters
|
||||
&& parameters.ProjectId != Guid.Empty)
|
||||
return parameters with { Media = parameters.Media ?? BundleMedia.Proxies };
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
throw new InvalidDataException(Loc.L("Errors.Workspace.Parameters"), ex);
|
||||
}
|
||||
throw new InvalidDataException(Loc.L("Errors.Workspace.Parameters"));
|
||||
}
|
||||
|
||||
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes));
|
||||
|
||||
private enum Stage
|
||||
{
|
||||
Copy,
|
||||
Done,
|
||||
}
|
||||
|
||||
private static string Checkpoint(Stage stage, int frame, int framesVersion, string folder) =>
|
||||
string.Create(CultureInfo.InvariantCulture, $"{stage.ToString().ToLowerInvariant()}:{frame}:{framesVersion}:{folder}");
|
||||
|
||||
/// <summary>
|
||||
/// The frames copied by an earlier run (<c>copy:<frame>[:<frames version>[:<folder>]]</c>, the folder
|
||||
/// last as it may hold a colon), the frames version and the sequence's folder they were of (null when the checkpoint does
|
||||
/// not say); nothing for a finished run, or another stage (an earlier build's archive proxy pass), whose files the copy
|
||||
/// checks again.
|
||||
/// </summary>
|
||||
private static (int Copied, int? FramesVersion, string? Folder) Parse(string? checkpoint)
|
||||
{
|
||||
var parts = checkpoint?.Split(':', 4);
|
||||
if (parts is not ([_, _] or [_, _, _] or [_, _, _, _]) || !Enum.TryParse<Stage>(parts[0], ignoreCase: true, out var stage) || stage != Stage.Copy
|
||||
|| !int.TryParse(parts[1], NumberStyles.None, CultureInfo.InvariantCulture, out var next))
|
||||
return (0, null, null);
|
||||
int? version = parts.Length >= 3 && int.TryParse(parts[2], NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture, out var v) ? v : null;
|
||||
return (next, version, parts.Length == 4 ? parts[3] : null);
|
||||
}
|
||||
|
||||
private sealed class Tally
|
||||
{
|
||||
public int Copied { get; set; }
|
||||
public int Made { get; set; }
|
||||
public int Kept { get; set; }
|
||||
public int Missing { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
using System.Text.Json.Serialization;
|
||||
using Chronos.Core.Imaging;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Core.Project;
|
||||
using Chronos.Core.Render;
|
||||
using Chronos.Data.Processing;
|
||||
using Chronos.Data.Projects;
|
||||
|
||||
namespace Chronos.Data.Workspaces;
|
||||
|
||||
/// <summary>
|
||||
/// One frame of a workspace's media (<c>media/media.json</c>): the frame's record at its home (what analyses and the
|
||||
/// editor read) and the file the workspace holds for it, with its SHA-256 and size as copied.
|
||||
/// </summary>
|
||||
/// <param name="File">The file relative to the media folder ("proxies/<frame id>.jpg"); null when the workspace holds none.</param>
|
||||
public sealed record WorkspaceFrame(
|
||||
Guid Id,
|
||||
int Index,
|
||||
string FileName,
|
||||
string? OriginalFileName,
|
||||
long Size,
|
||||
string? Sha256,
|
||||
DateTime? CapturedAt,
|
||||
int? UtcOffsetMinutes,
|
||||
TimestampSource TimestampSource,
|
||||
double? ExposureTime,
|
||||
double? FNumber,
|
||||
int? Iso,
|
||||
double? FocalLength,
|
||||
double? ExposureBias,
|
||||
int? WhiteBalance,
|
||||
int Orientation,
|
||||
int Width,
|
||||
int Height,
|
||||
ImageFileFormat Format,
|
||||
int PreviewWidth,
|
||||
int PreviewHeight,
|
||||
double? MeanLogLuminance,
|
||||
double? ClipLow,
|
||||
double? ClipHigh,
|
||||
FrameFlags Flags,
|
||||
string? File,
|
||||
string? FileSha256,
|
||||
long FileBytes)
|
||||
{
|
||||
/// <summary>The record of a home's frame, with the file the workspace holds for it.</summary>
|
||||
public static WorkspaceFrame Of(Frame frame, string? file, string? fileSha256, long fileBytes)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(frame);
|
||||
return new WorkspaceFrame(frame.Id, frame.Index, frame.FileName, frame.OriginalFileName, frame.Size, frame.Sha256, frame.CapturedAt, frame.UtcOffsetMinutes,
|
||||
frame.TimestampSource, frame.ExposureTime, frame.FNumber, frame.Iso, frame.FocalLength, frame.ExposureBias, frame.WhiteBalance, frame.Orientation,
|
||||
frame.Width, frame.Height, frame.Format, frame.PreviewWidth, frame.PreviewHeight, frame.MeanLogLuminance, frame.ClipLow, frame.ClipHigh, frame.Flags,
|
||||
file, fileSha256, fileBytes);
|
||||
}
|
||||
|
||||
/// <summary>The frame as the processing engine reads it; flagged missing while the workspace holds no file for it.</summary>
|
||||
public Frame ToFrame(Guid sequenceId) => new()
|
||||
{
|
||||
Id = Id,
|
||||
SequenceId = sequenceId,
|
||||
Index = Index,
|
||||
RelativePath = File ?? FileName,
|
||||
FileName = FileName,
|
||||
OriginalFileName = OriginalFileName,
|
||||
Size = Size,
|
||||
Sha256 = Sha256,
|
||||
CapturedAt = CapturedAt,
|
||||
UtcOffsetMinutes = UtcOffsetMinutes,
|
||||
TimestampSource = TimestampSource,
|
||||
ExposureTime = ExposureTime,
|
||||
FNumber = FNumber,
|
||||
Iso = Iso,
|
||||
FocalLength = FocalLength,
|
||||
ExposureBias = ExposureBias,
|
||||
WhiteBalance = WhiteBalance,
|
||||
Orientation = Orientation,
|
||||
Width = Width,
|
||||
Height = Height,
|
||||
Format = Format,
|
||||
PreviewWidth = PreviewWidth,
|
||||
PreviewHeight = PreviewHeight,
|
||||
MeanLogLuminance = MeanLogLuminance,
|
||||
ClipLow = ClipLow,
|
||||
ClipHigh = ClipHigh,
|
||||
Flags = File is null ? Flags | FrameFlags.Missing : Flags,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// What a workspace project holds of its sequence's photos (<c>media/media.json</c>, owner review 2, R8): the kind
|
||||
/// (<see cref="BundleMedia.Proxies"/>; originals in phase 5c), the sequence and its frames version at the home when they
|
||||
/// were copied, and every frame in index order. Written last by a copy, so a copy cut short leaves the previous index.
|
||||
/// </summary>
|
||||
/// <param name="Aliases">The encodings of the sequence's photos a conversion joined at the home (R7, its tombstones of reason
|
||||
/// "replaced") when the media were copied: a version parked on a photo before its file was converted still finds its frame.</param>
|
||||
public sealed record WorkspaceMedia(string Format, int FormatVersion, string Kind, Guid SequenceId, int FramesVersion, IReadOnlyList<WorkspaceFrame> Frames,
|
||||
DateTimeOffset Copied, string? AnalysisSha256 = null, IReadOnlyList<PhotoAlias>? Aliases = null)
|
||||
{
|
||||
public const string FormatName = "chronos-workspace-media";
|
||||
public const int CurrentFormatVersion = 1;
|
||||
|
||||
/// <summary>The frames' photos in index order, as project positions name them, with the encodings conversions joined.</summary>
|
||||
[JsonIgnore]
|
||||
public FramePhotos Photos => new(Frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, f.CapturedAt))), Aliases);
|
||||
|
||||
/// <summary>The frames held, as the processing engine reads them.</summary>
|
||||
public IReadOnlyList<Frame> ToFrames() => [.. Frames.Select(f => f.ToFrame(SequenceId))];
|
||||
|
||||
/// <summary>
|
||||
/// These media with their photos named as <paramref name="home"/> names them now (owner review 2, R8): a frame copied
|
||||
/// before the home hashed its photo (a folder adopted, verified since) takes the SHA-256 the home's frame of that id has
|
||||
/// for the same photo (the same size and capture time; a conversion that gave the frame another file is not the same
|
||||
/// photo), and the encodings the home's conversions joined since the copy are added to <see cref="Aliases"/>. Versions
|
||||
/// the home made since name those photos so; named alike here, one photo merges as one. The frames, their files and the
|
||||
/// frames version stay as they are. This instance when there is nothing to learn.
|
||||
/// </summary>
|
||||
public WorkspaceMedia NamedAsAt(HomeFrames home)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(home);
|
||||
var there = new Dictionary<Guid, Frame>(home.Frames.Count);
|
||||
foreach (var frame in home.Frames) there[frame.Id] = frame;
|
||||
var learned = false;
|
||||
var frames = new List<WorkspaceFrame>(Frames.Count);
|
||||
foreach (var frame in Frames)
|
||||
{
|
||||
if (string.IsNullOrEmpty(frame.Sha256) && there.TryGetValue(frame.Id, out var held) && !string.IsNullOrEmpty(held.Sha256)
|
||||
&& held.Size == frame.Size && Millisecond(held.CapturedAt) == Millisecond(frame.CapturedAt))
|
||||
{
|
||||
frames.Add(frame with { Sha256 = held.Sha256.ToLowerInvariant() });
|
||||
learned = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
frames.Add(frame);
|
||||
}
|
||||
}
|
||||
var known = Aliases ?? [];
|
||||
var links = known.Select(Link).ToHashSet();
|
||||
var added = home.Photos.Aliases.Where(a => links.Add(Link(a))).ToList();
|
||||
if (!learned && added.Count == 0) return this;
|
||||
return this with { Frames = learned ? frames : Frames, Aliases = added.Count > 0 ? [.. known, .. added] : Aliases };
|
||||
|
||||
static long? Millisecond(DateTime? time) => time is { } at ? at.Ticks / TimeSpan.TicksPerMillisecond : null;
|
||||
// A media index read back may hold a damaged link: compared as it is, and left out of the classes like any other.
|
||||
static (string?, string?) Link(PhotoAlias alias) => (alias.Sha256?.ToLowerInvariant(), alias.Replacement?.ToLowerInvariant());
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A workspace project's sequence opened for editing (like a server sequence opened with proxies): the home's sequence
|
||||
/// row, the frames as the workspace holds them and the absolute path of each frame's file (null, and the frame flagged
|
||||
/// missing, when it holds none).
|
||||
/// </summary>
|
||||
public sealed class WorkspaceSequence
|
||||
{
|
||||
public WorkspaceSequence(Sequence sequence, IReadOnlyList<Frame> frames, IReadOnlyList<string?> files)
|
||||
{
|
||||
if (files.Count != frames.Count) throw new ArgumentException("One file entry per frame is required.", nameof(files));
|
||||
Sequence = sequence;
|
||||
Frames = frames;
|
||||
Files = files;
|
||||
}
|
||||
|
||||
public Sequence Sequence { get; }
|
||||
|
||||
public IReadOnlyList<Frame> Frames { get; }
|
||||
|
||||
public IReadOnlyList<string?> Files { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The editor's frame source over the workspace's files, through the same read-ahead window as archive renders; it also
|
||||
/// implements <see cref="Core.Analysis.Deep.IFrameSource"/> and <see cref="IAsyncDisposable"/> (dispose it when done).
|
||||
/// </summary>
|
||||
public IRenderFrameSource OpenFrames(ImageDecoder decoder, FrameWindowOptions? window = null)
|
||||
{
|
||||
var frames = new SequenceFrames(Sequence, Frames, Files);
|
||||
return new SequenceFrameSource(frames, frames.LightFrames(), decoder, window);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,666 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.Diagnostics;
|
||||
using Chronos.Core.IO;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Core.Project;
|
||||
using Chronos.Data.Projects;
|
||||
using Chronos.Data.Sync;
|
||||
|
||||
namespace Chronos.Data.Workspaces;
|
||||
|
||||
/// <summary>The manifest of a workspace folder (<c>workspace.json</c>).</summary>
|
||||
public sealed record WorkspaceManifest(string Format, int FormatVersion, Guid Id, DateTimeOffset Created);
|
||||
|
||||
/// <summary>
|
||||
/// A workspace's sync state of one project besides its versions (<c>sync.json</c>): a restore to record, a conflict to
|
||||
/// settle, and the journal of a replay or a save-both under way, which the next sync finishes or undoes after a crash.
|
||||
/// </summary>
|
||||
/// <param name="RestoredFrom">The version a restore brought back; the next commit is a "restore" of it.</param>
|
||||
/// <param name="Conflict">The conflict a pull or push stopped at, until the user settles it.</param>
|
||||
/// <param name="Replay">A replay (or a base moved onto local commits the home holds) written in steps; null when none is under way.</param>
|
||||
/// <param name="SaveBoth">A save-both that made its copy at the home and has not finished; null when none is under way.</param>
|
||||
public sealed record WorkspaceSyncState(Guid? RestoredFrom, WorkspaceConflict? Conflict, WorkspaceReplay? Replay = null, WorkspaceSaveBoth? SaveBoth = null)
|
||||
{
|
||||
public static WorkspaceSyncState None { get; } = new(null, null);
|
||||
|
||||
/// <summary>Nothing to keep: the file can go.</summary>
|
||||
[JsonIgnore]
|
||||
public bool IsEmpty => RestoredFrom is null && Conflict is null && Replay is null && SaveBoth is null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The journal of a replay (<see cref="WorkspaceSync"/>): the base it moves the project onto, the local versions it
|
||||
/// replaces, the replayed ones it writes and the local versions the home already holds (their answer lost), whose files
|
||||
/// take the home's copy. The bundle's base is the commit point: once it is <see cref="OntoBase"/> the replay is finished
|
||||
/// (the replaced set aside, the held ones adopted), before it the replayed ones are set aside and nothing else moved.
|
||||
/// </summary>
|
||||
/// <param name="Settles">
|
||||
/// The replay settles the conflict recorded beside it (a pull, a push or a choice that merged): once its base is
|
||||
/// <see cref="OntoBase"/> that conflict is over, the choice made applied. False for a base moved onto versions the home
|
||||
/// held, which leaves the conflict to settle (and for a journal an earlier build wrote).
|
||||
/// </param>
|
||||
public sealed record WorkspaceReplay(Guid OntoBase, IReadOnlyList<Guid> Replaced, IReadOnlyList<Guid> Replayed, IReadOnlyList<Guid> Adopted,
|
||||
bool Settles = false);
|
||||
|
||||
/// <summary>
|
||||
/// A save-both under way (<see cref="WorkspaceSync"/>): the copy made at the home and its first version (the base the
|
||||
/// local versions are sent on top of), the base this project had when it started, and the id each local version is sent
|
||||
/// under, so a retry sends to the same copy what it lacks and the home answers "already stored" for the rest. Before the
|
||||
/// copy is asked of the home its name and the version it is made from are journaled (<see cref="Intent"/>, the copy's ids
|
||||
/// still empty): a retry after a cut finds the copy the home made by them instead of making another.
|
||||
/// </summary>
|
||||
public sealed record WorkspaceSaveBoth(Guid CopyId, Guid CopyBaseId, int CopyBaseNumber, Guid? FromBase, IReadOnlyList<WorkspaceMove> Moves,
|
||||
WorkspaceCopyIntent? Intent = null)
|
||||
{
|
||||
/// <summary>The copy is asked of the home but not known yet: only its <see cref="Intent"/> is.</summary>
|
||||
[JsonIgnore]
|
||||
public bool Intended => CopyId == Guid.Empty;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The copy a save-both asks the home for, journaled before it asks: its name, the version it is made from (null: Main's
|
||||
/// head) with that version's content hash, and when it was asked. A project of that name made since, whose only version
|
||||
/// holds that content, is that copy.
|
||||
/// </summary>
|
||||
public sealed record WorkspaceCopyIntent(string Name, Guid? From, string? FromHash, DateTimeOffset Asked);
|
||||
|
||||
/// <summary>A local version (<paramref name="LocalId"/>) and the id it is sent under to a save-both's copy.</summary>
|
||||
public sealed record WorkspaceMove(Guid LocalId, Guid CopyVersionId);
|
||||
|
||||
/// <summary>A version file of a workspace that cannot be read (<see cref="WorkspaceProject.UnreadableVersions"/>): the number and id its name carries, and why.</summary>
|
||||
public sealed record UnreadableVersion(int Number, Guid Id, string Path, string Reason);
|
||||
|
||||
/// <summary>
|
||||
/// A pull or push that met real conflicts (owner review 2, R8): the home's head it met, the version the local commits were
|
||||
/// made from, the local commits (oldest first) and the members both sides changed (mine, theirs). Settled by keeping mine,
|
||||
/// taking the server's (each member or all), or saving both (<see cref="WorkspaceSync.ResolveAsync"/>).
|
||||
/// </summary>
|
||||
public sealed record WorkspaceConflict(Guid TheirsVersionId, int TheirsNumber, Guid BaseVersionId, IReadOnlyList<Guid> Mine,
|
||||
IReadOnlyList<SettingsDifference> Differences, DateTimeOffset Found)
|
||||
{
|
||||
/// <summary>
|
||||
/// The home's head holds settings of a newer Chronos (a difference at <see cref="ProjectMerge.FormatPath"/>): nothing of
|
||||
/// this workspace's is merged into them. Only Save both settles it (this workspace's versions go to a new project in this
|
||||
/// Chronos's format), or a Chronos that reads them; keeping mine or taking the server's is refused.
|
||||
/// </summary>
|
||||
[JsonIgnore]
|
||||
public bool NewerFormat => Differences.Any(d => d.Path == ProjectMerge.FormatPath);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parameters of a <see cref="JobKind.WorkspaceCopy"/> job: target = the home's sequence; the workspace folder, the
|
||||
/// project whose media to copy and what to copy (<see cref="BundleMedia.Proxies"/>; originals in phase 5c).
|
||||
/// </summary>
|
||||
public sealed record WorkspaceCopyParameters(string WorkspaceRoot, Guid ProjectId, string Media);
|
||||
|
||||
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
UseStringEnumConverter = true, WriteIndented = true)]
|
||||
[JsonSerializable(typeof(WorkspaceManifest))]
|
||||
[JsonSerializable(typeof(WorkspaceMedia))]
|
||||
[JsonSerializable(typeof(WorkspaceSyncState))]
|
||||
[JsonSerializable(typeof(WorkspaceCopyParameters))]
|
||||
internal sealed partial class WorkspaceJson : JsonSerializerContext;
|
||||
|
||||
/// <summary>
|
||||
/// The workspace of this PC (owner review 2, R8): a folder on a fast disk where projects are edited, never inside the
|
||||
/// library. <c><root>/workspace.json</c> names it; each project has a folder named by its id
|
||||
/// (<see cref="WorkspaceProject"/>). There is no index: the store lists the folders holding a <c>project.chronos</c>. The
|
||||
/// desktop puts it in <c>%LOCALAPPDATA%\Chronos\workspace</c> unless set otherwise; the store takes any folder.
|
||||
/// </summary>
|
||||
public sealed class WorkspaceStore
|
||||
{
|
||||
public const string ManifestFile = "workspace.json";
|
||||
public const string FormatName = "chronos-workspace";
|
||||
public const int FormatVersion = 1;
|
||||
|
||||
/// <summary>The folder name of the workspace inside the program's data folder.</summary>
|
||||
public const string DefaultFolderName = "workspace";
|
||||
|
||||
/// <summary>
|
||||
/// Opens the workspace at <paramref name="root"/>, making the folder and its manifest when there is none. The manifest
|
||||
/// is written whole to a temporary file and moved into place, so a crash never leaves half of one; a manifest left
|
||||
/// empty, or unreadable in a folder that holds no project yet (an earlier build's interrupted first write), is made again.
|
||||
/// </summary>
|
||||
/// <exception cref="InvalidDataException">The folder's <c>workspace.json</c> is not a workspace manifest, or a newer one.</exception>
|
||||
/// <exception cref="IOException">The folder cannot be made or read.</exception>
|
||||
public WorkspaceStore(string root)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(root);
|
||||
Root = FullRoot(root);
|
||||
Directory.CreateDirectory(Root);
|
||||
var path = Path.Combine(Root, ManifestFile);
|
||||
if (!File.Exists(path)) Create(path, replace: false);
|
||||
else if (Unfinished(path)) Create(path, replace: true);
|
||||
Manifest = Read(path);
|
||||
}
|
||||
|
||||
public string Root { get; }
|
||||
|
||||
public WorkspaceManifest Manifest { get; }
|
||||
|
||||
public Guid Id => Manifest.Id;
|
||||
|
||||
/// <summary>The folder of a project, whether it exists or not.</summary>
|
||||
public WorkspaceProject Project(Guid projectId) => new(this, projectId);
|
||||
|
||||
/// <summary>The project's folder when it holds a working state; null otherwise.</summary>
|
||||
public WorkspaceProject? Find(Guid projectId)
|
||||
{
|
||||
var project = Project(projectId);
|
||||
return project.Exists ? project : null;
|
||||
}
|
||||
|
||||
/// <summary>The folder a workspace at <paramref name="root"/> has as its <see cref="Root"/> (the store opened or not).</summary>
|
||||
internal static string FullRoot(string root) => LongPath.Normalize(Path.TrimEndingDirectorySeparator(Path.GetFullPath(root)));
|
||||
|
||||
/// <summary>
|
||||
/// The projects of the workspace: every folder named by a project id that holds a <c>project.chronos</c>. A folder named
|
||||
/// by a project id that holds nothing but what a lease no one holds left (a removal that met a lease taken meanwhile)
|
||||
/// goes (<see cref="IsLeftover"/>).
|
||||
/// </summary>
|
||||
public IReadOnlyList<WorkspaceProject> List()
|
||||
{
|
||||
var found = new List<WorkspaceProject>();
|
||||
foreach (var directory in Directory.EnumerateDirectories(Root).Order(StringComparer.Ordinal))
|
||||
{
|
||||
if (!Guid.TryParseExact(Path.GetFileName(directory), "N", out var id)) continue;
|
||||
if (File.Exists(Path.Combine(directory, WorkspaceProject.BundleFile))) found.Add(Project(id));
|
||||
else RemoveLeftover(directory, DateTime.UtcNow);
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A project's folder that is no project and holds nothing a user or a writer has: no <c>project.chronos</c>, no folder,
|
||||
/// only lease files (the lease and those set aside as stale), none of them held, and nothing written there for as long
|
||||
/// as a lease lasts (a lease taken this moment makes its folder first).
|
||||
/// </summary>
|
||||
internal static bool IsLeftover(string folder, DateTime nowUtc)
|
||||
{
|
||||
var info = new DirectoryInfo(folder);
|
||||
if (!info.Exists || File.Exists(Path.Combine(folder, WorkspaceProject.BundleFile)) || info.LastWriteTimeUtc + LeaseFile.DefaultDuration > nowUtc) return false;
|
||||
foreach (var entry in info.EnumerateFileSystemInfos())
|
||||
{
|
||||
if (entry is not FileInfo file || !WorkspaceProject.IsLeaseFile(file.Name) || file.LastWriteTimeUtc + LeaseFile.DefaultDuration > nowUtc) return false;
|
||||
if (string.Equals(file.Name, WorkspaceProject.LeaseFileName, StringComparison.OrdinalIgnoreCase) && LeaseFile.Read(file.FullName) is { } held
|
||||
&& held.Expires >= nowUtc)
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private static void RemoveLeftover(string folder, DateTime nowUtc)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!IsLeftover(folder, nowUtc)) return;
|
||||
Directory.Delete(folder, recursive: true);
|
||||
Log.Info("workspace", $"{folder} held only an old lease and was removed");
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Debug("workspace", $"{folder} stays: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The working states of the workspace's projects (one unreadable is left out, and logged).</summary>
|
||||
public IReadOnlyList<(WorkspaceProject Project, ProjectBundle Bundle)> ListBundles()
|
||||
{
|
||||
var bundles = new List<(WorkspaceProject, ProjectBundle)>();
|
||||
foreach (var project in List())
|
||||
{
|
||||
try
|
||||
{
|
||||
bundles.Add((project, project.ReadBundle()));
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("workspace", $"The project in {project.Folder} cannot be read", ex);
|
||||
}
|
||||
}
|
||||
return bundles;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a new manifest. Without <paramref name="replace"/> another program opening the same folder at the same moment
|
||||
/// may have made one first: that one wins (the move never replaces a file).
|
||||
/// </summary>
|
||||
private static void Create(string path, bool replace)
|
||||
{
|
||||
var manifest = new WorkspaceManifest(FormatName, FormatVersion, Guid.CreateVersion7(), DateTimeOffset.UtcNow);
|
||||
var text = JsonSerializer.Serialize(manifest, WorkspaceJson.Default.WorkspaceManifest);
|
||||
if (replace)
|
||||
{
|
||||
Log.Warning("workspace", $"{path} was left unfinished; a new manifest is written");
|
||||
AtomicFile.WriteAllText(path, text);
|
||||
return;
|
||||
}
|
||||
var temp = AtomicFile.TempPathFor(path);
|
||||
try
|
||||
{
|
||||
using (var stream = new FileStream(temp, FileMode.CreateNew, FileAccess.Write, FileShare.None))
|
||||
{
|
||||
stream.Write(System.Text.Encoding.UTF8.GetBytes(text));
|
||||
stream.Flush(flushToDisk: true);
|
||||
}
|
||||
try
|
||||
{
|
||||
File.Move(temp, path, overwrite: false);
|
||||
}
|
||||
catch (IOException) when (File.Exists(path))
|
||||
{
|
||||
// Another program's manifest came first.
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(temp)) File.Delete(temp);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A manifest an interrupted first write left (an earlier build wrote it in place): empty, or not JSON in a folder that
|
||||
/// holds no project yet. A readable manifest, or any in a folder with projects, is never written over.
|
||||
/// </summary>
|
||||
private bool Unfinished(string path)
|
||||
{
|
||||
string text;
|
||||
try
|
||||
{
|
||||
text = AtomicFile.ReadAllText(path);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if (string.IsNullOrWhiteSpace(text.Trim('\0'))) return true;
|
||||
try
|
||||
{
|
||||
using var _ = JsonDocument.Parse(text);
|
||||
return false;
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return List().Count == 0;
|
||||
}
|
||||
}
|
||||
|
||||
private static WorkspaceManifest Read(string path)
|
||||
{
|
||||
WorkspaceManifest? manifest;
|
||||
try
|
||||
{
|
||||
manifest = JsonSerializer.Deserialize(AtomicFile.ReadAllText(path), WorkspaceJson.Default.WorkspaceManifest);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
throw new InvalidDataException($"{path} is not a workspace manifest.", ex);
|
||||
}
|
||||
if (manifest is null || manifest.Format != FormatName || manifest.Id == Guid.Empty) throw new InvalidDataException($"{path} is not a workspace manifest.");
|
||||
if (manifest.FormatVersion > FormatVersion)
|
||||
throw new InvalidDataException($"{path} uses format {manifest.FormatVersion}, newer than this Chronos ({FormatVersion}).");
|
||||
return manifest;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One project in the workspace (owner review 2, R8): <c><root>/<project id>/</c> with
|
||||
/// <c>project.chronos</c> (the working state: a project file of format 2, its settings placed on the media's frames),
|
||||
/// <c>media/media.json</c> with <c>media/proxies/<frame id>.jpg</c> (originals later) and <c>media/analysis.json</c>
|
||||
/// (the home's deep report), <c>versions/<number>-<id>.json</c> (commits not pushed yet, and the versions pulled),
|
||||
/// <c>sync.json</c>, <c>exports/</c> and the lease <c>.lock.lease</c> that one writer at a time holds.
|
||||
/// </summary>
|
||||
public sealed class WorkspaceProject
|
||||
{
|
||||
public const string BundleFile = "project.chronos";
|
||||
public const string MediaFolder = "media";
|
||||
public const string MediaIndexFile = "media.json";
|
||||
public const string ProxiesFolder = "proxies";
|
||||
public const string OriginalsFolder = "originals";
|
||||
public const string VersionsFolder = "versions";
|
||||
public const string ExportsFolder = "exports";
|
||||
public const string SyncFile = "sync.json";
|
||||
|
||||
/// <summary>
|
||||
/// Old local commits replaced by their replay on top of the home's head, files the home's copy replaced, and the working
|
||||
/// states a take of the server's values replaced: kept up to a month, the newest 20 at most (<see cref="PruneReplaced"/>).
|
||||
/// </summary>
|
||||
public const string ReplacedFolder = ".replaced";
|
||||
|
||||
/// <summary>
|
||||
/// Beside <see cref="ReplacedFolder"/> in <c>versions/</c>: when this Chronos first set a file aside there or pruned it
|
||||
/// (an ISO-8601 time). A file dated before it was set aside by an earlier build, which kept the time it was written: it
|
||||
/// counts as set aside then, at the earliest (<see cref="PruneReplaced"/>).
|
||||
/// </summary>
|
||||
public const string ReplacedSinceFile = ".replaced.since";
|
||||
|
||||
private const string LeaseName = ".lock";
|
||||
|
||||
/// <summary>The file of the project's lease (<see cref="LockAsync"/>) in <see cref="Folder"/>.</summary>
|
||||
internal const string LeaseFileName = LeaseName + ".lease";
|
||||
|
||||
/// <summary>The project's lease, or one set aside as stale (<c>.lock.lease.<tag>.stale</c>).</summary>
|
||||
internal static bool IsLeaseFile(string name) => name.StartsWith(LeaseFileName, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
internal WorkspaceProject(WorkspaceStore store, Guid projectId)
|
||||
{
|
||||
Store = store;
|
||||
ProjectId = projectId;
|
||||
Folder = Path.Combine(store.Root, projectId.ToString("N"));
|
||||
}
|
||||
|
||||
public WorkspaceStore Store { get; }
|
||||
public Guid ProjectId { get; }
|
||||
public string Folder { get; }
|
||||
public string BundlePath => Path.Combine(Folder, BundleFile);
|
||||
public string MediaDirectory => Path.Combine(Folder, MediaFolder);
|
||||
public string MediaIndexPath => Path.Combine(MediaDirectory, MediaIndexFile);
|
||||
public string ProxiesDirectory => Path.Combine(MediaDirectory, ProxiesFolder);
|
||||
public string OriginalsDirectory => Path.Combine(MediaDirectory, OriginalsFolder);
|
||||
public string AnalysisPath => Path.Combine(MediaDirectory, ArchiveLayout.AnalysisFile);
|
||||
public string VersionsDirectory => Path.Combine(Folder, VersionsFolder);
|
||||
public string ExportsDirectory => Path.Combine(Folder, ExportsFolder);
|
||||
public string SyncPath => Path.Combine(Folder, SyncFile);
|
||||
|
||||
public bool Exists => File.Exists(BundlePath);
|
||||
|
||||
/// <summary>The file a frame's proxy is copied to, relative to <see cref="MediaDirectory"/>.</summary>
|
||||
public static string ProxyName(Guid frameId) => ProxiesFolder + "/" + ProxyCache.FileName(frameId);
|
||||
|
||||
/// <summary>
|
||||
/// The file a frame's proxy of other bytes is copied to while the media name <see cref="ProxyName(Guid)"/> for the frame
|
||||
/// (a conversion since): the editor keeps reading that one until the copy commits.
|
||||
/// </summary>
|
||||
public static string ProxyName(Guid frameId, string sha256)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrEmpty(sha256);
|
||||
return ProxiesFolder + "/" + frameId.ToString("N") + "-" + sha256[..Math.Min(16, sha256.Length)].ToLowerInvariant() + ".jpg";
|
||||
}
|
||||
|
||||
/// <summary>The absolute path of a file named relative to <see cref="MediaDirectory"/>.</summary>
|
||||
public string MediaFile(string relative) => LongPath.FromPortableRelative(MediaDirectory, relative);
|
||||
|
||||
/// <summary>Holds the project for one writer (this program's sync, a copy job) until disposed.</summary>
|
||||
/// <exception cref="LeaseUnavailableException">Another writer kept it for longer than <paramref name="wait"/>.</exception>
|
||||
public async Task<IDisposable> LockAsync(TimeSpan wait, CancellationToken cancellationToken = default)
|
||||
{
|
||||
Directory.CreateDirectory(Folder);
|
||||
return await LeaseFile.AcquireAsync(Folder, LeaseName, "workspace@" + Environment.MachineName, wait, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Holds the project as <see cref="LockAsync"/> does, never making its folder: taken by a writer that only clears what is
|
||||
/// there, which a removal of the folder may race.
|
||||
/// </summary>
|
||||
/// <exception cref="DirectoryNotFoundException">The folder is not there, or went while the lease was awaited.</exception>
|
||||
/// <exception cref="LeaseUnavailableException">Another writer kept it for longer than <paramref name="wait"/>.</exception>
|
||||
internal async Task<IDisposable> LockExistingAsync(TimeSpan wait, CancellationToken cancellationToken = default) =>
|
||||
await LeaseFile.AcquireAsync(Folder, LeaseName, "workspace@" + Environment.MachineName, wait, cancellationToken: cancellationToken, createDirectory: false)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
/// <exception cref="InvalidDataException">Not a project file of a workspace, damaged or newer.</exception>
|
||||
/// <exception cref="IOException">The file cannot be read.</exception>
|
||||
public ProjectBundle ReadBundle()
|
||||
{
|
||||
var bundle = ProjectBundle.Load(BundlePath);
|
||||
if (bundle.Workspace is not { } workspace || workspace.ProjectId != ProjectId)
|
||||
throw new InvalidDataException($"{BundlePath} is not the working state of project {ProjectId}.");
|
||||
return bundle;
|
||||
}
|
||||
|
||||
public void WriteBundle(ProjectBundle bundle)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(bundle);
|
||||
if (bundle.Workspace?.ProjectId != ProjectId) throw new ArgumentException("The project file is another project's.", nameof(bundle));
|
||||
Directory.CreateDirectory(Folder);
|
||||
bundle.Save(BundlePath);
|
||||
}
|
||||
|
||||
/// <summary>The media index; null while the media were never copied, or were removed.</summary>
|
||||
/// <exception cref="InvalidDataException">The index is damaged or newer.</exception>
|
||||
public WorkspaceMedia? ReadMedia()
|
||||
{
|
||||
if (!File.Exists(MediaIndexPath)) return null;
|
||||
WorkspaceMedia? media;
|
||||
try
|
||||
{
|
||||
media = JsonSerializer.Deserialize(AtomicFile.ReadAllText(MediaIndexPath), WorkspaceJson.Default.WorkspaceMedia);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
throw new InvalidDataException($"{MediaIndexPath} cannot be read.", ex);
|
||||
}
|
||||
if (media is null || media.Format != WorkspaceMedia.FormatName) throw new InvalidDataException($"{MediaIndexPath} is not a media index.");
|
||||
if (media.FormatVersion > WorkspaceMedia.CurrentFormatVersion) throw new InvalidDataException($"{MediaIndexPath} is newer than this Chronos.");
|
||||
return media;
|
||||
}
|
||||
|
||||
internal void WriteMedia(WorkspaceMedia media)
|
||||
{
|
||||
Directory.CreateDirectory(MediaDirectory);
|
||||
AtomicFile.WriteAllText(MediaIndexPath, JsonSerializer.Serialize(media, WorkspaceJson.Default.WorkspaceMedia));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The versions this workspace holds, oldest first. One that cannot be read is left out here, logged, and listed by
|
||||
/// <see cref="UnreadableVersions"/>: the sync treats one above the base as a gap in the chain of versions to send, never
|
||||
/// as a version that is not there (<see cref="WorkspaceSync"/>).
|
||||
/// </summary>
|
||||
public IReadOnlyList<ProjectVersion> Versions() => ReadVersions(log: true).Versions;
|
||||
|
||||
/// <summary>
|
||||
/// The version files of the workspace that cannot be read now (damaged, held open, over the size a version may be, of
|
||||
/// a newer Chronos), by the number and id their names carry, oldest first.
|
||||
/// </summary>
|
||||
public IReadOnlyList<UnreadableVersion> UnreadableVersions() => ReadVersions(log: false).Unreadable;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="Versions"/> and <see cref="UnreadableVersions"/> from one read of the folder: each file (up to
|
||||
/// <see cref="ProjectSidecars.MaxVersionBytes"/>) is parsed once.
|
||||
/// </summary>
|
||||
internal (IReadOnlyList<ProjectVersion> Versions, IReadOnlyList<UnreadableVersion> Unreadable) ReadVersionFiles() => ReadVersions(log: true);
|
||||
|
||||
private (List<ProjectVersion> Versions, List<UnreadableVersion> Unreadable) ReadVersions(bool log)
|
||||
{
|
||||
var versions = new List<ProjectVersion>();
|
||||
var unreadable = new List<UnreadableVersion>();
|
||||
foreach (var (number, id, path) in ProjectSidecars.ListVersions(VersionsDirectory))
|
||||
{
|
||||
try
|
||||
{
|
||||
var version = ProjectSidecars.ReadVersion(path);
|
||||
if (version.ProjectId == ProjectId) versions.Add(version);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
|
||||
{
|
||||
// Said once per file as it is now: every sync step lists the versions again.
|
||||
if (log && (!Reported.TryGetValue(path, out var said) || said != ex.Message))
|
||||
{
|
||||
Reported[path] = ex.Message;
|
||||
Log.Warning("workspace", $"Version file {path} cannot be read", ex);
|
||||
}
|
||||
unreadable.Add(new UnreadableVersion(number, id, path, ex.Message));
|
||||
}
|
||||
}
|
||||
return (versions, unreadable);
|
||||
}
|
||||
|
||||
/// <summary>The version files whose read failure was logged, with why (logged again when the reason changes).</summary>
|
||||
private static readonly System.Collections.Concurrent.ConcurrentDictionary<string, string> Reported = new(StringComparer.Ordinal);
|
||||
|
||||
/// <summary>The path of a version in <see cref="VersionsDirectory"/>.</summary>
|
||||
public string VersionPath(ProjectVersion version) => Path.Combine(VersionsDirectory, ArchiveLayout.ProjectVersionFileName(version.Number, version.Id));
|
||||
|
||||
internal void WriteVersion(ProjectVersion version, bool replace = false) => ProjectSidecars.WriteVersion(VersionPath(version), version, replace);
|
||||
|
||||
/// <summary>
|
||||
/// Moves a version file aside into <c>versions/.replaced</c> (a local commit replayed under another id), trying again
|
||||
/// for a moment while another program (an indexer, an antivirus) holds it open.
|
||||
/// </summary>
|
||||
internal void SetAside(ProjectVersion version) => SetAside(VersionPath(version));
|
||||
|
||||
/// <summary>Moves the version file at <paramref name="path"/> aside into <c>versions/.replaced</c> (see <see cref="SetAside(ProjectVersion)"/>).</summary>
|
||||
internal void SetAside(string path)
|
||||
{
|
||||
if (!File.Exists(path)) return;
|
||||
var aside = Path.Combine(VersionsDirectory, ReplacedFolder);
|
||||
Directory.CreateDirectory(aside);
|
||||
ReplacedSince(DateTime.UtcNow);
|
||||
var target = Path.Combine(aside, Path.GetFileName(path));
|
||||
if (File.Exists(target)) target += "." + DateTime.UtcNow.Ticks.ToString(System.Globalization.CultureInfo.InvariantCulture);
|
||||
for (var attempt = 1; ; attempt++)
|
||||
{
|
||||
try
|
||||
{
|
||||
File.Move(path, target);
|
||||
break;
|
||||
}
|
||||
catch (IOException) when (attempt < SetAsideAttempts && File.Exists(path) && !File.Exists(target))
|
||||
{
|
||||
Thread.Sleep(10 * attempt);
|
||||
}
|
||||
}
|
||||
// Dated as set aside, which the pruning goes by (a move keeps the time the version was written).
|
||||
TryTouch(target);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a version straight into <c>versions/.replaced</c>: kept, never part of the chain of versions to send (the
|
||||
/// working state a take of the server's values replaced).
|
||||
/// </summary>
|
||||
internal void WriteAside(ProjectVersion version)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(version);
|
||||
ProjectSidecars.WriteVersion(Path.Combine(VersionsDirectory, ReplacedFolder, Path.GetFileName(VersionPath(version))), version);
|
||||
ReplacedSince(DateTime.UtcNow);
|
||||
}
|
||||
|
||||
/// <summary>Files <see cref="PruneReplaced"/> keeps in <c>versions/.replaced</c> whatever their age.</summary>
|
||||
public const int ReplacedKept = 20;
|
||||
|
||||
/// <summary>The age past which <see cref="PruneReplaced"/> removes a file of <c>versions/.replaced</c>.</summary>
|
||||
public static readonly TimeSpan ReplacedMaxAge = TimeSpan.FromDays(30);
|
||||
|
||||
/// <summary>
|
||||
/// Removes from <c>versions/.replaced</c> the files set aside more than <see cref="ReplacedMaxAge"/> ago and those past
|
||||
/// the newest <see cref="ReplacedKept"/>, never one of a version in <paramref name="referenced"/> (a journal's). Every
|
||||
/// file there is a copy of what the chain or the home holds, or what a take of the server's values replaced (its undo):
|
||||
/// called only after a sync that ended well. A file is set aside when this Chronos dated it (it touches what it sets
|
||||
/// aside), and no earlier than this Chronos first knew the folder (<see cref="ReplacedSinceFile"/>): one an earlier build
|
||||
/// set aside keeps the time its version was written, which says nothing of when it went aside. How many went.
|
||||
/// </summary>
|
||||
internal int PruneReplaced(IReadOnlySet<Guid> referenced, DateTime nowUtc)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(referenced);
|
||||
var aside = Path.Combine(VersionsDirectory, ReplacedFolder);
|
||||
if (!Directory.Exists(aside)) return 0;
|
||||
var since = ReplacedSince(nowUtc);
|
||||
var files = new DirectoryInfo(aside).EnumerateFiles()
|
||||
.Select(f => (File: f, Aside: f.LastWriteTimeUtc > since ? f.LastWriteTimeUtc : since))
|
||||
.OrderByDescending(f => f.Aside).ThenByDescending(f => f.File.LastWriteTimeUtc).ThenByDescending(f => f.File.Name, StringComparer.Ordinal).ToList();
|
||||
var removed = 0;
|
||||
for (var i = 0; i < files.Count; i++)
|
||||
{
|
||||
var (file, set) = files[i];
|
||||
if (i < ReplacedKept && nowUtc - set <= ReplacedMaxAge) continue;
|
||||
if (VersionIdOf(file.Name) is { } id && referenced.Contains(id)) continue;
|
||||
try
|
||||
{
|
||||
file.Delete();
|
||||
removed++;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("workspace", $"{file.FullName} could not be removed", ex);
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
|
||||
// "<number>-<id>.json", maybe with a suffix a second file of the same version took.
|
||||
static Guid? VersionIdOf(string name)
|
||||
{
|
||||
var end = name.IndexOf(".json", StringComparison.OrdinalIgnoreCase);
|
||||
return end > 0 && ProjectSidecars.TryParseName(name[..(end + 5)], out _, out var id) ? id : null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// When this Chronos first knew <c>versions/.replaced</c> (<see cref="ReplacedSinceFile"/>): read, else written as
|
||||
/// <paramref name="nowUtc"/> (also over a file that cannot be read). One that cannot be written counts as now: nothing
|
||||
/// set aside by an earlier build goes by its age then.
|
||||
/// </summary>
|
||||
private DateTime ReplacedSince(DateTime nowUtc)
|
||||
{
|
||||
var path = Path.Combine(VersionsDirectory, ReplacedSinceFile);
|
||||
try
|
||||
{
|
||||
if (File.Exists(path)
|
||||
&& DateTime.TryParse(File.ReadAllText(path).Trim(), System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.RoundtripKind,
|
||||
out var known)
|
||||
&& known.Kind == DateTimeKind.Utc && known <= nowUtc)
|
||||
return known;
|
||||
Directory.CreateDirectory(VersionsDirectory);
|
||||
AtomicFile.WriteAllText(path, nowUtc.ToString("O", System.Globalization.CultureInfo.InvariantCulture));
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Debug("workspace", $"{path} could not be kept: {ex.Message}");
|
||||
}
|
||||
return nowUtc;
|
||||
}
|
||||
|
||||
private static void TryTouch(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (File.Exists(path)) File.SetLastWriteTimeUtc(path, DateTime.UtcNow);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Debug("workspace", $"{path} keeps the time it was written: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Moves tried, some 1 s in all, before a version file held open fails to move aside.</summary>
|
||||
private const int SetAsideAttempts = 15;
|
||||
|
||||
public WorkspaceSyncState ReadSync()
|
||||
{
|
||||
if (!File.Exists(SyncPath)) return WorkspaceSyncState.None;
|
||||
try
|
||||
{
|
||||
return JsonSerializer.Deserialize(AtomicFile.ReadAllText(SyncPath), WorkspaceJson.Default.WorkspaceSyncState) ?? WorkspaceSyncState.None;
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
throw new InvalidDataException($"{SyncPath} cannot be read.", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The conflict <c>sync.json</c> records that still waits for a choice, as <see cref="WorkspaceSync.StatusAsync"/> reads
|
||||
/// it: one a replay that settles it took past its commit point (its base moved onto the replay's) is over, the choice
|
||||
/// made applied, even before the next sync clears it.
|
||||
/// </summary>
|
||||
/// <exception cref="InvalidDataException"><c>sync.json</c> or the working state cannot be read.</exception>
|
||||
/// <exception cref="IOException">A file cannot be read.</exception>
|
||||
public WorkspaceConflict? OpenConflict() => ReadSync() is { Conflict: not null } sync ? Unsettled(sync, ReadBundle()) : null;
|
||||
|
||||
/// <summary>The conflict of <paramref name="sync"/> that waits for a choice with the working state <paramref name="bundle"/> (<see cref="OpenConflict"/>).</summary>
|
||||
internal static WorkspaceConflict? Unsettled(WorkspaceSyncState sync, ProjectBundle bundle) =>
|
||||
sync.Replay is { Settles: true } replay && bundle.Workspace?.Base?.VersionId == replay.OntoBase ? null : sync.Conflict;
|
||||
|
||||
internal void WriteSync(WorkspaceSyncState state)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(state);
|
||||
if (state.IsEmpty)
|
||||
{
|
||||
if (File.Exists(SyncPath)) File.Delete(SyncPath);
|
||||
return;
|
||||
}
|
||||
Directory.CreateDirectory(Folder);
|
||||
AtomicFile.WriteAllText(SyncPath, JsonSerializer.Serialize(state, WorkspaceJson.Default.WorkspaceSyncState));
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user