fix: projects and the workspace copy follow phase 4's folder and conversion rules
Phase 5a was written on an early phase 4a state. On top of phase 4b it now keeps the rules every writer of a sequence's folder follows, and keeps project versions intact through Reorganize and conversions: - ProjectService writes project sidecars, project.json and the settings only under the sequence's lease and only in a folder that holds the sequence (SequenceFolders.Holds). A folder another index moved away, or one another sequence was moved into, is refused with SequenceFolderMovedException: reads answer from the index, so no project is taken for gone, nothing is written into the other sequence's folder and the old folder is never created again. The home reports it as out of reach. Full scans and rebuilds reconcile projects the same way, and a failure there is logged instead of failing the scan. - Stored versions park positions by photo and now follow the encodings a Replace conversion (or its undo) joined through the sequence's Replaced tombstones (PhotoAlias, HomePhotos): Main's mirror, renders of a version and workspace media place them on the converted frames. ProjectContent.Rebind/SameChoices name parked photos as the frames hold them now, so a conversion or a hash since is no "external" Main version and no spurious "Modified" in a workspace, and merges see one photo. - The workspace copy job only reads the archive: it no longer runs the archive's proxy pass, makes missing proxies in the workspace, reads only a folder that holds the sequence, starts again when the folder moved or files were renamed during the copy, resumes from a checkpoint of its frames version, and refuses a workspace folder inside any library before it writes anything (Errors.Workspace.InsideLibrary). The working state keeps its positions by frame where frames stay. - ArchiveProjectHome looks for a project or version the index does not know yet in the sequence's sidecars first (an index taken back from a backup, a sequence back from the trash), instead of calling it gone. Tests cover the moved and taken folder, conversions in the archive and in the workspace, the copy refusing a workspace in a library, resuming on other frames, the alias lookup and the rebinding. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b9d1c396a1
commit
50e28fd63d
18 files changed
+596
-91
No files matched your search
@@ -345,17 +345,28 @@ public sealed class ArchiveIndexer
|
||||
private SequenceStatus Uncovered => _writer.Options.ImagesAfterScan ? CoveringStatus : SequenceStatus.None;
|
||||
|
||||
/// <summary>
|
||||
/// The projects of a sequence brought in line with their sidecars (<see cref="ProjectIndexer"/>), under its lease; nothing
|
||||
/// for a sequence without a projects folder (its Main is made when a project is first asked for).
|
||||
/// The projects of a sequence brought in line with their sidecars (<see cref="ProjectIndexer"/>), under its lease, in the
|
||||
/// folder the index has for it then, which must hold it (<see cref="SequenceFolders.Holds"/>: another sequence's projects
|
||||
/// are never taken, and a folder another index moved away never marks the projects gone). Nothing for a sequence without
|
||||
/// a projects folder (its Main is made when a project is first asked for). A failure is logged and leaves the folder
|
||||
/// indexed: the next project operation on the sequence catches up again.
|
||||
/// </summary>
|
||||
private async Task ReconcileProjectsAsync(ArchivePaths paths, Guid sequenceId, CancellationToken cancellationToken)
|
||||
{
|
||||
if (await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) is not { } row
|
||||
|| !Directory.Exists(paths.Sequence(row.Folder).ProjectsDirectory)) return;
|
||||
await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false))
|
||||
try
|
||||
{
|
||||
var folder = await SequenceWriter.FolderAsync(_db, paths, row, cancellationToken).ConfigureAwait(false);
|
||||
await _projects.ReconcileAsync(folder, sequenceId, cancellationToken).ConfigureAwait(false);
|
||||
await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var folder = await SequenceWriter.FolderAsync(_db, paths, row, cancellationToken).ConfigureAwait(false);
|
||||
if (!SequenceFolders.Holds(folder, sequenceId) || !Directory.Exists(folder.ProjectsDirectory)) return;
|
||||
await _projects.ReconcileAsync(folder, sequenceId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("projects", $"The projects of {row.Folder} could not be read back; the next project operation tries again", ex);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,14 @@
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.Localization;
|
||||
|
||||
namespace Chronos.Data.Archiving;
|
||||
|
||||
/// <summary>
|
||||
/// The folder the index has for a sequence does not hold it (<see cref="SequenceFolders.Holds"/>): another index of the
|
||||
/// archive moved it away, maybe another sequence in; this index's next scan finds both. Nothing was written.
|
||||
/// </summary>
|
||||
public sealed class SequenceFolderMovedException() : IOException(Loc.L("Errors.Archive.FolderMoved"));
|
||||
|
||||
/// <summary>Whether a sequence is in the folder the index has for it, as writers that file into that folder ask.</summary>
|
||||
public static class SequenceFolders
|
||||
{
|
||||
|
||||
@@ -50,7 +50,8 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt
|
||||
if (options.Host == EncoderHost.Server) export = export with { Folder = null };
|
||||
var embedded = parameters.SettingsJson ?? await VersionSettingsAsync(parameters, sequence, cancellation).ConfigureAwait(false);
|
||||
var frames = await SequenceFrames.LoadAsync(db, archive, sequence, parameters.UseProxies, options.Archive.Decoder, cancellation).ConfigureAwait(false);
|
||||
var project = (embedded is not null ? ProjectStore.Parse(Placed(embedded, frames)) : ProjectStore.Read(sequence, folder)) with { Export = export };
|
||||
var aliases = embedded is not null ? await Projects.HomePhotos.AliasesAsync(db, sequence.Id, cancellation).ConfigureAwait(false) : [];
|
||||
var project = (embedded is not null ? ProjectStore.Parse(Placed(embedded, frames, aliases)) : ProjectStore.Read(sequence, folder)) with { Export = export };
|
||||
var settingsJson = project.ToJson();
|
||||
|
||||
using var names = new OutputNames();
|
||||
@@ -138,14 +139,15 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt
|
||||
|
||||
/// <summary>
|
||||
/// Settings whose positions wait for their photos (a project version's portable form, settings synced from another copy)
|
||||
/// with them placed on the frames rendered; others as they are.
|
||||
/// with them placed on the frames rendered, a photo parked on an encoding a conversion replaced since found through
|
||||
/// <paramref name="aliases"/> (the sequence's tombstones of reason "replaced"); others as they are.
|
||||
/// </summary>
|
||||
internal static string Placed(string settingsJson, SequenceFrames frames)
|
||||
internal static string Placed(string settingsJson, SequenceFrames frames, IReadOnlyList<PhotoAlias>? aliases = null)
|
||||
{
|
||||
if (!settingsJson.Contains("pendingFrames", StringComparison.OrdinalIgnoreCase)) return settingsJson;
|
||||
try
|
||||
{
|
||||
return ProjectContent.Place(settingsJson, new FramePhotos(frames.Frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, f.CapturedAt)))));
|
||||
return ProjectContent.Place(settingsJson, new FramePhotos(frames.Frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, f.CapturedAt))), aliases));
|
||||
}
|
||||
catch (System.Text.Json.JsonException)
|
||||
{
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Core.Project;
|
||||
using Chronos.Data.Repositories;
|
||||
|
||||
namespace Chronos.Data.Projects;
|
||||
|
||||
/// <summary>
|
||||
/// The photos of a sequence's frames at its home as project positions name them (owner review 2, R8), with the encodings
|
||||
/// a conversion joined (R7): a version parked on a photo before its file was converted, or after a conversion was undone,
|
||||
/// still finds its frame.
|
||||
/// </summary>
|
||||
internal static class HomePhotos
|
||||
{
|
||||
/// <summary>The links between encodings of the sequence's photos: its tombstones of reason "replaced".</summary>
|
||||
public static async Task<IReadOnlyList<PhotoAlias>> AliasesAsync(ChronosRepositories db, Guid sequenceId, CancellationToken cancellationToken) =>
|
||||
[
|
||||
.. (await db.FrameRemovals.ListBySequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false))
|
||||
.Where(r => r.Reason == RemovalReason.Replaced && !string.IsNullOrEmpty(r.ReplacementSha256))
|
||||
.Select(r => new PhotoAlias(r.Sha256, r.ReplacementSha256!)),
|
||||
];
|
||||
|
||||
/// <summary>The photos of <paramref name="frames"/> (rows of the index, any order) with <paramref name="aliases"/>.</summary>
|
||||
public static FramePhotos Of(IEnumerable<Frame> frames, IReadOnlyList<PhotoAlias> aliases) =>
|
||||
new(frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, f.CapturedAt))), aliases);
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Core.Project;
|
||||
using Chronos.Data.Archiving;
|
||||
|
||||
namespace Chronos.Data.Projects;
|
||||
|
||||
@@ -78,8 +79,17 @@ public sealed class ArchiveProjectHome(ChronosDatabase db, ProjectService projec
|
||||
return await projects.HeadAsync(projectId, cancellationToken).ConfigureAwait(false);
|
||||
});
|
||||
|
||||
public async Task<ProjectVersion?> GetVersionAsync(Guid projectId, Guid versionId, CancellationToken cancellationToken) =>
|
||||
await projects.GetVersionAsync(versionId, cancellationToken).ConfigureAwait(false) is { } version && version.ProjectId == projectId ? version : null;
|
||||
public Task<ProjectVersion?> GetVersionAsync(Guid projectId, Guid versionId, CancellationToken cancellationToken) =>
|
||||
Reach(async () =>
|
||||
{
|
||||
if (await projects.GetVersionAsync(versionId, cancellationToken).ConfigureAwait(false) is not { } version)
|
||||
{
|
||||
// A version the index does not know yet may be in the sequence's sidecars (see Own): they are read first.
|
||||
await projects.ListAsync(sequenceId, cancellationToken).ConfigureAwait(false);
|
||||
version = await projects.GetVersionAsync(versionId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
return version is not null && version.ProjectId == projectId ? version : null;
|
||||
});
|
||||
|
||||
public Task<IReadOnlyList<ProjectVersion>> VersionsAfterAsync(Guid projectId, int afterNumber, int limit, CancellationToken cancellationToken) =>
|
||||
Reach(async () =>
|
||||
@@ -104,11 +114,22 @@ public sealed class ArchiveProjectHome(ChronosDatabase db, ProjectService projec
|
||||
|
||||
public Task<HomeFrames> FramesAsync(CancellationToken cancellationToken) => projects.FramesAsync(sequenceId, cancellationToken);
|
||||
|
||||
/// <summary>The project is one of this sequence's.</summary>
|
||||
private async Task<SequenceProject> Own(Guid projectId, CancellationToken cancellationToken) =>
|
||||
await projects.GetAsync(projectId, cancellationToken).ConfigureAwait(false) is { } project && project.SequenceId == sequenceId
|
||||
/// <summary>
|
||||
/// The project is one of this sequence's. One the index does not know yet (an index taken back from an older backup, a
|
||||
/// sequence back from the trash, a project another program sharing the archive made) is looked for in the sequence's
|
||||
/// sidecars first, which every listing brings into the index.
|
||||
/// </summary>
|
||||
private async Task<SequenceProject> Own(Guid projectId, CancellationToken cancellationToken)
|
||||
{
|
||||
if (await projects.GetAsync(projectId, cancellationToken).ConfigureAwait(false) is not { } project)
|
||||
{
|
||||
await projects.ListAsync(sequenceId, cancellationToken).ConfigureAwait(false);
|
||||
project = await projects.GetAsync(projectId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
return project is not null && project.SequenceId == sequenceId
|
||||
? project
|
||||
: throw new ProjectException(ProjectError.NotFound, $"Project {projectId} is not a project of sequence {sequenceId}.");
|
||||
}
|
||||
|
||||
/// <summary>An archive offline, read-only or busy is a home out of reach for now.</summary>
|
||||
private static async Task<T> Reach<T>(Func<Task<T>> call)
|
||||
@@ -125,5 +146,10 @@ public sealed class ArchiveProjectHome(ChronosDatabase db, ProjectService projec
|
||||
{
|
||||
throw new HomeUnavailableException(ex.Message, ex);
|
||||
}
|
||||
catch (SequenceFolderMovedException ex)
|
||||
{
|
||||
// Another index of the archive moved the sequence: this one finds it at its next scan.
|
||||
throw new HomeUnavailableException(ex.Message, ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -317,13 +317,20 @@ public sealed class ProjectService
|
||||
{
|
||||
await UnderLeaseAsync(sequenceId, (_, _) => Task.FromResult(true), cancellationToken, ReadWait).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is ArchiveOfflineException or LeaseUnavailableException)
|
||||
catch (Exception ex) when (ex is ArchiveOfflineException or LeaseUnavailableException or SequenceFolderMovedException)
|
||||
{
|
||||
Log.Info("projects", $"The projects of sequence {sequenceId} are read from the index: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Runs <paramref name="work"/> under the sequence's lease, after catching its projects up.</summary>
|
||||
/// <summary>
|
||||
/// Runs <paramref name="work"/> under the sequence's lease, after catching its projects up, in the folder the index has
|
||||
/// for the sequence as read under the lease (a Reorganize moves it under the same lease), which must hold it.
|
||||
/// </summary>
|
||||
/// <exception cref="SequenceFolderMovedException">
|
||||
/// The folder does not hold the sequence (another index of the archive moved it, maybe another sequence in): nothing is
|
||||
/// read from it or written into it, so no project is taken for gone and no file lands in another sequence's folder.
|
||||
/// </exception>
|
||||
private async Task<T> UnderLeaseAsync<T>(Guid sequenceId, Func<LeaseContext, CancellationToken, Task<T>> work, CancellationToken cancellationToken,
|
||||
TimeSpan? wait = null)
|
||||
{
|
||||
@@ -334,7 +341,9 @@ public sealed class ProjectService
|
||||
await using (await _writer.LockAsync(paths, sequenceId, wait, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var row = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? throw SequenceMissing(sequenceId);
|
||||
var context = new LeaseContext(paths.Sequence(row.Folder), row);
|
||||
var folder = paths.Sequence(row.Folder);
|
||||
if (!SequenceFolders.Holds(folder, sequenceId)) throw new SequenceFolderMovedException();
|
||||
var context = new LeaseContext(folder, row);
|
||||
await CatchUpAsync(context, cancellationToken).ConfigureAwait(false);
|
||||
return await work(context, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
@@ -359,7 +368,8 @@ public sealed class ProjectService
|
||||
await _db.Projects.SetMirroredAsync(main.Id, main.HeadVersionId, context.Row.SettingsVersion, cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
var sameAsHead = ProjectContent.Hash(current) == head.ContentHash;
|
||||
// The head parks its photos as the frames held them when it was made: a conversion or a hash since names them anew.
|
||||
var sameAsHead = ProjectContent.SameChoices(head.SettingsJson, current, frames.Photos);
|
||||
if (main.MirroredVersionId == main.HeadVersionId || main.MirroredVersionId is null)
|
||||
{
|
||||
// Only the sequence's settings version moved (settings another path wrote, or only its title or tags), or the index
|
||||
@@ -375,14 +385,16 @@ public sealed class ProjectService
|
||||
return;
|
||||
}
|
||||
var mirrored = await _db.Projects.GetVersionAsync(main.MirroredVersionId.Value, cancellationToken).ConfigureAwait(false);
|
||||
if (main.MirroredSettingsVersion == context.Row.SettingsVersion || mirrored is null || ProjectContent.Hash(current) == mirrored.ContentHash)
|
||||
if (main.MirroredSettingsVersion == context.Row.SettingsVersion || mirrored is null || ProjectContent.SameChoices(mirrored.SettingsJson, current, frames.Photos))
|
||||
{
|
||||
// Nobody changed the settings since the last mirror: the mirror completes.
|
||||
await MirrorAsync(context, main, head, cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
// Both moved: the settings written meanwhile merge with the head (the version made in a workspace wins where both changed).
|
||||
var merged = ProjectMerge.Merge(ProjectMerge.Read(mirrored.SettingsJson), ProjectMerge.Read(head.SettingsJson), ProjectMerge.Read(current));
|
||||
// Both moved: the settings written meanwhile merge with the head (the version made in a workspace wins where both changed),
|
||||
// each photo named as the frames hold it now so the same photo merges as one.
|
||||
var merged = ProjectMerge.Merge(ProjectMerge.Read(ProjectContent.Rebind(mirrored.SettingsJson, frames.Photos)),
|
||||
ProjectMerge.Read(ProjectContent.Rebind(head.SettingsJson, frames.Photos)), ProjectMerge.Read(ProjectContent.Rebind(current, frames.Photos)));
|
||||
var version = await AppendDerivedAsync(context, main, head, ProjectContent.Canonical(merged.ToJson()), ProjectVersionKind.Merge, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
main = (await _db.Projects.GetAsync(main.Id, cancellationToken).ConfigureAwait(false))!;
|
||||
@@ -550,7 +562,8 @@ public sealed class ProjectService
|
||||
if (page.Count < PageSize) break;
|
||||
after = page[^1].Index;
|
||||
}
|
||||
return (frames, new FramePhotos(frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, f.CapturedAt)))));
|
||||
// With the encodings conversions joined: a version parked on a photo before its file was converted still finds its frame.
|
||||
return (frames, HomePhotos.Of(frames, await HomePhotos.AliasesAsync(_db, sequenceId, cancellationToken).ConfigureAwait(false)));
|
||||
}
|
||||
|
||||
private async Task EnsureNameFreeAsync(Guid sequenceId, Guid? projectId, string name, CancellationToken cancellationToken)
|
||||
|
||||
@@ -10,6 +10,7 @@ using Chronos.Core.Project;
|
||||
using Chronos.Data.Archiving;
|
||||
using Chronos.Data.Archiving.Jobs;
|
||||
using Chronos.Data.Jobs;
|
||||
using Chronos.Data.Projects;
|
||||
|
||||
namespace Chronos.Data.Workspaces;
|
||||
|
||||
@@ -37,12 +38,17 @@ public static class WorkspaceJobs
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.WorkspaceCopy"/> for a sequence of this PC's archive (owner review 2, R8): its proxies into the
|
||||
/// workspace project's <c>media/proxies/<frame id>.jpg</c>, the archive's own when current, the missing ones made
|
||||
/// first by the archive's proxy pass (made here from the original when the archive makes none), each written atomically
|
||||
/// and read back to check its SHA-256; then the deep report (<c>media/analysis.json</c>) and, last, <c>media/media.json</c>.
|
||||
/// Frames changed at the home during the copy make it start again (three times at most). The working state's positions
|
||||
/// follow the new frames by photo. Checkpoint: <c>proxies:<frame></c> during the archive's pass, then
|
||||
/// <c>copy:<frame></c>. Originals come in phase 5c: asked for, the job fails saying so.
|
||||
/// workspace project's <c>media/proxies/<frame id>.jpg</c>, the archive's own when current, else made here from the
|
||||
/// original (at the archive's proxy size, or the default one when the archive makes none), each written atomically and read
|
||||
/// back to check its SHA-256; then the deep report (<c>media/analysis.json</c>) and, last, <c>media/media.json</c> with the
|
||||
/// encodings conversions joined at the home. The job only reads the archive: it never writes there, not even proxies (the
|
||||
/// archive's own pass makes those), refuses a workspace folder inside any library of the index before it writes anything,
|
||||
/// and reads only a folder that holds the sequence (<see cref="SequenceFolders.Holds"/>).
|
||||
/// Frames changed at the home during the copy, or the folder moved (a Reorganize), make it start again (three times at
|
||||
/// most; a folder still moving fails it, to be retried). The working state's positions follow the new frames: by frame
|
||||
/// where a frame stays (one whose file a conversion replaced keeps its positions), by photo where frames came or went.
|
||||
/// Checkpoint: <c>copy:<frame>:<frames version></c>; a run resumed on other frames checks every file again.
|
||||
/// Originals come in phase 5c: asked for, the job fails saying so.
|
||||
/// </summary>
|
||||
internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService archives) : IJobHandler
|
||||
{
|
||||
@@ -62,35 +68,29 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
if (context.Job.TargetIds is not [var sequenceId] || await db.Sequences.GetAsync(sequenceId, ct).ConfigureAwait(false) is not { } sequence
|
||||
|| await db.Archives.GetAsync(sequence.ArchiveId, ct).ConfigureAwait(false) is not { } archive)
|
||||
throw new InvalidOperationException(Loc.L("Archive.Log.SequenceMissing", context.Job.TargetIds.FirstOrDefault()));
|
||||
// Asked before the store is opened, which makes its folder: editing happens outside every library (owner review 2,
|
||||
// R8), and a workspace inside one would put its copies among that library's own files.
|
||||
if (await InsideALibraryAsync(parameters.WorkspaceRoot, ct).ConfigureAwait(false))
|
||||
throw new InvalidOperationException(Loc.L("Errors.Workspace.InsideLibrary"));
|
||||
var store = new WorkspaceStore(parameters.WorkspaceRoot);
|
||||
var project = store.Find(parameters.ProjectId) ?? throw new InvalidOperationException(Loc.L("Errors.Workspace.NotCheckedOut"));
|
||||
if (project.ReadBundle().Workspace is not { Home: { IsArchive: true } home } || home.SequenceId != sequenceId)
|
||||
throw new InvalidOperationException(Loc.L("Errors.Workspace.OtherHome"));
|
||||
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
var (stage, next) = Parse(context.Checkpoint);
|
||||
// Read only: a read-only archive is copied from like any other.
|
||||
if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(archive.RootPath, ArchiveAvailability.Offline);
|
||||
var (copied, checkpointVersion) = Parse(context.Checkpoint);
|
||||
for (var pass = 1; ; pass++)
|
||||
{
|
||||
var (row, frames) = await FramesAsync(sequenceId, ct).ConfigureAwait(false);
|
||||
var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy;
|
||||
var makesProxies = policy.LongEdge != ProxyPolicy.Disabled && ArchiveVolume.Probe(paths) == ArchiveAvailability.Online;
|
||||
var total = (long)frames.Count * (makesProxies ? 2 : 1);
|
||||
long offset = 0;
|
||||
if (makesProxies)
|
||||
{
|
||||
if (stage == Stage.Proxies && (row.Status & SequenceStatus.ProxiesBuilt) == 0)
|
||||
{
|
||||
var result = await archives.Images.BuildProxiesAsync(archive, row, force: false, next,
|
||||
(frame, _) => context.CheckpointAsync(Checkpoint(Stage.Proxies, frame), frame, total), ct).ConfigureAwait(false);
|
||||
context.Log(Loc.L("Archive.Log.ProxiesDone", row.DisplayTitle, result.Built, result.Reused, result.Undecodable, result.Waiting));
|
||||
next = 0;
|
||||
}
|
||||
offset = frames.Count;
|
||||
}
|
||||
// Frames before a copy checkpoint were copied by an earlier run.
|
||||
var copied = stage == Stage.Copy ? next : 0;
|
||||
(stage, next) = (Stage.Copy, 0);
|
||||
var folder = paths.Sequence(row.Folder);
|
||||
// Another index moved the sequence away, maybe another one in: its files are not this sequence's photos.
|
||||
if (!SequenceFolders.Holds(folder, sequenceId)) throw new SequenceFolderMovedException();
|
||||
// Files kept from an earlier run are this frames version's only (a conversion keeps a frame's id, not its photo).
|
||||
if (checkpointVersion is { } version && version != row.FramesVersion) copied = 0;
|
||||
var total = (long)frames.Count;
|
||||
var tally = new Tally();
|
||||
var entries = new WorkspaceFrame[frames.Count];
|
||||
Directory.CreateDirectory(project.ProxiesDirectory);
|
||||
@@ -101,25 +101,32 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
entries[i] = Copy(project, folder, paths, frames[i], policy, done: i < copied, tally, ct);
|
||||
if (clock.Elapsed >= CheckpointEvery)
|
||||
{
|
||||
await context.CheckpointAsync(Checkpoint(Stage.Copy, i + 1), offset + i + 1, total).ConfigureAwait(false);
|
||||
await context.CheckpointAsync(Checkpoint(Stage.Copy, i + 1, row.FramesVersion), i + 1, total).ConfigureAwait(false);
|
||||
clock.Restart();
|
||||
}
|
||||
else
|
||||
{
|
||||
await context.ReportAsync(offset + i + 1, total).ConfigureAwait(false);
|
||||
await context.ReportAsync(i + 1, total).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
var analysis = CopyAnalysis(folder, project, frames.Count);
|
||||
var after = await db.Sequences.GetAsync(sequenceId, ct).ConfigureAwait(false)
|
||||
?? throw new InvalidOperationException(Loc.L("Archive.Log.SequenceMissing", sequenceId));
|
||||
if (after.FramesVersion != row.FramesVersion && pass < Passes)
|
||||
var (after, now) = await FramesAsync(sequenceId, ct).ConfigureAwait(false);
|
||||
var moved = after.Folder != row.Folder || !SequenceFolders.Holds(paths.Sequence(after.Folder), sequenceId) || Renamed(frames, now, entries);
|
||||
if (after.FramesVersion != row.FramesVersion || moved)
|
||||
{
|
||||
context.Log(Loc.L("Archive.Log.WorkspaceFramesChanged", row.DisplayTitle));
|
||||
(stage, next) = (Stage.Proxies, 0);
|
||||
continue;
|
||||
// The files read may be of other frames, or missing where a Reorganize took them: read them again.
|
||||
if (pass < Passes)
|
||||
{
|
||||
context.Log(Loc.L("Archive.Log.WorkspaceFramesChanged", row.DisplayTitle));
|
||||
(copied, checkpointVersion) = (0, null);
|
||||
continue;
|
||||
}
|
||||
// Frames still arriving are copied as they were read (Update media takes the rest); a folder still moving is not.
|
||||
if (moved) throw new SequenceFolderMovedException();
|
||||
}
|
||||
var aliases = await HomePhotos.AliasesAsync(db, sequenceId, ct).ConfigureAwait(false);
|
||||
var media = new WorkspaceMedia(WorkspaceMedia.FormatName, WorkspaceMedia.CurrentFormatVersion, BundleMedia.Proxies, sequenceId, row.FramesVersion, entries,
|
||||
DateTimeOffset.UtcNow, analysis);
|
||||
DateTimeOffset.UtcNow, analysis, aliases.Count > 0 ? aliases : null);
|
||||
// The working state is the editor's meanwhile: it is held only while the media index and the positions move.
|
||||
using (await project.LockAsync(LockWait, ct).ConfigureAwait(false))
|
||||
{
|
||||
@@ -127,11 +134,42 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
RemoveStale(project, media);
|
||||
}
|
||||
context.Log(Loc.L("Archive.Log.WorkspaceCopied", row.DisplayTitle, tally.Copied, tally.Made, tally.Kept, tally.Missing));
|
||||
await context.CheckpointAsync(Checkpoint(Stage.Done, frames.Count), total, total).ConfigureAwait(false);
|
||||
await context.CheckpointAsync(Checkpoint(Stage.Done, frames.Count, row.FramesVersion), total, total).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Whether <paramref name="folder"/> is an archive's root of this index or lies inside one.</summary>
|
||||
private async Task<bool> InsideALibraryAsync(string folder, CancellationToken cancellationToken)
|
||||
{
|
||||
var path = Plain(Path.GetFullPath(folder));
|
||||
foreach (var archive in await db.Archives.ListAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (DataLocation.IsUnder(path, Plain(Path.GetFullPath(archive.RootPath)))) return true;
|
||||
}
|
||||
return false;
|
||||
|
||||
// Without the long-path prefix, which only one of the two may carry.
|
||||
static string Plain(string path) =>
|
||||
path.StartsWith(@"\\?\UNC\", StringComparison.Ordinal) ? @"\\" + path[8..]
|
||||
: path.StartsWith(@"\\?\", StringComparison.Ordinal) ? path[4..]
|
||||
: path;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A frame the copy found no file for whose original the index has under another name now (a Reorganize renamed the
|
||||
/// files during the copy, keeping the frames version): the copy runs again.
|
||||
/// </summary>
|
||||
private static bool Renamed(List<Frame> read, List<Frame> now, WorkspaceFrame[] entries)
|
||||
{
|
||||
var paths = now.ToDictionary(f => f.Id, f => f.RelativePath);
|
||||
for (var i = 0; i < read.Count; i++)
|
||||
{
|
||||
if (entries[i].File is null && paths.TryGetValue(read[i].Id, out var path) && path != read[i].RelativePath) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The frame's file in the workspace: kept when an earlier run copied it (before the checkpoint, or the same bytes as the
|
||||
/// archive's proxy), else the archive's current proxy, else one made from the original; none for a frame without either.
|
||||
@@ -219,7 +257,9 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
/// <summary>
|
||||
/// The media index and the working state moved onto it: first the working state with every position parked by photo
|
||||
/// (valid whatever the frames), then the index, then the working state placed on the new frames. A crash in between
|
||||
/// leaves positions that wait for their photos, which the next open places.
|
||||
/// leaves positions that wait for their photos, which the next open places. A frame the new media still hold is parked
|
||||
/// as the photo they hold for it, so one whose file a conversion replaced (the same frame, another SHA-256) keeps its
|
||||
/// positions, as the archive's own settings do (<see cref="ProjectPositions"/>).
|
||||
/// </summary>
|
||||
private static void Commit(WorkspaceProject project, ProjectBundle bundle, WorkspaceMedia media, Sequence row)
|
||||
{
|
||||
@@ -230,7 +270,9 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
// Without the media index the positions were made against, they are taken as naming these frames.
|
||||
if (!working.IsNewerFormat && old is not null)
|
||||
{
|
||||
var portable = ProjectSettings.FromJson(ProjectContent.ToPortable(working, old.Photos), allowNewer: true);
|
||||
var held = media.Frames.ToDictionary(f => f.Id);
|
||||
var before = new FramePhotos(old.Frames.Select(f => (f.Index, Photo(held.TryGetValue(f.Id, out var now) ? now : f))), old.Aliases);
|
||||
var portable = ProjectSettings.FromJson(ProjectContent.ToPortable(working, before), allowNewer: true);
|
||||
project.WriteBundle(bundle with { Project = portable, Workspace = workspace, Sequence = sequence });
|
||||
project.WriteMedia(media);
|
||||
working = ProjectSettings.FromJson(ProjectContent.Place(portable.ToJson(), media.Photos), allowNewer: true);
|
||||
@@ -242,6 +284,8 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
project.WriteBundle(bundle with { Project = working, Workspace = workspace, Sequence = sequence, Saved = DateTimeOffset.UtcNow });
|
||||
}
|
||||
|
||||
private static FramePhoto Photo(WorkspaceFrame frame) => new(frame.Sha256, frame.Size, frame.CapturedAt);
|
||||
|
||||
/// <summary>Proxies of frames the sequence no longer has: regenerable copies, removed so the media folder holds the frames listed.</summary>
|
||||
private static void RemoveStale(WorkspaceProject project, WorkspaceMedia media)
|
||||
{
|
||||
@@ -313,21 +357,26 @@ internal sealed class WorkspaceCopyJobHandler(ChronosDatabase db, ArchiveService
|
||||
|
||||
private enum Stage
|
||||
{
|
||||
Proxies,
|
||||
Copy,
|
||||
Done,
|
||||
}
|
||||
|
||||
private static string Checkpoint(Stage stage, int frame) =>
|
||||
string.Create(CultureInfo.InvariantCulture, $"{stage.ToString().ToLowerInvariant()}:{frame}");
|
||||
private static string Checkpoint(Stage stage, int frame, int framesVersion) =>
|
||||
string.Create(CultureInfo.InvariantCulture, $"{stage.ToString().ToLowerInvariant()}:{frame}:{framesVersion}");
|
||||
|
||||
private static (Stage Stage, int Next) Parse(string? checkpoint)
|
||||
/// <summary>
|
||||
/// The frames copied by an earlier run (<c>copy:<frame>[:<frames version>]</c>) and the frames version they
|
||||
/// were of (null when the checkpoint does not say); nothing for a finished run, or another stage (an earlier build's
|
||||
/// archive proxy pass), whose files the copy checks again.
|
||||
/// </summary>
|
||||
private static (int Copied, int? FramesVersion) Parse(string? checkpoint)
|
||||
{
|
||||
var parts = checkpoint?.Split(':');
|
||||
return parts is [var name, var number] && Enum.TryParse<Stage>(name, ignoreCase: true, out var stage)
|
||||
&& int.TryParse(number, NumberStyles.None, CultureInfo.InvariantCulture, out var next)
|
||||
? (stage == Stage.Done ? Stage.Copy : stage, stage == Stage.Done ? 0 : next)
|
||||
: (Stage.Proxies, 0);
|
||||
if (parts is not ([_, _] or [_, _, _]) || !Enum.TryParse<Stage>(parts[0], ignoreCase: true, out var stage) || stage != Stage.Copy
|
||||
|| !int.TryParse(parts[1], NumberStyles.None, CultureInfo.InvariantCulture, out var next))
|
||||
return (0, null);
|
||||
int? version = parts.Length == 3 && int.TryParse(parts[2], NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture, out var v) ? v : null;
|
||||
return (next, version);
|
||||
}
|
||||
|
||||
private sealed class Tally
|
||||
|
||||
@@ -90,15 +90,17 @@ public sealed record WorkspaceFrame(
|
||||
/// (<see cref="BundleMedia.Proxies"/>; originals in phase 5c), the sequence and its frames version at the home when they
|
||||
/// were copied, and every frame in index order. Written last by a copy, so a copy cut short leaves the previous index.
|
||||
/// </summary>
|
||||
/// <param name="Aliases">The encodings of the sequence's photos a conversion joined at the home (R7, its tombstones of reason
|
||||
/// "replaced") when the media were copied: a version parked on a photo before its file was converted still finds its frame.</param>
|
||||
public sealed record WorkspaceMedia(string Format, int FormatVersion, string Kind, Guid SequenceId, int FramesVersion, IReadOnlyList<WorkspaceFrame> Frames,
|
||||
DateTimeOffset Copied, string? AnalysisSha256 = null)
|
||||
DateTimeOffset Copied, string? AnalysisSha256 = null, IReadOnlyList<PhotoAlias>? Aliases = null)
|
||||
{
|
||||
public const string FormatName = "chronos-workspace-media";
|
||||
public const int CurrentFormatVersion = 1;
|
||||
|
||||
/// <summary>The frames' photos in index order, as project positions name them.</summary>
|
||||
/// <summary>The frames' photos in index order, as project positions name them, with the encodings conversions joined.</summary>
|
||||
[JsonIgnore]
|
||||
public FramePhotos Photos => new(Frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, f.CapturedAt))));
|
||||
public FramePhotos Photos => new(Frames.Select(f => (f.Index, new FramePhoto(f.Sha256, f.Size, f.CapturedAt))), Aliases);
|
||||
|
||||
/// <summary>The frames held, as the processing engine reads them.</summary>
|
||||
public IReadOnlyList<Frame> ToFrames() => [.. Frames.Select(f => f.ToFrame(SequenceId))];
|
||||
|
||||
@@ -359,13 +359,15 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName)
|
||||
locals = [.. locals.Skip(ours.Count)];
|
||||
var media = project.ReadMedia();
|
||||
var photos = media?.Photos ?? NoFrames;
|
||||
var basisSettings = basis is null ? new ProjectSettings() : ProjectMerge.Read(basis.SettingsJson);
|
||||
var headSettings = ProjectMerge.Read(head.SettingsJson);
|
||||
var tip = tipVersion is null ? basisSettings : ProjectMerge.Read(tipVersion.SettingsJson);
|
||||
// Each side names a photo as the frames held it when it was saved: named as the media hold it now (a conversion or a
|
||||
// hash since), the same photo merges as one.
|
||||
var basisSettings = basis is null ? new ProjectSettings() : ProjectMerge.Read(ProjectContent.Rebind(basis.SettingsJson, photos));
|
||||
var headSettings = ProjectMerge.Read(ProjectContent.Rebind(head.SettingsJson, photos));
|
||||
var tip = tipVersion is null ? basisSettings : ProjectMerge.Read(ProjectContent.Rebind(tipVersion.SettingsJson, photos));
|
||||
var working = ProjectMerge.Read(Portable(bundle.Project, media) ?? throw MediaMissing(project.ProjectId));
|
||||
// "Take the server's" everywhere: the paths are those the merge finds, so it runs once to find them.
|
||||
var choice = takeTheirs is EveryPath ? null : takeTheirs;
|
||||
List<ProjectSettings> mine = [.. locals.Select(v => ProjectMerge.Read(v.SettingsJson))];
|
||||
List<ProjectSettings> mine = [.. locals.Select(v => ProjectMerge.Read(ProjectContent.Rebind(v.SettingsJson, photos)))];
|
||||
var replay = ProjectMerge.Replay(basisSettings, mine, headSettings, choice);
|
||||
if (takeTheirs is EveryPath) replay = ProjectMerge.Replay(basisSettings, mine, headSettings, Paths(replay.Conflicts));
|
||||
var newTip = replay.Commits.Count > 0 ? replay.Commits[^1] : headSettings;
|
||||
@@ -469,7 +471,7 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName)
|
||||
var (tipId, tipNumber, tipHash) = locals.Count > 0
|
||||
? (locals[^1].Id, locals[^1].Number, locals[^1].ContentHash)
|
||||
: (workspace.Base?.VersionId, workspace.Base?.Number ?? 0, workspace.Base?.ContentHash);
|
||||
if (hash == tipHash) return null;
|
||||
if (hash == tipHash || SameAsTip(project, bundle, locals, portable, media)) return null;
|
||||
var sync = project.ReadSync();
|
||||
var version = new ProjectVersion
|
||||
{
|
||||
@@ -509,7 +511,8 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName)
|
||||
var conflict = project.ReadSync().Conflict;
|
||||
var locals = Unpushed(project, bundle);
|
||||
var tipHash = locals.Count > 0 ? locals[^1].ContentHash : workspace.Base?.ContentHash;
|
||||
var modified = !bundle.IsReadOnly && (Portable(bundle.Project, media) is not { } portable || ProjectContent.Hash(portable) != tipHash);
|
||||
var modified = !bundle.IsReadOnly && (Portable(bundle.Project, media) is not { } portable
|
||||
|| (ProjectContent.Hash(portable) != tipHash && !SameAsTip(project, bundle, locals, portable, media)));
|
||||
ProjectVersion? head = null;
|
||||
var offline = false;
|
||||
if (home is not null)
|
||||
@@ -536,6 +539,18 @@ public sealed class WorkspaceSync(WorkspaceStore store, string clientName)
|
||||
return new WorkspaceStatus(state, modified, locals.Count, behind, offline, media is null, bundle.IsReadOnly, conflict, workspace.Base, head);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether the working state's portable form holds what the last version does (the last local one, else the base),
|
||||
/// once each names its photos as the media hold them now: a version saved before the media were copied again after a
|
||||
/// conversion or a hash at the home parks the same photos under their former names.
|
||||
/// </summary>
|
||||
private static bool SameAsTip(WorkspaceProject project, ProjectBundle bundle, List<ProjectVersion> locals, string portable, WorkspaceMedia? media)
|
||||
{
|
||||
if (media is null) return false;
|
||||
var tip = locals.Count > 0 ? locals[^1] : bundle.Workspace!.Base is { } basis ? project.Versions().FirstOrDefault(v => v.Id == basis.VersionId) : null;
|
||||
return tip is not null && ProjectContent.SameChoices(tip.SettingsJson, portable, media.Photos);
|
||||
}
|
||||
|
||||
/// <summary>The local versions after the base, oldest first, each the child of the one before (the base's for the first).</summary>
|
||||
private static List<ProjectVersion> Unpushed(WorkspaceProject project, ProjectBundle bundle)
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user