fix(archive): a rebase moves no row of the sequence that took the old folder; writers refuse a folder another sequence holds
Round 4 let every row under the old folder follow when that folder holds another sequence now. When the scan took the new holder first, its own row (already rebased into the old folder) then followed the moving sequence too and named that sequence's file, leaving its own export without a row; a second scan did not repair it. Only the moving sequence's own rows now follow because the old folder is taken, and a row of the sequence holding the old folder never follows: its export is in its own folder whatever the other one took along under the same name. The rebase callback now gets the owner of each row. Writers that file into the folder the index has for a sequence checked only that the folder existed. After another index moved the sequence away and another one into that folder, Take server wrote the server's settings into the other sequence's project.json, Save both wrote this PC's copy into its .chronos, and a render sent for the sequence was filed among its exports. SequenceFolders.Holds treats a folder whose sidecar names another sequence exactly like a missing one, and every such writer asks it: settings sync and conflict resolution (Errors.Archive.FolderMoved), Save both's copy and its clean-up, server render filing (503 folder_moved, the upload kept), export downloads, render jobs as they start, proxies sent and proxy passes, the deep analysis checkpoint and the editor's project save. A thumbnail pass refuses a sidecar of another sequence before it opens the pack. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> (cherry picked from commit b406a4f6a9a7a3121e9e40289663fb62ee31948e)
This commit is contained in:
1 parent
26b03d1c6d
commit
8c2b9f1bfe
17 files changed
+260
-57
No files matched your search
@@ -179,8 +179,9 @@ public sealed class LocalSequenceSource : ISequenceSource
|
||||
// it too: moved since the read, the project goes to the new folder once the editor read the sequence there,
|
||||
// and the folder it left is never created again.
|
||||
if (row.FramesVersion != framesVersion || row.Folder != relativeFolder) throw new FramesChangedException(Loc.L("Editor.Error.FramesArriving"));
|
||||
// Moved by another index of the archive (a server's Reorganize), which this index finds at its next scan.
|
||||
if (!Directory.Exists(folder.Folder)) throw Missing();
|
||||
// Moved by another index of the archive (a server's Reorganize), which this index finds at its next scan; the folder
|
||||
// it left may hold another sequence that index moved into it since.
|
||||
if (!SequenceFolders.Holds(folder, _sequenceId)) throw Missing();
|
||||
var stored = ReadProject(row, folder);
|
||||
var merged = ProjectMerge.SameValues(stored, basis) ? project : ProjectMerge.Merge(basis, project, stored);
|
||||
var json = merged.ToJson();
|
||||
|
||||
@@ -168,7 +168,8 @@ public sealed class ArchiveImages
|
||||
/// frame's (proxies carry it, so the proxy counts as current and cannot belong to another photo) and whose size is
|
||||
/// the policy's (<see cref="TooSmall"/>), written atomically under the sequence's lease into its folder as the index has
|
||||
/// it then: a Reorganize moves the folder under that lease, and the folder it left is never created again (one another
|
||||
/// index moved, which this index finds at its next scan, takes nothing: <see cref="ProxyStoreResult.Gone"/>).
|
||||
/// index moved, which this index finds at its next scan, takes nothing, nor does that folder when it holds another
|
||||
/// sequence now: <see cref="ProxyStoreResult.Gone"/>).
|
||||
/// </summary>
|
||||
/// <exception cref="ArchiveOfflineException">The archive is offline or read-only.</exception>
|
||||
/// <exception cref="LeaseUnavailableException">Another writer held the sequence for longer than the lease wait.</exception>
|
||||
@@ -191,7 +192,7 @@ public sealed class ArchiveImages
|
||||
{
|
||||
if (await _db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false) is not { } row) return ProxyStoreResult.Gone;
|
||||
var folder = paths.Sequence(row.Folder);
|
||||
if (!Directory.Exists(folder.Folder)) return ProxyStoreResult.Gone;
|
||||
if (!SequenceFolders.Holds(folder, row.Id)) return ProxyStoreResult.Gone;
|
||||
Directory.CreateDirectory(folder.Proxies);
|
||||
var path = folder.Proxy(frame.Index);
|
||||
AtomicFile.WriteAllBytes(path, jpeg);
|
||||
@@ -226,6 +227,8 @@ public sealed class ArchiveImages
|
||||
{
|
||||
var folder = await SequenceWriter.FolderAsync(_db, paths, sequence, cancellationToken).ConfigureAwait(false);
|
||||
manifest = ManifestStore.ReadSequence(folder.ManifestPath);
|
||||
// Another sequence's folder now (another index moved this one away and that one in): its pack is never opened.
|
||||
if (manifest.Id != sequence.Id) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
|
||||
var companions = await GprCompanions.FindAsync(_db, archive, manifest.Frames, cancellationToken).ConfigureAwait(false);
|
||||
using var pack = ThumbPackWriter.Open(folder.ThumbsPackPath);
|
||||
for (var i = 0; i < startFrame && i < manifest.Frames.Count; i++)
|
||||
@@ -298,8 +301,8 @@ public sealed class ArchiveImages
|
||||
/// originals are decoded at once (<see cref="PassParallelism"/>), and a checkpoint still means that every frame before
|
||||
/// it has its proxy written (<see cref="OrderedWork{T}"/>). The pass works in batches (<see cref="ProxyBatch"/>), each
|
||||
/// under the thumbnail lease with the sequence's folder and sidecar read again; a folder that is not where the index
|
||||
/// has it (another index moved it, until this index's next scan) stops the pass with an
|
||||
/// <see cref="InvalidOperationException"/>, nothing written there.
|
||||
/// has it (another index moved it, until this index's next scan), or that holds another sequence now, stops the pass
|
||||
/// with an <see cref="InvalidOperationException"/>, nothing written there.
|
||||
/// </remarks>
|
||||
public async Task<ImagePassResult> BuildProxiesAsync(Archive archive, Sequence sequence, bool force, int startFrame,
|
||||
Func<int, int, ValueTask> checkpoint, CancellationToken cancellationToken = default)
|
||||
@@ -324,7 +327,7 @@ public sealed class ArchiveImages
|
||||
_writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var folder = await SequenceWriter.FolderAsync(_db, paths, sequence, cancellationToken).ConfigureAwait(false);
|
||||
if (!Directory.Exists(folder.Folder)) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
|
||||
if (!SequenceFolders.Holds(folder, sequence.Id)) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
|
||||
var read = ManifestStore.ReadSequence(folder.ManifestPath);
|
||||
// Photos added, removed or renumbered since the last batch: what the earlier batches made may not be theirs.
|
||||
if (manifest is not null && !SameFrames(read.Frames, manifest.Frames, names: false)) complete = false;
|
||||
|
||||
@@ -188,33 +188,32 @@ public sealed class ArchiveIndexer
|
||||
/// <summary>
|
||||
/// The render rows of the archive that follow a sequence's folder from <paramref name="from"/> to <paramref name="to"/>,
|
||||
/// in the caller's transaction (<see cref="Repositories.RenderRepository.RebaseFolderAsync"/>). An export is a file, or
|
||||
/// a folder for an image sequence. It is at its old path still when it is there, the new path is not the same one (on a
|
||||
/// file system that ignores case it is, when only the case changed) and the old folder holds no other sequence now. The
|
||||
/// sequence's own exports inside the folder follow unless they are at the old path still and not at the new one:
|
||||
/// written there after the move by a writer that read the old folder, which created it again, so the row keeps naming
|
||||
/// where the export is. Another sequence's export follows only when it moved with the folder (at its new path and no
|
||||
/// longer at the old one). An export missing from both places follows if it is the sequence's own, and so does one whose
|
||||
/// old path is another sequence's: a sequence that moved into the old folder since brought an export of that name.
|
||||
/// a folder for an image sequence. It is at its old path still when it is there and the new path is not the same one
|
||||
/// (on a file system that ignores case it is, when only the case changed). The sequence's own exports inside the folder
|
||||
/// follow unless they are at the old path still and not at the new one: written there after the move by a writer that
|
||||
/// read the old folder, which created it again, so the row keeps naming where the export is. An export of its own
|
||||
/// missing from both places follows, and so does one at its old path when the old folder holds another sequence now:
|
||||
/// that sequence moved into it since and brought an export of that name. Another sequence's export follows only when it
|
||||
/// moved with the folder (at its new path and no longer at the old one), and an export of the sequence the old folder
|
||||
/// holds now never does: that one is in its own folder, whatever the moving sequence took along under the same name.
|
||||
/// </summary>
|
||||
internal static Task<int> RebaseRendersAsync(ChronosRepositories tx, Archive archive, Guid sequenceId, string from, string to,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
// Read once, and only for an export found at its old path.
|
||||
bool? taken = null;
|
||||
return tx.Renders.RebaseFolderAsync(archive.Id, sequenceId, from, to, (before, after, own) =>
|
||||
// The sequence the old folder holds now, when another one: read once, and only when a row asks.
|
||||
var holder = new Lazy<Guid?>(() => SequenceFolders.OtherSequence(paths.Sequence(from), sequenceId), LazyThreadSafetyMode.None);
|
||||
return tx.Renders.RebaseFolderAsync(archive.Id, sequenceId, from, to, (before, after, owner) =>
|
||||
{
|
||||
var own = owner == sequenceId;
|
||||
if (!own && owner == holder.Value) return false;
|
||||
var moved = Path.Exists(paths.FromRelative(after));
|
||||
var behind = !string.Equals(before, after, StringComparison.OrdinalIgnoreCase) && Path.Exists(paths.FromRelative(before))
|
||||
&& !(taken ??= TakenByAnother(paths.Sequence(from), sequenceId));
|
||||
&& !(own && holder.Value is not null);
|
||||
return own ? moved || !behind : moved && !behind;
|
||||
}, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>Whether <paramref name="folder"/> is another sequence's now: its sidecar names another id.</summary>
|
||||
private static bool TakenByAnother(SequencePaths folder, Guid sequenceId) =>
|
||||
File.Exists(folder.ManifestPath) && SequenceTrash.SidecarId(folder.ManifestPath) is { } id && id != sequenceId;
|
||||
|
||||
/// <summary>
|
||||
/// Remembers the current stamps of a folder whose sidecar and index were just updated together, so the next
|
||||
/// incremental scan does not re-read what is already indexed.
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
using Chronos.Core.Archiving;
|
||||
|
||||
namespace Chronos.Data.Archiving;
|
||||
|
||||
/// <summary>Whether a sequence is in the folder the index has for it, as writers that file into that folder ask.</summary>
|
||||
public static class SequenceFolders
|
||||
{
|
||||
/// <summary>
|
||||
/// Whether <paramref name="folder"/>, the one the index has for sequence <paramref name="sequenceId"/>, holds it: the
|
||||
/// folder is there and its sidecar, when it has one, names no other sequence. Another index of the archive that moved
|
||||
/// the sequence away may have moved another one into the folder it left since; until this index's next scan finds both,
|
||||
/// a writer of the sequence files nothing there, exactly as when the folder is missing, so none of its writes lands in
|
||||
/// the other sequence's folder.
|
||||
/// </summary>
|
||||
public static bool Holds(SequencePaths folder, Guid sequenceId)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(folder);
|
||||
return Directory.Exists(folder.Folder) && OtherSequence(folder, sequenceId) is null;
|
||||
}
|
||||
|
||||
/// <summary>The sequence other than <paramref name="sequenceId"/> whose sidecar <paramref name="folder"/> holds; null when none does.</summary>
|
||||
internal static Guid? OtherSequence(SequencePaths folder, Guid sequenceId) =>
|
||||
File.Exists(folder.ManifestPath) && SequenceTrash.SidecarId(folder.ManifestPath) is { } id && id != sequenceId ? id : null;
|
||||
}
|
||||
@@ -22,7 +22,8 @@ namespace Chronos.Data.Processing.Jobs;
|
||||
/// output is recorded at once (its row id derives from the job and the entry, so every attempt knows what is recorded).
|
||||
/// Checkpoint: the entries recorded and the output being written, so a crashed attempt's leftovers are removed and only
|
||||
/// unfinished entries render again. A failed entry fails the job after the others ran; retrying renders it alone. A
|
||||
/// sequence whose folder is not where the index has it (another index moved it, until this index's next scan) is refused.
|
||||
/// sequence whose folder is not where the index has it (another index moved it, until this index's next scan; the folder
|
||||
/// it left may hold another sequence now) is refused.
|
||||
/// </summary>
|
||||
internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions options, FrameTiers tiers) : IJobHandler
|
||||
{
|
||||
@@ -42,8 +43,8 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
// A folder another index moved is found by this index's next scan. Until then nothing renders: its photos would all be
|
||||
// stand-ins, and its outputs would create the old folder again.
|
||||
if (!Directory.Exists(folder.Folder)) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
|
||||
// stand-ins, and its outputs would create the old folder again, or land among the exports of a sequence moved into it.
|
||||
if (!SequenceFolders.Holds(folder, sequence.Id)) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
|
||||
// A server writes only inside its archive (BRIEF §14.2): an export folder chosen on a client means nothing there.
|
||||
if (options.Host == EncoderHost.Server) export = export with { Folder = null };
|
||||
var project = (parameters.SettingsJson is { } embedded ? ProjectStore.Parse(embedded) : ProjectStore.Read(sequence, folder)) with { Export = export };
|
||||
|
||||
@@ -359,14 +359,15 @@ internal sealed class SequenceAnalyser(ChronosDatabase db, ProcessingOptions opt
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The sequence lock, taken while <paramref name="folder"/>, where the pass reads the frames, is still there: a
|
||||
/// Reorganize, of this process or another sharing the archive, moves the folder under that lock's lease.
|
||||
/// The sequence lock, taken while <paramref name="folder"/>, where the pass reads the frames, still holds the sequence: a
|
||||
/// Reorganize, of this process or another sharing the archive, moves the folder under that lock's lease, and another one
|
||||
/// may move another sequence into the folder it left.
|
||||
/// </summary>
|
||||
/// <exception cref="FolderMovedException">The folder moved since the pass read the sequence.</exception>
|
||||
private async Task<IAsyncDisposable> InPlaceAsync(ArchivePaths paths, Sequence sequence, SequencePaths folder, CancellationToken cancellationToken)
|
||||
{
|
||||
var held = await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false);
|
||||
if (Directory.Exists(folder.Folder)) return held;
|
||||
if (SequenceFolders.Holds(folder, sequence.Id)) return held;
|
||||
await held.DisposeAsync().ConfigureAwait(false);
|
||||
throw new FolderMovedException();
|
||||
}
|
||||
|
||||
@@ -57,9 +57,9 @@ public sealed class RenderRepository
|
||||
/// into the old folder. Outputs elsewhere (absolute paths, other folders) stay as they are.
|
||||
/// </summary>
|
||||
/// <param name="follows">Whether the render at the first archive-relative path follows to the second one; the third
|
||||
/// argument says whether it is one of the sequence's own.</param>
|
||||
/// argument is the sequence it is an export of.</param>
|
||||
/// <remarks>Only paths still under <paramref name="oldFolder"/> change, so repeating it after a crash is harmless.</remarks>
|
||||
public async Task<int> RebaseFolderAsync(Guid archiveId, Guid sequenceId, string oldFolder, string newFolder, Func<string, string, bool, bool> follows,
|
||||
public async Task<int> RebaseFolderAsync(Guid archiveId, Guid sequenceId, string oldFolder, string newFolder, Func<string, string, Guid, bool> follows,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(follows);
|
||||
@@ -75,9 +75,8 @@ public sealed class RenderRepository
|
||||
foreach (var (id, owner, path) in under)
|
||||
{
|
||||
var to = newFolder.Length == 0 ? path[prefix.Length..] : newFolder + path[oldFolder.Length..];
|
||||
var own = owner == sequenceId;
|
||||
var taken = own && under.Any(o => o.SequenceId != sequenceId && string.Equals(o.Path, path, StringComparison.Ordinal));
|
||||
if (!taken && !follows(path, to, own)) continue;
|
||||
var taken = owner == sequenceId && under.Any(o => o.SequenceId != sequenceId && string.Equals(o.Path, path, StringComparison.Ordinal));
|
||||
if (!taken && !follows(path, to, owner)) continue;
|
||||
rebased += await _db.ExecuteAsync("UPDATE renders SET output_path = $2 WHERE id = $1", [id, to], cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
return rebased;
|
||||
|
||||
@@ -754,9 +754,10 @@ public sealed class SyncService
|
||||
if (await positions.ToLocalAsync(payload.SettingsJson, local.SettingsVersion, cancellationToken).ConfigureAwait(false) is not { } current) return null;
|
||||
var row = current.Local;
|
||||
// The folder too: a Reorganize moves it and its row under this lease. One another index moved is found by the next
|
||||
// scan; nothing is written until then, so the folder it left is not created again.
|
||||
// scan; nothing is written until then, so the folder it left is not created again, nor written in when it holds
|
||||
// another sequence now.
|
||||
var folder = paths.Sequence(row.Folder);
|
||||
if (!Directory.Exists(folder.Folder)) return null;
|
||||
if (!SequenceFolders.Holds(folder, row.Id)) return null;
|
||||
payload = payload with { SettingsJson = current.SettingsJson };
|
||||
var metadata = !payload.SameMetadata(SyncPayload.Of(row));
|
||||
var settings = !SyncRules.SameSettings(payload.SettingsJson, row.SettingsJson);
|
||||
@@ -808,7 +809,8 @@ public sealed class SyncService
|
||||
/// "Save both": this PC's project is kept next to <c>project.json</c> as <c>project.this-pc-<local time>.json</c>
|
||||
/// before the server's replaces it; returns that file. Written under the sequence's lease into its folder as the index
|
||||
/// has it then: a Reorganize may have moved it while the server was asked, and the folder it left is never created again.
|
||||
/// Null, with nothing written, when the sequence is not in that folder (another index moved it; the next scan finds it).
|
||||
/// Null, with nothing written, when the sequence is not in that folder (another index moved it, and maybe another
|
||||
/// sequence into it; the next scan finds them, <see cref="SequenceFolders.Holds"/>).
|
||||
/// </summary>
|
||||
private async Task<string?> SaveCopyAsync(Sequence local, CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -820,7 +822,7 @@ public sealed class SyncService
|
||||
await using (await _writer.LockAsync(paths, local.Id, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var folder = await SequenceWriter.FolderAsync(_db, paths, local, cancellationToken).ConfigureAwait(false);
|
||||
if (!Directory.Exists(folder.Folder)) return null;
|
||||
if (!SequenceFolders.Holds(folder, local.Id)) return null;
|
||||
Directory.CreateDirectory(folder.MetaDirectory);
|
||||
path = Path.Combine(folder.MetaDirectory, SavedCopyName(DateTime.Now));
|
||||
AtomicFile.WriteAllText(path, SyncRules.NormalizeSettings(local.SettingsJson));
|
||||
@@ -831,7 +833,8 @@ public sealed class SyncService
|
||||
|
||||
/// <summary>
|
||||
/// Whether the sequence is not in the folder the index has for it now, on an archive that is there: another index moved
|
||||
/// it, and this index's next scan finds it. Read without the lease, to tell why a resolution wrote nothing.
|
||||
/// it (and maybe another sequence into that folder), and this index's next scan finds it. Read without the lease, to
|
||||
/// tell why a resolution wrote nothing.
|
||||
/// </summary>
|
||||
private async Task<bool> FolderMovedAsync(Sequence local, CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -839,13 +842,14 @@ public sealed class SyncService
|
||||
|| await _db.Archives.GetAsync(row.ArchiveId, cancellationToken).ConfigureAwait(false) is not { } archive)
|
||||
return false;
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
return Directory.Exists(paths.Root) && !Directory.Exists(paths.Sequence(row.Folder).Folder);
|
||||
return Directory.Exists(paths.Root) && !SequenceFolders.Holds(paths.Sequence(row.Folder), row.Id);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes a "Save both" copy of a version the user did not keep from the sequence's folder as the index has it now: a
|
||||
/// Reorganize may have moved the folder, the copy with it, since the copy was written. Best effort: a copy that cannot
|
||||
/// be reached (the folder gone, the sequence busy, the archive offline) stays, as one more copy of this PC's version.
|
||||
/// be reached (the folder gone or another sequence's now, the sequence busy, the archive offline) stays, as one more
|
||||
/// copy of this PC's version.
|
||||
/// </summary>
|
||||
private async Task DeleteCopyAsync(Sequence local, string saved, CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -857,7 +861,8 @@ public sealed class SyncService
|
||||
await using (await _writer.LockAsync(paths, local.Id, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var folder = await SequenceWriter.FolderAsync(_db, paths, local, cancellationToken).ConfigureAwait(false);
|
||||
if (Directory.Exists(folder.MetaDirectory)) File.Delete(Path.Combine(folder.MetaDirectory, Path.GetFileName(saved)));
|
||||
if (SequenceFolders.Holds(folder, local.Id) && Directory.Exists(folder.MetaDirectory))
|
||||
File.Delete(Path.Combine(folder.MetaDirectory, Path.GetFileName(saved)));
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException)
|
||||
|
||||
@@ -379,12 +379,13 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
// Filed under the sequence's lease into its folder as the index has it then, as a server files a render: a
|
||||
// Reorganize may have moved it since the sequence was read, and moves the row's path along when it moves it later.
|
||||
// One another index moved is found by this index's next scan; until then nothing is filed (the download stays
|
||||
// staged for the next attempt), so the folder it left is not created again.
|
||||
// staged for the next attempt), so the folder it left is not created again, nor filed into when it holds another
|
||||
// sequence now.
|
||||
var filed = await sync.Archives.UnderLeaseAsync(local, async row =>
|
||||
{
|
||||
if (row is null) throw new InvalidOperationException(Loc.L("Errors.Sync.LocalMissing"));
|
||||
var folder = paths.Sequence(row.Folder);
|
||||
if (!Directory.Exists(folder.Folder)) return false;
|
||||
if (!SequenceFolders.Holds(folder, row.Id)) return false;
|
||||
Directory.CreateDirectory(folder.Exports);
|
||||
var target = Unique(folder.Exports, name);
|
||||
File.Move(part, target);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.IO;
|
||||
using Chronos.Core.Localization;
|
||||
using Chronos.Core.Model;
|
||||
@@ -20,8 +21,9 @@ public sealed class RenderStore(ChronosDatabase db, ServerArchive archive, TimeP
|
||||
/// path along when it moves it later. Null when the sequence left the index or has no folder in the archive yet; nothing
|
||||
/// is moved then.
|
||||
/// </summary>
|
||||
/// <exception cref="FolderMovedException">The sequence is not in the folder the index has (another index moved it):
|
||||
/// nothing is moved, and the filing can be done again once this index's next scan found the folder.</exception>
|
||||
/// <exception cref="FolderMovedException">The sequence is not in the folder the index has (another index moved it, and
|
||||
/// maybe another sequence into that folder): nothing is moved, and the filing can be done again once this index's next
|
||||
/// scan found the folder.</exception>
|
||||
public Task<Render?> CommitAsync(Sequence sequence, string stagedFile, RenderResultRequest output, string producedBy, CancellationToken cancellationToken) =>
|
||||
CommitAsync(sequence, stagedFile, output, producedBy, Guid.CreateVersion7(), cancellationToken);
|
||||
|
||||
@@ -33,8 +35,9 @@ public sealed class RenderStore(ChronosDatabase db, ServerArchive archive, TimeP
|
||||
{
|
||||
if (row is null || string.IsNullOrEmpty(row.Folder) || ArchivePathGuard.Resolve(archive.Root, row.Folder.TrimEnd('/') + "/exports") is not { } exports)
|
||||
return null;
|
||||
// A folder another index moved is found by this index's next scan; until then nothing is filed, so it is not created again.
|
||||
if (!Directory.Exists(Path.GetDirectoryName(exports))) throw new FolderMovedException();
|
||||
// A folder another index moved is found by this index's next scan; until then nothing is filed, so it is not created
|
||||
// again, nor filed among the exports of another sequence that index moved into it since.
|
||||
if (!SequenceFolders.Holds(new ArchivePaths(archive.Root).Sequence(row.Folder), row.Id)) throw new FolderMovedException();
|
||||
Directory.CreateDirectory(exports);
|
||||
var target = Unique(exports, output.FileName);
|
||||
File.Move(stagedFile, target);
|
||||
|
||||
@@ -289,6 +289,33 @@ public sealed class ProjectStoreTests(TestContext context)
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ASaveNeverWritesIntoAnotherSequenceMovedIntoTheFolder()
|
||||
{
|
||||
var (db, sequenceId, root) = await ArchiveAsync(4);
|
||||
await using (db)
|
||||
{
|
||||
await using var source = new LocalSequenceSource(db, sequenceId, DecoderFactory.CreateClient(null), null,
|
||||
new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) });
|
||||
var content = await source.LoadAsync(context.Cancellation);
|
||||
|
||||
// A server sharing the archive moved the folder away, then another sequence into the folder it left; this index
|
||||
// finds both at its next scan.
|
||||
var paths = new ArchivePaths(root);
|
||||
Directory.Move(paths.Sequence("Synthetic").Folder, paths.Sequence("Moved").Folder);
|
||||
var stranger = paths.Sequence("Synthetic");
|
||||
Directory.CreateDirectory(stranger.MetaDirectory);
|
||||
ManifestStore.WriteSequence(stranger.ManifestPath, ManifestStore.ReadSequence(paths.Sequence("Moved").ManifestPath) with { Id = Guid.NewGuid() });
|
||||
const string theirs = """{"exposure":{"deflickerWindow":{"isAuto":false,"manual":7}}}""";
|
||||
await File.WriteAllTextAsync(stranger.ProjectPath, theirs, context.Cancellation);
|
||||
var gone = await Assert.ThrowsAsync<EditorException>(() => source.SaveProjectAsync(content.Project with { HeroFrame = 1 }, content.Project, context.Cancellation));
|
||||
|
||||
Assert.Equal(Loc.L("Editor.Error.Missing"), gone.Message);
|
||||
Assert.Equal(theirs, await File.ReadAllTextAsync(stranger.ProjectPath, context.Cancellation), "the other sequence's project is untouched");
|
||||
Assert.Equal(content.Sequence.SettingsVersion, (await db.Sequences.GetAsync(sequenceId))!.SettingsVersion, "nothing recorded");
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task JobsGoToTheEngine()
|
||||
{
|
||||
|
||||
@@ -123,6 +123,13 @@ internal sealed class ArchiveFixture : IAsyncDisposable
|
||||
return (path, bytes.Length, Convert.ToHexStringLower(SHA256.HashData(bytes)));
|
||||
}
|
||||
|
||||
/// <summary>Every file under <paramref name="folder"/> with the hash of its content, in path order: what a test compares to tell that nothing there changed.</summary>
|
||||
public static List<string> FilesOf(string folder) =>
|
||||
[
|
||||
.. Directory.EnumerateFiles(folder, "*", SearchOption.AllDirectories).Order(StringComparer.Ordinal)
|
||||
.Select(p => $"{Path.GetRelativePath(folder, p).Replace('\\', '/')} {Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(p)))}"),
|
||||
];
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
foreach (var owned in _owned) await owned.DisposeAsync();
|
||||
|
||||
@@ -356,31 +356,38 @@ public sealed class MassRenameTests(TestContext context)
|
||||
[Test]
|
||||
public async Task AnExportNeverNamesTheOneAnotherSequenceBroughtIntoItsOldFolder()
|
||||
{
|
||||
// The new folder is scanned after "inbox/a", then before it.
|
||||
foreach (var template in new[] { "moved/{date}", "done/{date}" })
|
||||
// The new folder is scanned after "inbox/a", then before it. "inbox/a" and "inbox/c" each have an export of the
|
||||
// same name: the one of "inbox/a" gone, both there, or the one of "inbox/c" gone.
|
||||
var cases = from template in new[] { "moved/{date}", "done/{date}" }
|
||||
from files in new[] { (A: false, C: true), (A: true, C: true), (A: true, C: false) }
|
||||
select (template, files.A, files.C);
|
||||
foreach (var (template, a, c) in cases)
|
||||
{
|
||||
var label = $"{template}, a {(a ? "there" : "gone")}, c {(c ? "there" : "gone")}";
|
||||
await using var fx = await CreateAsync(context);
|
||||
var ids = await PopulateAsync(fx);
|
||||
// The export of "inbox/a" is gone; "inbox/c" has one of the same name.
|
||||
List<Guid> renders =
|
||||
[
|
||||
await RenderAsync(fx, ids[0], "inbox/a/exports/Clip.mp4", write: false),
|
||||
await RenderAsync(fx, ids[2], "inbox/c/exports/Clip.mp4"),
|
||||
await RenderAsync(fx, ids[0], "inbox/a/exports/Clip.mp4", write: a),
|
||||
await RenderAsync(fx, ids[2], "inbox/c/exports/Clip.mp4", write: c),
|
||||
];
|
||||
if (a) await File.WriteAllTextAsync(fx.Paths.FromRelative("inbox/a/exports/Clip.mp4"), "a", fx.Ct);
|
||||
if (c) await File.WriteAllTextAsync(fx.Paths.FromRelative("inbox/c/exports/Clip.mp4"), "c", fx.Ct);
|
||||
var (server, archive) = await fx.OtherWriterAsync();
|
||||
|
||||
// The other index moves "inbox/a", then "inbox/c" into the folder it left, its export along.
|
||||
// The other index moves "inbox/a", then "inbox/c" into the folder it left, their exports along.
|
||||
var first = await server.PlanRenameAsync(archive.Id, [ids[0]], template, null, fx.Ct);
|
||||
await server.ApplyRenameAsync(first, null, fx.Ct);
|
||||
await server.ApplyRenameAsync(await server.PlanRenameAsync(archive.Id, [ids[2]], "inbox/a", null, fx.Ct), null, fx.Ct);
|
||||
|
||||
// The first sequence's row follows its folder, missing there too, and never names the other one's file.
|
||||
// Each row follows its own sequence's folder, its file there or missing there too, and never names the other one's.
|
||||
await fx.ScanAsync();
|
||||
string[] expected = [first.Steps.Single().ToFolder + "/exports/Clip.mp4", "inbox/a/exports/Clip.mp4"];
|
||||
Assert.SequenceEqual(expected, await OutputsAsync(fx, renders), template);
|
||||
Assert.FileExists(fx.Paths.FromRelative(expected[1]));
|
||||
Assert.SequenceEqual(expected, await OutputsAsync(fx, renders), label);
|
||||
if (a) Assert.Equal("a", await File.ReadAllTextAsync(fx.Paths.FromRelative(expected[0]), fx.Ct), label);
|
||||
if (c) Assert.Equal("c", await File.ReadAllTextAsync(fx.Paths.FromRelative(expected[1]), fx.Ct), label);
|
||||
Assert.Equal(0, (await fx.ScanAsync()).Indexed);
|
||||
Assert.SequenceEqual(expected, await OutputsAsync(fx, renders), template);
|
||||
Assert.SequenceEqual(expected, await OutputsAsync(fx, renders), label);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -117,6 +117,33 @@ public sealed class ProxyTests(TestContext context)
|
||||
Assert.SequenceEqual(proxy, await File.ReadAllBytesAsync(moved.Proxy(0), fx.Ct));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ImagesOfAFolderAnotherSequenceTookAreNotWrittenThere()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var (sequence, frames) = await HeifSequenceAsync(fx);
|
||||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "inbox", "other"), Start.AddDays(3), 2, firstSeed: 70);
|
||||
await fx.ScanAsync();
|
||||
var stranger = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single(l => l.Folder == "inbox/other").Id;
|
||||
var images = fx.Service.Images;
|
||||
var (server, archive) = await fx.OtherWriterAsync();
|
||||
await server.ApplyRenameAsync(await server.PlanRenameAsync(archive.Id, [sequence.Id], "moved/{date}", null, fx.Ct), null, fx.Ct);
|
||||
await server.ApplyRenameAsync(await server.PlanRenameAsync(archive.Id, [stranger], sequence.Folder, null, fx.Ct), null, fx.Ct);
|
||||
var folder = fx.Paths.Sequence(sequence.Folder).Folder;
|
||||
var before = FilesOf(folder);
|
||||
|
||||
// This index still has the sequence in the folder the other sequence holds now: a proxy sent, a proxy pass and a
|
||||
// thumbnail pass write nothing there.
|
||||
Assert.Equal(ProxyStoreResult.Gone, await images.StoreProxyAsync(fx.Archive, sequence, frames[0], Proxy(frames[0].CapturedAt!.Value, 2), fx.Ct));
|
||||
foreach (var pass in new Func<Task<ImagePassResult>>[]
|
||||
{
|
||||
() => images.BuildProxiesAsync(fx.Archive, sequence, force: false, 0, (_, _) => ValueTask.CompletedTask, fx.Ct),
|
||||
() => images.BuildThumbnailsAsync(fx.Archive, sequence, force: true, 0, (_, _) => ValueTask.CompletedTask, fx.Ct),
|
||||
})
|
||||
Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), (await Assert.ThrowsAsync<InvalidOperationException>(pass)).Message);
|
||||
Assert.SequenceEqual(before, FilesOf(folder));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AProxyPassLetsAnotherProcessReorganizeBetweenItsBatches()
|
||||
{
|
||||
|
||||
@@ -9,6 +9,7 @@ using Chronos.Core.Project;
|
||||
using Chronos.Core.Video;
|
||||
using Chronos.Data.Jobs;
|
||||
using Chronos.Data.Processing;
|
||||
using Chronos.Data.Tests.Archiving;
|
||||
using Chronos.Testing;
|
||||
|
||||
namespace Chronos.Data.Tests.Processing;
|
||||
@@ -154,6 +155,27 @@ public sealed class RenderJobTests(TestContext context)
|
||||
Assert.False(Directory.Exists(left));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ARenderNeverWritesIntoAnotherSequenceMovedIntoItsFolder()
|
||||
{
|
||||
await using var fx = await ProcessingFixture.CreateAsync(context);
|
||||
var sequence = await fx.AddAsync(Scene());
|
||||
var stranger = await fx.AddAsync(Scene(), "Other");
|
||||
var (server, archive) = await fx.Archive.OtherWriterAsync();
|
||||
await server.ApplyRenameAsync(await server.PlanRenameAsync(archive.Id, [sequence.Id], "Moved/{date}", null, fx.Ct), null, fx.Ct);
|
||||
await server.ApplyRenameAsync(await server.PlanRenameAsync(archive.Id, [stranger.Id], sequence.Folder, null, fx.Ct), null, fx.Ct);
|
||||
var folder = fx.Folder(sequence).Folder;
|
||||
var before = ArchiveFixture.FilesOf(folder);
|
||||
await fx.StartEngineAsync();
|
||||
|
||||
// A server sharing the archive moved the folder away and the other sequence into it; this index has not scanned since.
|
||||
var job = await fx.EnqueueAsync(ProcessingJobs.Render(sequence, Mjpeg, 96, 64));
|
||||
var failed = await fx.WaitForAsync(job.Id, JobState.Failed);
|
||||
Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), failed.Error);
|
||||
Assert.SequenceEqual(before, ArchiveFixture.FilesOf(folder), "the other sequence's files are untouched");
|
||||
Assert.Empty(await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AFrameThatNoLongerDecodesIsReplacedByItsNeighbourAndLeftOutNextTime()
|
||||
{
|
||||
|
||||
@@ -530,6 +530,47 @@ public sealed class SyncServiceTests(TestContext context)
|
||||
"the moved project is this PC's version");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TakeServerAndSaveBothWriteNothingIntoAnotherSequenceMovedIntoTheFolder()
|
||||
{
|
||||
const string strangers = """{"exposure":{"deflickerWindow":{"isAuto":false,"manual":7}}}""";
|
||||
foreach (var resolution in new[] { SyncResolution.TakeServer, SyncResolution.SaveBoth })
|
||||
{
|
||||
await using var f = await SyncFixture.CreateAsync(context);
|
||||
var (local, remote) = await ConflictAsync(f);
|
||||
var stranger = await f.LocalAsync("Stranger", f.NewPhotos(2), strangers);
|
||||
var folder = f.Archive.Paths.Sequence(local.Folder);
|
||||
|
||||
// Another index moves the folder away while the server's version is read, then another sequence (first parked
|
||||
// elsewhere: its folder is the same name with a number) into the folder it left; this index has scanned neither.
|
||||
var (other, archive) = await f.Archive.OtherWriterAsync();
|
||||
List<string>? before = null;
|
||||
f.Remote.OnceBeforeManifest = async () =>
|
||||
{
|
||||
foreach (var (id, template) in new[] { (local.Id, "moved/{title}"), (stranger.Id, "parked/{title}"), (stranger.Id, local.Folder) })
|
||||
await other.ApplyRenameAsync(await other.PlanRenameAsync(archive.Id, [id], template, null, f.Ct), null, f.Ct);
|
||||
before = ArchiveFixture.FilesOf(folder.Folder);
|
||||
};
|
||||
var refused = await Assert.ThrowsAsync<SyncConflictChangedException>(() => f.Sync.ResolveAsync(f.Remote, remote.Id, resolution, f.Ct));
|
||||
Assert.Equal(Loc.L("Errors.Archive.FolderMoved"), refused.Message, $"{resolution}: the user hears to try again after the next scan");
|
||||
|
||||
// Neither version went into the other sequence's folder, nor replaced this PC's where the sequence is.
|
||||
Assert.Equal(stranger.Id, SequenceTrash.SidecarId(folder.ManifestPath), "the other sequence holds the folder");
|
||||
Assert.SequenceEqual(before!, ArchiveFixture.FilesOf(folder.Folder), $"{resolution}: the other sequence's files are untouched");
|
||||
Assert.True(SyncRules.SameSettings(SyncFixture.Window41, (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.SettingsJson), "nothing applied");
|
||||
var moved = f.Archive.Paths.Sequence("moved/Holy grail");
|
||||
Assert.True(SyncRules.SameSettings(SyncFixture.Window41, await File.ReadAllTextAsync(moved.ProjectPath, f.Ct)), "the moved project is this PC's version");
|
||||
Assert.Empty(Directory.GetFiles(moved.MetaDirectory, SyncService.SavedCopyPattern));
|
||||
|
||||
// Chosen again once this index's scan found both, it is done where the sequence is now.
|
||||
await f.Archive.ScanAsync();
|
||||
var copy = await f.Sync.ResolveAsync(f.Remote, remote.Id, resolution, f.Ct);
|
||||
Assert.True(SyncRules.SameSettings(SyncFixture.Window33, await File.ReadAllTextAsync(moved.ProjectPath, f.Ct)), resolution.ToString());
|
||||
Assert.Equal(resolution == SyncResolution.SaveBoth, copy is not null && Path.GetDirectoryName(copy) == moved.MetaDirectory);
|
||||
Assert.True(SyncRules.SameSettings(strangers, await File.ReadAllTextAsync(folder.ProjectPath, f.Ct)), "the other sequence keeps its own");
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task KeepMineRefusesWhenTheServerChangedAfterTheConflictWasShown()
|
||||
{
|
||||
|
||||
@@ -170,6 +170,41 @@ public sealed class ReorganizeTests(TestContext context)
|
||||
Assert.Empty(await server.Database.Renders.ListBySequenceAsync(id));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnExportIsNeverFiledAmongThoseOfAnotherSequenceMovedIntoItsFolder()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
var id = (await Uploads.UploadAsync(editor, Photo(1), "IMG_0001.JPG", sessionKey: "roof")).SequenceId;
|
||||
var other = (await Uploads.UploadAsync(editor, Photo(2), "IMG_0002.JPG", sessionKey: "cam")).SequenceId;
|
||||
await UntilIdleAsync(server);
|
||||
var sequence = (await server.Database.Sequences.GetAsync(id))!;
|
||||
var stranger = (await server.Database.Sequences.GetAsync(other))!;
|
||||
|
||||
// A desktop sharing the archive moved the folder away, then the other sequence into the folder it left; this server's
|
||||
// index still has both where they were.
|
||||
var folder = Path.Combine(server.Options.ArchivePath, sequence.Folder);
|
||||
Directory.Move(folder, Path.Combine(server.Options.ArchivePath, "elsewhere"));
|
||||
Directory.Move(Path.Combine(server.Options.ArchivePath, stranger.Folder), folder);
|
||||
var before = Files(folder);
|
||||
var video = Uploads.Photo(4096, 31);
|
||||
var staged = Path.Combine(context.CreateTempDirectory("staged"), "render.mp4");
|
||||
await File.WriteAllBytesAsync(staged, video);
|
||||
await Assert.ThrowsAsync<FolderMovedException>(() => server.Services.GetRequiredService<RenderStore>().CommitAsync(sequence, staged,
|
||||
new RenderResultRequest("Roof - 4K.mp4", video.Length, Uploads.Sha(video), "4K HEVC", "hevc", 3840, 2160, 25, 4), "studio-pc", CancellationToken.None));
|
||||
|
||||
Assert.FileExists(staged);
|
||||
Assert.SequenceEqual(before, Files(folder), "the other sequence's files are untouched");
|
||||
Assert.Empty(await server.Database.Renders.ListBySequenceAsync(id));
|
||||
}
|
||||
|
||||
/// <summary>Every file under <paramref name="folder"/> with the hash of its content, in path order.</summary>
|
||||
private static List<string> Files(string folder) =>
|
||||
[
|
||||
.. Directory.EnumerateFiles(folder, "*", SearchOption.AllDirectories).Order(StringComparer.Ordinal)
|
||||
.Select(p => $"{Path.GetRelativePath(folder, p).Replace('\\', '/')} {Uploads.Sha(File.ReadAllBytes(p))}"),
|
||||
];
|
||||
|
||||
[Test]
|
||||
public async Task AReorganizeAsksForTheArchiveAndAValidRequest()
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user