fix: keep render filing retryable and writers out of a folder another index moved
Third fix round of owner review 2, phase 4a (Reorganize), cross-cutting items. - Server render filing now runs under the sequence lease, so it can meet a busy sequence or an offline archive. Both completions (client render uploads and render-node results) used to answer 500 and leave the upload stuck in Committing until a restart. They now answer 503 sequence_busy or archive_offline, which clients retry. The upload goes back to receiving with its part file and the node keeps its job (UploadService.ReceiveAgainAsync). - Save both: when this PC's copy cannot be written (another index moved the folder), the server's version is no longer applied, even if this index's scan relocates the row before the apply would run. A copy that has to be dropped is deleted from where the folder is now, as a best effort. - Export rebase on a folder move (scan and Reorganize): the sequence's own export follows the folder unless its file is still at the old path and not at the new one. A file written into a re-created old folder therefore keeps a row that finds it, so the file is never stranded. - A render job refuses to start when its folder is not where the index has it, which also stops it from re-creating the old folder. Renders fetched by sync are now filed under the sequence lease into the current folder, or not at all (new string Errors.Archive.FolderMoved, en and it). - The proxy pass holds the thumbnail lease one batch at a time (a quarter of the lease wait), then lets it go for longer than a waiter's poll and reads the folder and sidecar again. Before, another process's Reorganize, sequence deletion or pack compaction could wait out its lease wait and fail. Updated the stale comments on that lease and ARCHITECTURE.md to match. Tests cover each fix, and each new test fails with its fix reverted. Also covered: proxy store and pass in a moved folder, RenderStore with the folder gone, Take server with the folder moved, and an editor save with the folder moved. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> (cherry picked from commit 811509d32a118ed50ad10d47d4b002e534da9d77)
This commit is contained in:
1 parent
0377040a19
commit
c3756d03c9
26 files changed
+666
-134
No files matched your search
@@ -177,7 +177,9 @@ public sealed class SyncService
|
||||
/// <summary>
|
||||
/// Settles a conflict as the user chose (BRIEF §12.7), against the versions the user was shown: when either side
|
||||
/// changed again since, nothing is written, the sequence is compared again and <see cref="SyncConflictChangedException"/>
|
||||
/// is thrown. Returns the file this PC's settings were saved to for <see cref="SyncResolution.SaveBoth"/>, else null.
|
||||
/// is thrown, as it is when <see cref="SyncResolution.SaveBoth"/> cannot keep this PC's version first (the folder is not
|
||||
/// where the index has it). Returns the file this PC's settings were saved to for <see cref="SyncResolution.SaveBoth"/>,
|
||||
/// else null.
|
||||
/// </summary>
|
||||
public async Task<string?> ResolveAsync(ISyncRemote remote, Guid remoteId, SyncResolution resolution, CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -222,9 +224,14 @@ public sealed class SyncService
|
||||
{
|
||||
changed = true;
|
||||
}
|
||||
else if (resolution == SyncResolution.SaveBoth && (saved = await SaveCopyAsync(local, cancellationToken).ConfigureAwait(false)) is null)
|
||||
{
|
||||
// This PC's version could not be kept (another index moved the folder): the server's does not replace it,
|
||||
// even where this index's scan finds the folder before the server's version would be written.
|
||||
changed = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (resolution == SyncResolution.SaveBoth) saved = await SaveCopyAsync(local, cancellationToken).ConfigureAwait(false);
|
||||
var theirs = SyncPayload.Of(dto, manifest!.SettingsJson);
|
||||
if (await ApplyLocallyAsync(local, theirs, Positions(remote, local, manifest), cancellationToken).ConfigureAwait(false) is { } applied)
|
||||
{
|
||||
@@ -234,7 +241,7 @@ public sealed class SyncService
|
||||
{
|
||||
// Edited on this PC since it was read: the copy of a version the user did not choose to keep goes.
|
||||
changed = true;
|
||||
if (saved is not null) File.Delete(saved);
|
||||
if (saved is not null) await DeleteCopyAsync(local, saved, cancellationToken).ConfigureAwait(false);
|
||||
saved = null;
|
||||
}
|
||||
}
|
||||
@@ -816,6 +823,30 @@ public sealed class SyncService
|
||||
return path;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes a "Save both" copy of a version the user did not keep from the sequence's folder as the index has it now: a
|
||||
/// Reorganize may have moved the folder, the copy with it, since the copy was written. Best effort: a copy that cannot
|
||||
/// be reached (the folder gone, the sequence busy, the archive offline) stays, as one more copy of this PC's version.
|
||||
/// </summary>
|
||||
private async Task DeleteCopyAsync(Sequence local, string saved, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
var archive = await _db.Archives.GetAsync(local.ArchiveId, cancellationToken).ConfigureAwait(false)
|
||||
?? throw new InvalidDataException($"Archive {local.ArchiveId} is not indexed.");
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
await using (await _writer.LockAsync(paths, local.Id, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var folder = await SequenceWriter.FolderAsync(_db, paths, local, cancellationToken).ConfigureAwait(false);
|
||||
if (Directory.Exists(folder.MetaDirectory)) File.Delete(Path.Combine(folder.MetaDirectory, Path.GetFileName(saved)));
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException)
|
||||
{
|
||||
Log.Warning(LogCategory, $"The copy {saved} of {local.DisplayTitle}'s settings could not be deleted", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The file name of a "Save both" copy kept at <paramref name="localTime"/>.</summary>
|
||||
public static string SavedCopyName(DateTime localTime) =>
|
||||
string.Create(CultureInfo.InvariantCulture, $"{SavedCopyPrefix}{localTime.ToString(SavedCopyTimeFormat, CultureInfo.InvariantCulture)}.json");
|
||||
|
||||
@@ -370,31 +370,43 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
EnsureSpace(paths.Root, wanted.Sum(r => r.Bytes));
|
||||
|
||||
var staging = Path.Combine(paths.StagingDirectory, "exports-" + remoteId.ToString("N"));
|
||||
var exports = paths.Sequence(local.Folder).Exports;
|
||||
Directory.CreateDirectory(staging);
|
||||
Directory.CreateDirectory(exports);
|
||||
var progress = new TransferProgress(context, wanted.Count, 0);
|
||||
await progress.ReportAsync().ConfigureAwait(false);
|
||||
foreach (var render in wanted)
|
||||
{
|
||||
var (part, name) = await FetchRenderAsync(remote, render, staging, ct).ConfigureAwait(false);
|
||||
var target = Unique(exports, name);
|
||||
File.Move(part, target);
|
||||
await db.Renders.InsertAsync(new Render
|
||||
// Filed under the sequence's lease into its folder as the index has it then, as a server files a render: a
|
||||
// Reorganize may have moved it since the sequence was read, and moves the row's path along when it moves it later.
|
||||
// One another index moved is found by this index's next scan; until then nothing is filed (the download stays
|
||||
// staged for the next attempt), so the folder it left is not created again.
|
||||
var filed = await sync.Archives.UnderLeaseAsync(local, async row =>
|
||||
{
|
||||
Id = render.Id,
|
||||
SequenceId = local.Id,
|
||||
PresetName = render.PresetName,
|
||||
OutputPath = paths.ToRelative(target),
|
||||
Codec = render.Codec,
|
||||
Width = render.Width,
|
||||
Height = render.Height,
|
||||
Fps = render.Fps,
|
||||
DurationSeconds = render.DurationSeconds,
|
||||
Bytes = new FileInfo(target).Length,
|
||||
Created = render.Created,
|
||||
ProducedBy = render.ProducedBy,
|
||||
if (row is null) throw new InvalidOperationException(Loc.L("Errors.Sync.LocalMissing"));
|
||||
var folder = paths.Sequence(row.Folder);
|
||||
if (!Directory.Exists(folder.Folder)) return false;
|
||||
Directory.CreateDirectory(folder.Exports);
|
||||
var target = Unique(folder.Exports, name);
|
||||
File.Move(part, target);
|
||||
// Recorded even if the job is being cancelled: a filed export is never left without its row.
|
||||
await db.Renders.InsertAsync(new Render
|
||||
{
|
||||
Id = render.Id,
|
||||
SequenceId = row.Id,
|
||||
PresetName = render.PresetName,
|
||||
OutputPath = paths.ToRelative(target),
|
||||
Codec = render.Codec,
|
||||
Width = render.Width,
|
||||
Height = render.Height,
|
||||
Fps = render.Fps,
|
||||
DurationSeconds = render.DurationSeconds,
|
||||
Bytes = new FileInfo(target).Length,
|
||||
Created = render.Created,
|
||||
ProducedBy = render.ProducedBy,
|
||||
}, CancellationToken.None).ConfigureAwait(false);
|
||||
return true;
|
||||
}, ct).ConfigureAwait(false);
|
||||
if (!filed) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
|
||||
await progress.StepAsync().ConfigureAwait(false);
|
||||
}
|
||||
TryDeleteDirectory(staging);
|
||||
|
||||
Reference in new issue
Block a user