A render queued before the editor had seen frames inserted under it (a sync download, an import of earlier photos, a scan without an event) embedded the project's positions in the old frame order. Render now checks first that the frames read are still the sequence's (ISequenceSource.FramesCurrentAsync: the row's frames version on this PC, the manifest on a server). On this PC the sequence is read again and the positions follow the frames; a server sequence whose frames changed turns outdated and the render is not queued. Reads of the sequence no longer overlap: a read waits for the one under way, saves that meet new frames read them through the same slot, and they save the project as it is once its positions followed. The frame on screen stays on its photo when frames are inserted or removed before it. Tests: the event-loss test waits for the listener's subscription before dropping the stream, and the conflict tests flush once more after the choice when a retry started before it was still failing. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
456 lines
22 KiB
C#
456 lines
22 KiB
C#
using System.Collections.Concurrent;
|
|
using System.IO;
|
|
using System.Net;
|
|
using System.Net.Http;
|
|
using System.Text.Json;
|
|
using Titano.Core.Advice;
|
|
using Titano.Core.Analysis;
|
|
using Titano.Core.Analysis.Deep;
|
|
using Titano.Core.Diagnostics;
|
|
using Titano.Core.Imaging;
|
|
using Titano.Core.Localization;
|
|
using Titano.Core.Model;
|
|
using Titano.Core.Project;
|
|
using Titano.Core.Render;
|
|
using Titano.Data.Archiving.Jobs;
|
|
using Titano.Data.Jobs;
|
|
using Titano.Data.Sync;
|
|
using Titano.Protocol;
|
|
using Titano.Protocol.Client;
|
|
|
|
namespace Titano.App.Editor;
|
|
|
|
/// <summary>
|
|
/// A server sequence opened with proxies (BRIEF §4.4, §14.5 "edit with proxies, render where the originals are"):
|
|
/// frames and thumbnails from this PC's proxy cache (<see cref="RemoteSequence.OpenFrames"/>), the project from the
|
|
/// server's settings with their version, the deep analysis from the server's report. Saves send the settings with the
|
|
/// version they were made from; a 409 (someone changed them meanwhile) reads the server's settings and merges again
|
|
/// (<see cref="ProjectMerge"/>) when the two changes touch different settings. A setting both changed to different
|
|
/// values is a conflict the user settles (BRIEF §14.5 "explicit conflict prompt"): <see cref="ProjectConflictException"/>,
|
|
/// and nothing is sent until the editor's next save names the server's settings as its basis. Nothing is written to a
|
|
/// local archive. Analyses and renders are server jobs; their state and progress come from the server's events.
|
|
/// Project settings name frames by position, so the cached frames must be the server's: a load that finds other frames
|
|
/// throws <see cref="ProxiesOutdatedException"/>, and frames that change on the server while the sequence is open make
|
|
/// it <see cref="IsOutdated"/> (read-only, nothing saved or rendered). Saves name the frames version the editor's frames
|
|
/// are at, so the server refuses one made against frames it no longer has even before its event arrives.
|
|
/// </summary>
|
|
public sealed class RemoteSequenceSource : ISequenceSource
|
|
{
|
|
private const int SaveAttempts = 3;
|
|
private static readonly TimeSpan MinBackoff = TimeSpan.FromSeconds(2);
|
|
private static readonly TimeSpan MaxBackoff = TimeSpan.FromMinutes(1);
|
|
|
|
private readonly RemoteSequence _remote;
|
|
private readonly IRemoteSequenceApi _api;
|
|
private readonly IRenderFrameSource _frames;
|
|
private readonly IFrameSource _pixels;
|
|
private readonly ConcurrentDictionary<Guid, Job> _jobs = new();
|
|
private readonly ConcurrentDictionary<Guid, byte> _foreign = new();
|
|
private readonly CancellationTokenSource _stopping = new();
|
|
private readonly Task _listening;
|
|
private readonly bool _noEditRight;
|
|
private Stored? _stored;
|
|
private int _framesVersion = -1;
|
|
private int _outdated;
|
|
private int _disposed;
|
|
|
|
/// <param name="readOnly">The account may look at the sequence but not change it (no Edit right on the server).</param>
|
|
public RemoteSequenceSource(RemoteSequence remote, IRemoteSequenceApi api, ImageDecoder decoder, bool readOnly = false)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(remote);
|
|
ArgumentNullException.ThrowIfNull(api);
|
|
ArgumentNullException.ThrowIfNull(decoder);
|
|
_remote = remote;
|
|
_api = api;
|
|
_noEditRight = readOnly;
|
|
_frames = remote.OpenFrames(decoder);
|
|
_pixels = (IFrameSource)_frames;
|
|
var stopping = _stopping.Token;
|
|
_listening = Task.Run(() => ListenAsync(stopping), CancellationToken.None);
|
|
}
|
|
|
|
public bool IsRemote => true;
|
|
|
|
public bool IsReadOnly => _noEditRight || IsOutdated;
|
|
|
|
public bool IsOutdated => Volatile.Read(ref _outdated) == 1;
|
|
|
|
public event Action? ReadOnlyChanged;
|
|
|
|
public event Action<Job>? JobChanged;
|
|
|
|
/// <summary>The server sequence's id.</summary>
|
|
public Guid Id => _remote.Sequence.Id;
|
|
|
|
public async Task<SequenceContent> LoadAsync(CancellationToken cancellationToken)
|
|
{
|
|
var dto = await Call(() => _api.GetSequenceAsync(Id, cancellationToken));
|
|
var manifest = await Call(() => _api.GetManifestAsync(Id, cancellationToken));
|
|
if (!SameFrames(manifest)) throw Outdated();
|
|
Volatile.Write(ref _framesVersion, manifest.FramesVersion);
|
|
var report = await Call(() => _api.GetAnalysisReportAsync(Id, cancellationToken));
|
|
return await Task.Run(() =>
|
|
{
|
|
var project = Parse(manifest.SettingsJson);
|
|
Volatile.Write(ref _stored, new Stored(project, manifest.SettingsVersion));
|
|
var frames = _remote.Frames;
|
|
var sequence = _remote.Sequence with
|
|
{
|
|
Title = dto.Title,
|
|
Tags = dto.Tags,
|
|
Rating = dto.Rating,
|
|
Notes = dto.Notes,
|
|
Status = (SequenceStatus)dto.Status,
|
|
RenderedCount = dto.RenderedCount,
|
|
HeroFrameId = dto.HeroFrameId,
|
|
WarningCount = dto.WarningCount,
|
|
PreviewOnlyCount = dto.PreviewOnlyCount,
|
|
SettingsJson = manifest.SettingsJson,
|
|
SettingsVersion = manifest.SettingsVersion,
|
|
UpdatedAt = dto.UpdatedAt,
|
|
};
|
|
var context = SequenceContext.FromFrames(frames);
|
|
var renders = manifest.Renders.OrderByDescending(r => r.Created).Select(r => new Render
|
|
{
|
|
Id = r.Id,
|
|
SequenceId = Id,
|
|
PresetName = r.PresetName,
|
|
OutputPath = string.Empty,
|
|
Codec = r.Codec,
|
|
Width = r.Width,
|
|
Height = r.Height,
|
|
Fps = r.Fps,
|
|
DurationSeconds = r.DurationSeconds,
|
|
Bytes = r.Bytes,
|
|
Created = r.Created,
|
|
ProducedBy = r.ProducedBy,
|
|
}).ToList();
|
|
return new SequenceContent(sequence, frames, project, LightAnalyzer.Analyze(frames), Report(report, frames.Count), context.SourceWidth,
|
|
context.SourceHeight, null, renders, []);
|
|
}, cancellationToken);
|
|
}
|
|
|
|
/// <remarks>Read from the server's manifest: a sequence whose frames are not this PC's becomes outdated.</remarks>
|
|
public async Task<bool> FramesCurrentAsync(CancellationToken cancellationToken)
|
|
{
|
|
if (IsOutdated) return false;
|
|
return Compare(await Call(() => _api.GetManifestAsync(Id, cancellationToken)));
|
|
}
|
|
|
|
public async ValueTask<ImageBuffer?> DecodeAsync(int index, int maxLongEdge, DecodePurpose purpose, CancellationToken cancellationToken)
|
|
{
|
|
if ((uint)index >= (uint)_remote.Frames.Count) return null;
|
|
return await Task.Run(() => _pixels.LoadAsync(index, maxLongEdge, cancellationToken).AsTask(), cancellationToken);
|
|
}
|
|
|
|
public async Task<ProjectSettings> SaveProjectAsync(ProjectSettings project, ProjectSettings basis, CancellationToken cancellationToken)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(project);
|
|
ArgumentNullException.ThrowIfNull(basis);
|
|
for (var attempt = 1; ; attempt++)
|
|
{
|
|
var (stored, version) = Volatile.Read(ref _stored) ?? throw new InvalidOperationException("LoadAsync first.");
|
|
var changedThere = !ProjectMerge.SameValues(stored, basis);
|
|
var merged = changedThere ? ProjectMerge.Merge(basis, project, stored) : project;
|
|
if (ProjectMerge.SameValues(merged, stored)) return merged;
|
|
if (IsOutdated) throw new ProxiesOutdatedException(Loc.L("Editor.Error.FramesChanged"));
|
|
if (_noEditRight) throw new EditorException(Loc.L("Editor.Error.RemoteReadOnly"));
|
|
if (changedThere && ProjectMerge.Conflicts(basis, project, stored) is { Count: > 0 } conflicts)
|
|
throw new ProjectConflictException(Loc.L("Editor.Error.SaveConflict"), stored, conflicts);
|
|
var hero = merged.HeroFrame == stored.HeroFrame ? (Guid?)null : HeroFrameId(merged);
|
|
// The positions name the frames this editor holds: the server refuses them (409 frames_conflict) once its
|
|
// frames changed, also before its event says so. A server that predates the check ignores it.
|
|
var request = new UpdateSequenceRequest(null, null, null, null, merged.ToJson(), version, hero, Volatile.Read(ref _framesVersion));
|
|
try
|
|
{
|
|
var updated = await Call(() => _api.UpdateSequenceAsync(Id, request, cancellationToken));
|
|
Volatile.Write(ref _stored, new Stored(merged, updated.SettingsVersion));
|
|
return merged;
|
|
}
|
|
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.Conflict)
|
|
{
|
|
// Changed on the server since this editor read it, its settings or its frames: other frames make the
|
|
// editor outdated (its proxies are not the server's), otherwise the edit merges again on top of the
|
|
// server's settings, at the frames version they are at.
|
|
var manifest = await Call(() => _api.GetManifestAsync(Id, cancellationToken));
|
|
if (!SameFrames(manifest)) throw Outdated();
|
|
if (attempt == SaveAttempts) throw new EditorException(Loc.L("Editor.Error.SaveConflict"), ex);
|
|
Volatile.Write(ref _framesVersion, manifest.FramesVersion);
|
|
Volatile.Write(ref _stored, new Stored(Parse(manifest.SettingsJson), manifest.SettingsVersion));
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <remarks>
|
|
/// The server queues every job at the user's priority; another priority ("Add to queue") is set right after. A
|
|
/// render embeds the project, whose frame positions mean nothing once the frames changed: refused while outdated.
|
|
/// </remarks>
|
|
public async Task<Guid> SubmitAsync(ArchiveJobRequest request, CancellationToken cancellationToken)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(request);
|
|
if (request.Kind == JobKind.Render && IsOutdated) throw new ProxiesOutdatedException(Loc.L("Editor.Error.FramesChanged"));
|
|
var dto = await Call(() => _api.CreateJobAsync(new CreateJobRequest(JobWire.Kind(request.Kind), request.Targets, request.ParametersJson), cancellationToken));
|
|
if (request.Priority != JobPriority.User) dto = await PrioritiseAsync(dto, request.Priority, cancellationToken);
|
|
// The server's events may have reported the job already; its first state never replaces a later one.
|
|
if (JobWire.FromDto(dto) is { } job && _jobs.TryAdd(job.Id, job)) JobChanged?.Invoke(job);
|
|
return dto.Id;
|
|
}
|
|
|
|
public async Task<IReadOnlyList<Job>> ListJobsAsync(CancellationToken cancellationToken)
|
|
{
|
|
var active = await Call(() => _api.ListActiveJobsAsync(cancellationToken));
|
|
var mine = active.Select(JobWire.FromDto).OfType<Job>().Where(Concerns).ToList();
|
|
foreach (var job in mine) _jobs[job.Id] = job;
|
|
return mine;
|
|
}
|
|
|
|
public async Task CancelJobAsync(Guid jobId, CancellationToken cancellationToken)
|
|
{
|
|
if (JobWire.FromDto(await _api.CancelJobAsync(jobId, cancellationToken)) is { } job) Publish(job);
|
|
}
|
|
|
|
public async Task RetryJobAsync(Guid jobId, CancellationToken cancellationToken)
|
|
{
|
|
if (JobWire.FromDto(await _api.RetryJobAsync(jobId, cancellationToken)) is { } job) Publish(job);
|
|
}
|
|
|
|
public async ValueTask DisposeAsync()
|
|
{
|
|
if (Interlocked.Exchange(ref _disposed, 1) == 1) return;
|
|
try
|
|
{
|
|
await _stopping.CancelAsync();
|
|
await _listening;
|
|
}
|
|
finally
|
|
{
|
|
_stopping.Dispose();
|
|
if (_frames is IAsyncDisposable frames) await frames.DisposeAsync();
|
|
_api.Dispose();
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Follows the server's events until the editor closes, reconnecting with a growing delay (job states lost meanwhile
|
|
/// are read back from the jobs, and the frames are compared again). Nothing that fails on the way ends it.
|
|
/// </summary>
|
|
private async Task ListenAsync(CancellationToken cancellationToken)
|
|
{
|
|
var failures = 0;
|
|
while (true)
|
|
{
|
|
try
|
|
{
|
|
await foreach (var e in _api.EventsAsync(cancellationToken).ConfigureAwait(false))
|
|
{
|
|
failures = 0;
|
|
await OnEventAsync(e, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
}
|
|
catch (Exception ex) when (ex is not OutOfMemoryException)
|
|
{
|
|
if (cancellationToken.IsCancellationRequested) return;
|
|
Log.Info("editor", $"Events of the server lost for {_remote.Sequence.DisplayTitle}: {ex.Message}");
|
|
}
|
|
try
|
|
{
|
|
var delay = Math.Min(MaxBackoff.Ticks, MinBackoff.Ticks << Math.Min(failures++, 10));
|
|
await Task.Delay(TimeSpan.FromTicks(delay), cancellationToken).ConfigureAwait(false);
|
|
await RefreshTrackedAsync(cancellationToken).ConfigureAwait(false);
|
|
// A frames change announced while the socket was down is found now rather than at the next save.
|
|
if (Volatile.Read(ref _framesVersion) >= 0) await CheckFramesAsync(cancellationToken).ConfigureAwait(false);
|
|
}
|
|
catch (Exception ex) when (ex is not OutOfMemoryException)
|
|
{
|
|
if (cancellationToken.IsCancellationRequested) return;
|
|
Log.Info("editor", $"Jobs of {_remote.Sequence.DisplayTitle} could not be read again: {ex.Message}");
|
|
}
|
|
}
|
|
}
|
|
|
|
private async Task OnEventAsync(ServerEvent e, CancellationToken cancellationToken)
|
|
{
|
|
switch (e)
|
|
{
|
|
case SequenceChangedEvent changed when changed.SequenceId == Id && changed.FramesVersion != Volatile.Read(ref _framesVersion):
|
|
await CheckFramesAsync(cancellationToken).ConfigureAwait(false);
|
|
break;
|
|
// A late progress event never brings a finished job back to running.
|
|
case JobProgressEvent progress when _jobs.TryGetValue(progress.JobId, out var job) && job.State is JobState.Queued or JobState.Running:
|
|
Publish(job with
|
|
{
|
|
State = JobState.Running,
|
|
ProgressDone = progress.Done,
|
|
ProgressTotal = progress.Total,
|
|
ItemsPerSecond = progress.ItemsPerSecond,
|
|
Eta = progress.EtaSeconds is { } eta ? TimeSpan.FromSeconds(eta) : null,
|
|
});
|
|
break;
|
|
case JobStateEvent state when !_foreign.ContainsKey(state.JobId):
|
|
await FetchAsync(state.JobId, cancellationToken).ConfigureAwait(false);
|
|
break;
|
|
}
|
|
}
|
|
|
|
/// <summary>Reads a job the events named: tracked when it concerns this sequence, remembered as foreign otherwise.</summary>
|
|
private async Task FetchAsync(Guid jobId, CancellationToken cancellationToken)
|
|
{
|
|
try
|
|
{
|
|
if (JobWire.FromDto(await _api.GetJobAsync(jobId, cancellationToken).ConfigureAwait(false)) is { } job && Concerns(job)) Publish(job);
|
|
else _foreign[jobId] = 0;
|
|
}
|
|
catch (Exception ex) when (ex is TitanoApiException or HttpRequestException or JsonException
|
|
|| (ex is OperationCanceledException && !cancellationToken.IsCancellationRequested))
|
|
{
|
|
Log.Info("editor", $"Job {jobId} could not be read: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
/// <summary>Frames added or removed on the server: whether this PC's frames are still the sequence's.</summary>
|
|
private async Task CheckFramesAsync(CancellationToken cancellationToken)
|
|
{
|
|
try
|
|
{
|
|
Compare(await _api.GetManifestAsync(Id, cancellationToken).ConfigureAwait(false));
|
|
}
|
|
catch (Exception ex) when (ex is TitanoApiException or HttpRequestException or JsonException
|
|
|| (ex is OperationCanceledException && !cancellationToken.IsCancellationRequested))
|
|
{
|
|
Log.Info("editor", $"The frames of {_remote.Sequence.DisplayTitle} could not be compared: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
/// <summary>Sets a priority other than the user's on a job the server just queued; the job stays queued if that fails.</summary>
|
|
private async Task<JobDto> PrioritiseAsync(JobDto dto, JobPriority priority, CancellationToken cancellationToken)
|
|
{
|
|
try
|
|
{
|
|
return await _api.SetJobPriorityAsync(dto.Id, JobWire.Priority(priority), cancellationToken);
|
|
}
|
|
catch (Exception ex) when (ex is TitanoApiException or HttpRequestException)
|
|
{
|
|
// Already claimed (409) or refused: the job runs at the user's priority, which is never later than asked.
|
|
Log.Info("editor", $"Job {dto.Id} keeps the user's priority on the server: {ex.Message}");
|
|
return dto;
|
|
}
|
|
}
|
|
|
|
/// <summary>After a lost event socket: the tracked jobs as the server has them now.</summary>
|
|
private async Task RefreshTrackedAsync(CancellationToken cancellationToken)
|
|
{
|
|
foreach (var job in _jobs.Values.Where(j => j.State is JobState.Queued or JobState.Running or JobState.Paused).ToList())
|
|
await FetchAsync(job.Id, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
private void Publish(Job job)
|
|
{
|
|
_jobs[job.Id] = job;
|
|
JobChanged?.Invoke(job);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Takes the server's frames as <paramref name="manifest"/> lists them: their frames version when they are this PC's
|
|
/// (saves name it), outdated otherwise. Returns whether they are.
|
|
/// </summary>
|
|
private bool Compare(SequenceManifest manifest)
|
|
{
|
|
if (!SameFrames(manifest))
|
|
{
|
|
MarkOutdated();
|
|
return false;
|
|
}
|
|
Volatile.Write(ref _framesVersion, manifest.FramesVersion);
|
|
return true;
|
|
}
|
|
|
|
/// <summary>The server's frames are the cached ones, in the same order (frame positions in the project mean the same frames).</summary>
|
|
private bool SameFrames(SequenceManifest manifest) =>
|
|
manifest.Frames.OrderBy(f => f.Index).Select(f => f.Id).SequenceEqual(_remote.Frames.Select(f => f.Id));
|
|
|
|
/// <summary>The frames changed on the server: from now on the project is only shown.</summary>
|
|
private ProxiesOutdatedException Outdated()
|
|
{
|
|
MarkOutdated();
|
|
return new ProxiesOutdatedException(Loc.L("Editor.Error.FramesChanged"));
|
|
}
|
|
|
|
private void MarkOutdated()
|
|
{
|
|
if (Interlocked.Exchange(ref _outdated, 1) == 1) return;
|
|
Log.Info("editor", $"The frames of {_remote.Sequence.DisplayTitle} changed on the server; its proxies are out of date");
|
|
ReadOnlyChanged?.Invoke();
|
|
}
|
|
|
|
private bool Concerns(Job job) => job.TargetIds.Contains(Id) && job.Kind is JobKind.DeepAnalyse or JobKind.LightAnalyse or JobKind.Render;
|
|
|
|
/// <summary>The frame of the project's hero, or <see cref="Guid.Empty"/> (the first frame shows) without one.</summary>
|
|
private Guid HeroFrameId(ProjectSettings project) =>
|
|
project.HeroFrame is { } hero && (uint)hero < (uint)_remote.Frames.Count ? _remote.Frames[hero].Id : Guid.Empty;
|
|
|
|
/// <exception cref="EditorException">The settings are not a project or come from a newer Titano.</exception>
|
|
private static ProjectSettings Parse(string? json)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(json)) return new ProjectSettings();
|
|
try
|
|
{
|
|
return ProjectSettings.FromJson(json);
|
|
}
|
|
catch (Exception ex) when (ex is JsonException or InvalidDataException)
|
|
{
|
|
throw new EditorException(Loc.L("Editor.Error.Project"), ex);
|
|
}
|
|
}
|
|
|
|
/// <summary>The server's deep report when it describes the frames this PC has.</summary>
|
|
private DeepAnalysisReport? Report(string? json, int frameCount)
|
|
{
|
|
if (json is null) return null;
|
|
try
|
|
{
|
|
var report = DeepAnalysisReport.FromJson(json);
|
|
return report.FrameCount == frameCount ? report : null;
|
|
}
|
|
catch (Exception ex) when (ex is JsonException or InvalidDataException)
|
|
{
|
|
Log.Warning("editor", $"The analysis of {_remote.Sequence.DisplayTitle} on the server cannot be read", ex);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/// <summary>A sequence deleted on the server reads as gone, like a local one removed from the index.</summary>
|
|
private static async Task<T> Call<T>(Func<Task<T>> call)
|
|
{
|
|
try
|
|
{
|
|
return await call();
|
|
}
|
|
catch (TitanoApiException ex) when (ex.Status == HttpStatusCode.NotFound)
|
|
{
|
|
throw new EditorException(Loc.L("Editor.Error.Missing"), ex);
|
|
}
|
|
}
|
|
|
|
/// <summary>The server's settings as last read or written, with the version a save must name.</summary>
|
|
private sealed record Stored(ProjectSettings Project, int Version);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Settings the editor changed were changed to other values on the server meanwhile (BRIEF §14.5): nothing is saved
|
|
/// until the user keeps the editor's values or takes the server's (<see cref="EditorSession.ResolveConflict"/>).
|
|
/// </summary>
|
|
/// <param name="server">The server's project now.</param>
|
|
/// <param name="differences">The settings both changed: the editor's value, then the server's.</param>
|
|
public sealed class ProjectConflictException(string message, ProjectSettings server, IReadOnlyList<SettingsDifference> differences) : IOException(message)
|
|
{
|
|
public ProjectSettings Server { get; } = server;
|
|
|
|
public IReadOnlyList<SettingsDifference> Differences { get; } = differences;
|
|
}
|
|
|
|
/// <summary>
|
|
/// The proxies of a server sequence are for frames the server no longer has (added or removed since they were fetched):
|
|
/// they must be fetched again before the sequence can be edited.
|
|
/// </summary>
|
|
public sealed class ProxiesOutdatedException(string message) : EditorException(message);
|