using System.Collections.Concurrent; using System.Net; using System.Net.Http; using System.Text.Json; using Chronos.Core.Diagnostics; using Chronos.Core.Localization; using Chronos.Core.Model; using Chronos.Data.Archiving.Jobs; using Chronos.Data.Jobs; using Chronos.Protocol; using Chronos.Protocol.Client; namespace Chronos.App.Editor; /// /// The analyses and renders of one server sequence as an editor follows them (BRIEF ยง14.5): queued through the API (the /// server queues each at the user's priority, another one is set right after), their states and progress taken from the /// server's events, the jobs followed read again whenever the event socket comes back after a loss (reconnecting with a /// growing delay; nothing that fails on the way ends it). What a server sequence opened with proxies /// () and a workspace project whose home is a server (, /// owner review 2, R8) run on the server. Events of other kinds go to other; reconnected runs after the jobs /// were read again. Owns the connection: disposing stops listening and disposes it. /// internal sealed class ServerJobs : IAsyncDisposable { private static readonly TimeSpan MinBackoff = TimeSpan.FromSeconds(2); private static readonly TimeSpan MaxBackoff = TimeSpan.FromMinutes(1); private readonly Guid _sequenceId; private readonly string _title; private readonly Action _changed; private readonly Func? _other; private readonly Func? _reconnected; private readonly ConcurrentDictionary _jobs = new(); private readonly ConcurrentDictionary _foreign = new(); private readonly CancellationTokenSource _stopping = new(); private readonly Task _listening; private int _disposed; /// A job of the sequence changed (any thread). public ServerJobs(IRemoteSequenceApi api, Guid sequenceId, string title, Action changed, Func? other = null, Func? reconnected = null) { ArgumentNullException.ThrowIfNull(api); ArgumentNullException.ThrowIfNull(changed); Api = api; _sequenceId = sequenceId; _title = title; _changed = changed; _other = other; _reconnected = reconnected; var stopping = _stopping.Token; _listening = Task.Run(() => ListenAsync(stopping), CancellationToken.None); } /// The connection the jobs travel on (the source's other calls use it too). public IRemoteSequenceApi Api { get; } /// Queues on the server (its priority set after, when it is not the user's); returns the job id. /// The sequence is gone from the server. public async Task SubmitAsync(ArchiveJobRequest request, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(request); var dto = await Call(() => Api.CreateJobAsync(new CreateJobRequest(JobWire.Kind(request.Kind), request.Targets, request.ParametersJson), cancellationToken)); if (request.Priority != JobPriority.User) dto = await PrioritiseAsync(dto, request.Priority, cancellationToken); // The server's events may have reported the job already; its first state never replaces a later one. if (JobWire.FromDto(dto) is { } job && _jobs.TryAdd(job.Id, job)) _changed(job); return dto.Id; } /// The analyses and renders of the sequence waiting, running or paused on the server, followed from now on. public async Task> ListAsync(CancellationToken cancellationToken) { var active = await Call(() => Api.ListActiveJobsAsync(cancellationToken)); var mine = active.Select(JobWire.FromDto).OfType().Where(Concerns).ToList(); foreach (var job in mine) _jobs[job.Id] = job; return mine; } public async Task CancelAsync(Guid jobId, CancellationToken cancellationToken) { if (JobWire.FromDto(await Api.CancelJobAsync(jobId, cancellationToken)) is { } job) Publish(job); } public async Task RetryAsync(Guid jobId, CancellationToken cancellationToken) { if (JobWire.FromDto(await Api.RetryJobAsync(jobId, cancellationToken)) is { } job) Publish(job); } public async ValueTask DisposeAsync() { if (Interlocked.Exchange(ref _disposed, 1) == 1) return; try { await _stopping.CancelAsync(); await _listening; } finally { _stopping.Dispose(); Api.Dispose(); } } /// /// Follows the server's events until disposed, reconnecting with a growing delay (job states lost meanwhile are read /// back from the jobs, then reconnected runs). Nothing that fails on the way ends it. /// private async Task ListenAsync(CancellationToken cancellationToken) { var failures = 0; while (true) { try { await foreach (var e in Api.EventsAsync(cancellationToken).ConfigureAwait(false)) { failures = 0; await OnEventAsync(e, cancellationToken).ConfigureAwait(false); } } catch (Exception ex) when (ex is not OutOfMemoryException) { if (cancellationToken.IsCancellationRequested) return; Log.Info("editor", $"Events of the server lost for {_title}: {ex.Message}"); } try { var delay = Math.Min(MaxBackoff.Ticks, MinBackoff.Ticks << Math.Min(failures++, 10)); await Task.Delay(TimeSpan.FromTicks(delay), cancellationToken).ConfigureAwait(false); await RefreshTrackedAsync(cancellationToken).ConfigureAwait(false); if (_reconnected is not null) await _reconnected(cancellationToken).ConfigureAwait(false); } catch (Exception ex) when (ex is not OutOfMemoryException) { if (cancellationToken.IsCancellationRequested) return; Log.Info("editor", $"Jobs of {_title} could not be read again: {ex.Message}"); } } } private async Task OnEventAsync(ServerEvent e, CancellationToken cancellationToken) { switch (e) { // A late progress event never brings a finished job back to running. case JobProgressEvent progress when _jobs.TryGetValue(progress.JobId, out var job) && job.State is JobState.Queued or JobState.Running: Publish(job with { State = JobState.Running, ProgressDone = progress.Done, ProgressTotal = progress.Total, ItemsPerSecond = progress.ItemsPerSecond, Eta = JobEta.FromSeconds(progress.EtaSeconds), }); break; case JobProgressEvent: break; case JobStateEvent state: if (!_foreign.ContainsKey(state.JobId)) await FetchAsync(state.JobId, cancellationToken).ConfigureAwait(false); break; default: if (_other is not null) await _other(e, cancellationToken).ConfigureAwait(false); break; } } /// Reads a job the events named: tracked when it concerns this sequence, remembered as foreign otherwise. private async Task FetchAsync(Guid jobId, CancellationToken cancellationToken) { try { if (JobWire.FromDto(await Api.GetJobAsync(jobId, cancellationToken).ConfigureAwait(false)) is { } job && Concerns(job)) Publish(job); else _foreign[jobId] = 0; } catch (Exception ex) when (ex is ChronosApiException or HttpRequestException or JsonException || (ex is OperationCanceledException && !cancellationToken.IsCancellationRequested)) { Log.Info("editor", $"Job {jobId} could not be read: {ex.Message}"); } } /// Sets a priority other than the user's on a job the server just queued; the job stays queued if that fails. private async Task PrioritiseAsync(JobDto dto, JobPriority priority, CancellationToken cancellationToken) { try { return await Api.SetJobPriorityAsync(dto.Id, JobWire.Priority(priority), cancellationToken); } catch (Exception ex) when (ex is ChronosApiException or HttpRequestException) { // Already claimed (409) or refused: the job runs at the user's priority, which is never later than asked. Log.Info("editor", $"Job {dto.Id} keeps the user's priority on the server: {ex.Message}"); return dto; } } /// After a lost event socket: the tracked jobs as the server has them now. private async Task RefreshTrackedAsync(CancellationToken cancellationToken) { foreach (var job in _jobs.Values.Where(j => j.State is JobState.Queued or JobState.Running or JobState.Paused).ToList()) await FetchAsync(job.Id, cancellationToken).ConfigureAwait(false); } private void Publish(Job job) { _jobs[job.Id] = job; _changed(job); } private bool Concerns(Job job) => job.TargetIds.Contains(_sequenceId) && SequenceJobs.IsSequenceKind(job.Kind); /// A sequence deleted on the server reads as gone, like a local one removed from the index. internal static async Task Call(Func> call) { try { return await call(); } catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.NotFound) { throw new EditorException(Loc.L("Editor.Error.Missing"), ex); } } }