using System.Collections.ObjectModel; using System.Net.Http; using System.Text.Json; using Chronos.App.Infrastructure; using Chronos.App.Jobs; using Chronos.Core.Diagnostics; using Chronos.Core.Localization; using Chronos.Core.Model; using Chronos.Data.Jobs; using Chronos.Protocol; using Chronos.Protocol.Client; using Chronos.Ui.Controls; using Chronos.Ui.Mvvm; namespace Chronos.App.Server; /// /// Who is signed in, as the Server section's job list sees it: names the session (a new sign-in is another /// list), whether the account may pause, resume and cancel any of the server's jobs. /// public sealed record ServerJobsAccount(string Key, bool CanControl); /// /// The jobs running or paused on the signed-in server, live (owner review 5, #65 and the owner's answer to Q2): read from the /// server's jobs API, then followed through its events ( with what each job does now, /// ), at most four updates a second, and read again whenever the event socket comes back after a /// loss (with a growing delay; nothing that fails on the way ends it). Queued jobs are the admin panel's (a busy server /// queues thumbnails by the thousand). Pause, Resume and Cancel are offered only to an account with the right to manage /// the archive, which the server requires to act on a job someone else started; otherwise the list is read-only. Empty and /// idle while signed out or without the right to read. UI thread, except where said. /// public sealed class ServerJobsViewModel : ObservableObject, IDisposable { private static readonly TimeSpan FlushInterval = TimeSpan.FromMilliseconds(250); private static readonly TimeSpan MinBackoff = TimeSpan.FromSeconds(2); private static readonly TimeSpan MaxBackoff = TimeSpan.FromMinutes(1); private static readonly string[] ListedStates = [JobWire.State(JobState.Running), JobWire.State(JobState.Paused)]; private readonly JobListModel _model = new(); private readonly Updates _updates = new(); private readonly Dictionary _rows = []; private readonly Func _account; private readonly Func _connect; private readonly Action? _failed; private readonly CoalescingThrottle _throttle; private Watch? _watch; private bool _canControl; private bool _disposed; /// The signed-in account; null while signed out or without the right to read the server's jobs. /// A connection for the signed-in session; null when there is none. /// Where updates are applied (the UI); null applies them on a timer thread. /// Says that an action failed: a title and the reason. public ServerJobsViewModel(Func account, Func connect, SynchronizationContext? context, Action? failed = null) { ArgumentNullException.ThrowIfNull(account); ArgumentNullException.ThrowIfNull(connect); _account = account; _connect = connect; _failed = failed; _throttle = new CoalescingThrottle(FlushInterval, context, Flush); } /// The account of : none while signed out or without the Read right; control with ManageArchive. public static ServerJobsAccount? AccountOf(ServerSession session) { ArgumentNullException.ThrowIfNull(session); if (session.User is not { } user || session.Flow.Address is not { } address || session.Flow.Token is not { } token) return null; var permissions = (Permissions)user.Permissions; if (!permissions.HasFlag(Permissions.Read)) return null; return new ServerJobsAccount($"{ServerAddress.Display(address)}|{user.Id}|{token}", permissions.HasFlag(Permissions.ManageArchive)); } /// A connection over for , pinned to its trusted certificates; null while signed out. public static IServerJobsApi? ConnectionOf(ServerSession session) { ArgumentNullException.ThrowIfNull(session); return session.IsConnected && session.Flow is { Address: { } address, Token: { } token } ? new ChronosServerJobsApi(new ChronosClient(address, new CertificatePinning(session.Fingerprints), $"Chronos ({Environment.MachineName})") { Token = token }) : null; } /// Raised after the listener posted an update (any thread; tests wait on it). internal event Action? Posted; /// The running jobs first (oldest start first), then the paused ones. public ObservableCollection Rows { get; } = []; /// The list shows: a session is followed and the server runs or holds something. public bool HasRows => Rows.Count > 0; /// The account may pause, resume and cancel the server's jobs; without it the list is read-only. public bool CanControl => _canControl; /// What the list shows, and whether the account can act on it. public string TitleToolTip => Loc.L(_canControl ? "Tooltips.Server.Jobs" : "Tooltips.Server.Jobs.ReadOnly"); /// A session is being followed. internal bool IsWatching => _watch is not null; /// Follows the signed-in session, or stops when it ended or changed (call when the session changes). public void Refresh() { if (_disposed) return; var account = _account(); if (account?.CanControl != _canControl) { _canControl = account?.CanControl ?? false; foreach (var row in _rows.Values) row.Refresh(); OnPropertyChanged(nameof(CanControl)); OnPropertyChanged(nameof(TitleToolTip)); } if (_watch?.Key == account?.Key) return; Stop(); if (account is not null && _connect() is { } api) { _watch = new Watch(account.Key, api, new CancellationTokenSource()); var watch = _watch; watch.Listening = Task.Run(() => ListenAsync(watch), CancellationToken.None); } } /// Reads the server's running and paused jobs again (a picture of the list as it is now). public async Task ReloadAsync() { if (_watch is not { } watch) return; await LoadAsync(watch.Api, watch.Stopping.Token); Flush(); } /// Applies what the listener collected since the last flush (UI thread). internal void Flush() { var (reset, jobs, removed, progress) = _updates.Drain(); if (reset is not null) { var listed = reset.Select(j => j.Id).ToHashSet(); foreach (var entry in _model.Ordered().Where(e => !listed.Contains(e.Id)).ToList()) _model.Remove(entry.Id); foreach (var job in reset) _model.Apply(job); } foreach (var job in jobs) { if (IsListed(job.State)) _model.Apply(job); else _model.Remove(job.Id); } foreach (var id in removed) _model.Remove(id); foreach (var p in progress) _model.Apply(p); SyncRows(); } public void Dispose() { if (_disposed) return; _disposed = true; Stop(); _throttle.Dispose(); } private static bool IsListed(JobState state) => state is JobState.Running or JobState.Paused; private void Stop() { if (Interlocked.Exchange(ref _watch, null) is not { } watch) return; watch.Stopping.Cancel(); // The listener disposes its connection once it has stopped. _updates.Reset([]); Flush(); } /// /// Lists the jobs, then follows the events until stopped; after a loss, waits with a growing delay and lists again /// (events missed meanwhile are in that list). Nothing that fails on the way ends it. Disposes the connection at the end. /// private async Task ListenAsync(Watch watch) { var ct = watch.Stopping.Token; var failures = 0; try { while (!ct.IsCancellationRequested) { try { await LoadAsync(watch.Api, ct).ConfigureAwait(false); await foreach (var e in watch.Api.EventsAsync(ct).ConfigureAwait(false)) { failures = 0; await OnEventAsync(watch.Api, e, ct).ConfigureAwait(false); } } catch (Exception ex) when (ex is not OutOfMemoryException) { if (ct.IsCancellationRequested) return; Log.Info("server", $"Jobs of the server lost: {ex.Message}"); } var delay = Math.Min(MaxBackoff.Ticks, MinBackoff.Ticks << Math.Min(failures++, 10)); await Task.Delay(TimeSpan.FromTicks(delay), ct).ConfigureAwait(false); } } catch (OperationCanceledException) when (ct.IsCancellationRequested) { } finally { watch.Api.Dispose(); watch.Stopping.Dispose(); } } /// Every running and paused job of the server, all pages (any thread). private async Task LoadAsync(IServerJobsApi api, CancellationToken ct) { var jobs = new List(); foreach (var state in ListedStates) { string? cursor = null; do { var page = await api.ListJobsAsync(state, cursor, ct).ConfigureAwait(false); jobs.AddRange(page.Items.Select(JobWire.FromDto).OfType().Where(j => IsListed(j.State))); cursor = page.NextCursor; } while (cursor is not null); } _updates.Reset(jobs); Signal(); } private async Task OnEventAsync(IServerJobsApi api, ServerEvent e, CancellationToken ct) { switch (e) { case JobProgressEvent p: // Kind and target come from the listed job; what matters here is the count, the pace and the detail. _updates.Post(new JobProgressInfo(p.JobId, default, null, p.Done, p.Total, p.ItemsPerSecond, JobEta.FromSeconds(p.EtaSeconds), null, JobWire.FromDto(p.Detail))); Signal(); break; case JobStateEvent s when JobWire.TryParseState(s.State, out var state) && IsListed(state): await FetchAsync(api, s.JobId, ct).ConfigureAwait(false); break; case JobStateEvent s: // Queued again, finished or of a state this build does not know: it leaves the list. _updates.Remove(s.JobId); Signal(); break; } } private async Task FetchAsync(IServerJobsApi api, Guid jobId, CancellationToken ct) { try { if (JobWire.FromDto(await api.GetJobAsync(jobId, ct).ConfigureAwait(false)) is { } job) _updates.Post(job); else _updates.Remove(jobId); Signal(); } catch (Exception ex) when (ex is ChronosApiException or HttpRequestException or JsonException || (ex is OperationCanceledException && !ct.IsCancellationRequested)) { Log.Info("server", $"Job {jobId} of the server could not be read: {ex.Message}"); } } private void Signal() { _throttle.Signal(); Posted?.Invoke(); } /// Pauses, resumes or cancels a job on the server; what the server answers shows at once (UI thread). internal async Task ActAsync(Guid jobId, Func> action) { if (_watch is not { } watch || !_canControl) return; try { var dto = await action(watch.Api, jobId, watch.Stopping.Token); if (JobWire.FromDto(dto) is { } job) _updates.Post(job); else _updates.Remove(jobId); Flush(); } catch (Exception ex) when (ex is ChronosApiException or HttpRequestException or JsonException or OperationCanceledException or ObjectDisposedException) { if (watch.Stopping.IsCancellationRequested) return; Log.Info("server", $"Job {jobId} of the server not changed: {ex.Message}"); // The server words its refusal in this PC's language (Accept-Language); anything else is a lost connection. _failed?.Invoke(Loc.L("Server.Jobs.NotChanged"), ex is ChronosApiException { Error.Message: { Length: > 0 } reason } ? reason : Loc.L("Server.Error.Unreachable")); } } /// Brings to the model's order with moves. private void SyncRows() { var ordered = _model.Ordered(); var wanted = ordered.Select(e => e.Id).ToHashSet(); for (var i = Rows.Count - 1; i >= 0; i--) { if (wanted.Contains(Rows[i].Id)) continue; _rows.Remove(Rows[i].Id); Rows.RemoveAt(i); } for (var i = 0; i < ordered.Count; i++) { var entry = ordered[i]; if (!_rows.TryGetValue(entry.Id, out var row)) { row = new ServerJobRowViewModel(entry, this); _rows[entry.Id] = row; } else { row.Update(entry); } if (i < Rows.Count && ReferenceEquals(Rows[i], row)) continue; var existing = Rows.IndexOf(row); if (existing >= 0) Rows.Move(existing, i); else Rows.Insert(i, row); } OnPropertyChanged(nameof(HasRows)); } /// The session followed: its key, its connection and what stops its listener. private sealed class Watch(string key, IServerJobsApi api, CancellationTokenSource stopping) { public string Key { get; } = key; public IServerJobsApi Api { get; } = api; public CancellationTokenSource Stopping { get; } = stopping; public Task? Listening { get; set; } } /// /// What the listener collected for the next flush (thread-safe): a new list, which drops everything collected before it; /// then per job its newest snapshot or its removal, and its newest progress. /// private sealed class Updates { private readonly Lock _gate = new(); private IReadOnlyList? _reset; private Dictionary _jobs = []; private HashSet _removed = []; private Dictionary _progress = []; public void Reset(IReadOnlyList jobs) { lock (_gate) { _reset = jobs; _jobs = []; _removed = []; _progress = []; } } public void Post(Job job) { lock (_gate) { _jobs[job.Id] = job; _removed.Remove(job.Id); _progress.Remove(job.Id); } } public void Remove(Guid id) { lock (_gate) { _jobs.Remove(id); _progress.Remove(id); _removed.Add(id); } } public void Post(JobProgressInfo progress) { lock (_gate) _progress[progress.JobId] = progress; } public (IReadOnlyList? Reset, IReadOnlyCollection Jobs, IReadOnlyCollection Removed, IReadOnlyCollection Progress) Drain() { lock (_gate) { var drained = (_reset, (IReadOnlyCollection)_jobs.Values, (IReadOnlyCollection)_removed, (IReadOnlyCollection)_progress.Values); _reset = null; _jobs = []; _removed = []; _progress = []; return drained; } } } } /// /// One job of the server as the Server section lists it: kind, target with what it does now (or where a paused one stopped), /// progress, speed, time left, state, and Pause / Resume / Cancel when the account may. /// public sealed class ServerJobRowViewModel : ObservableObject { private readonly ServerJobsViewModel _owner; private JobEntry _entry; internal ServerJobRowViewModel(JobEntry entry, ServerJobsViewModel owner) { _entry = entry; _owner = owner; } public Guid Id => _entry.Id; public JobEntry Entry => _entry; public string Kind => JobText.Kind(_entry.Job.Kind); public string Target => _entry.Job.TargetLabel ?? string.Empty; /// "Verify · Harbour sunset": the row's name for screen readers. public string AccessibleName => JobText.Name(_entry.Job); /// What it does now ("Verifying photo 812 of 2,400 · Harbour sunset · sequence 3 of 41"), or "Stopped at …". public string StatusText => JobText.RowStatus(_entry); public bool HasStatus => StatusText.Length > 0; public string StatusToolTip => JobText.RowStatusToolTip(_entry); public double Progress => _entry.Fraction ?? 0; public bool IsIndeterminate => _entry.State == JobState.Running && _entry.Fraction is null; public string PercentText => _entry.Fraction is { } f ? Formatting.Percent(f) : "—"; public string SpeedText => _entry.State == JobState.Running ? Blank(JobText.Speed(_entry.Job.Kind, _entry.ItemsPerSecond)) : "—"; public string LeftText => Blank(JobText.Left(_entry, 0)); public string StateText => JobText.State(_entry); public StatusKind StateKind => _entry.State == JobState.Paused ? StatusKind.Paused : StatusKind.Running; public string StateToolTip => string.Join(Loc.Separator, StateText, Loc.L("Tooltips.Jobs.State." + _entry.State)); /// "Verify · Harbour sunset: running": what a screen reader announces when the state changes (not progress). public string StateAnnouncement => Loc.L("Jobs.Row.State", AccessibleName, StateText); /// Pause and Cancel while it runs, Resume and Cancel while paused; none for a read-only account. public IReadOnlyList Actions { get { if (!_owner.CanControl) return []; var id = _entry.Id; return _entry.State == JobState.Paused ? [Action("Jobs.Action.Resume", "Tooltips.Jobs.Resume", (api, job, ct) => api.ResumeJobAsync(job, ct)), Action("Jobs.Action.Cancel", "Tooltips.Jobs.Cancel", (api, job, ct) => api.CancelJobAsync(job, ct))] : [Action("Jobs.Action.Pause", "Tooltips.Jobs.Pause", (api, job, ct) => api.PauseJobAsync(job, ct)), Action("Jobs.Action.Cancel", "Tooltips.Jobs.Cancel", (api, job, ct) => api.CancelJobAsync(job, ct))]; JobAction Action(string textKey, string toolTipKey, Func> run) { var text = Loc.L(textKey); return new JobAction(text, Loc.L(toolTipKey), Loc.L("Jobs.ActionFor", text, AccessibleName), new AsyncRelayCommand(() => _owner.ActAsync(id, run))); } } } internal void Update(JobEntry entry) { var stateChanged = entry.State != _entry.State; _entry = entry; if (stateChanged) { OnAllPropertiesChanged(); return; } OnPropertyChanged(nameof(StatusText)); OnPropertyChanged(nameof(HasStatus)); OnPropertyChanged(nameof(StatusToolTip)); OnPropertyChanged(nameof(Progress)); OnPropertyChanged(nameof(IsIndeterminate)); OnPropertyChanged(nameof(PercentText)); OnPropertyChanged(nameof(SpeedText)); OnPropertyChanged(nameof(LeftText)); } internal void Refresh() => OnAllPropertiesChanged(); private static string Blank(string text) => text.Length == 0 ? "—" : text; }