using System.Collections.ObjectModel;
using System.Net.Http;
using System.Text.Json;
using Chronos.App.Infrastructure;
using Chronos.App.Jobs;
using Chronos.Core.Diagnostics;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Data.Jobs;
using Chronos.Protocol;
using Chronos.Protocol.Client;
using Chronos.Ui.Controls;
using Chronos.Ui.Mvvm;
namespace Chronos.App.Server;
///
/// Who is signed in, as the Server section's job list sees it: names the session (a new sign-in is another
/// list), whether the account may pause, resume and cancel any of the server's jobs.
///
public sealed record ServerJobsAccount(string Key, bool CanControl);
///
/// The jobs running or paused on the signed-in server, live (owner review 5, #65 and the owner's answer to Q2): read from the
/// server's jobs API, then followed through its events ( with what each job does now,
/// ), at most four updates a second, and read again whenever the event socket comes back after a
/// loss (with a growing delay; nothing that fails on the way ends it). Queued jobs are the admin panel's (a busy server
/// queues thumbnails by the thousand). Pause, Resume and Cancel are offered only to an account with the right to manage
/// the archive, which the server requires to act on a job someone else started; otherwise the list is read-only. Empty and
/// idle while signed out or without the right to read. UI thread, except where said.
///
public sealed class ServerJobsViewModel : ObservableObject, IDisposable
{
private static readonly TimeSpan FlushInterval = TimeSpan.FromMilliseconds(250);
private static readonly TimeSpan MinBackoff = TimeSpan.FromSeconds(2);
private static readonly TimeSpan MaxBackoff = TimeSpan.FromMinutes(1);
private static readonly string[] ListedStates = [JobWire.State(JobState.Running), JobWire.State(JobState.Paused)];
private readonly JobListModel _model = new();
private readonly Updates _updates = new();
private readonly Dictionary _rows = [];
private readonly Func _account;
private readonly Func _connect;
private readonly Action? _failed;
private readonly CoalescingThrottle _throttle;
private Watch? _watch;
private bool _canControl;
private bool _disposed;
/// The signed-in account; null while signed out or without the right to read the server's jobs.
/// A connection for the signed-in session; null when there is none.
/// Where updates are applied (the UI); null applies them on a timer thread.
/// Says that an action failed: a title and the reason.
public ServerJobsViewModel(Func account, Func connect, SynchronizationContext? context, Action? failed = null)
{
ArgumentNullException.ThrowIfNull(account);
ArgumentNullException.ThrowIfNull(connect);
_account = account;
_connect = connect;
_failed = failed;
_throttle = new CoalescingThrottle(FlushInterval, context, Flush);
}
/// The account of : none while signed out or without the Read right; control with ManageArchive.
public static ServerJobsAccount? AccountOf(ServerSession session)
{
ArgumentNullException.ThrowIfNull(session);
if (session.User is not { } user || session.Flow.Address is not { } address || session.Flow.Token is not { } token) return null;
var permissions = (Permissions)user.Permissions;
if (!permissions.HasFlag(Permissions.Read)) return null;
return new ServerJobsAccount($"{ServerAddress.Display(address)}|{user.Id}|{token}", permissions.HasFlag(Permissions.ManageArchive));
}
/// A connection over for , pinned to its trusted certificates; null while signed out.
public static IServerJobsApi? ConnectionOf(ServerSession session)
{
ArgumentNullException.ThrowIfNull(session);
return session.IsConnected && session.Flow is { Address: { } address, Token: { } token }
? new ChronosServerJobsApi(new ChronosClient(address, new CertificatePinning(session.Fingerprints), $"Chronos ({Environment.MachineName})") { Token = token })
: null;
}
/// Raised after the listener posted an update (any thread; tests wait on it).
internal event Action? Posted;
/// The running jobs first (oldest start first), then the paused ones.
public ObservableCollection Rows { get; } = [];
/// The list shows: a session is followed and the server runs or holds something.
public bool HasRows => Rows.Count > 0;
/// The account may pause, resume and cancel the server's jobs; without it the list is read-only.
public bool CanControl => _canControl;
/// What the list shows, and whether the account can act on it.
public string TitleToolTip => Loc.L(_canControl ? "Tooltips.Server.Jobs" : "Tooltips.Server.Jobs.ReadOnly");
/// A session is being followed.
internal bool IsWatching => _watch is not null;
/// Follows the signed-in session, or stops when it ended or changed (call when the session changes).
public void Refresh()
{
if (_disposed) return;
var account = _account();
if (account?.CanControl != _canControl)
{
_canControl = account?.CanControl ?? false;
foreach (var row in _rows.Values) row.Refresh();
OnPropertyChanged(nameof(CanControl));
OnPropertyChanged(nameof(TitleToolTip));
}
if (_watch?.Key == account?.Key) return;
Stop();
if (account is not null && _connect() is { } api)
{
_watch = new Watch(account.Key, api, new CancellationTokenSource());
var watch = _watch;
watch.Listening = Task.Run(() => ListenAsync(watch), CancellationToken.None);
}
}
/// Reads the server's running and paused jobs again (a picture of the list as it is now).
public async Task ReloadAsync()
{
if (_watch is not { } watch) return;
await LoadAsync(watch.Api, watch.Stopping.Token);
Flush();
}
/// Applies what the listener collected since the last flush (UI thread).
internal void Flush()
{
var (reset, jobs, removed, progress) = _updates.Drain();
if (reset is not null)
{
var listed = reset.Select(j => j.Id).ToHashSet();
foreach (var entry in _model.Ordered().Where(e => !listed.Contains(e.Id)).ToList()) _model.Remove(entry.Id);
foreach (var job in reset) _model.Apply(job);
}
foreach (var job in jobs)
{
if (IsListed(job.State)) _model.Apply(job);
else _model.Remove(job.Id);
}
foreach (var id in removed) _model.Remove(id);
foreach (var p in progress) _model.Apply(p);
SyncRows();
}
public void Dispose()
{
if (_disposed) return;
_disposed = true;
Stop();
_throttle.Dispose();
}
private static bool IsListed(JobState state) => state is JobState.Running or JobState.Paused;
private void Stop()
{
if (Interlocked.Exchange(ref _watch, null) is not { } watch) return;
watch.Stopping.Cancel();
// The listener disposes its connection once it has stopped.
_updates.Reset([]);
Flush();
}
///
/// Lists the jobs, then follows the events until stopped; after a loss, waits with a growing delay and lists again
/// (events missed meanwhile are in that list). Nothing that fails on the way ends it. Disposes the connection at the end.
///
private async Task ListenAsync(Watch watch)
{
var ct = watch.Stopping.Token;
var failures = 0;
try
{
while (!ct.IsCancellationRequested)
{
try
{
await LoadAsync(watch.Api, ct).ConfigureAwait(false);
await foreach (var e in watch.Api.EventsAsync(ct).ConfigureAwait(false))
{
failures = 0;
await OnEventAsync(watch.Api, e, ct).ConfigureAwait(false);
}
}
catch (Exception ex) when (ex is not OutOfMemoryException)
{
if (ct.IsCancellationRequested) return;
Log.Info("server", $"Jobs of the server lost: {ex.Message}");
}
var delay = Math.Min(MaxBackoff.Ticks, MinBackoff.Ticks << Math.Min(failures++, 10));
await Task.Delay(TimeSpan.FromTicks(delay), ct).ConfigureAwait(false);
}
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
}
finally
{
watch.Api.Dispose();
watch.Stopping.Dispose();
}
}
/// Every running and paused job of the server, all pages (any thread).
private async Task LoadAsync(IServerJobsApi api, CancellationToken ct)
{
var jobs = new List();
foreach (var state in ListedStates)
{
string? cursor = null;
do
{
var page = await api.ListJobsAsync(state, cursor, ct).ConfigureAwait(false);
jobs.AddRange(page.Items.Select(JobWire.FromDto).OfType().Where(j => IsListed(j.State)));
cursor = page.NextCursor;
}
while (cursor is not null);
}
_updates.Reset(jobs);
Signal();
}
private async Task OnEventAsync(IServerJobsApi api, ServerEvent e, CancellationToken ct)
{
switch (e)
{
case JobProgressEvent p:
// Kind and target come from the listed job; what matters here is the count, the pace and the detail.
_updates.Post(new JobProgressInfo(p.JobId, default, null, p.Done, p.Total, p.ItemsPerSecond, JobEta.FromSeconds(p.EtaSeconds), null,
JobWire.FromDto(p.Detail)));
Signal();
break;
case JobStateEvent s when JobWire.TryParseState(s.State, out var state) && IsListed(state):
await FetchAsync(api, s.JobId, ct).ConfigureAwait(false);
break;
case JobStateEvent s:
// Queued again, finished or of a state this build does not know: it leaves the list.
_updates.Remove(s.JobId);
Signal();
break;
}
}
private async Task FetchAsync(IServerJobsApi api, Guid jobId, CancellationToken ct)
{
try
{
if (JobWire.FromDto(await api.GetJobAsync(jobId, ct).ConfigureAwait(false)) is { } job) _updates.Post(job);
else _updates.Remove(jobId);
Signal();
}
catch (Exception ex) when (ex is ChronosApiException or HttpRequestException or JsonException
|| (ex is OperationCanceledException && !ct.IsCancellationRequested))
{
Log.Info("server", $"Job {jobId} of the server could not be read: {ex.Message}");
}
}
private void Signal()
{
_throttle.Signal();
Posted?.Invoke();
}
/// Pauses, resumes or cancels a job on the server; what the server answers shows at once (UI thread).
internal async Task ActAsync(Guid jobId, Func> action)
{
if (_watch is not { } watch || !_canControl) return;
try
{
var dto = await action(watch.Api, jobId, watch.Stopping.Token);
if (JobWire.FromDto(dto) is { } job) _updates.Post(job);
else _updates.Remove(jobId);
Flush();
}
catch (Exception ex) when (ex is ChronosApiException or HttpRequestException or JsonException or OperationCanceledException or ObjectDisposedException)
{
if (watch.Stopping.IsCancellationRequested) return;
Log.Info("server", $"Job {jobId} of the server not changed: {ex.Message}");
// The server words its refusal in this PC's language (Accept-Language); anything else is a lost connection.
_failed?.Invoke(Loc.L("Server.Jobs.NotChanged"),
ex is ChronosApiException { Error.Message: { Length: > 0 } reason } ? reason : Loc.L("Server.Error.Unreachable"));
}
}
/// Brings to the model's order with moves.
private void SyncRows()
{
var ordered = _model.Ordered();
var wanted = ordered.Select(e => e.Id).ToHashSet();
for (var i = Rows.Count - 1; i >= 0; i--)
{
if (wanted.Contains(Rows[i].Id)) continue;
_rows.Remove(Rows[i].Id);
Rows.RemoveAt(i);
}
for (var i = 0; i < ordered.Count; i++)
{
var entry = ordered[i];
if (!_rows.TryGetValue(entry.Id, out var row))
{
row = new ServerJobRowViewModel(entry, this);
_rows[entry.Id] = row;
}
else
{
row.Update(entry);
}
if (i < Rows.Count && ReferenceEquals(Rows[i], row)) continue;
var existing = Rows.IndexOf(row);
if (existing >= 0) Rows.Move(existing, i);
else Rows.Insert(i, row);
}
OnPropertyChanged(nameof(HasRows));
}
/// The session followed: its key, its connection and what stops its listener.
private sealed class Watch(string key, IServerJobsApi api, CancellationTokenSource stopping)
{
public string Key { get; } = key;
public IServerJobsApi Api { get; } = api;
public CancellationTokenSource Stopping { get; } = stopping;
public Task? Listening { get; set; }
}
///
/// What the listener collected for the next flush (thread-safe): a new list, which drops everything collected before it;
/// then per job its newest snapshot or its removal, and its newest progress.
///
private sealed class Updates
{
private readonly Lock _gate = new();
private IReadOnlyList? _reset;
private Dictionary _jobs = [];
private HashSet _removed = [];
private Dictionary _progress = [];
public void Reset(IReadOnlyList jobs)
{
lock (_gate)
{
_reset = jobs;
_jobs = [];
_removed = [];
_progress = [];
}
}
public void Post(Job job)
{
lock (_gate)
{
_jobs[job.Id] = job;
_removed.Remove(job.Id);
_progress.Remove(job.Id);
}
}
public void Remove(Guid id)
{
lock (_gate)
{
_jobs.Remove(id);
_progress.Remove(id);
_removed.Add(id);
}
}
public void Post(JobProgressInfo progress)
{
lock (_gate) _progress[progress.JobId] = progress;
}
public (IReadOnlyList? Reset, IReadOnlyCollection Jobs, IReadOnlyCollection Removed, IReadOnlyCollection Progress) Drain()
{
lock (_gate)
{
var drained = (_reset, (IReadOnlyCollection)_jobs.Values, (IReadOnlyCollection)_removed, (IReadOnlyCollection)_progress.Values);
_reset = null;
_jobs = [];
_removed = [];
_progress = [];
return drained;
}
}
}
}
///
/// One job of the server as the Server section lists it: kind, target with what it does now (or where a paused one stopped),
/// progress, speed, time left, state, and Pause / Resume / Cancel when the account may.
///
public sealed class ServerJobRowViewModel : ObservableObject
{
private readonly ServerJobsViewModel _owner;
private JobEntry _entry;
internal ServerJobRowViewModel(JobEntry entry, ServerJobsViewModel owner)
{
_entry = entry;
_owner = owner;
}
public Guid Id => _entry.Id;
public JobEntry Entry => _entry;
public string Kind => JobText.Kind(_entry.Job.Kind);
public string Target => _entry.Job.TargetLabel ?? string.Empty;
/// "Verify · Harbour sunset": the row's name for screen readers.
public string AccessibleName => JobText.Name(_entry.Job);
/// What it does now ("Verifying photo 812 of 2,400 · Harbour sunset · sequence 3 of 41"), or "Stopped at …".
public string StatusText => JobText.RowStatus(_entry);
public bool HasStatus => StatusText.Length > 0;
public string StatusToolTip => JobText.RowStatusToolTip(_entry);
public double Progress => _entry.Fraction ?? 0;
public bool IsIndeterminate => _entry.State == JobState.Running && _entry.Fraction is null;
public string PercentText => _entry.Fraction is { } f ? Formatting.Percent(f) : "—";
public string SpeedText => _entry.State == JobState.Running ? Blank(JobText.Speed(_entry.Job.Kind, _entry.ItemsPerSecond)) : "—";
public string LeftText => Blank(JobText.Left(_entry, 0));
public string StateText => JobText.State(_entry);
public StatusKind StateKind => _entry.State == JobState.Paused ? StatusKind.Paused : StatusKind.Running;
public string StateToolTip => string.Join(Loc.Separator, StateText, Loc.L("Tooltips.Jobs.State." + _entry.State));
/// "Verify · Harbour sunset: running": what a screen reader announces when the state changes (not progress).
public string StateAnnouncement => Loc.L("Jobs.Row.State", AccessibleName, StateText);
/// Pause and Cancel while it runs, Resume and Cancel while paused; none for a read-only account.
public IReadOnlyList Actions
{
get
{
if (!_owner.CanControl) return [];
var id = _entry.Id;
return _entry.State == JobState.Paused
? [Action("Jobs.Action.Resume", "Tooltips.Jobs.Resume", (api, job, ct) => api.ResumeJobAsync(job, ct)),
Action("Jobs.Action.Cancel", "Tooltips.Jobs.Cancel", (api, job, ct) => api.CancelJobAsync(job, ct))]
: [Action("Jobs.Action.Pause", "Tooltips.Jobs.Pause", (api, job, ct) => api.PauseJobAsync(job, ct)),
Action("Jobs.Action.Cancel", "Tooltips.Jobs.Cancel", (api, job, ct) => api.CancelJobAsync(job, ct))];
JobAction Action(string textKey, string toolTipKey, Func> run)
{
var text = Loc.L(textKey);
return new JobAction(text, Loc.L(toolTipKey), Loc.L("Jobs.ActionFor", text, AccessibleName),
new AsyncRelayCommand(() => _owner.ActAsync(id, run)));
}
}
}
internal void Update(JobEntry entry)
{
var stateChanged = entry.State != _entry.State;
_entry = entry;
if (stateChanged)
{
OnAllPropertiesChanged();
return;
}
OnPropertyChanged(nameof(StatusText));
OnPropertyChanged(nameof(HasStatus));
OnPropertyChanged(nameof(StatusToolTip));
OnPropertyChanged(nameof(Progress));
OnPropertyChanged(nameof(IsIndeterminate));
OnPropertyChanged(nameof(PercentText));
OnPropertyChanged(nameof(SpeedText));
OnPropertyChanged(nameof(LeftText));
}
internal void Refresh() => OnAllPropertiesChanged();
private static string Blank(string text) => text.Length == 0 ? "—" : text;
}