CI / Build and test (push) Successful in 9m15s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 6s
Release / Build, test and publish (push) Successful in 15m35s
CI / PostgreSQL 16 integration (push) Successful in 7m26s
CI / PostgreSQL 17 integration (push) Successful in 6m47s
CI / Docker image (push) Successful in 59s
Release / Server image (push) Successful in 1m24s
- Reorganize… on the web panel and the desktop: folder, file and title templates applied to existing sequences, with a dry run and an explicit Apply that runs as a job. - One naming rule (FrameNaming) for import, upload and Reorganize. - Crash-safe plan and file journals, held plans, stale-plan refusal; exports move with their folder. - Writers refuse a folder another program moved or gave to another sequence. - Server: POST /api/v1/archive/reorganize with a bounded plan cache; template rules. - Phase 3 docs and translations merged; strings in all 38 languages. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
167 lines
8.7 KiB
C#
167 lines
8.7 KiB
C#
using System.Collections.Concurrent;
|
||
using Chronos.Core.Archiving;
|
||
using Chronos.Core.Imaging;
|
||
using Chronos.Core.Model;
|
||
|
||
namespace Chronos.Data.Archiving;
|
||
|
||
/// <summary>Settings shared by the archive services.</summary>
|
||
public sealed record ArchiveServiceOptions
|
||
{
|
||
/// <summary>Decoders for thumbnails and proxies beyond embedded previews and JPEG (WIC on Windows).</summary>
|
||
public ImageDecoder Decoder { get; init; } = new([]);
|
||
|
||
/// <summary>Name written into lease files so a waiting writer can say who holds a sequence.</summary>
|
||
public string Owner { get; init; } = "chronos@" + Environment.MachineName;
|
||
|
||
/// <summary>How long a writer waits for a sequence held by another process before giving up.</summary>
|
||
public TimeSpan LeaseWait { get; init; } = TimeSpan.FromMinutes(2);
|
||
|
||
public TimeSpan LeaseDuration { get; init; } = LeaseFile.DefaultDuration;
|
||
|
||
/// <summary>
|
||
/// Scans queue thumbnails and proxies of the sequences they index without them, as an upload does (the server, whose
|
||
/// clients load both from it: BRIEF §4.3, §14.4). The desktop decodes its thumbnails on demand and makes proxies when
|
||
/// asked.
|
||
/// </summary>
|
||
public bool ImagesAfterScan { get; init; }
|
||
|
||
/// <summary>
|
||
/// Photos a pass works on at once, asked at the start of each sequence (owner review 1, R2: one job runs at a time and
|
||
/// the host's parallelism decides how many photos it processes at once; Settings › Rendering › Parallelism on the
|
||
/// desktop, <c>CHRONOS_PARALLELISM</c> on the server). Default: half the logical cores.
|
||
/// </summary>
|
||
public Func<int> Parallelism { get; init; } = () => DefaultParallelism;
|
||
|
||
/// <summary>
|
||
/// Memory the photos a proxy pass decodes at once may take, in bytes, asked with <see cref="Parallelism"/>: fewer at once
|
||
/// for large originals. Default: a quarter of the memory available, at least 1 GiB.
|
||
/// </summary>
|
||
public Func<long> MemoryBudget { get; init; } = () => DefaultMemoryBudget;
|
||
|
||
/// <summary>
|
||
/// The host's own clean-up steps (the server's uploads sweep, the desktop's cache trim), run by
|
||
/// <see cref="Core.Model.JobKind.Cleanup"/> jobs after the shared ones of <see cref="Housekeeping.CleanupSteps"/> when a
|
||
/// schedule names them (owner review 2, R10). A name the shared steps use already is ignored.
|
||
/// </summary>
|
||
public IReadOnlyList<Housekeeping.ICleanupStep> CleanupSteps { get; init; } = [];
|
||
|
||
/// <summary>
|
||
/// Folder of the copies <see cref="Core.Model.JobKind.Backup"/> jobs make; null: the <c>backups</c> folder next to the
|
||
/// SQLite index (<see cref="DatabaseBackups.DefaultDirectory"/>), where the desktop and the server keep them.
|
||
/// </summary>
|
||
public string? BackupDirectory { get; init; }
|
||
|
||
/// <summary>
|
||
/// Told after a Reorganize changed a sequence's folder, title or file names (change <c>reorganized</c>), once its step is
|
||
/// journaled, or once its folder moved and its index row followed when the step then failed: by its job, or by an archive
|
||
/// open that completed a plan an interruption left (<see cref="ArchiveService.OpenAsync"/>).
|
||
/// A server tells its connected clients (BRIEF §14.3); the service it opens its archive with at start sets none, as no
|
||
/// client is connected yet. The desktop reloads an editor open on the sequence. A failure is logged and never fails the job.
|
||
/// </summary>
|
||
public Func<Guid, string, CancellationToken, Task>? SequenceChanged { get; init; }
|
||
|
||
/// <summary>Half the logical cores, at least one: the parallelism when the host sets none.</summary>
|
||
public static int DefaultParallelism => Math.Max(1, Environment.ProcessorCount / 2);
|
||
|
||
/// <summary>A quarter of the memory available to this process, at least 1 GiB.</summary>
|
||
public static long DefaultMemoryBudget => Math.Max(1L << 30, GC.GetGCMemoryInfo().TotalAvailableMemoryBytes / 4);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Read-modify-write of sidecars, serialised per sequence: an in-process gate for this process's writers plus a lease
|
||
/// file for writers in other processes on the same share (server and client). Every edit sees the latest sidecar,
|
||
/// with an interrupted ingest batch completed first.
|
||
/// </summary>
|
||
internal sealed class SequenceWriter(ArchiveServiceOptions options)
|
||
{
|
||
private readonly ConcurrentDictionary<Guid, SemaphoreSlim> _gates = new();
|
||
|
||
public ArchiveServiceOptions Options => options;
|
||
|
||
/// <summary>Holds the sequence exclusively until disposed.</summary>
|
||
public async Task<IAsyncDisposable> LockAsync(ArchivePaths archive, Guid sequenceId, CancellationToken cancellationToken)
|
||
{
|
||
var gate = _gates.GetOrAdd(sequenceId, _ => new SemaphoreSlim(1, 1));
|
||
await gate.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||
try
|
||
{
|
||
var lease = await LeaseFile.AcquireAsync(archive.LocksDirectory, $"seq-{sequenceId:N}", options.Owner, options.LeaseWait,
|
||
options.LeaseDuration, cancellationToken).ConfigureAwait(false);
|
||
return new Held(gate, lease);
|
||
}
|
||
catch
|
||
{
|
||
gate.Release();
|
||
throw;
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Reads the sidecar of a locked sequence, completing an interrupted Reorganize of its files (<see cref="FileRenameJournal"/>),
|
||
/// undoing an interrupted photo deletion (<see cref="FrameRemovalJournal"/>), settling an interrupted restore from the
|
||
/// trash (<see cref="FrameRestoreJournal"/>; left for later while its trash entry is busy) and completing an interrupted
|
||
/// ingest batch first. A sidecar a rename recovery rewrites leaves the index behind it until the rename resumes or the
|
||
/// next scan reads it.
|
||
/// </summary>
|
||
/// <param name="heldEntryLease">The trash entry lease the caller holds already (a restore of that entry), if any.</param>
|
||
/// <exception cref="InvalidDataException">The sidecar in the folder belongs to another sequence.</exception>
|
||
public static SequenceManifest ReadLocked(SequencePaths sequence, Guid sequenceId, string? heldEntryLease = null)
|
||
{
|
||
var manifest = ManifestStore.ReadSequence(sequence.ManifestPath);
|
||
if (manifest.Id != sequenceId)
|
||
throw new InvalidDataException($"{sequence.ManifestPath} belongs to sequence {manifest.Id}, not {sequenceId}.");
|
||
if (File.Exists(sequence.FileRenameJournalPath))
|
||
{
|
||
if (FileRenameJournal.Recover(sequence, manifest) is { } renamed)
|
||
{
|
||
ManifestStore.WriteSequence(sequence.ManifestPath, renamed);
|
||
manifest = renamed;
|
||
}
|
||
FileRenameJournal.Delete(sequence);
|
||
}
|
||
FrameRemovalJournal.Recover(sequence, manifest);
|
||
FrameTrash.RecoverRestore(sequence, manifest, heldEntryLease);
|
||
if (IngestJournal.Recover(sequence, manifest) is { } recovered)
|
||
{
|
||
ManifestStore.WriteSequence(sequence.ManifestPath, recovered);
|
||
manifest = recovered;
|
||
}
|
||
IngestJournal.Delete(sequence);
|
||
return manifest;
|
||
}
|
||
|
||
/// <summary>
|
||
/// The folder of a sequence whose lock the caller holds, as the index has it now. A Reorganize moves a folder and
|
||
/// updates its row under the sequence's lease (and its thumbnail lease), so a writer that read the row before taking
|
||
/// the lease writes where the sequence is, never into the folder it left, which the write would create again. The
|
||
/// folder of <paramref name="sequence"/> when the row is gone (reading its sidecar then fails as it did).
|
||
/// </summary>
|
||
public static async Task<SequencePaths> FolderAsync(ChronosDatabase db, ArchivePaths archive, Sequence sequence, CancellationToken cancellationToken) =>
|
||
archive.Sequence((await db.Sequences.GetAsync(sequence.Id, cancellationToken).ConfigureAwait(false))?.Folder ?? sequence.Folder);
|
||
|
||
/// <summary>Applies <paramref name="edit"/> to the sidecar under the lock; a null result writes nothing.</summary>
|
||
public async Task<SequenceManifest> UpdateAsync(ArchivePaths archive, Guid sequenceId, string folder,
|
||
Func<SequenceManifest, SequenceManifest?> edit, CancellationToken cancellationToken)
|
||
{
|
||
var sequence = archive.Sequence(folder);
|
||
await using (await LockAsync(archive, sequenceId, cancellationToken).ConfigureAwait(false))
|
||
{
|
||
var manifest = ReadLocked(sequence, sequenceId);
|
||
if (edit(manifest) is not { } updated) return manifest;
|
||
ManifestStore.WriteSequence(sequence.ManifestPath, updated);
|
||
return updated;
|
||
}
|
||
}
|
||
|
||
private sealed class Held(SemaphoreSlim gate, LeaseFile lease) : IAsyncDisposable
|
||
{
|
||
public ValueTask DisposeAsync()
|
||
{
|
||
lease.Dispose();
|
||
gate.Release();
|
||
return ValueTask.CompletedTask;
|
||
}
|
||
}
|
||
}
|