merge: server archive fixes (V-5, V-8, V-11, V-16, V-35, V-36)
Merge branch fix2/server-archive into the verification branch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
19 files changed
+671
-39
No files matched your search
@@ -29,6 +29,7 @@
|
||||
<data name="Server.Error.ArchiveBusy" xml:space="preserve"><value>Another program is changing the archive's settings; try again shortly.</value><comment>long</comment></data>
|
||||
<data name="Server.Error.ArchiveManifest" xml:space="preserve"><value>The archive's archive.json is damaged, from a newer Titano or of another archive; check the archive folder.</value><comment>long</comment></data>
|
||||
<data name="Server.Error.ArchiveOffline" xml:space="preserve"><value>The archive is offline or read-only; try again when it is back.</value><comment>long</comment></data>
|
||||
<data name="Server.Error.ArchivePolicy" xml:space="preserve"><value>The archive's archive.json has a thumbnail or proxy setting Titano cannot use; correct it in the archive folder.</value><comment>long</comment></data>
|
||||
<data name="Server.Error.ArchiveRoot" xml:space="preserve"><value>Its photos are the archive's root folder, which never moves to the trash.</value><comment>long</comment></data>
|
||||
<data name="Server.Error.BadCredentials" xml:space="preserve"><value>Wrong user name or password.</value><comment>long</comment></data>
|
||||
<data name="Server.Error.BadRequest" xml:space="preserve"><value>The request is not valid.</value><comment>long</comment></data>
|
||||
|
||||
@@ -27,6 +27,7 @@
|
||||
<data name="Server.Error.ArchiveBusy" xml:space="preserve"><value>Un altro programma sta modificando le impostazioni dell'archivio; riprova tra poco.</value></data>
|
||||
<data name="Server.Error.ArchiveManifest" xml:space="preserve"><value>Il file archive.json dell'archivio è danneggiato, di un Titano più recente o di un altro archivio; controlla la cartella dell'archivio.</value></data>
|
||||
<data name="Server.Error.ArchiveOffline" xml:space="preserve"><value>L'archivio non è raggiungibile o è di sola lettura; riprova quando torna disponibile.</value></data>
|
||||
<data name="Server.Error.ArchivePolicy" xml:space="preserve"><value>Il file archive.json dell'archivio ha un'impostazione delle miniature o dei proxy che Titano non può usare; correggila nella cartella dell'archivio.</value></data>
|
||||
<data name="Server.Error.ArchiveRoot" xml:space="preserve"><value>Le sue foto sono la cartella principale dell'archivio, che non va mai nel cestino.</value></data>
|
||||
<data name="Server.Error.BadCredentials" xml:space="preserve"><value>Nome utente o password errati.</value></data>
|
||||
<data name="Server.Error.BadRequest" xml:space="preserve"><value>La richiesta non è valida.</value></data>
|
||||
|
||||
@@ -20,7 +20,9 @@ public enum ScanMode
|
||||
public sealed record ScanProgress(int FoldersDone, int FoldersTotal, string Folder, bool Changed);
|
||||
|
||||
/// <param name="Failed">Folders that could not be indexed, with the reason; they are retried by the next scan.</param>
|
||||
public sealed record ScanSummary(int Folders, int Indexed, int Adopted, int Removed, IReadOnlyList<(string Folder, string Error)> Failed);
|
||||
/// <param name="Sequences">The sequences this run indexed (<paramref name="Indexed"/> of them), in folder order.</param>
|
||||
public sealed record ScanSummary(int Folders, int Indexed, int Adopted, int Removed, IReadOnlyList<(string Folder, string Error)> Failed,
|
||||
IReadOnlyList<Guid> Sequences);
|
||||
|
||||
/// <summary>
|
||||
/// Keeps the database index in line with the archive on disk (BRIEF §7, §12.5): scans by size/time stamps, adopts
|
||||
@@ -49,11 +51,15 @@ public sealed class ArchiveIndexer
|
||||
if (mode == ScanMode.Rebuild && resumeAfter is null) await _db.Archives.ClearFolderStatesAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
||||
var folders = ArchiveScanner.FindSequenceFolders(paths, cancellationToken);
|
||||
var states = await _db.Archives.GetFolderStatesAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
||||
var locations = new Locations(await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false));
|
||||
var rows = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
||||
// A sequence waiting for its first frame has no folder yet: the archive root, adopted or re-adopted, never takes its id.
|
||||
var awaiting = await AwaitingFolderAsync(paths, rows, cancellationToken).ConfigureAwait(false);
|
||||
var locations = new Locations(rows.Where(r => !awaiting.Contains(r.Id)));
|
||||
var snapshot = mode == ScanMode.Rebuild ? IndexSnapshot.TryLoad(paths.SnapshotPath) : null;
|
||||
using var snapshotWriter = mode != ScanMode.Incremental && resumeAfter is null ? new IndexSnapshot.Writer(paths.SnapshotPath) : null;
|
||||
|
||||
int indexed = 0, adopted = 0;
|
||||
int adopted = 0;
|
||||
var indexed = new List<Guid>();
|
||||
var failed = new List<(string, string)>();
|
||||
for (var i = 0; i < folders.Count; i++)
|
||||
{
|
||||
@@ -67,14 +73,14 @@ public sealed class ArchiveIndexer
|
||||
try
|
||||
{
|
||||
var reconcile = mode == ScanMode.Full || (state is not null && state.Photos != folder.Photos);
|
||||
var bytes = await IndexFolderAsync(archive, paths, folder, reconcile, snapshot, locations, cancellationToken).ConfigureAwait(false);
|
||||
var (id, bytes) = await IndexFolderAsync(archive, paths, folder, reconcile, snapshot, locations, cancellationToken).ConfigureAwait(false);
|
||||
if (snapshotWriter is not null)
|
||||
{
|
||||
var sidecar = paths.Sequence(folder.RelativeFolder).ManifestPath;
|
||||
snapshotWriter.Add(folder.RelativeFolder, FileStamp.Of(sidecar)!.Value,
|
||||
bytes ?? await File.ReadAllBytesAsync(sidecar, cancellationToken).ConfigureAwait(false));
|
||||
}
|
||||
indexed++;
|
||||
indexed.Add(id);
|
||||
if (folder.Sidecar is null) adopted++;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
|
||||
@@ -88,7 +94,7 @@ public sealed class ArchiveIndexer
|
||||
|
||||
var removed = mode == ScanMode.Rebuild ? await RemoveUnbackedAsync(archive, paths, folders, locations, cancellationToken).ConfigureAwait(false) : 0;
|
||||
if (failed.Count == 0) snapshotWriter?.Commit();
|
||||
return new ScanSummary(folders.Count, indexed, adopted, removed, failed);
|
||||
return new ScanSummary(folders.Count, indexed.Count, adopted, removed, failed, indexed);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -156,8 +162,8 @@ public sealed class ArchiveIndexer
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Indexes one folder; returns the sidecar bytes when they were read and not rewritten since (for the snapshot).</summary>
|
||||
private async Task<byte[]?> IndexFolderAsync(Archive archive, ArchivePaths paths, SequenceFolder folder, bool reconcile, IndexSnapshot? snapshot,
|
||||
/// <summary>Indexes one folder; returns its sequence and the sidecar bytes when they were read and not rewritten since (for the snapshot).</summary>
|
||||
private async Task<(Guid Id, byte[]? Bytes)> IndexFolderAsync(Archive archive, ArchivePaths paths, SequenceFolder folder, bool reconcile, IndexSnapshot? snapshot,
|
||||
Locations locations, CancellationToken cancellationToken)
|
||||
{
|
||||
var sequence = paths.Sequence(folder.RelativeFolder);
|
||||
@@ -189,10 +195,11 @@ public sealed class ArchiveIndexer
|
||||
}
|
||||
if (reconcile || File.Exists(sequence.IngestJournalPath))
|
||||
{
|
||||
// Photos added, gone or back: the light analysis no longer describes the frames, so the scan's follow-up runs it again.
|
||||
// Photos added, gone or back: the light analysis, thumbnails and proxies no longer cover the frames, so the scan's
|
||||
// follow-ups make them again (the images on a server, ArchiveServiceOptions.ImagesAfterScan).
|
||||
manifest = await UpdateAndSyncAsync(archive, paths, sequence, manifest.Id,
|
||||
m => SequenceAdopter.Reconcile(sequence, m, cancellationToken) is { } reconciled
|
||||
? reconciled with { Status = reconciled.Status & ~SequenceStatus.AnalyzedLight }
|
||||
? reconciled with { Status = reconciled.Status & ~(SequenceStatus.AnalyzedLight | SequenceStatus.ThumbsBuilt | SequenceStatus.ProxiesBuilt) }
|
||||
: null,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
bytes = null;
|
||||
@@ -208,7 +215,45 @@ public sealed class ArchiveIndexer
|
||||
await SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
locations.Set(manifest.Id, folder.RelativeFolder);
|
||||
return bytes;
|
||||
return (manifest.Id, bytes);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The sequences among <paramref name="locations"/> created without photos that still wait for their first frame: no
|
||||
/// folder, no frames, and not the sequence the archive root's sidecar names. Their empty folder is not the archive
|
||||
/// root: scans do not adopt or rebuild the root under their id, a rebuild keeps them although they have no sidecar yet,
|
||||
/// orphan checks do not count them missing, and maintenance leaves the root's files to the root's own sequence. Unlike
|
||||
/// <see cref="ArchiveIngest.AwaitsFolder"/>, whose commit reports it, a root sidecar that cannot be read names nobody
|
||||
/// here: a scan rebuilds a damaged one under the id of the sequence that holds the root's frames, and one it may not
|
||||
/// read fails the root folder alone rather than the whole scan or rebuild.
|
||||
/// </summary>
|
||||
internal async Task<HashSet<Guid>> AwaitingFolderAsync(ArchivePaths paths, IEnumerable<SequenceLocation> locations, CancellationToken cancellationToken)
|
||||
{
|
||||
var awaiting = new HashSet<Guid>();
|
||||
(Guid? Id, bool Read) root = (null, false);
|
||||
foreach (var location in locations.Where(l => l.Folder.Length == 0))
|
||||
{
|
||||
if (await _db.Sequences.GetAsync(location.Id, cancellationToken).ConfigureAwait(false) is not { Folder.Length: 0, FrameCount: 0 } sequence) continue;
|
||||
// Read once, and only when a row may be waiting: a root sidecar can list a hundred thousand frames.
|
||||
if (!root.Read) root = (RootSidecarId(paths), true);
|
||||
if (root.Id != sequence.Id) awaiting.Add(sequence.Id);
|
||||
}
|
||||
return awaiting;
|
||||
}
|
||||
|
||||
/// <summary>The id the archive root's sidecar names; null when there is none or it cannot be read.</summary>
|
||||
private static Guid? RootSidecarId(ArchivePaths paths)
|
||||
{
|
||||
var sidecar = paths.Sequence(string.Empty).ManifestPath;
|
||||
try
|
||||
{
|
||||
return File.Exists(sidecar) ? ManifestStore.ReadSequence(sidecar).Id : null;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
|
||||
{
|
||||
// Indexing the root reports it (a damaged sidecar is rebuilt, an unreadable one fails that folder alone).
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The index holds a project for this sequence and other frames than the sidecar: its positions may move.</summary>
|
||||
@@ -261,16 +306,21 @@ public sealed class ArchiveIndexer
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
};
|
||||
|
||||
/// <summary>Rebuild only: rows whose folder no longer holds a sidecar leave the index (their files are untouched).</summary>
|
||||
/// <summary>
|
||||
/// Rebuild only: rows whose folder no longer holds a sidecar leave the index (their files are untouched). A sequence
|
||||
/// waiting for its first frame has no sidecar yet and stays.
|
||||
/// </summary>
|
||||
private async Task<int> RemoveUnbackedAsync(Archive archive, ArchivePaths paths, IReadOnlyList<SequenceFolder> folders, Locations locations,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var present = folders.Where(f => File.Exists(paths.Sequence(f.RelativeFolder).ManifestPath)).Select(f => f.RelativeFolder).ToHashSet(StringComparer.Ordinal);
|
||||
var rows = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
||||
var awaiting = await AwaitingFolderAsync(paths, rows, cancellationToken).ConfigureAwait(false);
|
||||
var removed = 0;
|
||||
foreach (var location in await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false))
|
||||
foreach (var location in rows)
|
||||
{
|
||||
if (present.Contains(location.Folder) && locations.AtFolder(location.Folder) == location.Id) continue;
|
||||
if (awaiting.Contains(location.Id) || (present.Contains(location.Folder) && locations.AtFolder(location.Folder) == location.Id)) continue;
|
||||
if (await _db.Sequences.DeleteAsync(location.Id, cancellationToken).ConfigureAwait(false)) removed++;
|
||||
await _db.Archives.DeleteFolderStateAsync(archive.Id, location.Folder, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
@@ -71,7 +71,10 @@ public sealed class ArchiveMaintenance
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(paths.Root, ArchiveAvailability.Offline);
|
||||
var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
||||
var all = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
||||
// A sequence waiting for its first frame has no folder yet: none of it is missing, and the archive root is not its folder.
|
||||
var awaiting = await _indexer.AwaitingFolderAsync(paths, all, cancellationToken).ConfigureAwait(false);
|
||||
var locations = all.Where(l => !awaiting.Contains(l.Id)).ToList();
|
||||
var sequences = new List<SequenceOrphans>();
|
||||
var missing = new List<SequenceLocation>();
|
||||
foreach (var location in locations)
|
||||
@@ -91,7 +94,10 @@ public sealed class ArchiveMaintenance
|
||||
return new OrphanReport(sequences, missing, unindexed);
|
||||
}
|
||||
|
||||
/// <summary>Drops index rows of sequences whose sidecar is gone (an orphans fix action); rows whose sidecar is back are kept.</summary>
|
||||
/// <summary>
|
||||
/// Drops index rows of sequences whose sidecar is gone (an orphans fix action); rows whose sidecar is back are kept, and
|
||||
/// so are sequences waiting for their first frame (<see cref="ArchiveIngest.AwaitsFolder"/>), which have none yet.
|
||||
/// </summary>
|
||||
public async Task<int> RemoveMissingSequencesAsync(Archive archive, IReadOnlyList<Guid> sequenceIds, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
@@ -100,7 +106,7 @@ public sealed class ArchiveMaintenance
|
||||
foreach (var id in sequenceIds)
|
||||
{
|
||||
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } sequence || sequence.ArchiveId != archive.Id) continue;
|
||||
if (File.Exists(paths.Sequence(sequence.Folder).ManifestPath)) continue;
|
||||
if (File.Exists(paths.Sequence(sequence.Folder).ManifestPath) || ArchiveIngest.AwaitsFolder(sequence, paths)) continue;
|
||||
if (await _db.Sequences.DeleteAsync(id, cancellationToken).ConfigureAwait(false)) removed++;
|
||||
await _db.Archives.DeleteFolderStateAsync(archive.Id, sequence.Folder, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
@@ -117,7 +123,11 @@ public sealed class ArchiveMaintenance
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
if (!dryRun) ArchiveVolume.EnsureWritable(paths);
|
||||
var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
||||
var all = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
||||
// A sequence waiting for its first frame has no folder yet: the archive root's pack is compacted under the leases of
|
||||
// the root's own sequence, which its thumbnail passes hold, never under another id's.
|
||||
var awaiting = await _indexer.AwaitingFolderAsync(paths, all, cancellationToken).ConfigureAwait(false);
|
||||
var locations = all.Where(l => !awaiting.Contains(l.Id)).ToList();
|
||||
IReadOnlyList<string> leases = [], temps = [], packs = [];
|
||||
int groups = 0, flagChanges = 0;
|
||||
long? reclaimable = null;
|
||||
|
||||
@@ -54,8 +54,8 @@ public sealed class ArchiveService : IJobHandlerProvider
|
||||
/// <summary>Handlers for scan, rebuild, verify, import, thumbs, proxies and maintenance jobs; register them on the <see cref="JobEngine"/>.</summary>
|
||||
public IReadOnlyList<IJobHandler> CreateJobHandlers() =>
|
||||
[
|
||||
new ScanJobHandler(_db, Indexer, JobKind.Scan),
|
||||
new ScanJobHandler(_db, Indexer, JobKind.Rebuild),
|
||||
new ScanJobHandler(_db, Indexer, JobKind.Scan, _writer.Options.ImagesAfterScan),
|
||||
new ScanJobHandler(_db, Indexer, JobKind.Rebuild, _writer.Options.ImagesAfterScan),
|
||||
new VerifyJobHandler(_db, Verifier),
|
||||
new ImportJobHandler(_db, Imports),
|
||||
new ImageJobHandler(_db, Images, JobKind.Thumbs),
|
||||
|
||||
@@ -44,7 +44,7 @@ public sealed class ArchiveVerifier
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var targets = await TargetsAsync(archive, sequenceIds, cancellationToken).ConfigureAwait(false);
|
||||
var targets = await TargetsAsync(archive, paths, sequenceIds, cancellationToken).ConfigureAwait(false);
|
||||
var bytesTotal = targets.Sum(t => t.TotalBytes);
|
||||
var bytesDone = targets.Take(resume.Sequence).Sum(t => t.TotalBytes);
|
||||
long checkedFrames = 0, ok = 0, hashed = 0;
|
||||
@@ -82,13 +82,15 @@ public sealed class ArchiveVerifier
|
||||
return new VerifyReport(targets.Count, checkedFrames, ok, hashed, problems);
|
||||
}
|
||||
|
||||
private async Task<List<Sequence>> TargetsAsync(Archive archive, IReadOnlyList<Guid>? sequenceIds, CancellationToken cancellationToken)
|
||||
/// <summary>The sequences to check; one waiting for its first frame (<see cref="ArchiveIngest.AwaitsFolder"/>) has no sidecar to check yet.</summary>
|
||||
private async Task<List<Sequence>> TargetsAsync(Archive archive, ArchivePaths paths, IReadOnlyList<Guid>? sequenceIds, CancellationToken cancellationToken)
|
||||
{
|
||||
var ids = sequenceIds ?? (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)).Select(l => l.Id).ToList();
|
||||
var targets = new List<Sequence>(ids.Count);
|
||||
foreach (var id in ids)
|
||||
{
|
||||
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } s && s.ArchiveId == archive.Id) targets.Add(s);
|
||||
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } s && s.ArchiveId == archive.Id && !ArchiveIngest.AwaitsFolder(s, paths))
|
||||
targets.Add(s);
|
||||
}
|
||||
return targets;
|
||||
}
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Jobs;
|
||||
using Titano.Data.Processing;
|
||||
using Titano.Data.Repositories;
|
||||
|
||||
namespace Titano.Data.Archiving.Jobs;
|
||||
|
||||
@@ -18,9 +20,12 @@ internal static class JobArchive
|
||||
/// <see cref="JobKind.Scan"/> (incremental, or full with <see cref="ScanParameters.Full"/>) and
|
||||
/// <see cref="JobKind.Rebuild"/>. Checkpoint: <c>after:<folder></c>, the last folder indexed. When anything was
|
||||
/// indexed, the sequences without a current light analysis (adopted folders, new sidecars, photos added to a folder) are
|
||||
/// queued for it, and for a background deep analysis too when <see cref="ScanParameters.DeepAnalysis"/> asks.
|
||||
/// queued for it, and for a background deep analysis too when <see cref="ScanParameters.DeepAnalysis"/> asks. On the
|
||||
/// server (<paramref name="imagesAfterScan"/>, <see cref="ArchiveServiceOptions.ImagesAfterScan"/>) the sequences it indexed
|
||||
/// without thumbnails or proxies are queued for them too, one pass per sequence as after an upload. Sequences waiting for
|
||||
/// their first frame (<see cref="ArchiveIndexer.AwaitingFolderAsync"/>) have nothing to analyse or draw yet and are left out.
|
||||
/// </summary>
|
||||
internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, JobKind kind) : IJobHandler
|
||||
internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, JobKind kind, bool imagesAfterScan = false) : IJobHandler
|
||||
{
|
||||
private const string Prefix = "after:";
|
||||
|
||||
@@ -42,9 +47,51 @@ internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer,
|
||||
// A resumed scan may have indexed its folders before the interruption.
|
||||
if (summary.Indexed > 0 || resume is not null)
|
||||
{
|
||||
var unanalysed = await AnalysisFollowUp.UnanalysedAsync(db, archive.Id, context.Cancellation).ConfigureAwait(false);
|
||||
var awaiting = await indexer.AwaitingFolderAsync(new ArchivePaths(archive.RootPath),
|
||||
await db.Sequences.ListLocationsAsync(archive.Id, context.Cancellation).ConfigureAwait(false), context.Cancellation).ConfigureAwait(false);
|
||||
var unanalysed = (await AnalysisFollowUp.UnanalysedAsync(db, archive.Id, context.Cancellation).ConfigureAwait(false))
|
||||
.Where(id => !awaiting.Contains(id)).ToList();
|
||||
await AnalysisFollowUp.QueueAsync(db, context, unanalysed).ConfigureAwait(false);
|
||||
if (kind == JobKind.Scan && parameters.DeepAnalysis) await AnalysisFollowUp.QueueDeepAsync(db, context, unanalysed).ConfigureAwait(false);
|
||||
if (imagesAfterScan)
|
||||
{
|
||||
var resumed = resume is not null;
|
||||
await QueueImagesAsync(context, archive, JobKind.Thumbs, SequenceStatus.ThumbsBuilt, summary.Sequences, resumed, awaiting).ConfigureAwait(false);
|
||||
// An archive whose policy makes no proxies would get a pass that does nothing.
|
||||
if (archive.ProxyLongEdge != ProxyPolicy.Disabled)
|
||||
await QueueImagesAsync(context, archive, JobKind.Proxies, SequenceStatus.ProxiesBuilt, summary.Sequences, resumed, awaiting).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queues a background <paramref name="pass"/> (thumbnails or proxies) of each sequence this run indexed whose
|
||||
/// <paramref name="built"/> flag is clear: new and adopted sequences, and those whose photos changed (the scan clears the
|
||||
/// flag then). After a resumed run, which may have indexed sequences before the interruption, of every sequence of the
|
||||
/// archive without the flag but those in <paramref name="awaiting"/>. One pass per sequence, as after an upload: a folder
|
||||
/// renamed, deleted or damaged before its pass runs fails that pass alone, and the other sequences still get their
|
||||
/// images. A sequence whose own pass of that kind is already queued is left out.
|
||||
/// </summary>
|
||||
private async Task QueueImagesAsync(JobContext context, Archive archive, JobKind pass, SequenceStatus built, IReadOnlyList<Guid> indexed,
|
||||
bool resumed, IReadOnlySet<Guid> awaiting)
|
||||
{
|
||||
var missing = new Dictionary<Guid, Sequence>();
|
||||
var query = new SequenceQuery { ArchiveId = archive.Id, WithoutStatus = built, Sort = SequenceSort.Oldest, Limit = 1000 };
|
||||
do
|
||||
{
|
||||
var page = await db.Sequences.ListAsync(query, context.Cancellation).ConfigureAwait(false);
|
||||
foreach (var sequence in page.Items) missing[sequence.Id] = sequence;
|
||||
query = query with { Cursor = page.NextCursor };
|
||||
}
|
||||
while (query.Cursor is not null);
|
||||
var targets = resumed ? missing.Values.ToList() : indexed.Distinct().Where(missing.ContainsKey).Select(id => missing[id]).ToList();
|
||||
foreach (var sequence in targets)
|
||||
{
|
||||
if (awaiting.Contains(sequence.Id) || await db.Jobs.IsQueuedAsync(pass, sequence.Id, context.Cancellation).ConfigureAwait(false)) continue;
|
||||
var request = pass == JobKind.Thumbs
|
||||
? ArchiveJobs.Thumbnails([sequence.Id], sequence.DisplayTitle)
|
||||
: ArchiveJobs.Proxies([sequence.Id], sequence.DisplayTitle);
|
||||
await context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,10 +2,14 @@ using Titano.Core.Archiving;
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Repositories;
|
||||
|
||||
namespace Titano.Data.Archiving;
|
||||
|
||||
/// <summary>The plan no longer matches the archive (a sequence moved or vanished since it was made).</summary>
|
||||
/// <summary>
|
||||
/// The plan no longer matches the archive: a sequence moved or vanished since it was made, or one was indexed inside a
|
||||
/// folder the plan moves.
|
||||
/// </summary>
|
||||
public sealed class StaleRenamePlanException(Guid sequenceId)
|
||||
: InvalidOperationException(Loc.L("Archive.Error.StalePlan"))
|
||||
{
|
||||
@@ -62,7 +66,8 @@ public sealed class MassRenamer
|
||||
private static RenameSubject Subject(Sequence s) => new(s.Id, s.Folder, s.Title, s.FirstCapture, s.CameraMake, s.CameraModel, s.Lens);
|
||||
|
||||
/// <summary>Applies a plan; each completed step is journaled before the next starts.</summary>
|
||||
/// <exception cref="StaleRenamePlanException">A sequence is no longer where the plan expects it; nothing of that step was changed.</exception>
|
||||
/// <exception cref="StaleRenamePlanException">A sequence is no longer where the plan expects it, or a folder that moves holds
|
||||
/// a sequence the plan does not take along; nothing of that step was changed.</exception>
|
||||
public Task ApplyAsync(Archive archive, RenamePlan plan, IProgress<int>? progress = null, CancellationToken cancellationToken = default) =>
|
||||
RunAsync(archive, new RenameJournal(plan, []), resuming: false, progress, cancellationToken);
|
||||
|
||||
@@ -85,12 +90,17 @@ public sealed class MassRenamer
|
||||
var completed = journal.Completed.ToHashSet();
|
||||
if (!resuming)
|
||||
{
|
||||
// A plan made before another change is refused before anything moves.
|
||||
// A plan made before another change is refused before anything moves: a sequence moved, or one was indexed
|
||||
// inside a folder that moves (a scan adopted a subfolder copied in since the dry run).
|
||||
foreach (var step in journal.Plan.Steps)
|
||||
{
|
||||
if ((await _db.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false))?.Folder != step.FromFolder)
|
||||
throw new StaleRenamePlanException(step.SequenceId);
|
||||
}
|
||||
var planned = journal.Plan.Steps.Select(s => s.SequenceId).ToHashSet();
|
||||
if (await StrangerInsideAsync(archive, journal.Plan.Steps.Where(s => s.MovesFolder && !s.Carried), planned, cancellationToken)
|
||||
.ConfigureAwait(false) is { } stranger)
|
||||
throw new StaleRenamePlanException(stranger.Id);
|
||||
}
|
||||
RenameJournal.Write(paths, journal);
|
||||
var steps = journal.Plan.Steps;
|
||||
@@ -147,6 +157,7 @@ public sealed class MassRenamer
|
||||
{
|
||||
if (await StandsAsync(c, resuming, cancellationToken).ConfigureAwait(false)) follow.Add(c);
|
||||
}
|
||||
if (!await CarriesAllInsideAsync(archive, paths, step, carried, resuming, cancellationToken).ConfigureAwait(false)) return;
|
||||
RenamePlanner.MoveFolder(paths, step);
|
||||
await RecordAsync(archive, paths, step, cancellationToken).ConfigureAwait(false);
|
||||
// Moved with their carrier: only their sidecars and index rows follow.
|
||||
@@ -170,6 +181,35 @@ public sealed class MassRenamer
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether every sequence the index has inside the folder of a moving step moves with it as planned, checked again under
|
||||
/// the step's leases just before the folder moves. One indexed there since the dry run would move without its index
|
||||
/// row following, and was never shown in the plan: the step is refused, or dropped when resuming a folder that has not
|
||||
/// moved yet (a folder moved before the interruption goes on; the next scan indexes the newcomer where it ended up).
|
||||
/// </summary>
|
||||
/// <exception cref="StaleRenamePlanException">Not resuming, and such a sequence is inside the folder.</exception>
|
||||
private async Task<bool> CarriesAllInsideAsync(Archive archive, ArchivePaths paths, RenameStep step, IReadOnlyList<RenameStep> carried, bool resuming,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!step.MovesFolder) return true;
|
||||
var taken = carried.Select(c => c.SequenceId).ToHashSet();
|
||||
if (await StrangerInsideAsync(archive, [step], taken, cancellationToken).ConfigureAwait(false) is not { } stranger) return true;
|
||||
if (!resuming) throw new StaleRenamePlanException(stranger.Id);
|
||||
if (!Directory.Exists(paths.Sequence(step.FromFolder).Folder)) return true;
|
||||
Log.Warning("archive", $"Rename of {step.FromFolder} dropped: {stranger.Folder} was indexed inside it after the plan was made");
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>A sequence the index has inside the folder of one of <paramref name="moving"/> that is not in <paramref name="taken"/>; null when there is none.</summary>
|
||||
private async Task<SequenceLocation?> StrangerInsideAsync(Archive archive, IEnumerable<RenameStep> moving, IReadOnlySet<Guid> taken,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var folders = moving.Select(s => s.FromFolder).ToList();
|
||||
if (folders.Count == 0) return null;
|
||||
return (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false))
|
||||
.FirstOrDefault(l => !taken.Contains(l.Id) && folders.Any(f => RenamePlanner.IsInside(f, l.Folder)));
|
||||
}
|
||||
|
||||
/// <summary>Sidecar title, then index (folder, frames' paths); each part is idempotent so a resumed step can repeat it.</summary>
|
||||
private async Task RecordAsync(Archive archive, ArchivePaths paths, RenameStep step, CancellationToken cancellationToken)
|
||||
{
|
||||
|
||||
@@ -17,6 +17,13 @@ public sealed record ArchiveServiceOptions
|
||||
public TimeSpan LeaseWait { get; init; } = TimeSpan.FromMinutes(2);
|
||||
|
||||
public TimeSpan LeaseDuration { get; init; } = LeaseFile.DefaultDuration;
|
||||
|
||||
/// <summary>
|
||||
/// Scans queue thumbnails and proxies of the sequences they index without them, as an upload does (the server, whose
|
||||
/// clients load both from it: BRIEF §4.3, §14.4). The desktop decodes its thumbnails on demand and makes proxies when
|
||||
/// asked.
|
||||
/// </summary>
|
||||
public bool ImagesAfterScan { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -67,7 +67,9 @@ public static class ArchiveEndpoints
|
||||
/// Writes the name, templates and proxy policy into the archive's <c>archive.json</c> through the archive engine (so a
|
||||
/// restart and the proxy jobs see them), then into the index. 503 <c>archive_offline</c> while the archive cannot be
|
||||
/// written; 409 <c>archive_busy</c> while another process holds its settings; 409 <c>archive_manifest</c> when its
|
||||
/// archive.json is damaged, written by a newer Titano or of another archive (nothing is written).
|
||||
/// archive.json is damaged, written by a newer Titano or of another archive; 409 <c>archive_policy</c> when it holds a
|
||||
/// thumbnail or proxy setting this API cannot set and Titano cannot use (details: <c>thumbnailPolicy</c> or
|
||||
/// <c>proxyPolicy</c>, the property to correct in the file). Nothing is written.
|
||||
/// </summary>
|
||||
private static async Task<IResult> UpdateAsync(HttpContext context, Caller caller, ArchiveSettingsDto request, TitanoDatabase db, ServerArchive archive,
|
||||
[FromServices] ArchiveService? archives, AuditLog audit)
|
||||
@@ -99,6 +101,17 @@ public static class ArchiveEndpoints
|
||||
{
|
||||
return ApiErrors.Conflict(context, "archive_manifest");
|
||||
}
|
||||
catch (ArgumentException ex)
|
||||
{
|
||||
// The request's fields were checked above: what fails is a value read from archive.json (a thumbnail size, or a
|
||||
// proxy quality, written by hand or by another tool), which the admin corrects in the file, not in this form.
|
||||
return ApiErrors.Conflict(context, "archive_policy", ex.ParamName switch
|
||||
{
|
||||
"thumbnailLongEdge" => "thumbnailPolicy",
|
||||
"proxyLongEdge" => "proxyPolicy",
|
||||
var other => other,
|
||||
});
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
// ArchiveOfflineException included: nothing was written.
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using Microsoft.AspNetCore.Http.Metadata;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Protocol;
|
||||
using Titano.Server.Security;
|
||||
@@ -14,11 +15,14 @@ public static class UploadEndpoints
|
||||
{
|
||||
private const int MaxCardName = 200;
|
||||
|
||||
/// <summary>The largest chunk size a server can be set to (<see cref="Configuration.ServerOptions.ChunkSizeBytes"/>).</summary>
|
||||
private const long MaxChunkBytes = 64L * 1024 * 1024;
|
||||
|
||||
public static void Map(RouteGroupBuilder api)
|
||||
{
|
||||
var uploads = api.MapGroup("/uploads").Requires(Permissions.Upload | Permissions.RenderNode);
|
||||
uploads.MapPost("", StartAsync).Requires(Permissions.Upload);
|
||||
uploads.MapPut("/{id:guid}/chunks/{index:int}", ChunkAsync);
|
||||
uploads.MapPut("/{id:guid}/chunks/{index:int}", ChunkAsync).WithMetadata(new ChunkBodyLimit());
|
||||
uploads.MapGet("/{id:guid}", async (HttpContext context, Caller caller, Guid id, UploadService service) =>
|
||||
(await service.GetStateAsync(caller, id, context.RequestAborted)).ToResult(context, s => Results.Json(s, ProtocolJson.Default.UploadState)));
|
||||
uploads.MapDelete("/{id:guid}", async (HttpContext context, Caller caller, Guid id, UploadService service) =>
|
||||
@@ -53,12 +57,23 @@ public static class UploadEndpoints
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Chunks are the only large request bodies; every other endpoint keeps Kestrel's small default limit. The service
|
||||
/// raises it to the chunk size of this upload (fixed when it started) before it reads the body.
|
||||
/// Chunks are the only large request bodies; every other endpoint keeps Kestrel's small default limit. Routing raises
|
||||
/// it for this route (<see cref="ChunkBodyLimit"/>), and the service keeps it at least at the chunk size of this upload
|
||||
/// (fixed when it started) and of this server, so a body left unread by any answer can still be drained.
|
||||
/// </summary>
|
||||
private static async Task<IResult> ChunkAsync(HttpContext context, Caller caller, Guid id, int index, UploadService service)
|
||||
{
|
||||
var outcome = await service.ReceiveChunkAsync(caller, id, index, context.Request, context.RequestAborted);
|
||||
return outcome.ToResult(context, r => Results.Json(r, ProtocolJson.Default.ChunkReceived));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The chunk route's body limit, applied by routing as it selects the endpoint, before authentication can answer 401
|
||||
/// or 403 with the chunk unread. Kestrel drains an unread body only within the limit; above it, it closes the connection
|
||||
/// and the client sees a network error instead of the answer (an Uploader would retry rather than ask to sign in).
|
||||
/// </summary>
|
||||
private sealed class ChunkBodyLimit : IRequestSizeLimitMetadata
|
||||
{
|
||||
public long? MaxRequestBodySize => MaxChunkBytes;
|
||||
}
|
||||
}
|
||||
@@ -8,8 +8,9 @@ namespace Titano.Server.Storage;
|
||||
/// <summary>
|
||||
/// The archive engine behind the server (BRIEF §7, §14.4): ingest of verified uploads, lookup of proxies and
|
||||
/// thumbnails, its job handlers (scan, rebuild, verify, import, thumbnails, proxies, maintenance) and the periodic
|
||||
/// incremental scan (<see cref="ArchivePolling"/>). Decoding is managed only (BRIEF §4.2). The daily backup of an
|
||||
/// SQLite index runs with it (<see cref="DatabaseBackupService"/>).
|
||||
/// incremental scan (<see cref="ArchivePolling"/>); a scan queues thumbnails and proxies of what it indexes, as an upload
|
||||
/// does. Decoding is managed only (BRIEF §4.2). The daily backup of an SQLite index runs with it
|
||||
/// (<see cref="DatabaseBackupService"/>).
|
||||
/// </summary>
|
||||
public static class ArchiveModule
|
||||
{
|
||||
@@ -19,7 +20,7 @@ public static class ArchiveModule
|
||||
public static void Register(IServiceCollection services)
|
||||
{
|
||||
services.AddSingleton(sp => new ArchiveService(sp.GetRequiredService<TitanoDatabase>(),
|
||||
new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer(), Owner = Owner }));
|
||||
new ArchiveServiceOptions { Decoder = DecoderFactory.CreateServer(), Owner = Owner, ImagesAfterScan = true }));
|
||||
services.AddSingleton<IArchiveIngest>(sp => sp.GetRequiredService<ArchiveService>().Ingest);
|
||||
services.AddSingleton<IJobHandlerProvider>(sp => sp.GetRequiredService<ArchiveService>());
|
||||
services.AddSingleton<IFrameAssetLocator, ArchiveAssetLocator>();
|
||||
|
||||
@@ -101,14 +101,17 @@ public sealed class UploadService(
|
||||
public async Task<Outcome<ChunkReceived>> ReceiveChunkAsync(Caller caller, Guid id, int index, HttpRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
var upload = await db.Uploads.GetAsync(id, cancellationToken);
|
||||
// A body left unread by an answer below is drained after it: one of a chunk's size must not count as too large, or
|
||||
// Kestrel would close the connection while the client still sends it and the client would never see the answer. The
|
||||
// upload keeps the chunk size it started with: a smaller setting since a restart must not refuse its chunks (413).
|
||||
if (request.HttpContext.Features.Get<IHttpMaxRequestBodySizeFeature>() is { IsReadOnly: false, MaxRequestBodySize: { } current } limit)
|
||||
limit.MaxRequestBodySize = Math.Max(current, Math.Max(upload?.ChunkSize ?? 0, options.ChunkSizeBytes));
|
||||
if (upload is null || upload.UserId != caller.Id) return Fail<ChunkReceived>(404, "not_found", "upload");
|
||||
if (upload.State != UploadState.Receiving) return Fail<ChunkReceived>(409, "upload_state");
|
||||
if (index < 0 || index >= upload.ChunkCount) return Fail<ChunkReceived>(400, "chunk_out_of_range");
|
||||
var expected = Math.Min(upload.ChunkSize, upload.Size - (long)index * upload.ChunkSize);
|
||||
if (request.ContentLength is { } announced && announced != expected) return Fail<ChunkReceived>(400, "chunk_size", expected.ToString(CultureInfo.InvariantCulture));
|
||||
if (upload.HasChunk(index)) return new ChunkReceived(index, UploadRules.ReceivedChunks(upload));
|
||||
// The upload keeps the chunk size it started with: a smaller setting since a restart must not refuse its chunks (413).
|
||||
if (request.HttpContext.Features.Get<IHttpMaxRequestBodySizeFeature>() is { IsReadOnly: false } limit) limit.MaxRequestBodySize = upload.ChunkSize;
|
||||
|
||||
if (_chunksInFlight.AddOrUpdate(caller.Id, 1, (_, n) => n + 1) > options.ConcurrentChunksPerUser)
|
||||
{
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Runtime.Versioning;
|
||||
using System.Text.Json.Nodes;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.IO;
|
||||
@@ -17,6 +18,54 @@ public sealed class ArchiveIngestTests(TestContext context)
|
||||
Guid? sequenceId = null, string? title = null, string name = "IMG_0001.JPG") =>
|
||||
new(fx.Archive.Id, sequenceId, title, sequenceId is null && title is null ? sessionKey : null, staged.Path, name, staged.Size, staged.Sha256, null, null);
|
||||
|
||||
/// <summary>A sequence as <c>POST /api/v1/sequences</c> creates it: a row without a folder or frames.</summary>
|
||||
private static async Task<Sequence> CreateWithoutPhotosAsync(ArchiveFixture fx, string title)
|
||||
{
|
||||
var created = new Sequence
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
ArchiveId = fx.Archive.Id,
|
||||
Title = title,
|
||||
Folder = string.Empty,
|
||||
Rating = 4,
|
||||
Origin = SequenceOrigin.Server,
|
||||
OwnerUserId = Guid.CreateVersion7(),
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
};
|
||||
await fx.Db.Sequences.UpsertAsync(created, fx.Ct);
|
||||
return created;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Makes <paramref name="path"/> unreadable until disposed: without read permission where that stops the test process
|
||||
/// (UnauthorizedAccessException), otherwise (Windows, root) held open without sharing (IOException).
|
||||
/// </summary>
|
||||
private static IDisposable Unreadable(string path)
|
||||
{
|
||||
if (!OperatingSystem.IsWindows())
|
||||
{
|
||||
var mode = File.GetUnixFileMode(path);
|
||||
File.SetUnixFileMode(path, UnixFileMode.None);
|
||||
try
|
||||
{
|
||||
File.OpenRead(path).Dispose();
|
||||
File.SetUnixFileMode(path, mode);
|
||||
}
|
||||
catch (UnauthorizedAccessException)
|
||||
{
|
||||
return new UnixPermissions(path, mode);
|
||||
}
|
||||
}
|
||||
return new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.None);
|
||||
}
|
||||
|
||||
/// <summary>Gives a file its permissions back.</summary>
|
||||
[UnsupportedOSPlatform("windows")]
|
||||
private sealed class UnixPermissions(string path, UnixFileMode mode) : IDisposable
|
||||
{
|
||||
public void Dispose() => File.SetUnixFileMode(path, mode);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ASessionKeyCreatesOneSequenceNamedByTheTemplates()
|
||||
{
|
||||
@@ -242,6 +291,166 @@ public sealed class ArchiveIngestTests(TestContext context)
|
||||
Assert.FileExists(staged.Path);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ASequenceWaitingForItsFirstFrameNeverLendsItsIdToTheArchiveRoot()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset");
|
||||
// Loose photos at the root, which a scan adopts in place.
|
||||
Photos.WriteSeries(fx.Archive.RootPath, Start.AddDays(-1), 2);
|
||||
|
||||
await fx.ScanAsync();
|
||||
|
||||
var root = ManifestStore.ReadSequence(fx.Paths.Sequence(string.Empty).ManifestPath);
|
||||
Assert.NotEqual(waiting.Id, root.Id);
|
||||
var rootRow = (await fx.Db.Sequences.GetAsync(root.Id))!;
|
||||
Assert.Equal((string.Empty, 2), (rootRow.Folder, rootRow.FrameCount));
|
||||
var row = (await fx.Db.Sequences.GetAsync(waiting.Id))!;
|
||||
Assert.Equal((string.Empty, 0, "Harbour sunset"), (row.Folder, row.FrameCount, row.Title));
|
||||
|
||||
// Its first frame still gets a folder of its own, and the root keeps its photos.
|
||||
var first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 1), sequenceId: waiting.Id), fx.Ct);
|
||||
Assert.Equal(waiting.Id, first.SequenceId);
|
||||
var placed = (await fx.Db.Sequences.GetAsync(waiting.Id))!;
|
||||
Assert.Equal(("2026/2026-09-28 Canon EOS R5", 1), (placed.Folder, placed.FrameCount));
|
||||
Assert.Equal(root.Id, ManifestStore.ReadSequence(fx.Paths.Sequence(string.Empty).ManifestPath).Id);
|
||||
Assert.Equal(2, (await fx.Db.Sequences.GetAsync(root.Id))!.FrameCount);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ARootSequenceWhoseSidecarIsGoneIsAdoptedAgainUnderItsOwnId()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
Photos.WriteSeries(fx.Archive.RootPath, Start, 2);
|
||||
await fx.ScanAsync();
|
||||
var adopted = Assert.NotNull((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).SingleOrDefault());
|
||||
// Created later, so its id sorts after the root sequence's.
|
||||
var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset");
|
||||
File.Delete(fx.Paths.Sequence(string.Empty).ManifestPath);
|
||||
|
||||
await fx.ScanAsync();
|
||||
|
||||
Assert.Equal(adopted.Id, ManifestStore.ReadSequence(fx.Paths.Sequence(string.Empty).ManifestPath).Id);
|
||||
Assert.Equal(2, (await fx.Db.Sequences.GetAsync(adopted.Id))!.FrameCount);
|
||||
var row = (await fx.Db.Sequences.GetAsync(waiting.Id))!;
|
||||
Assert.Equal((string.Empty, 0), (row.Folder, row.FrameCount));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task RebuildOrphansAndVerifyLeaveASequenceWaitingForItsFirstFrame()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
Photos.WriteSeries(Path.Combine(fx.Archive.RootPath, "Trips", "Comet"), Start, 2);
|
||||
await fx.ScanAsync();
|
||||
var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset");
|
||||
|
||||
// It has no sidecar yet, which is no reason to drop it.
|
||||
var rebuilt = await fx.ScanAsync(ScanMode.Rebuild);
|
||||
Assert.Equal(0, rebuilt.Removed);
|
||||
var orphans = await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct);
|
||||
Assert.Empty(orphans.MissingSequences);
|
||||
Assert.Empty(orphans.UnindexedFolders);
|
||||
Assert.Equal(0, await fx.Service.RemoveMissingSequencesAsync(fx.Archive.Id, [waiting.Id], fx.Ct));
|
||||
var report = await fx.Service.Verifier.VerifyAsync(fx.Archive, null, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
|
||||
Assert.Equal((1, 2L), (report.Sequences, report.Checked));
|
||||
var kept = Assert.NotNull(await fx.Db.Sequences.GetAsync(waiting.Id));
|
||||
Assert.Equal(("Harbour sunset", 4), (kept.Title, kept.Rating));
|
||||
|
||||
// The archive root is not its folder: loose photos there are a folder the index does not know yet.
|
||||
Photos.WriteSeries(fx.Archive.RootPath, Start.AddDays(-1), 1, firstSeed: 50);
|
||||
Assert.SequenceEqual([string.Empty], (await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct)).UnindexedFolders);
|
||||
|
||||
var first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddDays(1), 1), sequenceId: waiting.Id), fx.Ct);
|
||||
Assert.Equal(waiting.Id, first.SequenceId);
|
||||
Assert.Equal("2026/2026-09-29 Canon EOS R5", (await fx.Db.Sequences.GetAsync(waiting.Id))!.Folder);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ADamagedRootSidecarIsRebuiltUnderTheRootSequenceNotOneWaitingForItsFirstFrame()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
Photos.WriteSeries(fx.Archive.RootPath, Start, 2);
|
||||
await fx.ScanAsync();
|
||||
var root = Assert.NotNull((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).SingleOrDefault());
|
||||
// Created later, so its id sorts after the root sequence's; then another tool tears the root sidecar.
|
||||
var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset");
|
||||
var sidecar = fx.Paths.Sequence(string.Empty).ManifestPath;
|
||||
File.WriteAllText(sidecar, "{");
|
||||
|
||||
var summary = await fx.ScanAsync();
|
||||
|
||||
Assert.Empty(summary.Failed);
|
||||
Assert.Equal(root.Id, ManifestStore.ReadSequence(sidecar).Id);
|
||||
Assert.Count(1, Directory.GetFiles(Path.GetDirectoryName(sidecar)!, $"{Path.GetFileName(sidecar)}.damaged-*"));
|
||||
Assert.Equal(2, (await fx.Db.Sequences.GetAsync(root.Id))!.FrameCount);
|
||||
var row = (await fx.Db.Sequences.GetAsync(waiting.Id))!;
|
||||
Assert.Equal((string.Empty, 0, "Harbour sunset"), (row.Folder, row.FrameCount, row.Title));
|
||||
Assert.Empty(await fx.Db.Frames.ListBySequenceAsync(waiting.Id));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ARootSidecarThatCannotBeReadFailsTheRootFolderAloneAndDropsNoSequence()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
Photos.WriteSeries(fx.Archive.RootPath, Start, 2);
|
||||
await fx.ScanAsync();
|
||||
var root = Assert.NotNull((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).SingleOrDefault());
|
||||
var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset");
|
||||
Photos.WriteSeries(Path.Combine(fx.Archive.RootPath, "Trips", "Comet"), Start.AddDays(1), 2, firstSeed: 50);
|
||||
var sidecar = fx.Paths.Sequence(string.Empty).ManifestPath;
|
||||
|
||||
ScanSummary rebuilt;
|
||||
using (Unreadable(sidecar)) rebuilt = await fx.ScanAsync(ScanMode.Rebuild);
|
||||
|
||||
Assert.SequenceEqual([string.Empty], rebuilt.Failed.Select(f => f.Folder));
|
||||
Assert.Equal(0, rebuilt.Removed);
|
||||
var locations = await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct);
|
||||
Assert.SequenceEqual(new[] { root.Id, waiting.Id }.Order(), locations.Where(l => l.Folder.Length == 0).Select(l => l.Id).Order());
|
||||
Assert.SequenceEqual(["Trips/Comet"], locations.Where(l => l.Folder.Length > 0).Select(l => l.Folder));
|
||||
|
||||
// Readable again, the root is indexed under its own sequence.
|
||||
Assert.Empty((await fx.ScanAsync()).Failed);
|
||||
Assert.Equal(root.Id, ManifestStore.ReadSequence(sidecar).Id);
|
||||
var row = (await fx.Db.Sequences.GetAsync(waiting.Id))!;
|
||||
Assert.Equal((string.Empty, 0), (row.Folder, row.FrameCount));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task MaintenanceCompactsTheRootPackUnderTheRootSequencesLeaseOnly()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
// Left over from before the root was adopted, so its id sorts before the root sequence's.
|
||||
var waiting = await CreateWithoutPhotosAsync(fx, "Harbour sunset");
|
||||
Photos.WriteSeries(fx.Archive.RootPath, Start, 2);
|
||||
await fx.ScanAsync();
|
||||
var root = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).Single(l => l.Id != waiting.Id);
|
||||
var packPath = fx.Paths.Sequence(string.Empty).ThumbsPackPath;
|
||||
var hash = new byte[ThumbPack.HashLength];
|
||||
using (var pack = ThumbPackWriter.Open(packPath))
|
||||
{
|
||||
// The same entry written again and again: everything but the last copy is garbage.
|
||||
for (var i = 0; i < 4; i++) pack.Append(0, 48, 32, hash, Photos.Jpeg(Start, 70 + i));
|
||||
pack.Commit();
|
||||
}
|
||||
var length = new FileInfo(packPath).Length;
|
||||
|
||||
// A thumbnail pass of the root sequence appends under its own lease: the pack waits for it.
|
||||
Task<MaintenanceReport> maintenance;
|
||||
using (await LeaseFile.AcquireAsync(fx.Paths.LocksDirectory, $"thumbs-{root.Id:N}", "thumbnails", TimeSpan.FromSeconds(5), cancellationToken: fx.Ct))
|
||||
{
|
||||
maintenance = fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct);
|
||||
await Task.Delay(500, fx.Ct);
|
||||
Assert.False(maintenance.IsCompleted);
|
||||
Assert.Equal(length, new FileInfo(packPath).Length);
|
||||
}
|
||||
|
||||
var report = await maintenance;
|
||||
Assert.SequenceEqual([string.Empty], report.CompactedPacks);
|
||||
using var compacted = ThumbPackReader.Open(packPath);
|
||||
Assert.Equal(0L, compacted.GarbageBytes);
|
||||
Assert.NotNull(compacted.Read(0));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task FramesInsertedBeforeOthersTakeTheProjectPositionsAlong()
|
||||
{
|
||||
|
||||
@@ -3,6 +3,7 @@ using Titano.Core.Import;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Data.Archiving.Jobs;
|
||||
using Titano.Data.Jobs;
|
||||
using Titano.Data.Repositories;
|
||||
using Titano.Testing;
|
||||
using static Titano.Data.Tests.Archiving.ArchiveFixture;
|
||||
@@ -22,6 +23,23 @@ public sealed class ArchiveJobTests(TestContext context)
|
||||
private static async Task<IReadOnlyList<SequenceLocation>> LocationsAsync(ArchiveFixture fx) =>
|
||||
await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id);
|
||||
|
||||
/// <summary>The server's archive engine (ArchiveModule), not started; the desktop's scans queue no image passes.</summary>
|
||||
private static JobEngine ServerEngine(ArchiveFixture fx, params IJobHandler[] others)
|
||||
{
|
||||
var service = new ArchiveService(fx.Db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20), ImagesAfterScan = true });
|
||||
return new JobEngine(fx.Db, [.. service.CreateJobHandlers(), .. others],
|
||||
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
|
||||
}
|
||||
|
||||
private static bool Built(SequenceStatus status) => status.HasFlag(SequenceStatus.ThumbsBuilt) && status.HasFlag(SequenceStatus.ProxiesBuilt);
|
||||
|
||||
private sealed class CallbackHandler(JobKind kind, Func<JobContext, Task> run) : IJobHandler
|
||||
{
|
||||
public JobKind Kind => kind;
|
||||
|
||||
public Task RunAsync(JobContext context) => run(context);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ScanAndRebuildJobsRunAndAScanResumesAfterItsCheckpoint()
|
||||
{
|
||||
@@ -40,6 +58,9 @@ public sealed class ArchiveJobTests(TestContext context)
|
||||
var rebuild = await engine.EnqueueAsync(ArchiveJobs.Rebuild(fx.Archive), cancellationToken: fx.Ct);
|
||||
await fx.WaitForAsync(rebuild.Id, JobState.Done);
|
||||
Assert.Equal(2, (await LocationsAsync(fx)).Count);
|
||||
// The desktop decodes its thumbnails on demand and makes proxies when asked.
|
||||
Assert.Empty((await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items);
|
||||
Assert.Empty((await fx.Db.Jobs.ListAsync(null, JobKind.Proxies, cancellationToken: fx.Ct)).Items);
|
||||
}
|
||||
|
||||
[Test]
|
||||
@@ -88,6 +109,94 @@ public sealed class ArchiveJobTests(TestContext context)
|
||||
Assert.Contains("made 3, reused 3", done.LogTail);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AServerScanQueuesThumbnailsAndProxiesOfTheSequencesItIndexesWithoutThem()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
Populate(fx);
|
||||
await using var engine = ServerEngine(fx);
|
||||
await engine.StartAsync(fx.Ct);
|
||||
|
||||
var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
|
||||
await fx.WaitForAsync(scan.Id, JobState.Done);
|
||||
var locations = await LocationsAsync(fx);
|
||||
var first = new HashSet<Guid>();
|
||||
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
|
||||
{
|
||||
// One pass per sequence, named after it, as after an upload.
|
||||
var passes = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items;
|
||||
Assert.SequenceEqual(locations.Select(l => l.Id).Order(), passes.SelectMany(j => j.TargetIds).Order());
|
||||
foreach (var pass in passes)
|
||||
{
|
||||
Assert.Count(1, pass.TargetIds);
|
||||
var target = Assert.NotNull(await fx.Db.Sequences.GetAsync(pass.TargetIds[0]));
|
||||
Assert.Equal((JobPriority.Background, target.DisplayTitle), (pass.Priority, pass.TargetLabel));
|
||||
await fx.WaitForAsync(pass.Id, JobState.Done);
|
||||
first.Add(pass.Id);
|
||||
}
|
||||
}
|
||||
foreach (var location in locations) Assert.True(Built((await fx.Db.Sequences.GetAsync(location.Id))!.Status), location.Folder);
|
||||
|
||||
// Every folder indexed again, but none without its images: nothing is queued.
|
||||
var full = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive, full: true), cancellationToken: fx.Ct);
|
||||
await fx.WaitForAsync(full.Id, JobState.Done);
|
||||
Assert.Count(2, (await fx.Db.Jobs.ListAsync(null, JobKind.Thumbs, cancellationToken: fx.Ct)).Items);
|
||||
Assert.Count(2, (await fx.Db.Jobs.ListAsync(null, JobKind.Proxies, cancellationToken: fx.Ct)).Items);
|
||||
|
||||
// A photo added to a folder: its sequence lacks the images of that frame until the passes queued for it alone ran.
|
||||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start.AddMinutes(5), 1, firstSeed: 90, firstNumber: 7);
|
||||
var added = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
|
||||
await fx.WaitForAsync(added.Id, JobState.Done);
|
||||
var a = locations.Single(l => l.Folder == "a");
|
||||
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
|
||||
{
|
||||
var pass = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items.Single(j => !first.Contains(j.Id));
|
||||
Assert.SequenceEqual([a.Id], pass.TargetIds);
|
||||
await fx.WaitForAsync(pass.Id, JobState.Done);
|
||||
}
|
||||
Assert.True(Built((await fx.Db.Sequences.GetAsync(a.Id))!.Status), "built again");
|
||||
using var pack = ThumbPackReader.Open(fx.Paths.Sequence("a").ThumbsPackPath);
|
||||
Assert.Count(7, pack.Entries);
|
||||
Assert.Count(7, Directory.GetFiles(fx.Paths.Sequence("a").Proxies));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AFolderGoneBeforeItsImagePassesFailsThemAloneAndTheOtherSequencesGetTheirImages()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
Populate(fx);
|
||||
// The only worker runs the scan, then this job ahead of the background passes the scan queued: the folder goes
|
||||
// meanwhile, as when someone renames or deletes it on the NAS between a poll and its passes.
|
||||
var holding = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
await using var engine = ServerEngine(fx, new CallbackHandler(JobKind.Render, async c =>
|
||||
{
|
||||
holding.SetResult();
|
||||
await release.Task.WaitAsync(c.Cancellation);
|
||||
}));
|
||||
var scan = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive), cancellationToken: fx.Ct);
|
||||
var hold = await engine.EnqueueAsync(JobKind.Render, [fx.Archive.Id], priority: JobPriority.Normal, cancellationToken: fx.Ct);
|
||||
await engine.StartAsync(fx.Ct);
|
||||
await holding.Task.WaitAsync(TimeSpan.FromSeconds(30), fx.Ct);
|
||||
Assert.Equal(JobState.Done, (await fx.Db.Jobs.GetAsync(scan.Id, fx.Ct))!.State);
|
||||
var locations = await LocationsAsync(fx);
|
||||
var (a, b) = (locations.Single(l => l.Folder == "a"), locations.Single(l => l.Folder == "b"));
|
||||
Directory.Delete(Path.Combine(fx.Paths.Root, "a"), recursive: true);
|
||||
release.SetResult();
|
||||
await fx.WaitForAsync(hold.Id, JobState.Done);
|
||||
|
||||
foreach (var kind in new[] { JobKind.Thumbs, JobKind.Proxies })
|
||||
{
|
||||
var passes = (await fx.Db.Jobs.ListAsync(null, kind, cancellationToken: fx.Ct)).Items;
|
||||
await fx.WaitForAsync(passes.Single(j => j.TargetIds.SequenceEqual([a.Id])).Id, JobState.Failed);
|
||||
await fx.WaitForAsync(passes.Single(j => j.TargetIds.SequenceEqual([b.Id])).Id, JobState.Done);
|
||||
}
|
||||
Assert.True(Built((await fx.Db.Sequences.GetAsync(b.Id))!.Status), "b built");
|
||||
using var pack = ThumbPackReader.Open(fx.Paths.Sequence("b").ThumbsPackPath);
|
||||
Assert.Count(6, pack.Entries);
|
||||
Assert.Count(6, Directory.GetFiles(fx.Paths.Sequence("b").Proxies));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AThumbnailPassFlagsFramesNothingDecodesAndClearsThemOnceTheyDecode()
|
||||
{
|
||||
|
||||
@@ -105,6 +105,57 @@ public sealed class MassRenameTests(TestContext context)
|
||||
Assert.True(Directory.Exists(fx.Paths.FromRelative(other.Steps[0].ToFolder)));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ASequenceIndexedInsideAMovingFolderAfterTheDryRunMakesThePlanStale()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "trips"), Start, 2);
|
||||
await fx.ScanAsync();
|
||||
var trips = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single();
|
||||
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [trips.Id], "moved/{date}", null, fx.Ct);
|
||||
Assert.Count(1, plan.Steps);
|
||||
|
||||
// Photos copied into a subfolder and indexed by the next scan, after the dry run was shown.
|
||||
Photos.WriteSeries(Path.Combine(fx.Paths.Root, "trips", "day2"), Start.AddDays(1), 2, firstSeed: 10);
|
||||
await fx.ScanAsync();
|
||||
var day2 = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "trips/day2");
|
||||
|
||||
var stale = await Assert.ThrowsAsync<StaleRenamePlanException>(() => fx.Service.ApplyRenameAsync(plan, null, fx.Ct));
|
||||
Assert.Equal(day2.Id, stale.SequenceId);
|
||||
Assert.Empty(RenameJournal.ListPending(fx.Paths));
|
||||
Assert.SequenceEqual(["trips|trips|trips/IMG_0001.JPG,trips/IMG_0002.JPG", "trips/day2|day2|trips/day2/IMG_0001.JPG,trips/day2/IMG_0002.JPG"],
|
||||
await StateAsync(fx, [trips.Id, day2.Id]));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AFolderThatGainedASequenceWhileTheRenameRanStaysWhereItIs()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var ids = await PopulateAsync(fx);
|
||||
var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [ids[0], ids[1]], "renamed/{date}", null, fx.Ct);
|
||||
Assert.SequenceEqual(["inbox/a", "inbox/b"], plan.Steps.Select(s => s.FromFolder));
|
||||
|
||||
// Once the first folder moved, a scan indexes photos copied into the second one.
|
||||
var late = Path.Combine(fx.Paths.Root, "inbox", "b", "late");
|
||||
var progress = new SyncProgress(done =>
|
||||
{
|
||||
if (done != 1) return;
|
||||
Photos.WriteSeries(late, Start.AddDays(5), 2, firstSeed: 40);
|
||||
fx.ScanAsync().GetAwaiter().GetResult();
|
||||
});
|
||||
await Assert.ThrowsAsync<StaleRenamePlanException>(() => fx.Service.ApplyRenameAsync(plan, progress, fx.Ct));
|
||||
var newcomer = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "inbox/b/late");
|
||||
Assert.True(Directory.Exists(late), "the folder did not move");
|
||||
Assert.Count(1, RenameJournal.ListPending(fx.Paths).Single().Completed);
|
||||
|
||||
// Resumed, the rest of the plan runs but that step is dropped: every index row still names its files.
|
||||
Assert.Equal(1, await fx.Service.Renamer.ResumePendingAsync(fx.Archive, fx.Ct));
|
||||
Assert.Empty(RenameJournal.ListPending(fx.Paths));
|
||||
var state = await StateAsync(fx, [ids[0], ids[1], newcomer.Id]);
|
||||
Assert.SequenceEqual(["renamed/2026-09-28", "inbox/b", "inbox/b/late"], state.Select(s => s.Split('|')[0]));
|
||||
Assert.Equal(0, (await fx.ScanAsync()).Indexed);
|
||||
}
|
||||
|
||||
/// <summary>"trips" with photos of its own, "trips/day2" inside it and "trips/day2/dawn" inside that: three nested sequences.</summary>
|
||||
private static async Task<List<Guid>> NestedAsync(ArchiveFixture fx)
|
||||
{
|
||||
|
||||
@@ -177,5 +177,21 @@ public sealed class AdminTests(TestContext context)
|
||||
{
|
||||
await File.WriteAllBytesAsync(path, saved, context.Cancellation);
|
||||
}
|
||||
|
||||
// A policy this form cannot set, written out of range by another tool, is named so the admin can correct the file.
|
||||
var current = Titano.Core.Archiving.ManifestStore.ReadArchive(path);
|
||||
foreach (var (odd, property) in new[]
|
||||
{
|
||||
(current with { ThumbnailPolicy = new Titano.Core.Archiving.ThumbnailPolicy { LongEdge = 2048 } }, "thumbnailPolicy"),
|
||||
(current with { ProxyPolicy = current.ProxyPolicy with { Quality = 0 } }, "proxyPolicy"),
|
||||
})
|
||||
{
|
||||
Titano.Core.Archiving.ManifestStore.WriteArchive(path, odd);
|
||||
var before = await File.ReadAllBytesAsync(path, context.Cancellation);
|
||||
using var refused = await admin.PutAsJsonAsync("/api/v1/archive/settings", changed with { ProxyLongEdge = 1280 }, ProtocolJson.Default.ArchiveSettingsDto);
|
||||
Assert.Equal(property, (await refused.ExpectErrorAsync(HttpStatusCode.Conflict, "archive_policy")).Details);
|
||||
Assert.SequenceEqual(before, await File.ReadAllBytesAsync(path, context.Cancellation));
|
||||
}
|
||||
await File.WriteAllBytesAsync(path, saved, context.Cancellation);
|
||||
}
|
||||
}
|
||||
@@ -68,7 +68,7 @@ public sealed class ArchiveModuleTests(TestContext context)
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task PhotosAlreadyInTheArchiveAndAddedLaterAreIndexedByPolling()
|
||||
public async Task PhotosAlreadyInTheArchiveAndAddedLaterAreIndexedByPollingWithThumbnailsAndProxies()
|
||||
{
|
||||
var photo = Photo();
|
||||
// An adopted folder that already holds photos.
|
||||
@@ -78,8 +78,11 @@ public sealed class ArchiveModuleTests(TestContext context)
|
||||
File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0001.JPG"), photo);
|
||||
}, ArchiveModule.Register);
|
||||
var archive = server.Services.GetRequiredService<ServerArchive>();
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
await UntilIndexedAsync(server, archive.Id, "roof");
|
||||
await UntilScannedAsync(server);
|
||||
// The scan queues the thumbnails and proxies clients load from the server, as an upload does.
|
||||
await UntilImagesAsync(server, editor, archive.Id, "roof");
|
||||
|
||||
// Another writer adds a folder: the next poll queues a scan, but not while one is pending.
|
||||
Directory.CreateDirectory(Path.Combine(server.Options.ArchivePath, "garden"));
|
||||
@@ -100,6 +103,7 @@ public sealed class ArchiveModuleTests(TestContext context)
|
||||
var scan = Assert.NotNull(await polling.PollAsync(context.Cancellation));
|
||||
Assert.Equal(JobPriority.Background, scan.Priority);
|
||||
await UntilIndexedAsync(server, archive.Id, "garden");
|
||||
await UntilImagesAsync(server, editor, archive.Id, "garden");
|
||||
}
|
||||
|
||||
private static async Task UntilIndexedAsync(TestServer server, Guid archiveId, string folder)
|
||||
@@ -112,6 +116,30 @@ public sealed class ArchiveModuleTests(TestContext context)
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Waits until the first frame of the sequence in <paramref name="folder"/> has its thumbnail and proxy served.</summary>
|
||||
private static async Task UntilImagesAsync(TestServer server, HttpClient client, Guid archiveId, string folder)
|
||||
{
|
||||
var sequence = (await server.Database.Sequences.ListLocationsAsync(archiveId)).Single(l => l.Folder == folder);
|
||||
var frame = (await server.Database.Frames.ListBySequenceAsync(sequence.Id)).First();
|
||||
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
||||
foreach (var asset in new[] { "thumb", "proxy" })
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
using var response = await client.GetAsync($"/api/v1/frames/{frame.Id}/{asset}");
|
||||
if (response.StatusCode == HttpStatusCode.OK)
|
||||
{
|
||||
Assert.True(await response.Content.ReadAsByteArrayAsync() is [0xFF, 0xD8, ..], $"a JPEG {asset} of {folder}");
|
||||
break;
|
||||
}
|
||||
Assert.True(DateTime.UtcNow < until, $"no {asset} of {folder}: {(int)response.StatusCode}");
|
||||
await Task.Delay(100);
|
||||
}
|
||||
}
|
||||
var built = Assert.NotNull(await server.Database.Sequences.GetAsync(sequence.Id)).Status;
|
||||
Assert.True(built.HasFlag(SequenceStatus.ThumbsBuilt) && built.HasFlag(SequenceStatus.ProxiesBuilt), $"{folder}: {built}");
|
||||
}
|
||||
|
||||
/// <summary>Waits for the scan queued at start to finish: its sequences are indexed before it ends.</summary>
|
||||
private static async Task UntilScannedAsync(TestServer server)
|
||||
{
|
||||
|
||||
@@ -170,6 +170,35 @@ public sealed class UploadTests(TestContext context)
|
||||
await oversized.ExpectErrorAsync(HttpStatusCode.BadRequest, "chunk_size");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnswersThatLeaveALargeChunkUnreadReachTheClient()
|
||||
{
|
||||
// Chunks far above the 1 MiB other requests may send: more than the connection buffers while the server reads nothing.
|
||||
const int large = 32 * 1024 * 1024;
|
||||
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = large);
|
||||
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
|
||||
var data = Uploads.Photo(24 * 1024 * 1024, 16);
|
||||
var id = (await Uploads.StartAsync(uploader, data)).UploadId!.Value;
|
||||
using (var first = await Uploads.PutChunkAsync(uploader, id, 0, data)) await first.ExpectAsync(HttpStatusCode.OK);
|
||||
|
||||
// The client lost the answer and sends the chunk again: the same answer, not a closed connection.
|
||||
using (var again = await Uploads.PutChunkAsync(uploader, id, 0, data))
|
||||
{
|
||||
await again.ExpectAsync(HttpStatusCode.OK);
|
||||
Assert.SequenceEqual([0], (await again.ReadAsync(ProtocolJson.Default.ChunkReceived)).Received);
|
||||
}
|
||||
using (var unknown = await Uploads.PutChunkAsync(uploader, Guid.CreateVersion7(), 0, data)) await unknown.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found");
|
||||
using (var complete = await uploader.PostAsync($"/api/v1/uploads/{id}/complete", null)) await complete.ExpectAsync(HttpStatusCode.OK);
|
||||
using (var late = await Uploads.PutChunkAsync(uploader, id, 0, data)) await late.ExpectErrorAsync(HttpStatusCode.Conflict, "upload_state");
|
||||
|
||||
// Authentication answers before the endpoint runs: a user who may not upload, and a session signed out meanwhile.
|
||||
var (viewer, _, _) = await server.UserAsync("marco", "viewer");
|
||||
using (var forbidden = await Uploads.PutChunkAsync(viewer, id, 0, data)) await forbidden.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
|
||||
using (var logout = await uploader.PostAsync("/api/v1/auth/logout", null)) await logout.ExpectAsync(HttpStatusCode.NoContent);
|
||||
using var revoked = await Uploads.PutChunkAsync(uploader, id, 0, data);
|
||||
await revoked.ExpectErrorAsync(HttpStatusCode.Unauthorized, "unauthorized");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task RenderUploadsNeedTheEditPermission()
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user