diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 5ec84a9a..6b05e0d6 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -88,7 +88,7 @@ Dependencies point one way: `Core` ← `Core.Windows`, `Data`, `Protocol` ← `S | `Server/Api`, `Server/Events` | Endpoint groups (`XxxEndpoints.Map`; `ScheduleEndpoints`: `/api/v1/schedules`; `LibraryEndpoints`: the Library filters' facets, sequence thumbnail, photo delete, bulk delete/retag/jobs; `TrashEndpoints`: list, restore, empty), API errors, the WebSocket `EventHub` | | `Server/Security` | Accounts, password hashing, tokens, session cookies, login guard, access rules, audit log | | `Server/Storage`, `Server/Jobs` | Archive and processing modules, uploads (`UploadMaintenance`: recovery at start, and the sweep that is the `uploads` clean-up step), render store; `JobEngineHost`, `SchedulerHost` (the `Scheduler` on the engine, started after it; archive.json read every 10 minutes whatever the scan schedule does; every schedule change and run published as a `scheduleChanged` event) and `ServerScheduleHost` (time zone, archive refreshed from archive.json, system schedules), render node service, monitor and availability | -| `Server/Hosting`, `Server/Admin/web` | `ServerHost`, `ServerLogging` (console and file levels the logging filters read at every line, the file sink's limits, the engine's level; applied again when Settings › Logs is saved), certificates, discovery beacon, embedded static files (`EmbeddedStaticFiles`: `no-cache` and a content ETag, so an upgrade never mixes old and new modules), Windows service (start and stop failures in the Application event log); the admin panel (ES modules, one per page; `schedules.js` is the Jobs page's Scheduled tab; the Library's views in `pages/library/`: `sequence.js` a sequence and its photo viewer, `trash.js` the archive's trash, `shared.js` cards, selection and grid keyboard; `upload.js` the upload dialog the Library opens on demand, hashing in the module worker `hash-worker.js` with the in-house incremental SHA-256 of `sha256.js`, self-tested before its first use) | +| `Server/Hosting`, `Server/Admin/web` | `ServerHost`, `ServerLogging` (console and file levels the logging filters read at every line, the file sink's limits, the engine's level; applied again when Settings › Logs is saved), certificates, discovery beacon, embedded static files (`EmbeddedStaticFiles`: `no-cache` and a content ETag, so an upgrade never mixes old and new modules), Windows service (start and stop failures in the Application event log); the admin panel (ES modules, one per page; `schedules.js` is the Jobs page's Scheduled tab; the Library's views in `pages/library/`: `sequence.js` a sequence and its photo viewer, `trash.js` the archive's trash, `shared.js` cards, selection and grid keyboard; `upload.js` the upload dialog the Library opens on demand, hashing in the module worker `hash-worker.js` with the in-house incremental SHA-256 of `sha256.js`, self-tested before its first use, and sending the capture time `capture-time.js` reads: the Exif DateTimeOriginal of a JPEG or TIFF-based file, else the file's time) | ### 2.5 Hosts diff --git a/i18n/en/Admin.resx b/i18n/en/Admin.resx index 38b1ab6e..d75b9771 100644 --- a/i18n/en/Admin.resx +++ b/i18n/en/Admin.resx @@ -781,6 +781,8 @@ Trash Add photos Choose photos on this computer; you can also drop them on this dialog + Upload {0} again + Upload again: add this photo to the sequence again, although it was deleted from it; Upload sends it {0} of {1} Cancel Stop now: the photos not finished yet are discarded on the server @@ -812,14 +814,14 @@ Folders skipped: {0} Folders are imported with the desktop app; here only photos are uploaded Not photos: {0} - Files in a format the server does not read as photos were left out + Hidden files and files in a format the server does not read as photos were left out Too large: {0} Files larger than the server's upload limit ({0}) were left out Upload Send the photos not on the server yet; the server checks each one against its SHA-256 Cancelled Deleted before - Deleted from this sequence earlier: restore it from the trash to bring it back + Deleted from this sequence earlier: restore it from the trash while it is there, or upload it again Uploaded On the server and verified against its SHA-256 Already there diff --git a/i18n/en/Server.resx b/i18n/en/Server.resx index 45467137..9e11c2b0 100644 --- a/i18n/en/Server.resx +++ b/i18n/en/Server.resx @@ -73,6 +73,7 @@ Not found.long That address is not a Chronos Server.long Change your password first.long + The photo was deleted from the sequence while it was being uploaded, so it was not added.long This page did not come through a trusted proxy over HTTPS. Open it through your proxy, or add the proxy's address to CHRONOS_TRUSTED_PROXIES.long This archive makes no proxies.long The proxy was made from another photo.long diff --git a/i18n/it/Admin.it.resx b/i18n/it/Admin.it.resx index ddb1488d..f4d521af 100644 --- a/i18n/it/Admin.it.resx +++ b/i18n/it/Admin.it.resx @@ -778,6 +778,8 @@ Cestino Aggiungi foto Scegli le foto su questo computer; puoi anche trascinarle in questa finestra + Carica di nuovo {0} + Carica di nuovo: aggiungi questa foto alla sequenza anche se ne era stata eliminata; Carica la invia {0} di {1} Annulla Ferma ora: le foto non ancora finite vengono scartate sul server @@ -809,14 +811,14 @@ Cartelle saltate: {0} Le cartelle si importano con l'app desktop; qui si caricano solo foto Non foto: {0} - Esclusi i file in un formato che il server non legge come foto + Esclusi i file nascosti e quelli in un formato che il server non legge come foto Troppo grandi: {0} Esclusi i file oltre il limite di caricamento del server ({0}) Carica Invia le foto non ancora sul server; il server controlla ognuna con il suo SHA-256 Annullata Eliminata prima - Eliminata in precedenza da questa sequenza: ripristinala dal cestino per riaverla + Eliminata in precedenza da questa sequenza: ripristinala dal cestino finché c'è, o caricala di nuovo Caricata Sul server e verificata con il suo SHA-256 Già presente diff --git a/i18n/it/Server.it.resx b/i18n/it/Server.it.resx index b0c716cd..057accba 100644 --- a/i18n/it/Server.it.resx +++ b/i18n/it/Server.it.resx @@ -71,6 +71,7 @@ Non trovato. Questo indirizzo non è un Chronos Server. Cambia prima la password. + La foto è stata eliminata dalla sequenza mentre veniva caricata, quindi non è stata aggiunta.long Questa pagina non è arrivata da un proxy attendibile in HTTPS. Aprila tramite il proxy o aggiungi l'indirizzo del proxy a CHRONOS_TRUSTED_PROXIES. Questo archivio non crea proxy. Il proxy è stato creato da un'altra foto. diff --git a/src/Chronos.Data/Sync/SyncRules.cs b/src/Chronos.Data/Sync/SyncRules.cs index aaead0cf..440026c5 100644 --- a/src/Chronos.Data/Sync/SyncRules.cs +++ b/src/Chronos.Data/Sync/SyncRules.cs @@ -167,7 +167,9 @@ public static class SyncRules /// /// honouring the photos /// either side removed (owner review 2, R6): a photo the server deleted is not missing there, one this PC deleted is not - /// missing here, and a photo replaced by a converted file on either side is compared as its replacement. + /// missing here, and a photo replaced by a converted file on either side is compared as its replacement. A tombstone + /// leaves out only a photo its side no longer holds: one left beside a frame of the same content (the other copy of a + /// duplicate, a file put back by hand) would otherwise keep the other side's copy missing for good. /// public static (int MissingRemote, int MissingLocal) CompareOriginals(IReadOnlyCollection local, IReadOnlyCollection remote, SyncTombstones tombstones) @@ -176,8 +178,12 @@ public static class SyncRules ArgumentNullException.ThrowIfNull(remote); ArgumentNullException.ThrowIfNull(tombstones); if (tombstones.IsEmpty) return CompareOriginals(local, remote); - return CompareOriginals([.. local.Where(p => !tombstones.DeletedRemotely(p.Sha256)).Select(tombstones.Canonical)], - [.. remote.Where(p => !tombstones.DeletedLocally(p.Sha256)).Select(tombstones.Canonical)]); + var mine = local.Select(p => (p.Sha256, Photo: tombstones.Canonical(p))).ToList(); + var theirs = remote.Select(p => (p.Sha256, Photo: tombstones.Canonical(p))).ToList(); + var minePhotos = new FramePhotos(mine.Select((p, i) => (i, p.Photo))); + var theirPhotos = new FramePhotos(theirs.Select((p, i) => (i, p.Photo))); + return CompareOriginals([.. mine.Where(p => !tombstones.DeletedRemotely(p.Sha256) || theirPhotos.IndexOf(p.Photo) is not null).Select(p => p.Photo)], + [.. theirs.Where(p => !tombstones.DeletedLocally(p.Sha256) || minePhotos.IndexOf(p.Photo) is not null).Select(p => p.Photo)]); } /// diff --git a/src/Chronos.Data/Sync/SyncTransfers.cs b/src/Chronos.Data/Sync/SyncTransfers.cs index d8b279ff..9db45a7d 100644 --- a/src/Chronos.Data/Sync/SyncTransfers.cs +++ b/src/Chronos.Data/Sync/SyncTransfers.cs @@ -70,8 +70,8 @@ internal sealed class SyncTransfers(SyncService sync) // server has filed one, which creates the sequence (an original it already holds elsewhere creates nothing). while (remoteId is null && pending.TryDequeue(out var first)) { - var sent = await UploadFrameAsync(context, remote, null, SyncService.SessionKey(local.Id), paths.FromRelative(first.RelativePath), first, hashes, ct) - .ConfigureAwait(false); + var sent = await UploadFrameAsync(context, remote, null, SyncService.SessionKey(local.Id), paths.FromRelative(first.RelativePath), first, tombstones, + hashes, ct).ConfigureAwait(false); if (sent is null) skipped++; else if (sent.SequenceId != Guid.Empty) { @@ -85,11 +85,15 @@ internal sealed class SyncTransfers(SyncService sync) await Parallel.ForEachAsync(pending, Workers(sync.Options.Uploads, ct), async (frame, token) => { - if (await UploadFrameAsync(context, remote, target, null, paths.FromRelative(frame.RelativePath), frame, hashes, token).ConfigureAwait(false) is null) - Interlocked.Increment(ref skipped); + var sent = await UploadFrameAsync(context, remote, target, null, paths.FromRelative(frame.RelativePath), frame, tombstones, hashes, token) + .ConfigureAwait(false); + if (sent is null) Interlocked.Increment(ref skipped); await progress.StepAsync().ConfigureAwait(false); }).ConfigureAwait(false); await SendProxiesAsync(context, remote, target, local, paths, frames, hashes).ConfigureAwait(false); + // An original the index has no hash for turned out to be a photo the server deleted: the verify gives the index + // the hashes (as before a download), so the comparison leaves it out instead of offering its upload again and again. + if (hashes.Values.Any(tombstones.DeletedRemotely)) await VerifyAsync(archive, local.Id, ct).ConfigureAwait(false); if (created) await sync.PushAllAsync(remote, target, ct).ConfigureAwait(false); await sync.RecompareAsync(remote, target, ct).ConfigureAwait(false); @@ -240,7 +244,7 @@ internal sealed class SyncTransfers(SyncService sync) await progress.StepAsync().ConfigureAwait(false); } var sequenceId = local?.Id; - var restored = 0; + int restored = 0, refused = 0; await Parallel.ForEachAsync(rest, Workers(sync.Options.Downloads, ct), async (frame, token) => { var (staged, sha) = await FetchOriginalAsync(remote, frame, staging, token).ConfigureAwait(false); @@ -249,6 +253,14 @@ internal sealed class SyncTransfers(SyncService sync) Restore(staged, place); Interlocked.Increment(ref restored); } + else if (frame.Sha256.Length == 0 && (tombstones.DeletedLocally(sha) || have.Contains(tombstones.Canonical(sha)!))) + { + // An original the server has not hashed is known by content only now: one this PC deleted (or holds as + // the converted file that replaced it) is not filed again, which would also drop its tombstone. + File.Delete(staged); + Interlocked.Increment(ref refused); + if (tombstones.DeletedLocally(sha)) context.Log($"{frame.FileName}: deleted on this PC, not downloaded"); + } else { await CommitAsync(remoteId, archive.Id, sequenceId, frame, staged, sha, captured, token).ConfigureAwait(false); @@ -261,7 +273,7 @@ internal sealed class SyncTransfers(SyncService sync) if (created) await sync.TakeAllAsync(remote, remoteId, ct).ConfigureAwait(false); await sync.RecompareAsync(remote, remoteId, ct).ConfigureAwait(false); - if (sequenceId is { } downloaded && wanted.Count > restored) + if (sequenceId is { } downloaded && wanted.Count > restored + refused) { var label = local!.DisplayTitle; await EnqueueFollowUpAsync(context, ArchiveJobs.Thumbnails([downloaded], label)).ConfigureAwait(false); @@ -497,10 +509,11 @@ internal sealed class SyncTransfers(SyncService sync) /// /// Hashes the original again and sends it into (or the sequence of /// ); null (logged) when it is gone or no longer the archived file. The hash of an - /// original the index has none for goes into . + /// original the index has none for goes into ; known by content only now, such an original + /// the server deleted () is not sent back. /// private static async Task UploadFrameAsync(JobContext context, ISyncRemote remote, Guid? sequenceId, string? sessionKey, string path, - Frame frame, ConcurrentDictionary hashes, CancellationToken cancellationToken) + Frame frame, SyncTombstones tombstones, ConcurrentDictionary hashes, CancellationToken cancellationToken) { if (!File.Exists(path)) { @@ -508,14 +521,22 @@ internal sealed class SyncTransfers(SyncService sync) return null; } var sha = await ChronosClient.HashFileAsync(path, cancellationToken).ConfigureAwait(false); - if (frame.Sha256 is null) hashes[frame.Id] = sha; + var notSent = new CompleteUploadResponse(Guid.Empty, sequenceId ?? Guid.Empty, -1); + if (frame.Sha256 is null) + { + hashes[frame.Id] = sha; + if (tombstones.DeletedRemotely(sha)) + { + context.Log($"{frame.FileName}: deleted on the server, not uploaded"); + return notSent; + } + } if (frame.Sha256 is { } archived && !archived.Equals(sha, StringComparison.OrdinalIgnoreCase)) { context.Log($"{frame.FileName}: changed since it was archived, not uploaded"); return null; } - return await remote.UploadFileAsync(path, sequenceId, sessionKey, sha, frame.CapturedAt, cancellationToken).ConfigureAwait(false) - ?? new CompleteUploadResponse(Guid.Empty, sequenceId ?? Guid.Empty, -1); + return await remote.UploadFileAsync(path, sequenceId, sessionKey, sha, frame.CapturedAt, cancellationToken).ConfigureAwait(false) ?? notSent; } /// Commits one original from staging into the local sequence (created from the server id when null). diff --git a/src/Chronos.Protocol/Client/ChronosClient.cs b/src/Chronos.Protocol/Client/ChronosClient.cs index 0d8e354f..6eba815b 100644 --- a/src/Chronos.Protocol/Client/ChronosClient.cs +++ b/src/Chronos.Protocol/Client/ChronosClient.cs @@ -165,7 +165,9 @@ public sealed class ChronosClient : IDisposable /// /// Uploads one file with resumable, idempotent chunks; the server verifies the SHA-256 before committing it - /// (BRIEF §14.4). Returns the committed frame id. Transient failures retry with exponential backoff. + /// (BRIEF §14.4). Returns the committed frame id; null when the server holds the photo without a frame to name: it was + /// deleted from the target sequence, before the upload or while it was on its way (owner review 2, R6). Transient + /// failures retry with exponential backoff. /// public async Task UploadFileAsync( string path, @@ -189,7 +191,15 @@ public sealed class ChronosClient : IDisposable var uploadId = start.UploadId ?? throw new InvalidOperationException("Server returned no upload id."); await SendChunksAsync(path, info.Length, uploadId, start.ChunkSize, start.Received, progress, ct).ConfigureAwait(false); - return await RetryAsync(() => SendAsync(HttpMethod.Post, $"api/v1/uploads/{uploadId}/complete", ProtocolJson.Default.CompleteUploadResponse, ct), ct).ConfigureAwait(false); + try + { + return await RetryAsync(() => SendAsync(HttpMethod.Post, $"api/v1/uploads/{uploadId}/complete", ProtocolJson.Default.CompleteUploadResponse, ct), ct) + .ConfigureAwait(false); + } + catch (ChronosApiException ex) when (ex is { Status: HttpStatusCode.Conflict, Error.Code: UploadErrors.PhotoRemoved }) + { + return null; + } } /// diff --git a/src/Chronos.Protocol/Uploads.cs b/src/Chronos.Protocol/Uploads.cs index d7223c17..8b2cf6fd 100644 --- a/src/Chronos.Protocol/Uploads.cs +++ b/src/Chronos.Protocol/Uploads.cs @@ -2,12 +2,26 @@ namespace Chronos.Protocol; /// /// POST /api/v1/uploads (BRIEF §14.4). The client hashes the file first; when the server already holds that -/// SHA-256 it answers 200 with and no upload is needed. +/// SHA-256 it answers 200 with and no upload is needed. A photo deleted +/// from the target sequence is answered the same way, without a frame (owner review 2, R6), unless +/// asks to file it there again: only an explicit user action sends it (the admin panel's "Upload again"), never sync or a +/// watch folder, and the completion repeats it (complete?restore=true). Filing it drops the photo's tombstone. /// -public sealed record StartUploadRequest(Guid? SequenceId, string FileName, long Size, string Sha256, DateTime? CapturedAt, string? SessionKey); +public sealed record StartUploadRequest(Guid? SequenceId, string FileName, long Size, string Sha256, DateTime? CapturedAt, string? SessionKey, + bool Restore = false); public sealed record StartUploadResponse(Guid? UploadId, int ChunkSize, IReadOnlyList Received, bool AlreadyExists, Guid? FrameId); +/// Error codes of the frame upload calls that clients act on. +public static class UploadErrors +{ + /// + /// 409 from POST /api/v1/uploads/{id}/complete: the photo was deleted from the target sequence while it was on its + /// way and the completion did not ask to restore it, so it was not filed and the upload is cancelled (owner review 2, R6). + /// + public const string PhotoRemoved = "photo_removed"; +} + /// PUT /api/v1/uploads/{id}/chunks/{n} response. public sealed record ChunkReceived(int Index, IReadOnlyList Received); diff --git a/src/Chronos.Server/Admin/web/capture-time.js b/src/Chronos.Server/Admin/web/capture-time.js new file mode 100644 index 00000000..dd9252a2 --- /dev/null +++ b/src/Chronos.Server/Admin/web/capture-time.js @@ -0,0 +1,91 @@ +// The capture time the upload dialog sends with a photo (owner review 2, R3), as local wall time without an offset +// ("2025-05-01T06:30:00", the way the desktop clients send it): the Exif DateTimeOriginal of a JPEG or of a TIFF-based file +// (TIFF, DNG and most RAW formats), else the file's modification time. The server keeps the time a photo records itself +// and uses this one only for a photo that records none (a PNG, an export stripped of its Exif data), so such photos keep +// their order instead of taking the moment they arrived. The reader is minimal: it follows one IFD pointer, checks every +// offset against the bytes it read and gives up (null) on anything it does not expect. + +/** Bytes read from the start of a file: every camera writes the Exif block within them. */ +const READ = 256 * 1024; + +/** The capture time of a picked File; null when neither the photo nor the file gives a usable one. */ +export async function captureTime(file) { + try { + const taken = exifCaptureTime(new Uint8Array(await file.slice(0, READ).arrayBuffer())); + if (taken) return taken; + } catch { + // Unreadable here: the file's own time stands in, and the server reads the photo again anyway. + } + const modified = file.lastModified; + return Number.isFinite(modified) && modified > 0 ? localIso(new Date(modified)) : null; +} + +/** The Exif DateTimeOriginal of the JPEG or TIFF whose first bytes are given, as "yyyy-MM-ddTHH:mm:ss"; null when there is none. */ +export function exifCaptureTime(bytes) { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const size = view.byteLength; + let tiff = -1; + if (size >= 4 && view.getUint16(0) === 0xffd8) { + // JPEG: the segments before the image data, up to the APP1 that starts with "Exif\0\0". + for (let at = 2; at + 4 <= size;) { + const marker = view.getUint16(at); + if ((marker & 0xff00) !== 0xff00 || marker === 0xffda) break; + const length = view.getUint16(at + 2); + if (marker === 0xffe1 && length >= 16 && at + 10 <= size && view.getUint32(at + 4) === 0x45786966 && view.getUint16(at + 8) === 0) { + tiff = at + 10; + break; + } + if (length < 2) break; + at += 2 + length; + } + } else { + tiff = 0; + } + if (tiff < 0 || tiff + 8 > size) return null; + const order = view.getUint16(tiff); + if (order !== 0x4949 && order !== 0x4d4d) return null; + const little = order === 0x4949; + const u16 = (at) => view.getUint16(at, little); + const u32 = (at) => view.getUint32(at, little); + if (u16(tiff + 2) !== 42) return null; + + // The 12-byte entry of a tag in the IFD at an offset from the TIFF header; -1 when it is not within reach. + const entry = (ifd, tag) => { + const at = tiff + ifd; + if (ifd < 8 || at + 2 > size) return -1; + const count = u16(at); + for (let i = 0; i < count; i++) { + const e = at + 2 + i * 12; + if (e + 12 > size) return -1; + if (u16(e) === tag) return e; + } + return -1; + }; + const exif = entry(u32(tiff + 4), 0x8769); + if (exif < 0) return null; + const original = entry(u32(exif + 8), 0x9003); + // ASCII "YYYY:MM:DD HH:MM:SS" and its terminator, stored after the IFD. + if (original < 0 || u16(original + 2) !== 2 || u32(original + 4) < 19) return null; + const text = tiff + u32(original + 8); + if (text + 19 > size) return null; + let value = ""; + for (let i = 0; i < 19; i++) value += String.fromCharCode(view.getUint8(text + i)); + const m = /^(\d{4}):(\d{2}):(\d{2}) (\d{2}):(\d{2}):(\d{2})$/.exec(value); + if (!m) return null; + const [year, month, day, hour, minute, second] = m.slice(1).map(Number); + // A real date and time only ("0000:00:00 00:00:00" is what cameras write without a clock): checked in UTC, where every + // wall time exists. + const check = new Date(Date.UTC(year, month - 1, day, hour, minute, second)); + if (year < 1900 || check.getUTCFullYear() !== year || check.getUTCMonth() !== month - 1 || check.getUTCDate() !== day || + check.getUTCHours() !== hour || check.getUTCMinutes() !== minute || check.getUTCSeconds() !== second) return null; + return `${m[1]}-${m[2]}-${m[3]}T${m[4]}:${m[5]}:${m[6]}`; +} + +/** Local wall time with milliseconds and no offset; null outside the years a server date takes. */ +export function localIso(date) { + const year = date.getFullYear(); + if (!(year >= 1 && year <= 9999)) return null; + const p = (n, width = 2) => String(n).padStart(width, "0"); + return `${p(year, 4)}-${p(date.getMonth() + 1)}-${p(date.getDate())}T${p(date.getHours())}:${p(date.getMinutes())}:${p(date.getSeconds())}` + + `.${p(date.getMilliseconds(), 3)}`; +} diff --git a/src/Chronos.Server/Admin/web/upload.js b/src/Chronos.Server/Admin/web/upload.js index 7e78d045..31323142 100644 --- a/src/Chronos.Server/Admin/web/upload.js +++ b/src/Chronos.Server/Admin/web/upload.js @@ -1,18 +1,22 @@ // Upload dialog of the admin panel (owner review 2, R3; BRIEF §14.4, §14.8): photos from this computer into an existing // or a new sequence, chosen with the file picker or dropped on the dialog. Folders are not taken: importing a whole folder // stays a feature of the desktop app. Each photo is hashed in a module worker (hash-worker.js, which self-tests its -// SHA-256 first and refuses to hash when a check fails), started, sent in the chunks the server does not have yet and -// completed; the server verifies the hash again before it files the photo. +// SHA-256 first and refuses to hash when a check fails), started with its capture time (capture-time.js), sent in the +// chunks the server does not have yet and completed; the server verifies the hash again before it files the photo. A +// photo deleted from the sequence before is not filed again unless the user asks for it on its row (Upload again). import { ApiFailure } from "./api.js"; import { t, plural, applyStrings, culture } from "./i18n.js"; import { escapeHtml, attr, bar, formatBytes, formatNumber, errorText } from "./ui.js"; import { icons } from "./icons.js"; +import { captureTime } from "./capture-time.js"; /** Photos sent at once, and chunks in flight for each: four requests, half of what the server takes from one account. */ const FILES_AT_ONCE = 2; const CHUNKS_AT_ONCE = 2; /** Retries of a call that failed for a passing reason (no answer, 429, 503), after 1, 2 and 4 seconds. */ const RETRIES = 3; +/** Checks of an upload another completion is filing, after 1, 2, 4 and then every 8 seconds: about a minute in all. */ +const SETTLE_CHECKS = 10; const SEARCH_DELAY = 250; const LIST_LIMIT = 50; /** States of a photo already on the server: a later run leaves it alone. */ @@ -48,6 +52,8 @@ export function openUpload({ api, sequence, onDone } = {}) { let searchTimer = null; let searchToken = 0; let frame = 0; + // Per run: by SHA-256, the send of the last photo of that content, which the next copy waits for. + let sameContent = new Map(); const items = []; const skipped = { folders: 0, formats: 0, size: 0 }; const titles = new Map(); @@ -103,6 +109,17 @@ export function openUpload({ api, sequence, onDone } = {}) { else dialog.close(); }); list.addEventListener("click", (e) => { + // Upload again: a photo deleted from the sequence before goes back into it with the next run, on the user's word. + const again = e.target.closest('[data-act="again"]'); + if (again && !running) { + const item = items.find((i) => i.key === again.closest("[data-key]").dataset.key); + if (item?.state !== "deleted") return; + item.restore = true; + setState(item, "waiting"); + drawButtons(); + startButton.focus(); + return; + } const button = e.target.closest('[data-act="remove"]'); if (!button || running) return; const index = items.findIndex((i) => i.key === button.closest("[data-key]").dataset.key); @@ -134,7 +151,12 @@ export function openUpload({ api, sequence, onDone } = {}) { take(files, folders); }); - // Esc closes only while nothing is being sent; Cancel stops a run. + // Esc closes only while nothing is being sent; Cancel stops a run. Chromium lets a dialog's cancel be prevented only + // once per user activation, so during a run Esc is stopped at its keydown; the cancel handler stays for close requests + // that do not come from the keyboard (a back gesture). + dialog.addEventListener("keydown", (e) => { + if (running && e.key === "Escape") e.preventDefault(); + }); dialog.addEventListener("cancel", (e) => { if (running) e.preventDefault(); }); @@ -234,13 +256,17 @@ export function openUpload({ api, sequence, onDone } = {}) { ``; } - /** Adds picked or dropped files: photos once each; folders, other files and files above the limit are counted. */ + /** + * Adds picked or dropped files: photos once each; folders, other files and files above the limit are counted. Hidden + * dot files (the AppleDouble "._" companions a Mac leaves on cards among them) are left out like the archive scanner + * leaves them out, and the server refuses them too. + */ function take(files, folders) { skipped.folders += folders; for (const file of files) { const dot = file.name.lastIndexOf("."); const extension = dot < 0 ? "" : file.name.slice(dot + 1).toLowerCase(); - if (file.size === 0 || !extensions?.has(extension)) { + if (file.size === 0 || file.name.startsWith(".") || !extensions?.has(extension)) { skipped.formats++; continue; } @@ -250,14 +276,16 @@ export function openUpload({ api, sequence, onDone } = {}) { } const key = `${file.name}\u0000${file.size}\u0000${file.lastModified}`; if (items.some((i) => i.key === key)) continue; - const item = { key, file, state: "waiting", hashed: 0, sent: 0, uploadId: null, message: "", row: null }; + const item = { key, file, state: "waiting", hashed: 0, sent: 0, uploadId: null, restore: false, message: "", row: null }; item.row = document.createElement("li"); item.row.className = "upload-file"; item.row.dataset.key = key; item.row.innerHTML = `${escapeHtml(file.name)} ${escapeHtml(formatBytes(file.size))} `; + aria-label="${attr(t("Admin.Upload.Remove", file.name))}">${icons.close} + `; list.appendChild(item.row); items.push(item); drawItem(item); @@ -341,7 +369,7 @@ export function openUpload({ api, sequence, onDone } = {}) { } status.innerHTML = (fraction === null ? "" : bar(fraction)) + `${escapeHtml(text)}`; - item.row.querySelector('[data-act="remove"]').hidden = !removable(item); + drawRowButtons(item); } /** A photo comes off the list only between runs, and only while it is not on the server. */ @@ -349,6 +377,12 @@ export function openUpload({ api, sequence, onDone } = {}) { return !running && !FINISHED.has(item.state) && !ACTIVE.has(item.state); } + /** Remove, or for a photo deleted from the sequence before, Upload again; both only between runs. */ + function drawRowButtons(item) { + item.row.querySelector('[data-act="remove"]').hidden = !removable(item); + item.row.querySelector('[data-act="again"]').hidden = running || item.state !== "deleted"; + } + /** Progress redraws at most once a frame: the worker reports every 8 MiB, chunks finish four at a time. */ function update(item) { item.dirty = true; @@ -400,7 +434,7 @@ export function openUpload({ api, sequence, onDone } = {}) { addButton.disabled = running || extensions === null; closeButton.textContent = t(running ? "Admin.Upload.Cancel" : "Admin.Upload.Close"); closeButton.title = t(running ? "Admin.Upload.Cancel.Tip" : "Admin.Upload.Close.Tip"); - for (const item of items) item.row.querySelector('[data-act="remove"]').hidden = !removable(item); + for (const item of items) drawRowButtons(item); targetHost.querySelectorAll("input, select").forEach((field) => { field.disabled = running; }); @@ -419,6 +453,7 @@ export function openUpload({ api, sequence, onDone } = {}) { showError(""); running = true; controller = new AbortController(); + sameContent = new Map(); const signal = controller.signal; window.addEventListener("beforeunload", guard); drawButtons(); @@ -520,7 +555,9 @@ export function openUpload({ api, sequence, onDone } = {}) { /** * One photo: hashed, started (the server may already have it, or have had it before it was deleted), the missing - * chunks sent two at a time, completed. True when the library changed. + * chunks sent two at a time, completed. True when the library changed. The same photo picked under another name + * waits for the first copy, which settles it for both: two lanes would otherwise complete the one upload the server + * gives them and refuse each other. */ async function send(item, sequenceId, signal) { const file = item.file; @@ -533,12 +570,32 @@ export function openUpload({ api, sequence, onDone } = {}) { }); if (signal.aborted) throw signal.reason; setState(item, "starting"); + const earlier = sameContent.get(sha256); + let settle; + sameContent.set(sha256, new Promise((resolve) => { + settle = resolve; + })); + try { + await earlier; + if (signal.aborted) throw signal.reason; + return await sendHashed(item, sha256, sequenceId, signal); + } finally { + settle(); + } + } + + /** A hashed photo: started (asking to bring it back when the user chose Upload again), its chunks sent, completed. */ + async function sendHashed(item, sha256, sequenceId, signal) { + const file = item.file; + const restore = item.restore; + const capturedAt = await captureTime(file); const start = await retry(() => api.post("/api/v1/uploads", { - sequenceId, fileName: file.name, size: file.size, sha256, capturedAt: null, sessionKey: null, + sequenceId, fileName: file.name, size: file.size, sha256, capturedAt, sessionKey: null, restore, }), signal); if (start.alreadyExists) { - // No frame: the photo was deleted from this sequence; a restore from the trash brings it back (R6). - setState(item, start.frameId ? "exists" : "deleted"); + // No frame: the photo was deleted from this sequence; a restore from the trash brings it back while the trash + // holds it, or Upload again on its row (R6). + finish(item, start.frameId ? "exists" : "deleted"); return false; } item.uploadId = start.uploadId; @@ -567,16 +624,57 @@ export function openUpload({ api, sequence, onDone } = {}) { } }; await Promise.all(Array.from({ length: Math.min(CHUNKS_AT_ONCE, missing.length) }, lane)); - if (failure !== null) throw failure; + // A chunk refused because the upload is no longer receiving: another completion of it is filing it (or it ended), + // which the completion below waits for and reports. + if (failure !== null && !filing(failure)) throw failure; setState(item, "verifying"); - await retry(() => api.post(`/api/v1/uploads/${item.uploadId}/complete`), signal); + try { + await complete(item.uploadId, restore, signal); + } catch (err) { + if (!(err instanceof ApiFailure && err.status === 409 && err.code === "photo_removed")) throw err; + // Deleted from the sequence while it was on its way: not filed, and the server dropped the upload (R6). + item.uploadId = null; + finish(item, "deleted"); + return false; + } item.uploadId = null; - setState(item, "done"); + finish(item, "done"); return true; } + + /** A photo is settled on the server: a later run leaves it alone, until Upload again asks to bring it back. */ + function finish(item, state) { + item.restore = false; + setState(item, state); + } }); } +/** 409 upload_state: the upload is not receiving now, because a completion is filing it or it ended. */ +function filing(err) { + return err instanceof ApiFailure && err.status === 409 && err.code === "upload_state"; +} + +/** + * Completes an upload (with restore, a photo deleted from the sequence meanwhile is filed anyway). A completion refused + * while another one is filing the upload (the same photo started twice, a retry after a lost answer) waits for it: once + * the upload is committed, completing again answers its frame. Only an upload that failed or was cancelled, or one that + * stays busy for about a minute, fails the photo. + */ +async function complete(uploadId, restore, signal) { + const url = `/api/v1/uploads/${uploadId}/complete${restore ? "?restore=true" : ""}`; + for (let check = 0; ; check++) { + try { + return await retry(() => api.post(url), signal); + } catch (err) { + if (signal.aborted || !filing(err) || check >= SETTLE_CHECKS) throw err; + await sleep(1000 * Math.min(2 ** check, 8), signal); + const upload = await retry(() => api.get(`/api/v1/uploads/${uploadId}`), signal); + if (upload?.state === "failed" || upload?.state === "cancelled") throw err; + } + } +} + /** What dropped: the files, and how many folders were left out (the entries API tells them apart where it exists). */ function dropped(transfer) { const files = []; diff --git a/src/Chronos.Server/Api/UploadEndpoints.cs b/src/Chronos.Server/Api/UploadEndpoints.cs index 967f07fd..27a12725 100644 --- a/src/Chronos.Server/Api/UploadEndpoints.cs +++ b/src/Chronos.Server/Api/UploadEndpoints.cs @@ -27,8 +27,9 @@ public static class UploadEndpoints (await service.GetStateAsync(caller, id, context.RequestAborted)).ToResult(context, s => Results.Json(s, ProtocolJson.Default.UploadState))); uploads.MapDelete("/{id:guid}", async (HttpContext context, Caller caller, Guid id, UploadService service) => (await service.CancelAsync(caller, id, context.RequestAborted)).ToResult(context, _ => Results.NoContent())); - uploads.MapPost("/{id:guid}/complete", async (HttpContext context, Caller caller, Guid id, UploadService service) => - (await service.CompleteAsync(caller, id)).ToResult(context, r => Results.Json(r, ProtocolJson.Default.CompleteUploadResponse))) + // ?restore=true repeats the start's StartUploadRequest.Restore: a photo deleted from the sequence meanwhile is filed anyway. + uploads.MapPost("/{id:guid}/complete", async (HttpContext context, Caller caller, Guid id, bool? restore, UploadService service) => + (await service.CompleteAsync(caller, id, restore == true)).ToResult(context, r => Results.Json(r, ProtocolJson.Default.CompleteUploadResponse))) .Requires(Permissions.Upload); uploads.MapPost("/cards", ReportCard).Requires(Permissions.Upload); } diff --git a/src/Chronos.Server/Storage/UploadRules.cs b/src/Chronos.Server/Storage/UploadRules.cs index cd3eea2b..687fbf95 100644 --- a/src/Chronos.Server/Storage/UploadRules.cs +++ b/src/Chronos.Server/Storage/UploadRules.cs @@ -56,13 +56,24 @@ public static class UploadRules return intact.FirstOrDefault(f => f.SequenceId == target) ?? intact.FirstOrDefault(); } + /// + /// A name the archive scanner would never take as a photo (ArchiveScanner: hidden dot files, among them the + /// AppleDouble ._ companions a Mac leaves on exFAT cards): a frame upload refuses it, so no client files what an + /// import of the same folder would leave out. + /// + public static bool IsHiddenName(string fileName) => fileName.StartsWith('.'); + /// /// Whether tombstoned (owner review 2, R6): a photo deleted from that /// sequence (or replaced by a converted file) that an upload must not bring back. The upload is answered as - /// alreadyExists without a frame, so a client neither sends the file nor treats the answer as a failure. + /// alreadyExists without a frame, so a client neither sends the file nor treats the answer as a failure. A + /// tombstone of content a frame of the target still holds (: another copy of a duplicate, a + /// file put back by hand) is stale and refuses nothing, so such a frame can still be repaired. /// - public static async Task RemovedFromTargetAsync(ChronosDatabase db, string sha256, Guid? target, CancellationToken cancellationToken) => - target is { } sequence && await db.FrameRemovals.IsRemovedAsync(sequence, sha256, cancellationToken).ConfigureAwait(false); + public static async Task RemovedFromTargetAsync(ChronosDatabase db, string sha256, Guid? target, IReadOnlyCollection copies, + CancellationToken cancellationToken) => + target is { } sequence && !copies.Any(f => f.SequenceId == sequence) + && await db.FrameRemovals.IsRemovedAsync(sequence, sha256, cancellationToken).ConfigureAwait(false); /// /// The sequence an upload files into: its own, or the one its session key names (the user's own key without one); diff --git a/src/Chronos.Server/Storage/UploadService.cs b/src/Chronos.Server/Storage/UploadService.cs index 90671c68..3663e2d4 100644 --- a/src/Chronos.Server/Storage/UploadService.cs +++ b/src/Chronos.Server/Storage/UploadService.cs @@ -49,7 +49,8 @@ public sealed class UploadService( public async Task> StartAsync(Caller caller, StartUploadRequest request, CancellationToken cancellationToken) { - if (UploadRules.SafeFileName(request.FileName) is not { } fileName) return Fail(400, "invalid_file_name"); + if (UploadRules.SafeFileName(request.FileName) is not { } fileName || UploadRules.IsHiddenName(fileName)) + return Fail(400, "invalid_file_name"); if (!UploadRules.IsSha256(request.Sha256)) return Fail(400, "invalid_hash"); if (request.Size <= 0) return Fail(400, "bad_request", "size"); if (request.Size > options.MaxUploadBytes) return Fail(413, "too_large", options.MaxUploadBytes.ToString(CultureInfo.InvariantCulture)); @@ -62,11 +63,14 @@ public sealed class UploadService( var copies = await db.Frames.FindBySha256Async(sha, cancellationToken); var target = await UploadRules.TargetAsync(db, archive.Id, request.SequenceId, sessionKey, caller.Id, cancellationToken); // A photo deleted from the target sequence is not filed into it again: an old client syncing a copy that still holds - // it, or a watch folder sending it again, hears that the server has it (owner review 2, R6). A restore brings it back. - if (await UploadRules.RemovedFromTargetAsync(db, sha, target, cancellationToken)) - return new StartUploadResponse(null, options.ChunkSizeBytes, [], AlreadyExists: true, null); - // Only a copy that is still whole stands for the upload: a client may delete its file on this answer. - if (UploadRules.StandIn(archive.Root, copies, target) is { } known) + // it, or a watch folder sending it again, hears that the server has it (owner review 2, R6). A restore from the trash + // brings it back while the trash holds it; a user who uploads it again on purpose asks with Restore, and filing it + // drops its tombstone. + var removed = await UploadRules.RemovedFromTargetAsync(db, sha, target, copies, cancellationToken); + if (removed && !request.Restore) return new StartUploadResponse(null, options.ChunkSizeBytes, [], AlreadyExists: true, null); + // Only a copy that is still whole stands for the upload: a client may delete its file on this answer. A photo + // brought back is filed into its sequence, whatever other sequences hold it. + if (UploadRules.StandIn(archive.Root, removed ? [] : copies, target) is { } known) return new StartUploadResponse(null, options.ChunkSizeBytes, [], AlreadyExists: true, known.Id); var resumable = await db.Uploads.FindReceivingAsync(caller.Id, sha, cancellationToken); @@ -158,9 +162,11 @@ public sealed class UploadService( /// /// Verifies and commits a frame upload. Completing an already committed upload answers its frame again, so a client - /// may retry a completion whose response it lost. Runs to the end even if the client disconnects. + /// may retry a completion whose response it lost. Runs to the end even if the client disconnects. A photo deleted from + /// the target sequence since the upload started is not filed (409 , the upload + /// cancelled) unless repeats the start's . /// - public async Task> CompleteAsync(Caller caller, Guid id) + public async Task> CompleteAsync(Caller caller, Guid id, bool restore = false) { var stopping = lifetime.ApplicationStopping; var upload = await db.Uploads.GetAsync(id, stopping); @@ -174,6 +180,15 @@ public sealed class UploadService( if (!ArchiveHasRoom(upload)) return Fail(507, "insufficient_storage"); if (!await db.Uploads.TryTransitionAsync(id, UploadState.Receiving, UploadState.Committing, cancellationToken: stopping)) return Fail(409, "upload_state"); + // Deleted from the target while its bytes were on the way (the start checked before): the upload does not bring the + // photo back (owner review 2, R6). + if (!restore && await RemovedFromTargetAsync(upload, stopping)) + { + logger.LogInformation("Upload {Upload} ({File}) not filed: the photo was deleted from its sequence meanwhile", id, upload.FileName); + await db.Uploads.TryTransitionAsync(id, UploadState.Committing, UploadState.Cancelled, cancellationToken: CancellationToken.None); + store.Delete(id); + return Fail(409, UploadErrors.PhotoRemoved); + } var user = await db.Users.GetAsync(caller.Id, stopping); if (user is { QuotaBytes: > 0 } && user.UsedBytes + upload.Size > user.QuotaBytes && !await HeldByTargetAsync(upload, stopping)) @@ -305,6 +320,14 @@ public sealed class UploadService( return copies.Any(f => f.SequenceId == target); } + /// The sequence the upload files into tombstoned its content (). + private async Task RemovedFromTargetAsync(Upload upload, CancellationToken cancellationToken) + { + var target = await UploadRules.TargetAsync(db, archive.Id, upload.SequenceId, upload.SessionKey, upload.UserId, cancellationToken); + return target is not null && await UploadRules.RemovedFromTargetAsync(db, upload.Sha256, target, + await db.Frames.FindBySha256Async(upload.Sha256, cancellationToken), cancellationToken); + } + private StartUploadResponse Started(Upload upload) => new(upload.Id, upload.ChunkSize, UploadRules.ReceivedChunks(upload), AlreadyExists: false, FrameId: null); diff --git a/tests/Chronos.Data.Tests/Sync/SyncRulesTests.cs b/tests/Chronos.Data.Tests/Sync/SyncRulesTests.cs index 74e2af64..0956c865 100644 --- a/tests/Chronos.Data.Tests/Sync/SyncRulesTests.cs +++ b/tests/Chronos.Data.Tests/Sync/SyncRulesTests.cs @@ -97,6 +97,27 @@ public sealed class SyncRulesTests new FrameRemoval(localId, "bb", "b2", RemovalReason.Replaced, at)]).Select(r => r.Reason)); } + [Test] + public void ATombstoneBesideAPhotoItsSideStillHoldsHidesNothing() + { + var time = new DateTime(2026, 9, 28, 18, 0, 0); + var at = new DateTimeOffset(2026, 10, 27, 10, 0, 0, TimeSpan.Zero); + FramePhoto Photo(string sha, int second = 0) => new(sha, 100 + second, time.AddSeconds(second)); + // The server tombstoned "xx" and still holds it (it deleted one of two copies, or the file came back by hand): this + // PC's copy is not missing there, and the server's is not missing here. + var server = new SyncTombstones([], [new RemovedPhotoDto("xx", null, "deleted", at)]); + Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa"), Photo("xx", 1)], [Photo("aa"), Photo("XX", 1)], server)); + // Also when the server has not hashed its copy: it holds it by size and capture time. + Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa"), Photo("xx", 1)], [Photo("aa"), Photo("", 1)], server)); + // The same on this side. + var here = new SyncTombstones([new FrameRemoval(Guid.CreateVersion7(), "xx", null, RemovalReason.Deleted, at)], null); + Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa"), Photo("xx", 1)], [Photo("aa"), Photo("xx", 1)], here)); + // A tombstone of a photo its side no longer holds still leaves the other side's copy out. + Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa"), Photo("xx", 1)], [Photo("aa")], server)); + Assert.Equal((0, 0), SyncRules.CompareOriginals([Photo("aa")], [Photo("aa"), Photo("xx", 1)], here)); + Assert.Equal((1, 0), SyncRules.CompareOriginals([Photo("aa"), Photo("xx", 1)], [Photo("aa")], SyncTombstones.None)); + } + [Test] public void FramesEitherSideHasNotHashedAreMatchedBySizeAndCaptureTime() { diff --git a/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs b/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs index d7f1b5ba..7419a590 100644 --- a/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs +++ b/tests/Chronos.Data.Tests/Sync/SyncTransferTests.cs @@ -151,6 +151,90 @@ public sealed class SyncTransferTests(TestContext context) Assert.SequenceEqual(photos[1..].Select(Sha), kept.Select(k => k.Sha256!)); } + [Test] + public async Task APhotoDeletedHereIsNotFetchedAgainFromAServerThatHasNotHashedIt() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(3); + // A folder the server adopted and never verified (its manifest lists no SHA-256), downloaded here. + var remote = f.Remote.Add("Adopted on the server", photos); + remote.Unhashed = true; + await f.RefreshAsync(); + var engine = await f.StartEngineAsync(); + var first = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, f.Archive.Archive.Id); + await f.Archive.WaitForAsync(first.Id, JobState.Done); + var local = Assert.NotNull((await f.EntryAsync("Adopted on the server")).Local); + + // This PC deletes a photo the server still lists, unhashed: only its content tells it is the one deleted here. + await UntilIdleAsync(f); + var gone = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).Single(fr => fr.Sha256 == Sha(photos[0])); + var sequence = (await f.Db.Sequences.GetAsync(local.Id, f.Ct))!; + var plan = await f.Archive.Service.FrameTrash.PlanDeleteAsync(local.Id, [gone.Id], sequence.FramesVersion, f.Ct); + await f.Archive.Service.FrameTrash.DeleteAsync(plan, null, null, f.Ct); + await f.RefreshAsync(); + + var again = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Adopted on the server", SyncDownloadMode.Originals, null); + var done = await f.Archive.WaitForAsync(again.Id, JobState.Done); + + var kept = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct); + Assert.True(kept.Select(k => k.Sha256).ToHashSet().SetEquals(photos[1..].Select(Sha)), "the photo deleted here stays deleted"); + Assert.True(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(photos[0])), "and keeps its tombstone"); + Assert.Contains("deleted on this PC, not downloaded", done.LogTail); + Assert.False(Directory.Exists(Path.Combine(f.Archive.Paths.StagingDirectory, "sync-" + remote.Id.ToString("N"))), "staging cleaned up"); + } + + [Test] + public async Task AnUnverifiedCopyDoesNotSendBackAPhotoTheServerDeleted() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(4); + // An adopted folder (never verified, so no hashes here) that still holds a photo the server has deleted since. + var folder = Path.Combine(f.Archive.Paths.Root, "Adopted"); + Directory.CreateDirectory(folder); + for (var i = 0; i < 4; i++) await File.WriteAllBytesAsync(Path.Combine(folder, $"DSC_{i + 1:0000}.JPG"), photos[i], f.Ct); + await f.Archive.ScanAsync(); + var local = Assert.Contains((await f.Db.Sequences.ListAsync(new Chronos.Data.Repositories.SequenceQuery(), f.Ct)).Items, s => s.Folder == "Adopted"); + var remote = f.Remote.Add("Adopted", photos[..3]); + remote.Removed.Add(new RemovedPhotoDto(Sha(photos[3]), null, "deleted", DateTimeOffset.UtcNow)); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + // Without its hash the photo cannot be told from one the server lacks: the upload finds out. + Assert.Equal(1, (await f.EntryAsync(local.DisplayTitle)).Link!.MissingRemote); + var engine = await f.StartEngineAsync(); + + var job = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); + var done = await f.Archive.WaitForAsync(job.Id, JobState.Done); + + Assert.Equal(3, remote.Frames.Count); + Assert.Contains("deleted on the server, not uploaded", done.LogTail); + // Its hash is in the index now: the comparison leaves it out, so the row settles instead of offering Upload for good. + Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is not null), "hashed by the verify"); + var entry = await f.EntryAsync(local.DisplayTitle); + Assert.Equal((SyncStatus.InSync, 0, 0), (entry.Status, entry.Link!.MissingRemote, entry.Link.MissingLocal)); + var calls = remote.Frames.Count; + var next = await f.Sync.EnqueueUploadAsync(engine, f.Server, local); + var settled = await f.Archive.WaitForAsync(next.Id, JobState.Done); + Assert.Equal((4L, 4L, calls), (settled.ProgressDone, settled.ProgressTotal, remote.Frames.Count)); + } + + [Test] + public async Task ATombstoneBesideAPhotoTheServerStillHoldsLeavesTheCopiesInSync() + { + await using var f = await SyncFixture.CreateAsync(context); + var photos = f.NewPhotos(2); + var local = await f.LocalAsync("Comet", photos); + // The server holds both photos and a tombstone of the second: it deleted one of two copies of it, or the file came + // back by hand. Neither side lacks anything. + var remote = f.Remote.Add("Comet", photos); + remote.Removed.Add(new RemovedPhotoDto(Sha(photos[1]), null, "deleted", DateTimeOffset.UtcNow)); + await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct); + await f.RefreshAsync(); + + var entry = await f.EntryAsync("Comet"); + Assert.Equal((0, 0), (entry.Link!.MissingRemote, entry.Link.MissingLocal)); + Assert.Equal(SyncStatus.InSync, entry.Status); + } + [Test] public async Task AResumedUploadSendsOnlyWhatTheServerLacks() { @@ -634,6 +718,20 @@ public sealed class SyncTransferTests(TestContext context) private static string Sha(byte[] bytes) => Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(bytes)); + /// Waits until no job is waiting or running (the thumbnails and analyses a download queues after it). + private static async Task UntilIdleAsync(SyncFixture f) + { + var until = DateTime.UtcNow + TimeSpan.FromSeconds(TestTiming.Budget(30)); + foreach (var state in new[] { JobState.Queued, JobState.Running, JobState.Queued, JobState.Running }) + { + while ((await f.Db.Jobs.ListAsync(state, cancellationToken: f.Ct)).Items.Count > 0) + { + Assert.True(DateTime.UtcNow < until, $"jobs still {state}"); + await Task.Delay(50, f.Ct); + } + } + } + /// The photos (SHA-256) the local copy's project excludes, shows as hero, keys the camera on, and waits for. private static async Task<(string[] Excluded, string? Hero, string[] Keyframes, string[] Pending)> NamedAsync(SyncFixture f, Guid localId) { diff --git a/tests/Chronos.Server.Tests/Storage/ArchiveModuleTests.cs b/tests/Chronos.Server.Tests/Storage/ArchiveModuleTests.cs index cabfb3ad..12465473 100644 --- a/tests/Chronos.Server.Tests/Storage/ArchiveModuleTests.cs +++ b/tests/Chronos.Server.Tests/Storage/ArchiveModuleTests.cs @@ -1,4 +1,5 @@ using System.Net; +using System.Net.Http.Json; using System.Security.Cryptography; using Microsoft.Extensions.DependencyInjection; using Chronos.Core.Archiving; @@ -419,6 +420,59 @@ public sealed class ArchiveModuleTests(TestContext context) Assert.Equal(photo.Length, (await me.ReadAsync(ProtocolJson.Default.UserDto)).UsedBytes); } + [Test] + public async Task TheProtocolClientTakesAPhotoDeletedWhileOnItsWayAsNotSent() + { + // Owner review 2, R6: a repair upload whose photo the Library deletes meanwhile is refused at its completion + // (photo_removed, nothing filed); the protocol client reports it like a photo deleted before: null, not a failure. + await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); + var (_, _, token) = await server.UserAsync("alberto", "editor"); + var admin = await server.AdminAsync(); + var path = Path.Combine(context.CreateTempDirectory("source"), "IMG_0001.JPG"); + var photo = Photo(5); + await File.WriteAllBytesAsync(path, photo); + using var client = new ChronosClient(server.HttpBase, new CertificatePinning([]), "Chronos Uploader (tests)") { Token = token }; + var sequence = await client.CreateSequenceAsync(new CreateSequenceRequest("Roof", null), context.Cancellation); + // A sequence keeps one photo at least: the other is the one deleted. + var keeper = Path.Combine(Path.GetDirectoryName(path)!, "IMG_0000.JPG"); + await File.WriteAllBytesAsync(keeper, Photo(6)); + var kept = Assert.NotNull(await client.UploadFileAsync(keeper, sequence.Id, null, ct: context.Cancellation)); + var first = Assert.NotNull(await client.UploadFileAsync(path, sequence.Id, null, ct: context.Cancellation)); + await UntilIdleAsync(server); + File.Delete(Path.Combine(server.Options.ArchivePath, Assert.NotNull(await server.Database.Frames.GetAsync(first.FrameId)).RelativePath)); + + using var deleting = new BeforeCompletion(async () => + { + await UntilIdleAsync(server); + var current = Assert.NotNull(await server.Database.Sequences.GetAsync(sequence.Id)); + using var delete = await admin.PostAsJsonAsync($"/api/v1/sequences/{sequence.Id}/frames/delete", + new FrameDeleteRequest([first.FrameId], current.FramesVersion, false), ProtocolJson.Default.FrameDeleteRequest); + await delete.ExpectAsync(HttpStatusCode.OK); + }); + using var racing = new ChronosClient(server.HttpBase, new CertificatePinning([]), "Chronos Uploader (tests)", deleting) { Token = token }; + Assert.Null(await racing.UploadFileAsync(path, sequence.Id, null, ct: context.Cancellation)); + + Assert.True(deleting.Ran, "the completion was reached"); + Assert.SequenceEqual([kept.FrameId], (await server.Database.Frames.ListBySequenceAsync(sequence.Id)).Select(f => f.Id)); + Assert.True(await server.Database.FrameRemovals.IsRemovedAsync(sequence.Id, Convert.ToHexStringLower(SHA256.HashData(photo))), "the tombstone stays"); + } + + /// Runs a step on the server before the first upload completion goes through, then sends it on. + private sealed class BeforeCompletion(Func step) : DelegatingHandler(new SocketsHttpHandler()) + { + public bool Ran { get; private set; } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + if (!Ran && request.Method == HttpMethod.Post && request.RequestUri?.AbsolutePath.EndsWith("/complete", StringComparison.Ordinal) == true) + { + Ran = true; + await step(); + } + return await base.SendAsync(request, cancellationToken); + } + } + [Test] public async Task AnIntactCopyElsewhereDoesNotStandForTheDamagedOneOfTheTargetSequence() { diff --git a/tests/Chronos.Server.Tests/Storage/BrowserUploadTests.cs b/tests/Chronos.Server.Tests/Storage/BrowserUploadTests.cs index 4122281c..7192cf88 100644 --- a/tests/Chronos.Server.Tests/Storage/BrowserUploadTests.cs +++ b/tests/Chronos.Server.Tests/Storage/BrowserUploadTests.cs @@ -3,6 +3,7 @@ using System.Net; using System.Net.Http.Headers; using System.Net.Http.Json; using System.Text; +using System.Text.RegularExpressions; using Microsoft.Extensions.DependencyInjection; using Chronos.Core.Codecs.Jpeg; using Chronos.Core.Localization; @@ -89,12 +90,9 @@ public sealed class BrowserUploadTests(TestContext context) Assert.True(again.AlreadyExists); Assert.Equal(frames[0].FrameId, again.FrameId); - // Deleted from the sequence: also answered as there, without a frame ("deleted before; restore it from the trash"). - await UntilIdleAsync(server); - var current = Assert.NotNull(await server.Database.Sequences.GetAsync(sequence.Id)); - using (var delete = await browser.SendAsync(HttpMethod.Post, $"/api/v1/sequences/{sequence.Id}/frames/delete", - Json($$"""{"frameIds":["{{frames[1].FrameId}}"],"expectedFramesVersion":{{current.FramesVersion}},"dryRun":false}"""))) - Assert.NotNull((await delete.ReadAsync(ProtocolJson.Default.FrameDeleteResultDto)).TrashEntry); + // Deleted from the sequence: also answered as there, without a frame ("deleted before": restore it from the trash, or + // upload it again). + await DeleteAsync(server, browser, sequence.Id, frames[1].FrameId); var deleted = await StartAsync(browser, sequence.Id, "IMG_0002.JPG", photos[1], HttpStatusCode.OK); Assert.True(deleted.AlreadyExists); Assert.Null(deleted.UploadId); @@ -130,6 +128,181 @@ public sealed class BrowserUploadTests(TestContext context) Assert.Equal(Loc.Get("Server.Error.QuotaExceeded", CultureInfo.GetCultureInfo("it")), error.Message); } + [Test] + public async Task UploadAgainBringsBackAPhotoDeletedBeforeEvenAfterTheTrashIsEmptied() + { + // Owner review 2, R6 and R3: a tombstone outlives the trash and keeps sync and old clients from filing the photo + // again, but the user who picks it in the dialog and chooses Upload again gets it back into the sequence. + await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk, ArchiveModule.Register); + var browser = await Browser.SignInAsync(server); + var sequence = await CreateAsync(browser, "Roof"); + var photo = Photo(11); + var sha = Uploads.Sha(photo); + // A sequence keeps one photo at least: the other is the one deleted. + var keeper = await UploadAsync(browser, sequence.Id, "IMG_0000.JPG", Photo(10)); + var first = await UploadAsync(browser, sequence.Id, "IMG_0001.JPG", photo); + await DeleteAsync(server, browser, sequence.Id, first.FrameId); + await EmptyTrashAsync(server, browser); + Assert.True(await server.Database.FrameRemovals.IsRemovedAsync(sequence.Id, sha), "the tombstone outlives the trash"); + + // Picked again: deleted before, nothing started. + var refused = await StartAsync(browser, sequence.Id, "IMG_0001.JPG", photo, HttpStatusCode.OK); + Assert.True(refused is { AlreadyExists: true, UploadId: null, FrameId: null }, "answered as deleted before"); + + // Upload again whose completion does not repeat the restore (an older page): refused at the end, the upload dropped. + var started = await StartAsync(browser, sequence.Id, "IMG_0001.JPG", photo, HttpStatusCode.Created, restore: true); + var dropped = Assert.NotNull(started.UploadId); + await SendChunksAsync(browser, dropped, photo); + using (var complete = await CompleteAsync(browser, dropped)) + { + var error = await complete.ExpectErrorAsync(HttpStatusCode.Conflict, UploadErrors.PhotoRemoved); + Assert.Equal(Loc.Get("Server.Error.PhotoRemoved", CultureInfo.GetCultureInfo("it")), error.Message); + } + using (var state = await browser.SendAsync(HttpMethod.Get, $"/api/v1/uploads/{dropped}")) + Assert.Equal("cancelled", (await state.ReadAsync(ProtocolJson.Default.UploadState)).State); + Assert.False(server.Services.GetRequiredService().Exists(dropped), "the partial file is gone"); + Assert.True(await server.Database.FrameRemovals.IsRemovedAsync(sequence.Id, sha), "still deleted"); + + // Upload again, start and completion: filed into the sequence, and its tombstone is gone. + var back = await UploadAsync(browser, sequence.Id, "IMG_0001.JPG", photo, restore: true); + Assert.Equal(sequence.Id, back.SequenceId); + var frames = await server.Database.Frames.ListBySequenceAsync(sequence.Id); + Assert.SequenceEqual([keeper.FrameId, back.FrameId], frames.Select(f => f.Id).Order()); + Assert.Equal(sha, frames.Single(f => f.Id == back.FrameId).Sha256); + Assert.False(await server.Database.FrameRemovals.IsRemovedAsync(sequence.Id, sha), "the tombstone went with the upload"); + var again = await StartAsync(browser, sequence.Id, "IMG_0001.JPG", photo, HttpStatusCode.OK); + Assert.Equal(back.FrameId, again.FrameId); + } + + [Test] + public async Task UploadAgainFilesThePhotoIntoItsSequenceWhateverOtherSequencesHold() + { + // A copy in another sequence stands for an ordinary upload; one brought back on purpose goes where the user put it. + await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk, ArchiveModule.Register); + var browser = await Browser.SignInAsync(server); + var roof = await CreateAsync(browser, "Roof"); + var yard = await CreateAsync(browser, "Yard"); + var photo = Photo(12); + await UploadAsync(browser, roof.Id, "IMG_0000.JPG", Photo(10)); + var first = await UploadAsync(browser, roof.Id, "IMG_0001.JPG", photo); + await DeleteAsync(server, browser, roof.Id, first.FrameId); + var elsewhere = await UploadAsync(browser, yard.Id, "IMG_0001.JPG", photo); + + var deleted = await StartAsync(browser, roof.Id, "IMG_0001.JPG", photo, HttpStatusCode.OK); + Assert.Null(deleted.FrameId); + var back = await UploadAsync(browser, roof.Id, "IMG_0001.JPG", photo, restore: true); + Assert.Equal(roof.Id, back.SequenceId); + Assert.NotEqual(elsewhere.FrameId, back.FrameId); + Assert.Count(1, await server.Database.Frames.ListBySequenceAsync(yard.Id)); + } + + [Test] + public async Task AnUploadCompletedAfterItsPhotoWasDeletedDoesNotBringItBack() + { + // Owner review 2, R6: the start checked the tombstones, but the photo can be deleted while its bytes are on the way + // (here a repair of a copy whose original is gone, deleted from the Library meanwhile). + await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk, ArchiveModule.Register); + var browser = await Browser.SignInAsync(server); + var sequence = await CreateAsync(browser, "Roof"); + var photo = Photo(13); + var sha = Uploads.Sha(photo); + var keeper = await UploadAsync(browser, sequence.Id, "IMG_0000.JPG", Photo(10)); + var first = await UploadAsync(browser, sequence.Id, "IMG_0001.JPG", photo); + await UntilIdleAsync(server); + var frame = Assert.NotNull(await server.Database.Frames.GetAsync(first.FrameId)); + File.Delete(Path.Combine(server.Options.ArchivePath, frame.RelativePath)); + + var repair = await StartAsync(browser, sequence.Id, "IMG_0001.JPG", photo, HttpStatusCode.Created); + var id = Assert.NotNull(repair.UploadId); + await SendChunksAsync(browser, id, photo); + await DeleteAsync(server, browser, sequence.Id, first.FrameId); + + using (var complete = await CompleteAsync(browser, id)) + await complete.ExpectErrorAsync(HttpStatusCode.Conflict, UploadErrors.PhotoRemoved); + Assert.SequenceEqual([keeper.FrameId], (await server.Database.Frames.ListBySequenceAsync(sequence.Id)).Select(f => f.Id)); + Assert.True(await server.Database.FrameRemovals.IsRemovedAsync(sequence.Id, sha), "the tombstone stays"); + Assert.False(server.Services.GetRequiredService().Exists(id), "the partial file is gone"); + // Completing again changes nothing: the upload is over. + using (var late = await CompleteAsync(browser, id)) + await late.ExpectErrorAsync(HttpStatusCode.Conflict, "upload_state"); + Assert.SequenceEqual([keeper.FrameId], (await server.Database.Frames.ListBySequenceAsync(sequence.Id)).Select(f => f.Id)); + } + + [Test] + public async Task AStaleTombstoneBesideAFrameOfTheSameContentRefusesNoRepair() + { + // A tombstone left beside a frame that still holds the content (the other copy of a duplicate, a file put back by + // hand) does not stand for a deletion: the frame's original can still be repaired by uploading it. + await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk, ArchiveModule.Register); + var browser = await Browser.SignInAsync(server); + var sequence = await CreateAsync(browser, "Roof"); + var photo = Photo(14); + var sha = Uploads.Sha(photo); + var first = await UploadAsync(browser, sequence.Id, "IMG_0001.JPG", photo); + await UntilIdleAsync(server); + await server.Database.FrameRemovals.SyncSequenceAsync(sequence.Id, [new FrameRemoval(sequence.Id, sha, null, RemovalReason.Deleted, DateTimeOffset.UtcNow)]); + var frame = Assert.NotNull(await server.Database.Frames.GetAsync(first.FrameId)); + var original = Path.Combine(server.Options.ArchivePath, frame.RelativePath); + File.Delete(original); + + var repaired = await UploadAsync(browser, sequence.Id, "IMG_0001.JPG", photo); + Assert.Equal(first.FrameId, repaired.FrameId); + await Http.BytesEqualAsync(photo, File.ReadAllBytesAsync(original)); + } + + [Test] + public async Task TheDialogsCaptureTimeDatesAPhotoThatRecordsNone() + { + // Owner review 2, R3: a photo without Exif (a PNG, a JPEG stripped on export) takes the time the dialog sends, as local + // wall time, not the moment the server staged it; frames sent out of order still end up in capture order. + await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk, ArchiveModule.Register); + var browser = await Browser.SignInAsync(server); + var sequence = await CreateAsync(browser, "Timelapse"); + var later = await UploadAsync(browser, sequence.Id, "frame_0002.jpg", Photo(16), "2025-05-01T06:31:00.000"); + var earlier = await UploadAsync(browser, sequence.Id, "frame_0001.jpg", Photo(15), "2025-05-01T06:30:00"); + + var frames = await server.Database.Frames.ListBySequenceAsync(sequence.Id); + Assert.SequenceEqual([earlier.FrameId, later.FrameId], frames.OrderBy(f => f.Index).Select(f => f.Id)); + Assert.Equal(new DateTime(2025, 5, 1, 6, 30, 0), frames.Single(f => f.Id == earlier.FrameId).CapturedAt); + Assert.Equal(new DateTime(2025, 5, 1, 6, 31, 0), frames.Single(f => f.Id == later.FrameId).CapturedAt); + } + + [Test] + public async Task HiddenDotFilesAreRefusedLikeTheScannerLeavesThemOut() + { + // The AppleDouble "._" companion a Mac leaves on an exFAT card, and any hidden dot file, is no photo for the scanner: + // the dialog leaves them out, and the server refuses them from any client. + await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk, ArchiveModule.Register); + var browser = await Browser.SignInAsync(server); + var sequence = await CreateAsync(browser, "Card"); + foreach (var name in new[] { "._IMG_0001.JPG", ".IMG_0002.JPG" }) + { + using var start = await browser.SendAsync(HttpMethod.Post, "/api/v1/uploads", StartBody(sequence.Id, name, Uploads.Photo(4096, 21))); + await start.ExpectErrorAsync(HttpStatusCode.BadRequest, "invalid_file_name"); + } + var upload = await server.Client.GetStringAsync("/admin/upload.js"); + Assert.Contains(@"file.name.startsWith(""."")", upload); + } + + [Test] + public async Task TheDialogKeepsItselfOpenAndSettlesWhatAnotherCompletionFiles() + { + // Owner review 2, R3 (upload.js): Esc is stopped at its keydown during a run, since Chromium lets a dialog's cancel be + // prevented only once per user activation; a completion refused while another one files the upload (409 + // upload_state) waits for it instead of failing the photo; the same photo picked twice waits for its first copy. + await using var server = await TestServer.StartAsync(context); + var upload = await server.Client.GetStringAsync("/admin/upload.js"); + Assert.True(Regex.IsMatch(upload, @"addEventListener\(""keydown"", \(e\) => \{\s*if \(running && e\.key === ""Escape""\) e\.preventDefault\(\);"), "Esc at keydown"); + Assert.Contains(@"err.code === ""upload_state""", upload); + Assert.Contains(@"api.get(`/api/v1/uploads/${uploadId}`)", upload); + Assert.Contains("sameContent.get(sha256)", upload); + Assert.Contains(@"?restore=true", upload); + Assert.Contains(@"err.code === ""photo_removed""", upload); + Assert.Contains(@"import { captureTime } from ""./capture-time.js"";", upload); + var capture = await server.Client.GetStringAsync("/admin/capture-time.js"); + Assert.Contains("export function exifCaptureTime(bytes)", capture); + } + /// The bootstrap administrator signed in to the panel (a cookie session over HTTPS), calling as api.js does. private sealed class Browser(HttpClient client, string csrfToken) { @@ -165,17 +338,80 @@ public sealed class BrowserUploadTests(TestContext context) return content; } - /// The start body upload.js writes: no capture time (the server reads it from the photo), no session key. - private static StringContent StartBody(Guid? sequenceId, string fileName, byte[] data) => - Json($$"""{"sequenceId":{{(sequenceId is { } id ? $"\"{id}\"" : "null")}},"fileName":"{{fileName}}","size":{{data.Length}},"sha256":"{{Uploads.Sha(data)}}","capturedAt":null,"sessionKey":null}"""); + /// + /// The start body upload.js writes: the capture time capture-time.js read (local wall time without an offset, or null), + /// no session key, and restore true only after the user chose Upload again on the photo's row. + /// + private static StringContent StartBody(Guid? sequenceId, string fileName, byte[] data, string? capturedAt = null, bool restore = false) => + Json($$"""{"sequenceId":{{(sequenceId is { } id ? $"\"{id}\"" : "null")}},"fileName":"{{fileName}}","size":{{data.Length}},"sha256":"{{Uploads.Sha(data)}}","capturedAt":{{(capturedAt is null ? "null" : $"\"{capturedAt}\"")}},"sessionKey":null,"restore":{{(restore ? "true" : "false")}}}"""); - private static async Task StartAsync(Browser browser, Guid sequenceId, string fileName, byte[] data, HttpStatusCode expected) + private static async Task StartAsync(Browser browser, Guid sequenceId, string fileName, byte[] data, HttpStatusCode expected, + string? capturedAt = null, bool restore = false) { - using var response = await browser.SendAsync(HttpMethod.Post, "/api/v1/uploads", StartBody(sequenceId, fileName, data)); + using var response = await browser.SendAsync(HttpMethod.Post, "/api/v1/uploads", StartBody(sequenceId, fileName, data, capturedAt, restore)); await response.ExpectAsync(expected); return await response.ReadAsync(ProtocolJson.Default.StartUploadResponse); } + /// A sequence made as the dialog's New sequence makes it. + private static async Task CreateAsync(Browser browser, string title) + { + using var created = await browser.SendAsync(HttpMethod.Post, "/api/v1/sequences", Json($$"""{"title":"{{title}}","cameraHint":null}""")); + await created.ExpectAsync(HttpStatusCode.Created); + return await created.ReadAsync(ProtocolJson.Default.SequenceDto); + } + + /// Every chunk of an upload the dialog started, one after the other. + private static async Task SendChunksAsync(Browser browser, Guid id, byte[] data) + { + for (var n = 0; n < (data.Length + Chunk - 1) / Chunk; n++) + { + using var put = await browser.SendAsync(HttpMethod.Put, $"/api/v1/uploads/{id}/chunks/{n}", Bytes(data, n)); + await put.ExpectAsync(HttpStatusCode.OK); + } + } + + /// The dialog's completion: a body of {} (api.post), ?restore=true after Upload again. + private static Task CompleteAsync(Browser browser, Guid id, bool restore = false) => + browser.SendAsync(HttpMethod.Post, $"/api/v1/uploads/{id}/complete{(restore ? "?restore=true" : string.Empty)}", Json("{}")); + + /// Started, sent and completed as the dialog does it; returns the frame. + private static async Task UploadAsync(Browser browser, Guid sequenceId, string fileName, byte[] data, string? capturedAt = null, + bool restore = false) + { + var start = await StartAsync(browser, sequenceId, fileName, data, HttpStatusCode.Created, capturedAt, restore); + var id = Assert.NotNull(start.UploadId); + await SendChunksAsync(browser, id, data); + using var complete = await CompleteAsync(browser, id, restore); + await complete.ExpectAsync(HttpStatusCode.OK); + return await complete.ReadAsync(ProtocolJson.Default.CompleteUploadResponse); + } + + /// Deletes a photo from its sequence as the Library does, once no job is working on it. + private static async Task DeleteAsync(TestServer server, Browser browser, Guid sequenceId, Guid frameId) + { + await UntilIdleAsync(server); + var current = Assert.NotNull(await server.Database.Sequences.GetAsync(sequenceId)); + using var delete = await browser.SendAsync(HttpMethod.Post, $"/api/v1/sequences/{sequenceId}/frames/delete", + Json($$"""{"frameIds":["{{frameId}}"],"expectedFramesVersion":{{current.FramesVersion}},"dryRun":false}""")); + Assert.NotNull((await delete.ReadAsync(ProtocolJson.Default.FrameDeleteResultDto)).TrashEntry); + } + + /// Empties the whole trash as the Trash page does: the dry run, then the job with its fingerprint. + private static async Task EmptyTrashAsync(TestServer server, Browser browser) + { + TrashEmptyPlanDto plan; + using (var dry = await browser.SendAsync(HttpMethod.Post, "/api/v1/archive/trash/empty", Json("""{"names":null,"dryRun":true,"fingerprint":null}"""))) + plan = await dry.ReadAsync(ProtocolJson.Default.TrashEmptyPlanDto); + Assert.True(plan.Entries.Count > 0, "the trash holds the deleted photo"); + using (var empty = await browser.SendAsync(HttpMethod.Post, "/api/v1/archive/trash/empty", + Json($$"""{"names":null,"dryRun":false,"fingerprint":"{{plan.Fingerprint}}"}"""))) + await empty.ExpectAsync(HttpStatusCode.Accepted); + await UntilIdleAsync(server); + using var listed = await browser.SendAsync(HttpMethod.Get, "/api/v1/archive/trash"); + Assert.Empty((await listed.ReadAsync(ProtocolJson.Default.TrashListDto)).Entries); + } + /// Waits until no job is queued or running (the thumbnails, proxies and analyses uploads queue). private static async Task UntilIdleAsync(TestServer server) {