Files
Chronos/src/Chronos.Data/Sync/SyncTransfers.cs
T
Alby96andClaude Opus 5.5 2e1c67f531 feat(workspace): lay the phase 5c foundation
Phase 5c (owner review 2, R8) is built by six packages in parallel. This
commit settles everything they share, so each one touches its own files:

- job kinds WorkspaceRender (21), WorkspaceSend (22) and WorkspaceMove (23)
  with their wire names, default priorities, notices and parameter records;
  their handlers are registered as skeletons that throw until their
  package lands
- WorkspaceJobOptions and a new CreateHandlers overload; the earlier ones
  stay and default the options
- originals in the media index (an index of phase 5b reads as proxies
  only), their file names, and the exports/renders.json ledger of local
  renders, written atomically
- the optional project and version of an uploaded render
- the workspace settings (size limit, default media, version on render,
  clean-up age, project to open) and when each project was opened, which
  Reset and Import keep and Export leaves out
- WorkspaceHost split into partial files, with stubs for usage, trash and
  the folder move, removal hooks for unsent renders and last originals,
  and a guard that holds work while the workspace moves
- every new string of the phase in English and Italian, with tooltips

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-04 15:05:09 +01:00

957 lines
55 KiB
C#

using System.Collections.Concurrent;
using System.Globalization;
using System.Net;
using Chronos.Core.Archiving;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Data.Archiving;
using Chronos.Data.Archiving.Jobs;
using Chronos.Data.Jobs;
using Chronos.Data.Processing;
using Chronos.Protocol;
using Chronos.Protocol.Client;
using SequenceFrames = Chronos.Data.Processing.SequenceFrames;
using SequenceManifest = Chronos.Protocol.SequenceManifest;
namespace Chronos.Data.Sync;
/// <summary>
/// The work of upload and download jobs. Uploads send every local original the server lacks through the chunked,
/// resumable protocol after hashing it again (an original that no longer matches its archived hash is never sent),
/// creating the server sequence first and sending its settings when the upload made it, then the proxies the server asks
/// for (originals it cannot decode, BRIEF §2.3), made here from this PC's originals. Downloads fetch originals the
/// local copy lacks into the archive's staging folder (resuming with Range), require the manifest's SHA-256 (or, for an
/// original the server has not hashed yet, its size, and hash it here) and commit them through the archive ingest,
/// which names them and writes the sidecar; the first original creates the local
/// sequence, which then takes the server's settings. Frames of the local copy not hashed yet are hashed first (so the
/// server's copies of them are recognised), and originals flagged missing here are restored to their recorded place in
/// the folder the sequence is in (never one it left; one the server holds in another encoding, a conversion on either
/// side, takes the frame's place under its id instead).
/// Proxy downloads fill the proxy cache. Renders travel on request: this PC's to the server (hash-verified chunked
/// upload) and the server's into the local copy's exports. A resumed job skips what is already on the other side, so
/// pausing or a crash loses at most the files in flight.
/// </summary>
internal sealed class SyncTransfers(SyncService sync)
{
private const long SpaceReserve = 512L << 20;
/// <summary>Space a cached proxy is assumed to take before it is downloaded (1920 px JPEG, BRIEF §8.5).</summary>
private const long ProxyEstimate = 3L << 19;
public async Task UploadAsync(JobContext context, ISyncRemote remote, Guid localId)
{
var ct = context.Cancellation;
var db = sync.Database;
var local = await db.Sequences.GetAsync(localId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.LocalMissing"));
var archive = await db.Archives.GetAsync(local.ArchiveId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.NoArchive"));
var paths = new ArchivePaths(archive.RootPath);
var (remoteId, created) = await TargetAsync(context, remote, local).ConfigureAwait(false);
var frames = (await SequenceFrames.ListAsync(db, local.Id, ct).ConfigureAwait(false)).Where(f => (f.Flags & FrameFlags.Missing) == 0).ToList();
var manifest = remoteId is { } known ? await remote.GetManifestAsync(known, ct).ConfigureAwait(false) : null;
// A photo the server deleted is not sent back; one it replaced by a converted file is there as its replacement (owner review 2, R6).
var tombstones = new SyncTombstones(await db.FrameRemovals.ListBySequenceAsync(local.Id, ct).ConfigureAwait(false), manifest?.Removed);
var present = manifest?.Frames.Where(f => f.Sha256.Length > 0).Select(f => tombstones.Canonical(f.Sha256)!).ToHashSet(StringComparer.Ordinal) ?? [];
// Originals the server holds without a hash (adopted there, not verified) are the ones of their size and capture
// time: the server could not recognise them by content, so they are not sent again.
var adopted = manifest is not null && manifest.Frames.Any(f => f.Sha256.Length == 0)
? await UnhashedAsync(remote, manifest, ct).ConfigureAwait(false)
: new FramePhotos([]);
var pending = new Queue<Frame>(frames.Where(f => (f.Sha256 is null || (!present.Contains(tombstones.Canonical(f.Sha256)!) && !tombstones.DeletedRemotely(f.Sha256)))
&& adopted.IndexOf(new FramePhoto(f.Sha256, f.Size, f.CapturedAt)) is null));
var progress = new TransferProgress(context, frames.Count, frames.Count - pending.Count);
await progress.ReportAsync().ConfigureAwait(false);
var skipped = 0;
// The SHA-256 of each original the index has none for (a folder adopted and not verified), as its upload computes
// it: the proxies the server asks for are matched to this PC's originals by content. Every such original is
// pending, so each one still here is hashed.
var hashes = new ConcurrentDictionary<Guid, string>();
// Without a server sequence the originals go out one by one under this sequence's session key until the
// server has filed one, which creates the sequence (an original it already holds elsewhere creates nothing).
while (remoteId is null && pending.TryDequeue(out var first))
{
var sent = await UploadFrameAsync(context, remote, null, SyncService.SessionKey(local.Id), paths.FromRelative(first.RelativePath), first, tombstones,
hashes, ct).ConfigureAwait(false);
if (sent is null) skipped++;
else if (sent.SequenceId != Guid.Empty)
{
remoteId = sent.SequenceId;
await context.CheckpointAsync(sent.SequenceId.ToString(), progress.Done + 1, frames.Count).ConfigureAwait(false);
await sync.LinkAsync(remote.Server, sent.SequenceId, local, ct).ConfigureAwait(false);
}
await progress.StepAsync().ConfigureAwait(false);
}
if (remoteId is not { } target) throw new InvalidOperationException(Loc.L("Errors.Sync.NothingSent"));
await Parallel.ForEachAsync(pending, Workers(sync.Options.Uploads, ct), async (frame, token) =>
{
var sent = await UploadFrameAsync(context, remote, target, null, paths.FromRelative(frame.RelativePath), frame, tombstones, hashes, token)
.ConfigureAwait(false);
if (sent is null) Interlocked.Increment(ref skipped);
await progress.StepAsync().ConfigureAwait(false);
}).ConfigureAwait(false);
await SendProxiesAsync(context, remote, target, local, paths, frames, hashes).ConfigureAwait(false);
// An original the index has no hash for turned out to be a photo the server deleted: the verify gives the index
// the hashes (as before a download), so the comparison leaves it out instead of offering its upload again and again.
if (hashes.Values.Any(tombstones.DeletedRemotely)) await VerifyAsync(archive, local.Id, ct).ConfigureAwait(false);
if (created) await sync.PushAllAsync(remote, target, ct).ConfigureAwait(false);
await sync.RecompareAsync(remote, target, ct).ConfigureAwait(false);
if (skipped > 0) throw new InvalidDataException(Loc.Plural("Errors.Sync.Unreadable", skipped));
}
/// <summary>The server's frames it has not hashed yet, by size and capture time, at their indices.</summary>
private static async Task<FramePhotos> UnhashedAsync(ISyncRemote remote, SequenceManifest manifest, CancellationToken cancellationToken)
{
var captured = (await remote.ListFramesAsync(manifest.SequenceId, cancellationToken).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt);
return new FramePhotos(manifest.Frames.Where(f => f.Sha256.Length == 0).Select(f => (f.Index, new FramePhoto(null, f.Size, captured.GetValueOrDefault(f.Id)))));
}
/// <summary>
/// The proxies the server asks for (<see cref="ManifestFrame.WantsProxy"/>) that this PC's decoders can make from its
/// originals (by their archived SHA-256, or the one <paramref name="hashes"/> holds for an original the index has none
/// for, compared as the photo either side's conversions make it, <see cref="SyncTombstones"/>: a frame this PC converted
/// makes the proxy of the server's original; by size and capture time for a frame the server has not hashed,
/// <see cref="SyncRules.ProxyOriginal"/>), made as the server's proxy policy says with the server's capture data, and
/// sent. One the server refuses is counted in the job log; a server that makes no proxies, or an account that may not
/// send them, ends the pass; a server busy or out of reach fails the job like an original would.
/// </summary>
private async Task SendProxiesAsync(JobContext context, ISyncRemote remote, Guid remoteId, Sequence local, ArchivePaths paths, IReadOnlyList<Frame> frames,
IReadOnlyDictionary<Guid, string> hashes)
{
var ct = context.Cancellation;
var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false);
var wanted = manifest.Frames.Where(f => f.WantsProxy).ToList();
if (manifest.Proxies is not { } spec || wanted.Count == 0) return;
var tombstones = new SyncTombstones(await sync.Database.FrameRemovals.ListBySequenceAsync(local.Id, ct).ConfigureAwait(false), manifest.Removed);
var originals = new Dictionary<string, Frame>(StringComparer.Ordinal);
foreach (var frame in frames)
{
if ((frame.Sha256 ?? hashes.GetValueOrDefault(frame.Id)) is { } sha) originals.TryAdd(tombstones.Canonical(sha)!, frame);
}
var records = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id);
var photos = SyncRules.Photos(frames);
var camera = new SidecarCamera { Make = local.CameraMake, Model = local.CameraModel, Serial = local.CameraSerial };
var decoder = sync.Options.Archive.Decoder;
var progress = new TransferProgress(context, wanted.Count, 0);
await progress.ReportAsync().ConfigureAwait(false);
int sent = 0, unsent = 0;
using var refused = CancellationTokenSource.CreateLinkedTokenSource(ct);
try
{
await Parallel.ForEachAsync(wanted, Workers(sync.Options.Uploads, refused.Token), async (frame, token) =>
{
records.TryGetValue(frame.Id, out var record);
// A frame the server has not hashed yet is this PC's original of its size and capture time.
var original = frame.Sha256.Length == 0
? SyncRules.ProxyOriginal(frame, record?.CapturedAt, frames, photos)
: originals.GetValueOrDefault(tombstones.Canonical(frame.Sha256)!);
var jpeg = original is not null && record is not null
? await Task.Run(() => MakeProxy(decoder, paths.FromRelative(original.RelativePath), RenderNodeClient.ToFrame(record), camera, spec, frame.FileName,
token), token).ConfigureAwait(false)
: null;
try
{
if (jpeg is not null)
{
await remote.UploadProxyAsync(frame.Id, jpeg, token).ConfigureAwait(false);
Interlocked.Increment(ref sent);
}
else
{
Interlocked.Increment(ref unsent);
}
}
catch (Exception ex) when (ex is NotSupportedException or ChronosApiException { Status: HttpStatusCode.Conflict or HttpStatusCode.Forbidden })
{
// The server takes no proxies (or none from this account): nothing more to send.
Core.Diagnostics.Log.Info("sync", $"Proxies of {local.DisplayTitle} not sent: {ex.Message}");
await refused.CancelAsync().ConfigureAwait(false);
}
catch (ChronosApiException ex) when (!ex.IsTransient)
{
// This proxy only (not a JPEG of that frame, a frame gone meanwhile): the others still go.
Core.Diagnostics.Log.Warning("sync", $"The proxy of {frame.FileName} was refused", ex);
Interlocked.Increment(ref unsent);
}
await progress.StepAsync().ConfigureAwait(false);
}).ConfigureAwait(false);
}
catch (OperationCanceledException) when (refused.IsCancellationRequested && !ct.IsCancellationRequested)
{
unsent = wanted.Count - sent;
}
context.Log(Loc.L("Archive.Log.ProxiesSent", sent, unsent));
}
/// <summary>A proxy made here for the server; null (logged) when this PC cannot decode or read the original either.</summary>
private static byte[]? MakeProxy(Core.Imaging.ImageDecoder decoder, string path, Frame frame, SidecarCamera camera, ProxySpec spec, string name,
CancellationToken cancellationToken)
{
try
{
return ArchiveImages.MakeProxy(decoder, path, frame, camera, spec.LongEdge, spec.Quality, cancellationToken);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException or NotSupportedException)
{
Core.Diagnostics.Log.Warning("sync", $"No proxy of {name} could be made", ex);
return null;
}
}
public async Task DownloadOriginalsAsync(JobContext context, ISyncRemote remote, Guid remoteId, Guid? targetArchive)
{
var ct = context.Cancellation;
var db = sync.Database;
var link = await sync.Links.GetAsync(remote.Server, remoteId, ct).ConfigureAwait(false);
var local = link?.LocalId is { } id ? await db.Sequences.GetAsync(id, ct).ConfigureAwait(false) : null;
var archive = (local?.ArchiveId ?? targetArchive) is { } archiveId ? await db.Archives.GetAsync(archiveId, ct).ConfigureAwait(false) : null;
if (archive is null) throw new InvalidOperationException(Loc.L("Errors.Sync.NoArchive"));
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false);
var captured = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt);
var (have, restore, photos, missing) = local is null ? ([], [], new FramePhotos([]), []) : await LocalContentAsync(archive, local, ct).ConfigureAwait(false);
// A photo this PC deleted is not fetched again; one replaced here by a converted file is here as its replacement (owner review 2, R6).
var tombstones = local is null ? SyncTombstones.None
: new SyncTombstones(await db.FrameRemovals.ListBySequenceAsync(local.Id, ct).ConfigureAwait(false), manifest.Removed);
if (!tombstones.IsEmpty) have = [.. have.Select(h => tombstones.Canonical(h)!)];
// A frame flagged missing here whose photo the server holds in another encoding (a conversion on either side, R7): the
// copy fetched takes that frame's place under its id, rather than coming in as a frame of its own beside it.
// One whose own encoding the server holds too is restored from that one instead, and one whose photo another frame
// here holds intact stays as it is, as a restore leaves it.
var byAlias = new ConcurrentDictionary<string, Frame>(StringComparer.Ordinal);
if (!tombstones.IsEmpty)
{
var served = manifest.Frames.Where(f => f.Sha256.Length > 0).Select(f => f.Sha256.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
foreach (var frame in missing.Where(f => !served.Contains(f.Sha256!.ToLowerInvariant()) && !have.Contains(tombstones.Canonical(f.Sha256)!)))
byAlias.TryAdd(tombstones.Canonical(frame.Sha256)!, frame);
}
// An original the server has not hashed yet (adopted there, not verified) is here when an original of its size and
// capture time is; each one is fetched on its own and hashed here.
var wanted = manifest.Frames
.Where(f => f.Sha256.Length == 0
? photos.IndexOf(new FramePhoto(null, f.Size, captured.GetValueOrDefault(f.Id))) is null
: !have.Contains(tombstones.Canonical(f.Sha256)!) && !tombstones.DeletedLocally(f.Sha256))
.DistinctBy(f => f.Sha256.Length == 0 ? f.Id.ToString("N") : f.Sha256.ToLowerInvariant()).ToList();
EnsureSpace(paths.Root, wanted.Sum(f => f.Size));
var staging = Path.Combine(paths.StagingDirectory, "sync-" + remoteId.ToString("N"));
Directory.CreateDirectory(staging);
var progress = new TransferProgress(context, manifest.Frames.Count, manifest.Frames.Count - wanted.Count);
await progress.ReportAsync().ConfigureAwait(false);
var created = false;
var rest = wanted;
if (local is null && wanted.Count > 0)
{
// The first original creates the local sequence (grouped by the server id); the others join it.
var (staged, sha) = await FetchOriginalAsync(remote, wanted[0], staging, ct).ConfigureAwait(false);
var first = await CommitAsync(remoteId, archive.Id, null, wanted[0], staged, sha, captured, ct).ConfigureAwait(false);
local = await db.Sequences.GetAsync(first.SequenceId, ct).ConfigureAwait(false)
?? throw new InvalidOperationException(Loc.L("Errors.Sync.LocalMissing"));
await sync.LinkAsync(remote.Server, remoteId, local, ct).ConfigureAwait(false);
await sync.Links.SetOriginAsync(local.Id, SequenceOrigin.Server, ct).ConfigureAwait(false);
created = true;
rest = wanted.GetRange(1, wanted.Count - 1);
await progress.StepAsync().ConfigureAwait(false);
}
var sequenceId = local?.Id;
var refused = 0;
var restores = new ConcurrentBag<(string Staged, Guid FrameId)>();
var aliases = new ConcurrentBag<AliasCopy>();
await Parallel.ForEachAsync(rest, Workers(sync.Options.Downloads, ct), async (frame, token) =>
{
var (staged, sha) = await FetchOriginalAsync(remote, frame, staging, token).ConfigureAwait(false);
if (restore.TryGetValue(sha, out var frameId))
{
restores.Add((staged, frameId));
}
else if (byAlias.TryRemove(tombstones.Canonical(sha)!, out var lost))
{
// This encoding takes the place of the one missing here, after the transfers as the restores do.
aliases.Add(new AliasCopy(staged, sha, lost, frame.FileName));
}
else if (frame.Sha256.Length == 0 && (tombstones.DeletedLocally(sha) || have.Contains(tombstones.Canonical(sha)!)))
{
// An original the server has not hashed is known by content only now: one this PC deleted (or holds as
// the converted file that replaced it) is not filed again, which would also drop its tombstone.
File.Delete(staged);
Interlocked.Increment(ref refused);
if (tombstones.DeletedLocally(sha)) context.Log(Loc.L("Archive.Log.SyncDeletedHere", frame.FileName));
}
else
{
await CommitAsync(remoteId, archive.Id, sequenceId, frame, staged, sha, captured, token).ConfigureAwait(false);
}
await progress.StepAsync().ConfigureAwait(false);
}).ConfigureAwait(false);
// Null when the sequence is not in the folder the index has: the copies stay staged for the next attempt.
var restored = restores.IsEmpty && aliases.IsEmpty ? 0 : await RestoreAsync(local!, paths, restores, ct).ConfigureAwait(false);
var (swapped, moved) = restored is null || aliases.IsEmpty ? (0, restored is null)
: await SwapAliasesAsync(context, archive, local!, paths, aliases, ct).ConfigureAwait(false);
if (!moved) TryDeleteDirectory(staging);
// Restored originals are checked against their records, which clears their missing flag (and reads the tier of a
// frame that took another encoding).
if (restored + swapped > 0) await VerifyAsync(archive, local!.Id, ct).ConfigureAwait(false);
if (created) await sync.TakeAllAsync(remote, remoteId, ct).ConfigureAwait(false);
await sync.RecompareAsync(remote, remoteId, ct).ConfigureAwait(false);
// Originals filed into the sequence (not restores, nor frames swapped to another encoding, whose passes are queued as
// they swap) need their thumbnails and light analysis.
if (sequenceId is { } downloaded && wanted.Count > restores.Count + aliases.Count + refused)
{
var label = local!.DisplayTitle;
await QueuePassesAsync(context, downloaded, [ArchiveJobs.Thumbnails([downloaded], label), ProcessingJobs.LightAnalysis([downloaded], label)])
.ConfigureAwait(false);
}
if (moved) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
}
/// <summary>
/// The originals a local copy holds (lower-case SHA-256, and their photos for server frames not hashed yet), the frame
/// each original flagged missing restores, and the frames flagged missing whose content no intact frame holds. Frames
/// not hashed yet (an adopted folder never verified) are hashed first through the archive's verify, so the server's
/// copies of them are recognised instead of committed again as duplicates.
/// </summary>
private async Task<(HashSet<string> Have, Dictionary<string, Guid> Restore, FramePhotos Photos, IReadOnlyList<Frame> Missing)> LocalContentAsync(
Archive archive, Sequence local, CancellationToken ct)
{
var frames = await SequenceFrames.ListAsync(sync.Database, local.Id, ct).ConfigureAwait(false);
if (frames.Any(f => f.Sha256 is null && (f.Flags & FrameFlags.Missing) == 0))
{
await VerifyAsync(archive, local.Id, ct).ConfigureAwait(false);
frames = await SequenceFrames.ListAsync(sync.Database, local.Id, ct).ConfigureAwait(false);
}
var have = frames.Where(f => (f.Flags & FrameFlags.Missing) == 0 && f.Sha256 is not null).Select(f => f.Sha256!.ToLowerInvariant())
.ToHashSet(StringComparer.Ordinal);
var restore = new Dictionary<string, Guid>(StringComparer.Ordinal);
var missing = new List<Frame>();
foreach (var frame in frames.Where(f => (f.Flags & FrameFlags.Missing) != 0 && f.Sha256 is not null))
{
var sha = frame.Sha256!.ToLowerInvariant();
if (have.Contains(sha)) continue;
restore.TryAdd(sha, frame.Id);
missing.Add(frame);
}
var photos = SyncRules.Photos([.. frames.Where(f => (f.Flags & FrameFlags.Missing) == 0)]);
return (have, restore, photos, missing);
}
private Task VerifyAsync(Archive archive, Guid sequenceId, CancellationToken ct) =>
sync.Archives.Verifier.VerifyAsync(archive, [sequenceId], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: ct);
/// <summary>
/// Puts the verified copies of originals flagged missing (<paramref name="staged"/>, each with the frame it restores)
/// back where their records say, under the sequence's lease in its folder as the index has it then: a Reorganize moves
/// the folder and the frames' paths under that lease. Returns how many went back; a file that reappeared meanwhile is
/// kept, and the copy of a frame that left the index is dropped. Null, with nothing restored and every copy kept in
/// staging for the next attempt, when the sequence is not in that folder (another index moved it, and maybe another
/// sequence into it; this index's next scan finds them): the folder it left is not created again, nor given an
/// original of this sequence when it holds another one now.
/// </summary>
private Task<int?> RestoreAsync(Sequence local, ArchivePaths paths, IReadOnlyCollection<(string Staged, Guid FrameId)> staged, CancellationToken ct) =>
sync.Archives.UnderLeaseAsync(local, async row =>
{
if (row is null) throw new InvalidOperationException(Loc.L("Errors.Sync.LocalMissing"));
if (!SequenceFolders.Holds(paths.Sequence(row.Folder), row.Id)) return (int?)null;
var restored = 0;
foreach (var (copy, frameId) in staged)
{
// Read under the lease, as the folder: where the photo belongs in the folder the sequence is in now.
if (await sync.Database.Frames.GetAsync(frameId, ct).ConfigureAwait(false) is not { } frame || frame.SequenceId != row.Id)
{
File.Delete(copy);
continue;
}
Restore(copy, paths.FromRelative(frame.RelativePath));
restored++;
}
return restored;
}, ct);
/// <summary>
/// Gives each frame flagged missing here whose photo the server holds only in another encoding (a conversion on either
/// side, owner review 2, R7) the verified copy fetched (<see cref="FrameTrash.ReplaceMissingAsync"/>: under the sequence's
/// lease, in its folder as the index has it then, whose sidecar must be the sequence's, <see cref="SequenceWriter.ReadLocked"/>).
/// Run after the transfers, once <see cref="RestoreAsync"/> found the sequence in that folder. A swap clears what covered
/// the sequence, as any replacement does, so the passes it had (<see cref="FrameTrash.FollowUps"/>: the light analysis, a
/// background deep analysis, the thumbnails and proxies) are queued after each one, whatever stops the download later.
/// Returns how many took their frame's place; the copy of a frame that changed meanwhile, or whose file came back, is
/// dropped (logged). Moved, with the copies not swapped yet kept in staging for the next attempt, when the sequence left
/// that folder meanwhile.
/// </summary>
private async Task<(int Swapped, bool Moved)> SwapAliasesAsync(JobContext context, Archive archive, Sequence local, ArchivePaths paths,
IEnumerable<AliasCopy> aliases, CancellationToken ct)
{
var swapped = 0;
// What the sequence had before the first swap cleared it: the passes that make it again.
var had = SequenceStatus.None;
foreach (var alias in aliases)
{
ReplaceResult? result;
try
{
result = await sync.Archives.FrameTrash.ReplaceMissingAsync(archive, local.Id, alias.Lost.Id, alias.Lost.Sha256!, alias.Staged, alias.Sha256,
alias.FileName, ct).ConfigureAwait(false);
}
catch (Exception ex) when (ex is InvalidDataException or DirectoryNotFoundException or FileNotFoundException)
{
// Another index moved the sequence away, maybe another one into the folder it left: nothing is filed there.
if (await sync.Database.Sequences.GetAsync(local.Id, ct).ConfigureAwait(false) is { } row && SequenceFolders.Holds(paths.Sequence(row.Folder), row.Id))
throw;
return (swapped, true);
}
if (result is null)
{
File.Delete(alias.Staged);
context.Log(Loc.L("Archive.Log.SyncFrameChanged", alias.FileName));
continue;
}
swapped++;
had |= result.Had;
await QueuePassesAsync(context, local.Id, sync.Archives.FrameTrash.FollowUps(archive, result.Sequence, had)).ConfigureAwait(false);
}
return (swapped, false);
}
/// <summary>
/// Queues <paramref name="requests"/>, passes over the sequence <paramref name="sequenceId"/>, as follow-ups of this job,
/// whatever stops it meanwhile, as a restore from the trash queues its own: the files they cover have changed already, and
/// the next download, which finds nothing missing, would not queue them again. One already queued for the sequence (not
/// started yet, so it covers what changed) is left out.
/// </summary>
private async Task QueuePassesAsync(JobContext context, Guid sequenceId, IEnumerable<ArchiveJobRequest> requests)
{
foreach (var request in requests)
{
if (await sync.Database.Jobs.IsQueuedAsync(request.Kind, sequenceId, CancellationToken.None).ConfigureAwait(false)) continue;
await EnqueueFollowUpAsync(context, request, CancellationToken.None).ConfigureAwait(false);
}
}
/// <summary>A verified copy fetched in another encoding of a frame flagged missing here (<see cref="SwapAliasesAsync"/>).</summary>
private sealed record AliasCopy(string Staged, string Sha256, Frame Lost, string FileName);
/// <summary>Puts a verified copy of an original flagged missing back where its record says; a file that reappeared meanwhile is kept.</summary>
private static void Restore(string staged, string place)
{
if (File.Exists(place))
{
File.Delete(staged);
return;
}
Directory.CreateDirectory(Path.GetDirectoryName(place)!);
File.Move(staged, place, overwrite: false);
}
public async Task SendExportsAsync(JobContext context, ISyncRemote remote, Guid localId)
{
var ct = context.Cancellation;
var db = sync.Database;
var local = await db.Sequences.GetAsync(localId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.LocalMissing"));
var archive = await db.Archives.GetAsync(local.ArchiveId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.NoArchive"));
var link = await sync.Links.FindByLocalAsync(remote.Server, local.Id, ct).ConfigureAwait(false)
?? throw new InvalidOperationException(Loc.L("Errors.Sync.NotLinked"));
var there = (await remote.GetManifestAsync(link.RemoteId, ct).ConfigureAwait(false)).Renders.Select(r => r.Id).ToHashSet();
var renders = (await db.Renders.ListBySequenceAsync(local.Id, ct).ConfigureAwait(false)).Where(r => !there.Contains(r.Id)).ToList();
var progress = new TransferProgress(context, renders.Count, 0);
await progress.ReportAsync().ConfigureAwait(false);
var gone = 0;
foreach (var render in renders)
{
var file = SyncService.RenderFile(archive, render);
if (File.Exists(file))
{
var sha = await ChronosClient.HashFileAsync(file, ct).ConfigureAwait(false);
await remote.UploadRenderAsync(link.RemoteId, file, new RenderUploadRequest(render.Id, Path.GetFileName(file), new FileInfo(file).Length, sha,
render.PresetName, render.Codec, render.Width, render.Height, render.Fps, render.DurationSeconds, render.ProducedBy), ct).ConfigureAwait(false);
}
else
{
gone++;
}
await progress.StepAsync().ConfigureAwait(false);
}
await sync.RecompareAsync(remote, link.RemoteId, ct).ConfigureAwait(false);
if (gone > 0) throw new FileNotFoundException(Loc.Plural("Errors.Sync.ExportsGone", gone));
}
public async Task GetExportsAsync(JobContext context, ISyncRemote remote, Guid remoteId)
{
var ct = context.Cancellation;
var db = sync.Database;
var link = await sync.Links.GetAsync(remote.Server, remoteId, ct).ConfigureAwait(false);
var local = (link?.LocalId is { } id ? await db.Sequences.GetAsync(id, ct).ConfigureAwait(false) : null)
?? throw new InvalidOperationException(Loc.L("Errors.Sync.LocalMissing"));
var archive = await db.Archives.GetAsync(local.ArchiveId, ct).ConfigureAwait(false) ?? throw new InvalidOperationException(Loc.L("Errors.Sync.NoArchive"));
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var here = (await db.Renders.ListBySequenceAsync(local.Id, ct).ConfigureAwait(false)).Select(r => r.Id).ToHashSet();
var wanted = (await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false)).Renders.Where(r => !here.Contains(r.Id)).ToList();
EnsureSpace(paths.Root, wanted.Sum(r => r.Bytes));
var staging = Path.Combine(paths.StagingDirectory, "exports-" + remoteId.ToString("N"));
Directory.CreateDirectory(staging);
var progress = new TransferProgress(context, wanted.Count, 0);
await progress.ReportAsync().ConfigureAwait(false);
foreach (var render in wanted)
{
var (part, name) = await FetchRenderAsync(remote, render, staging, ct).ConfigureAwait(false);
// Filed under the sequence's lease into its folder as the index has it then, as a server files a render: a
// Reorganize may have moved it since the sequence was read, and moves the row's path along when it moves it later.
// One another index moved is found by this index's next scan; until then nothing is filed (the download stays
// staged for the next attempt), so the folder it left is not created again, nor filed into when it holds another
// sequence now.
var filed = await sync.Archives.UnderLeaseAsync(local, async row =>
{
if (row is null) throw new InvalidOperationException(Loc.L("Errors.Sync.LocalMissing"));
var folder = paths.Sequence(row.Folder);
if (!SequenceFolders.Holds(folder, row.Id)) return false;
Directory.CreateDirectory(folder.Exports);
var target = Unique(folder.Exports, name);
File.Move(part, target);
// Recorded even if the job is being cancelled: a filed export is never left without its row.
await db.Renders.InsertAsync(new Render
{
Id = render.Id,
SequenceId = row.Id,
PresetName = render.PresetName,
OutputPath = paths.ToRelative(target),
Codec = render.Codec,
Width = render.Width,
Height = render.Height,
Fps = render.Fps,
DurationSeconds = render.DurationSeconds,
Bytes = new FileInfo(target).Length,
Created = render.Created,
ProducedBy = render.ProducedBy,
}, CancellationToken.None).ConfigureAwait(false);
return true;
}, ct).ConfigureAwait(false);
if (!filed) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
await progress.StepAsync().ConfigureAwait(false);
}
TryDeleteDirectory(staging);
await sync.RecompareAsync(remote, remoteId, ct).ConfigureAwait(false);
}
/// <summary>A render's file in staging (resumed with Range; started over once if the partial copy no longer fits) and the name the server gives it.</summary>
private static async Task<(string Part, string Name)> FetchRenderAsync(ISyncRemote remote, RenderDto render, string staging, CancellationToken cancellationToken)
{
var part = Path.Combine(staging, render.Id.ToString("N") + ".part");
for (var attempt = 1; ; attempt++)
{
try
{
string? name;
await using (var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None))
name = await remote.DownloadRenderAsync(render.Id, file, cancellationToken).ConfigureAwait(false);
return (part, SafeFileName(name) ?? render.Id.ToString("N"));
}
catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.RequestedRangeNotSatisfiable && attempt == 1)
{
File.Delete(part);
}
}
}
internal static string? SafeFileName(string? name)
{
var file = Path.GetFileName(name?.Trim());
return string.IsNullOrEmpty(file) || file.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0 || file is "." or ".." ? null : file;
}
/// <summary>"name.mp4", else "name (2).mp4", … — an existing export is never replaced.</summary>
internal static string Unique(string folder, string fileName)
{
var candidate = Path.Combine(folder, fileName);
var stem = Path.GetFileNameWithoutExtension(fileName);
var extension = Path.GetExtension(fileName);
for (var n = 2; File.Exists(candidate); n++) candidate = Path.Combine(folder, string.Create(CultureInfo.InvariantCulture, $"{stem} ({n}){extension}"));
return candidate;
}
public async Task DownloadProxiesAsync(JobContext context, ISyncRemote remote, Guid remoteId)
{
var ct = context.Cancellation;
var dto = await remote.GetSequenceAsync(remoteId, ct).ConfigureAwait(false);
var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false);
var frames = await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false);
var assets = manifest.Frames.ToDictionary(f => f.Id);
var folder = sync.Proxies.Folder(remote.Server, remoteId);
Directory.CreateDirectory(folder);
var cachedKinds = sync.Proxies.Kinds(remote.Server, remoteId);
ProxyCache.Prune(folder, frames.Select(f => ProxyCache.FileName(f.Id)).ToHashSet(StringComparer.Ordinal));
EnsureSpace(folder, frames.Count(f => !File.Exists(Path.Combine(folder, ProxyCache.FileName(f.Id)))) * ProxyEstimate);
var files = new string?[frames.Count];
var kinds = new string?[frames.Count];
var progress = new TransferProgress(context, frames.Count, 0);
await progress.ReportAsync().ConfigureAwait(false);
await Parallel.ForEachAsync(Enumerable.Range(0, frames.Count), Workers(sync.Options.Downloads, ct), async (i, token) =>
{
IReadOnlyList<string> offered = assets.TryGetValue(frames[i].Id, out var asset) ? Kinds(asset) : [];
var fetched = await FetchProxyAsync(remote, frames[i].Id, offered, cachedKinds.GetValueOrDefault(frames[i].Id), folder, token).ConfigureAwait(false);
files[i] = fetched?.Name;
kinds[i] = fetched?.Kind;
await progress.StepAsync().ConfigureAwait(false);
}).ConfigureAwait(false);
sync.Proxies.Save(remote.Server, remoteId, new ProxyCacheIndex(dto, manifest.SettingsJson, manifest.FramesVersion, frames, files, kinds));
var cached = files.Count(f => f is not null);
var standIns = Enumerable.Range(0, frames.Count).Count(i => kinds[i] != ProxyCache.Proxy && !NeverProxied(frames[i]));
await sync.SetProxiesAsync(remote.Server, dto, cached, standIns, manifest.FramesVersion, ct).ConfigureAwait(false);
if (cached < frames.Count) context.Log(Loc.Plural("Archive.Log.SyncNoPreview", frames.Count - cached));
}
/// <summary>
/// The server sequence an upload fills and whether this job creates it: the linked one, the one an interrupted
/// attempt created, or none yet (the first original creates it).
/// </summary>
private async Task<(Guid? RemoteId, bool Created)> TargetAsync(JobContext context, ISyncRemote remote, Sequence local)
{
var ct = context.Cancellation;
Guid? started = Guid.TryParse(context.Checkpoint, out var saved) ? saved : null;
var link = await sync.Links.FindByLocalAsync(remote.Server, local.Id, ct).ConfigureAwait(false);
foreach (var candidate in new[] { link?.RemoteId, started })
{
if (candidate is not { } id) continue;
if (await ExistsAsync(remote, id, ct).ConfigureAwait(false))
{
if (link?.RemoteId != id) await sync.LinkAsync(remote.Server, id, local, ct).ConfigureAwait(false);
return (id, id == started);
}
// Deleted on the server meanwhile: the originals go into a new sequence.
if (id == link?.RemoteId) await sync.ForgetGoneAsync(remote.Server, id, ct).ConfigureAwait(false);
}
return (null, true);
}
private static async Task<bool> ExistsAsync(ISyncRemote remote, Guid id, CancellationToken cancellationToken)
{
try
{
await remote.GetSequenceAsync(id, cancellationToken).ConfigureAwait(false);
return true;
}
catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.NotFound)
{
return false;
}
}
/// <summary>
/// Hashes the original again and sends it into <paramref name="sequenceId"/> (or the sequence of
/// <paramref name="sessionKey"/>); null (logged) when it is gone or no longer the archived file. The hash of an
/// original the index has none for goes into <paramref name="hashes"/>; known by content only now, such an original
/// the server deleted (<paramref name="tombstones"/>) is not sent back.
/// </summary>
private static async Task<CompleteUploadResponse?> UploadFrameAsync(JobContext context, ISyncRemote remote, Guid? sequenceId, string? sessionKey, string path,
Frame frame, SyncTombstones tombstones, ConcurrentDictionary<Guid, string> hashes, CancellationToken cancellationToken)
{
if (!File.Exists(path))
{
context.Log(Loc.L("Archive.Log.SyncMissing", frame.FileName));
return null;
}
var sha = await ChronosClient.HashFileAsync(path, cancellationToken).ConfigureAwait(false);
var notSent = new CompleteUploadResponse(Guid.Empty, sequenceId ?? Guid.Empty, -1);
if (frame.Sha256 is null)
{
hashes[frame.Id] = sha;
if (tombstones.DeletedRemotely(sha))
{
context.Log(Loc.L("Archive.Log.SyncDeletedThere", frame.FileName));
return notSent;
}
}
if (frame.Sha256 is { } archived && !archived.Equals(sha, StringComparison.OrdinalIgnoreCase))
{
context.Log(Loc.L("Archive.Log.SyncChanged", frame.FileName));
return null;
}
return await remote.UploadFileAsync(path, sequenceId, sessionKey, sha, frame.CapturedAt, cancellationToken).ConfigureAwait(false) ?? notSent;
}
/// <summary>Commits one original from staging into the local sequence (created from the server id when null).</summary>
private async Task<IngestResult> CommitAsync(Guid remoteId, Guid archiveId, Guid? sequenceId, ManifestFrame frame, string staged, string sha,
IReadOnlyDictionary<Guid, DateTime?> captured, CancellationToken cancellationToken)
{
var request = new IngestRequest(archiveId, sequenceId, null, sequenceId is null ? SyncService.SessionKey(remoteId) : null, staged, frame.FileName,
frame.Size, sha, captured.GetValueOrDefault(frame.Id), null);
var result = await sync.Archives.Ingest.CommitAsync(request, cancellationToken).ConfigureAwait(false);
// A duplicate stays in staging (the sequence already holds that content).
if (result.Duplicate) File.Delete(staged);
return result;
}
/// <summary>
/// A verified copy in staging and its lower-case SHA-256: resumed from a partial download, accepted only with the
/// manifest's SHA-256 (two attempts). A part as long as the file (a job stopped while hashing it) is only hashed again;
/// a part the server can no longer resume (416) starts over.
/// </summary>
internal static async Task<(string Path, string Sha256)> FetchOriginalAsync(ISyncRemote remote, ManifestFrame frame, string staging,
CancellationToken cancellationToken)
{
if (frame.Sha256.Length == 0) return await FetchUnhashedAsync(remote, frame, staging, cancellationToken).ConfigureAwait(false);
var sha = frame.Sha256.ToLowerInvariant();
if (sha.Length != 64 || !sha.All(char.IsAsciiHexDigitLower)) throw new InvalidDataException(Loc.L("Errors.Sync.Hash", frame.FileName));
var target = Path.Combine(staging, sha + Path.GetExtension(frame.FileName).ToLowerInvariant());
if (File.Exists(target)) return (target, sha);
var part = target + ".part";
for (var attempt = 1; ; attempt++)
{
if (!await FetchPartAsync(remote, frame, part, attempt, cancellationToken).ConfigureAwait(false)) continue;
if (await ChronosClient.HashFileAsync(part, cancellationToken).ConfigureAwait(false) == sha)
{
File.Move(part, target, overwrite: true);
return (target, sha);
}
File.Delete(part);
if (attempt == 2) throw new InvalidDataException(Loc.L("Errors.Sync.Hash", frame.FileName));
}
}
/// <summary>
/// An original the server has not hashed yet (a folder it adopted and did not verify): nothing to check it against
/// but its size, so a copy of another size is fetched again once; it is hashed here, and the archive ingest files it
/// by that hash like any other. Kept in staging under the frame's id, as several such originals may share content.
/// </summary>
private static async Task<(string Path, string Sha256)> FetchUnhashedAsync(ISyncRemote remote, ManifestFrame frame, string staging,
CancellationToken cancellationToken)
{
var target = Path.Combine(staging, frame.Id.ToString("N") + Path.GetExtension(frame.FileName).ToLowerInvariant());
var part = target + ".part";
for (var attempt = 1; !File.Exists(target); attempt++)
{
if (!await FetchPartAsync(remote, frame, part, attempt, cancellationToken).ConfigureAwait(false)) continue;
if (new FileInfo(part).Length == frame.Size)
{
File.Move(part, target, overwrite: true);
break;
}
File.Delete(part);
if (attempt == 2) throw new InvalidDataException(Loc.L("Errors.Sync.Hash", frame.FileName));
}
return (target, await ChronosClient.HashFileAsync(target, cancellationToken).ConfigureAwait(false));
}
/// <summary>
/// Downloads an original into <paramref name="part"/>, resuming after what it holds; a part as long as the file is
/// kept as it is. False when the server can no longer resume it (416) on the first attempt: the part is gone and the
/// next attempt starts over.
/// </summary>
private static async Task<bool> FetchPartAsync(ISyncRemote remote, ManifestFrame frame, string part, int attempt, CancellationToken cancellationToken)
{
if (File.Exists(part) && new FileInfo(part).Length >= frame.Size) return true;
try
{
await using (var file = new FileStream(part, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None))
await remote.DownloadFrameAsync(frame.Id, "original", file, cancellationToken).ConfigureAwait(false);
return true;
}
catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.RequestedRangeNotSatisfiable && attempt == 1)
{
File.Delete(part);
return false;
}
}
/// <summary>
/// The cached file of a frame and what it is, downloading the first asset kind the server has
/// (<paramref name="offered"/>, the proxy first); null when it has none. A cached proxy stays. A cached thumbnail,
/// or a file of a cache that did not record its kind (<paramref name="cachedKind"/> null), stands in only until the
/// server has a proxy of the frame, which then replaces it; until then it stays as the thumbnail.
/// </summary>
private static async Task<(string Name, string Kind)?> FetchProxyAsync(ISyncRemote remote, Guid frameId, IReadOnlyList<string> offered, string? cachedKind,
string folder, CancellationToken cancellationToken)
{
var name = ProxyCache.FileName(frameId);
var path = Path.Combine(folder, name);
var cached = new FileInfo(path) is { Exists: true, Length: > 0 };
if (cached && (cachedKind == ProxyCache.Proxy || !offered.Contains(ProxyCache.Proxy))) return (name, cachedKind ?? ProxyCache.Thumb);
var part = path + ".part";
// The cached file is as good as the server's thumbnail.
var kind = await DownloadFirstAsync(remote, frameId, offered.TakeWhile(k => !(cached && k == ProxyCache.Thumb)), part, null, cancellationToken)
.ConfigureAwait(false);
if (kind is not null)
{
File.Move(part, path, overwrite: true);
return (name, kind);
}
return cached ? (name, ProxyCache.Thumb) : null;
}
/// <summary>
/// Downloads the first of <paramref name="kinds"/> a frame has on the server (a 404 tries the next) into a fresh
/// <paramref name="part"/> file, written through <paramref name="through"/> when given (a hash of what arrives, which
/// leaves the file open) and flushed to the disk when <paramref name="durable"/>; the kind it got, or null when the server
/// has none of them, no part file left then. The proxy cache and the workspace copy of a server project (owner review 2,
/// R8) download alike.
/// </summary>
internal static async Task<string?> DownloadFirstAsync(ISyncRemote remote, Guid frameId, IEnumerable<string> kinds, string part, Func<Stream, Stream>? through,
CancellationToken cancellationToken, bool durable = false)
{
foreach (var kind in kinds)
{
try
{
await using (var file = new FileStream(part, FileMode.Create, FileAccess.ReadWrite, FileShare.None))
{
await remote.DownloadFrameAsync(frameId, kind, through?.Invoke(file) ?? file, cancellationToken).ConfigureAwait(false);
if (durable) file.Flush(flushToDisk: true);
}
return kind;
}
catch (ChronosApiException ex) when (ex.Status == HttpStatusCode.NotFound)
{
File.Delete(part);
}
}
return null;
}
/// <summary>
/// A frame missing or damaged on the server: its manifest never asks for its proxy and a pass makes none of a missing
/// one, so the thumbnail (or the lack of a file) that stands in for it does not keep the cache stale.
/// </summary>
internal static bool NeverProxied(FrameDto frame) => ((FrameFlags)frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0;
/// <summary>The kinds of a frame's assets the server offers, the proxy before the thumbnail that stands in while it has none.</summary>
internal static string[] Kinds(ManifestFrame frame) => (frame.HasProxy, frame.HasThumb) switch
{
(true, true) => [ProxyCache.Proxy, ProxyCache.Thumb],
(true, false) => [ProxyCache.Proxy],
(false, true) => [ProxyCache.Thumb],
_ => [],
};
/// <summary>Queues a follow-up on this job's engine for the same user; skipped when the engine has no handler for it.</summary>
private static Task<Job?> EnqueueFollowUpAsync(JobContext context, ArchiveJobRequest request, CancellationToken cancellationToken) =>
context.EnqueueFollowUpAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, cancellationToken);
private static void EnsureSpace(string root, long bytes)
{
long? free;
try
{
var full = Path.GetFullPath(root);
// Windows wants a drive root; Unix accepts any directory and reports the volume it is mounted on.
free = new DriveInfo(OperatingSystem.IsWindows() ? Path.GetPathRoot(full)! : full).AvailableFreeSpace;
}
catch (Exception ex) when (ex is ArgumentException or IOException or UnauthorizedAccessException)
{
free = null; // network shares: the copy itself reports a full disk
}
if (free is { } available && available < bytes + SpaceReserve)
throw new IOException(Loc.L("Errors.Sync.DiskFull", ((bytes + SpaceReserve - available) / 1e9).ToString("0.0", CultureInfo.CurrentCulture)));
}
private static void TryDeleteDirectory(string folder)
{
try
{
Directory.Delete(folder, recursive: true);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Core.Diagnostics.Log.Warning("sync", $"Staging folder {folder} could not be removed", ex);
}
}
private static ParallelOptions Workers(int degree, CancellationToken cancellationToken) =>
new() { MaxDegreeOfParallelism = Math.Max(1, degree), CancellationToken = cancellationToken };
/// <summary>Files done out of the total, reported in order from parallel workers.</summary>
private sealed class TransferProgress(JobContext context, long total, long done)
{
private readonly SemaphoreSlim _gate = new(1, 1);
public async Task StepAsync()
{
await _gate.WaitAsync(CancellationToken.None).ConfigureAwait(false);
try
{
done++;
await context.ReportAsync(done, total).ConfigureAwait(false);
}
finally
{
_gate.Release();
}
}
public long Done => done;
public ValueTask ReportAsync() => context.ReportAsync(done, total);
}
}
/// <summary>
/// A write-only stream that hashes (SHA-256) what passes through it into <paramref name="inner"/>, which it leaves open: the
/// hash of a download as it arrived, to check the file it was written to against (owner review 2, R8). It cannot seek,
/// so a download through it always starts from the beginning.
/// </summary>
internal sealed class HashingStream(Stream inner) : Stream
{
private readonly System.Security.Cryptography.IncrementalHash _hash = System.Security.Cryptography.IncrementalHash.CreateHash(
System.Security.Cryptography.HashAlgorithmName.SHA256);
private long _written;
/// <summary>The bytes written through.</summary>
public long Written => _written;
/// <summary>The lower-case hex SHA-256 of what was written through (once; it resets the hash).</summary>
public string Sha256() => Convert.ToHexStringLower(_hash.GetHashAndReset());
public override bool CanRead => false;
public override bool CanSeek => false;
public override bool CanWrite => true;
public override long Length => throw new NotSupportedException();
public override long Position
{
get => throw new NotSupportedException();
set => throw new NotSupportedException();
}
public override void Flush() => inner.Flush();
public override Task FlushAsync(CancellationToken cancellationToken) => inner.FlushAsync(cancellationToken);
public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException();
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
public override void SetLength(long value) => throw new NotSupportedException();
public override void Write(byte[] buffer, int offset, int count) => Write(buffer.AsSpan(offset, count));
public override void Write(ReadOnlySpan<byte> buffer)
{
_hash.AppendData(buffer);
_written += buffer.Length;
inner.Write(buffer);
}
public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) =>
WriteAsync(buffer.AsMemory(offset, count), cancellationToken).AsTask();
public override ValueTask WriteAsync(ReadOnlyMemory<byte> buffer, CancellationToken cancellationToken = default)
{
_hash.AppendData(buffer.Span);
_written += buffer.Length;
return inner.WriteAsync(buffer, cancellationToken);
}
protected override void Dispose(bool disposing)
{
if (disposing) _hash.Dispose();
base.Dispose(disposing);
}
}