feat: owner review 2 phase 3: Library on the web panel, photo trash and web upload (#11)
CI / Build and test (push) Successful in 7m25s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 10s
CI / PostgreSQL 16 integration (push) Successful in 5m20s
CI / PostgreSQL 17 integration (push) Successful in 5m22s
CI / Docker image (push) Successful in 50s
Release / Build, test and publish (push) Successful in 12m1s
Release / Server image (push) Successful in 1m26s
CI / Build and test (push) Successful in 7m25s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 10s
CI / PostgreSQL 16 integration (push) Successful in 5m20s
CI / PostgreSQL 17 integration (push) Successful in 5m22s
CI / Docker image (push) Successful in 50s
Release / Build, test and publish (push) Successful in 12m1s
Release / Server image (push) Successful in 1m26s
- Web panel Library: browse, view, delete sequences and photos, mass operations (analyse, proxies, thumbnails, verify, retag, delete, CSV). It never edits the video. - Photo trash with tombstones (migration 0008): crash-safe delete and restore, hash-checked restore with reasons, and emptying only by an explicit fingerprinted job. Sync and uploads keep deleted photos deleted; an explicit "Upload again" brings one back. - Upload from the web panel: in-house SHA-256 in a worker with a fail-closed self-test, capture time from EXIF, chunked, resumable; no folders. - Desktop trash dialog with photo entries and Empty. - Admin files are revalidated (no-cache + ETag). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
This commit was merged in pull request #11.
This commit is contained in:
1 parent
af9eeaab9b
commit
9aa818ed47
283 files changed
+24922
-293
No files matched your search
@@ -48,13 +48,15 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
|
||||
var frames = (await SequenceFrames.ListAsync(db, local.Id, ct).ConfigureAwait(false)).Where(f => (f.Flags & FrameFlags.Missing) == 0).ToList();
|
||||
var manifest = remoteId is { } known ? await remote.GetManifestAsync(known, ct).ConfigureAwait(false) : null;
|
||||
var present = manifest?.Frames.Where(f => f.Sha256.Length > 0).Select(f => f.Sha256.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal) ?? [];
|
||||
// A photo the server deleted is not sent back; one it replaced by a converted file is there as its replacement (owner review 2, R6).
|
||||
var tombstones = new SyncTombstones(await db.FrameRemovals.ListBySequenceAsync(local.Id, ct).ConfigureAwait(false), manifest?.Removed);
|
||||
var present = manifest?.Frames.Where(f => f.Sha256.Length > 0).Select(f => tombstones.Canonical(f.Sha256)!).ToHashSet(StringComparer.Ordinal) ?? [];
|
||||
// Originals the server holds without a hash (adopted there, not verified) are the ones of their size and capture
|
||||
// time: the server could not recognise them by content, so they are not sent again.
|
||||
var adopted = manifest is not null && manifest.Frames.Any(f => f.Sha256.Length == 0)
|
||||
? await UnhashedAsync(remote, manifest, ct).ConfigureAwait(false)
|
||||
: new FramePhotos([]);
|
||||
var pending = new Queue<Frame>(frames.Where(f => (f.Sha256 is null || !present.Contains(f.Sha256.ToLowerInvariant()))
|
||||
var pending = new Queue<Frame>(frames.Where(f => (f.Sha256 is null || (!present.Contains(tombstones.Canonical(f.Sha256)!) && !tombstones.DeletedRemotely(f.Sha256)))
|
||||
&& adopted.IndexOf(new FramePhoto(f.Sha256, f.Size, f.CapturedAt)) is null));
|
||||
var progress = new TransferProgress(context, frames.Count, frames.Count - pending.Count);
|
||||
await progress.ReportAsync().ConfigureAwait(false);
|
||||
@@ -68,8 +70,8 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
// server has filed one, which creates the sequence (an original it already holds elsewhere creates nothing).
|
||||
while (remoteId is null && pending.TryDequeue(out var first))
|
||||
{
|
||||
var sent = await UploadFrameAsync(context, remote, null, SyncService.SessionKey(local.Id), paths.FromRelative(first.RelativePath), first, hashes, ct)
|
||||
.ConfigureAwait(false);
|
||||
var sent = await UploadFrameAsync(context, remote, null, SyncService.SessionKey(local.Id), paths.FromRelative(first.RelativePath), first, tombstones,
|
||||
hashes, ct).ConfigureAwait(false);
|
||||
if (sent is null) skipped++;
|
||||
else if (sent.SequenceId != Guid.Empty)
|
||||
{
|
||||
@@ -83,11 +85,15 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
|
||||
await Parallel.ForEachAsync(pending, Workers(sync.Options.Uploads, ct), async (frame, token) =>
|
||||
{
|
||||
if (await UploadFrameAsync(context, remote, target, null, paths.FromRelative(frame.RelativePath), frame, hashes, token).ConfigureAwait(false) is null)
|
||||
Interlocked.Increment(ref skipped);
|
||||
var sent = await UploadFrameAsync(context, remote, target, null, paths.FromRelative(frame.RelativePath), frame, tombstones, hashes, token)
|
||||
.ConfigureAwait(false);
|
||||
if (sent is null) Interlocked.Increment(ref skipped);
|
||||
await progress.StepAsync().ConfigureAwait(false);
|
||||
}).ConfigureAwait(false);
|
||||
await SendProxiesAsync(context, remote, target, local, paths, frames, hashes).ConfigureAwait(false);
|
||||
// An original the index has no hash for turned out to be a photo the server deleted: the verify gives the index
|
||||
// the hashes (as before a download), so the comparison leaves it out instead of offering its upload again and again.
|
||||
if (hashes.Values.Any(tombstones.DeletedRemotely)) await VerifyAsync(archive, local.Id, ct).ConfigureAwait(false);
|
||||
|
||||
if (created) await sync.PushAllAsync(remote, target, ct).ConfigureAwait(false);
|
||||
await sync.RecompareAsync(remote, target, ct).ConfigureAwait(false);
|
||||
@@ -205,12 +211,16 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
var manifest = await remote.GetManifestAsync(remoteId, ct).ConfigureAwait(false);
|
||||
var captured = (await remote.ListFramesAsync(remoteId, ct).ConfigureAwait(false)).ToDictionary(f => f.Id, f => f.CapturedAt);
|
||||
var (have, restore, photos) = local is null ? ([], [], new FramePhotos([])) : await LocalContentAsync(archive, local, ct).ConfigureAwait(false);
|
||||
// A photo this PC deleted is not fetched again; one replaced here by a converted file is here as its replacement (owner review 2, R6).
|
||||
var tombstones = local is null ? SyncTombstones.None
|
||||
: new SyncTombstones(await db.FrameRemovals.ListBySequenceAsync(local.Id, ct).ConfigureAwait(false), manifest.Removed);
|
||||
if (!tombstones.IsEmpty) have = [.. have.Select(h => tombstones.Canonical(h)!)];
|
||||
// An original the server has not hashed yet (adopted there, not verified) is here when an original of its size and
|
||||
// capture time is; each one is fetched on its own and hashed here.
|
||||
var wanted = manifest.Frames
|
||||
.Where(f => f.Sha256.Length == 0
|
||||
? photos.IndexOf(new FramePhoto(null, f.Size, captured.GetValueOrDefault(f.Id))) is null
|
||||
: !have.Contains(f.Sha256.ToLowerInvariant()))
|
||||
: !have.Contains(tombstones.Canonical(f.Sha256)!) && !tombstones.DeletedLocally(f.Sha256))
|
||||
.DistinctBy(f => f.Sha256.Length == 0 ? f.Id.ToString("N") : f.Sha256.ToLowerInvariant()).ToList();
|
||||
EnsureSpace(paths.Root, wanted.Sum(f => f.Size));
|
||||
|
||||
@@ -234,7 +244,7 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
await progress.StepAsync().ConfigureAwait(false);
|
||||
}
|
||||
var sequenceId = local?.Id;
|
||||
var restored = 0;
|
||||
int restored = 0, refused = 0;
|
||||
await Parallel.ForEachAsync(rest, Workers(sync.Options.Downloads, ct), async (frame, token) =>
|
||||
{
|
||||
var (staged, sha) = await FetchOriginalAsync(remote, frame, staging, token).ConfigureAwait(false);
|
||||
@@ -243,6 +253,14 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
Restore(staged, place);
|
||||
Interlocked.Increment(ref restored);
|
||||
}
|
||||
else if (frame.Sha256.Length == 0 && (tombstones.DeletedLocally(sha) || have.Contains(tombstones.Canonical(sha)!)))
|
||||
{
|
||||
// An original the server has not hashed is known by content only now: one this PC deleted (or holds as
|
||||
// the converted file that replaced it) is not filed again, which would also drop its tombstone.
|
||||
File.Delete(staged);
|
||||
Interlocked.Increment(ref refused);
|
||||
if (tombstones.DeletedLocally(sha)) context.Log($"{frame.FileName}: deleted on this PC, not downloaded");
|
||||
}
|
||||
else
|
||||
{
|
||||
await CommitAsync(remoteId, archive.Id, sequenceId, frame, staged, sha, captured, token).ConfigureAwait(false);
|
||||
@@ -255,7 +273,7 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
|
||||
if (created) await sync.TakeAllAsync(remote, remoteId, ct).ConfigureAwait(false);
|
||||
await sync.RecompareAsync(remote, remoteId, ct).ConfigureAwait(false);
|
||||
if (sequenceId is { } downloaded && wanted.Count > restored)
|
||||
if (sequenceId is { } downloaded && wanted.Count > restored + refused)
|
||||
{
|
||||
var label = local!.DisplayTitle;
|
||||
await EnqueueFollowUpAsync(context, ArchiveJobs.Thumbnails([downloaded], label)).ConfigureAwait(false);
|
||||
@@ -491,10 +509,11 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
/// <summary>
|
||||
/// Hashes the original again and sends it into <paramref name="sequenceId"/> (or the sequence of
|
||||
/// <paramref name="sessionKey"/>); null (logged) when it is gone or no longer the archived file. The hash of an
|
||||
/// original the index has none for goes into <paramref name="hashes"/>.
|
||||
/// original the index has none for goes into <paramref name="hashes"/>; known by content only now, such an original
|
||||
/// the server deleted (<paramref name="tombstones"/>) is not sent back.
|
||||
/// </summary>
|
||||
private static async Task<CompleteUploadResponse?> UploadFrameAsync(JobContext context, ISyncRemote remote, Guid? sequenceId, string? sessionKey, string path,
|
||||
Frame frame, ConcurrentDictionary<Guid, string> hashes, CancellationToken cancellationToken)
|
||||
Frame frame, SyncTombstones tombstones, ConcurrentDictionary<Guid, string> hashes, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
@@ -502,14 +521,22 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
return null;
|
||||
}
|
||||
var sha = await ChronosClient.HashFileAsync(path, cancellationToken).ConfigureAwait(false);
|
||||
if (frame.Sha256 is null) hashes[frame.Id] = sha;
|
||||
var notSent = new CompleteUploadResponse(Guid.Empty, sequenceId ?? Guid.Empty, -1);
|
||||
if (frame.Sha256 is null)
|
||||
{
|
||||
hashes[frame.Id] = sha;
|
||||
if (tombstones.DeletedRemotely(sha))
|
||||
{
|
||||
context.Log($"{frame.FileName}: deleted on the server, not uploaded");
|
||||
return notSent;
|
||||
}
|
||||
}
|
||||
if (frame.Sha256 is { } archived && !archived.Equals(sha, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
context.Log($"{frame.FileName}: changed since it was archived, not uploaded");
|
||||
return null;
|
||||
}
|
||||
return await remote.UploadFileAsync(path, sequenceId, sessionKey, sha, frame.CapturedAt, cancellationToken).ConfigureAwait(false)
|
||||
?? new CompleteUploadResponse(Guid.Empty, sequenceId ?? Guid.Empty, -1);
|
||||
return await remote.UploadFileAsync(path, sequenceId, sessionKey, sha, frame.CapturedAt, cancellationToken).ConfigureAwait(false) ?? notSent;
|
||||
}
|
||||
|
||||
/// <summary>Commits one original from staging into the local sequence (created from the server id when null).</summary>
|
||||
|
||||
Reference in new issue
Block a user