feat(copies): matches between this PC and the server, Integrate and merge (owner review 6, v1.2.0) (#74)
CI / Build and test (push) Successful in 13m44s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 16s
Release / Build, test and publish (push) Successful in 26m34s
CI / PostgreSQL 16 integration (push) Failing after 17m36s
CI / PostgreSQL 17 integration (push) Failing after 15m36s
CI / Docker image (push) Successful in 1m12s
Release / Server image (push) Failing after 15s
CI / Build and test (push) Successful in 13m44s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 16s
Release / Build, test and publish (push) Successful in 26m34s
CI / PostgreSQL 16 integration (push) Failing after 17m36s
CI / PostgreSQL 17 integration (push) Failing after 15m36s
CI / Docker image (push) Successful in 1m12s
Release / Server image (push) Failing after 15s
The same time-lapse on a server and on this PC under different ids, or twice on this PC, is recognised, compared and settled as the owner asked in owner review 6. Matches are found after each sync refresh, through the server's new photo lookup, and after the jobs that change photos. The default is to ask every time. Server › Matches offers: - Keep apart, which is remembered; - Link only and Unlink; - the Compare dialog: photos, details line by line, projects and the plan. Integrate completes both sides. Photos default to Union, projects are copied across, and a losing version goes to the trash only after a hash check. Afterwards new photos move by themselves, and deletions wait for the user. A merge of two local sequences verifies every photo before the other sequence goes to its archive's trash. Upload all local skips suspected matches and flags them. Migration 0012 adds the found_copies and copy_probes tables. Jobs 25 (Integrate) and 26 (MergeSequences) are new, and so are the server capabilities photo-lookup and upload-into. Refs #73 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
This commit was merged in pull request #74.
This commit is contained in:
1 parent
43719bb5c5
commit
f33bcac32e
586 files changed
+41546
-280
No files matched your search
@@ -17,7 +17,8 @@ internal sealed class FakeSyncRemote : ISyncRemote
|
||||
private readonly Channel<ServerEvent> _events = Channel.CreateUnbounded<ServerEvent>();
|
||||
private readonly Dictionary<string, FakeSequence> _sessions = [];
|
||||
|
||||
public string Server => "https://nas.test:8443";
|
||||
/// <summary>The server's key: an address key unless a test names one by id (<c>id:<N></c>).</summary>
|
||||
public string Server { get; init; } = "https://nas.test:8443";
|
||||
|
||||
public Dictionary<Guid, FakeSequence> Sequences { get; } = [];
|
||||
|
||||
@@ -59,6 +60,155 @@ internal sealed class FakeSyncRemote : ISyncRemote
|
||||
|
||||
public Task<int> SettingsFormatAsync(CancellationToken cancellationToken) => Task.FromResult(SettingsFormat);
|
||||
|
||||
/// <summary>
|
||||
/// The capabilities the server announces (<see cref="ServerCapabilities"/>): none by default, an older server's path
|
||||
/// (owner review 6, #73); tests of the lookup and of uploads into a target opt in.
|
||||
/// </summary>
|
||||
public HashSet<string> Capabilities { get; } = new(StringComparer.Ordinal);
|
||||
|
||||
public Task<IReadOnlySet<string>> CapabilitiesAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
lock (_gate) return Task.FromResult<IReadOnlySet<string>>(new HashSet<string>(Capabilities, StringComparer.Ordinal));
|
||||
}
|
||||
|
||||
/// <summary>Lookup requests so far, as received.</summary>
|
||||
public List<PhotoLookupRequest> Lookups { get; } = [];
|
||||
|
||||
/// <summary>Uploads into a target so far (<see cref="UploadFileIntoAsync"/>), and how many the server filed from a copy it held elsewhere.</summary>
|
||||
public int IntoUploads { get; private set; }
|
||||
|
||||
public int CopiedInto { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Uploads into a target refused with 403, as for an account with Read and Upload but not Edit that does not own the
|
||||
/// target; <see cref="RefusedInto"/> counts them.
|
||||
/// </summary>
|
||||
public bool ForbidInto { get; set; }
|
||||
|
||||
public int RefusedInto { get; private set; }
|
||||
|
||||
/// <summary>Runs once, before the next frame deletion is answered (the server sequence changing meanwhile).</summary>
|
||||
public Action? OnceBeforeDelete { get; set; }
|
||||
|
||||
/// <summary>Frame deletions refused with 403, as for an account that is neither the owner nor an admin.</summary>
|
||||
public bool ForbidFrameDeletes { get; set; }
|
||||
|
||||
/// <summary>Frames moved to the server's trash so far, by sequence.</summary>
|
||||
public List<(Guid SequenceId, FakeFrame Frame)> Trashed { get; } = [];
|
||||
|
||||
/// <summary>Files <paramref name="sequence"/> under <paramref name="sessionKey"/>, as an upload with that key would.</summary>
|
||||
public void FileUnder(string sessionKey, FakeSequence sequence)
|
||||
{
|
||||
lock (_gate) _sessions[sessionKey] = sequence;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Like the server (owner review 6, #73): per probe, the sequences holding its hashes in a frame (a sequence it never
|
||||
/// hashed lists none), tombstoning them (by the photo's or its replacement's SHA-256), holding a frame of its unhashed keys
|
||||
/// by size and capture millisecond, or filed under its session key. 404 without <see cref="ServerCapabilities.PhotoLookup"/>,
|
||||
/// 413 beyond the limits.
|
||||
/// </summary>
|
||||
public Task<PhotoLookupResponse> LookupPhotosAsync(PhotoLookupRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (!Capabilities.Contains(ServerCapabilities.PhotoLookup)) throw new ChronosApiException(HttpStatusCode.NotFound, new ApiError("not_found", "route"));
|
||||
if (request.Probes.Count > PhotoLookupRequest.MaxProbes || request.CountItems() > PhotoLookupRequest.MaxItems)
|
||||
throw new ChronosApiException(HttpStatusCode.RequestEntityTooLarge, new ApiError("too_many", "probes"));
|
||||
Lookups.Add(request);
|
||||
var results = new List<ProbeResult>();
|
||||
foreach (var probe in request.Probes)
|
||||
{
|
||||
var hashes = probe.Sha256.Select(h => h.ToLowerInvariant()).ToHashSet(StringComparer.Ordinal);
|
||||
var keys = (probe.Unhashed ?? []).Select(k => (k.Size, Ms(k.CapturedAt))).ToHashSet();
|
||||
var filed = probe.SessionKey is { } key && _sessions.TryGetValue(key, out var held) ? held : null;
|
||||
var hits = new List<PhotoLookupHit>();
|
||||
foreach (var sequence in Sequences.Values)
|
||||
{
|
||||
var inFrames = sequence.Unhashed ? 0 : sequence.Frames.Select(f => f.Sha256).Where(hashes.Contains).Distinct(StringComparer.Ordinal).Count();
|
||||
var removed = hashes.Count(h => sequence.Removed.Any(r => r.Sha256 == h || r.Replacement == h));
|
||||
var unhashed = keys.Count(k => sequence.Frames.Any(f => f.CapturedAt is { } at && f.Bytes.Length == k.Size && Ms(at) == k.Item2));
|
||||
var session = ReferenceEquals(filed, sequence);
|
||||
if (inFrames + removed + unhashed > 0 || session) hits.Add(new PhotoLookupHit(sequence.Id, inFrames, removed, unhashed, session));
|
||||
}
|
||||
results.Add(new ProbeResult(probe.Key, hits));
|
||||
}
|
||||
return Task.FromResult(new PhotoLookupResponse(results));
|
||||
}
|
||||
|
||||
static long Ms(DateTime at) => at.Ticks / TimeSpan.TicksPerMillisecond;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Like the server with <see cref="ServerCapabilities.UploadInto"/>: the photo is filed into the target even when another
|
||||
/// sequence holds it (a server-side copy, counted in <see cref="CopiedInto"/>); one the target holds answers its frame, one
|
||||
/// it tombstoned answers none. Without the capability the server ignores the flag: an upload as before.
|
||||
/// </summary>
|
||||
public async Task<CompleteUploadResponse?> UploadFileIntoAsync(string path, Guid sequenceId, string sha256, DateTime? capturedAt,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
bool capable;
|
||||
lock (_gate) capable = Capabilities.Contains(ServerCapabilities.UploadInto);
|
||||
if (!capable) return await UploadFileAsync(path, sequenceId, null, sha256, capturedAt, cancellationToken);
|
||||
if (ForbidInto)
|
||||
{
|
||||
lock (_gate) RefusedInto++;
|
||||
throw new ChronosApiException(HttpStatusCode.Forbidden, new ApiError("forbidden", "into"));
|
||||
}
|
||||
var bytes = await File.ReadAllBytesAsync(path, cancellationToken);
|
||||
if (Convert.ToHexStringLower(SHA256.HashData(bytes)) != sha256)
|
||||
throw new ChronosApiException(HttpStatusCode.BadRequest, new ApiError("hash_mismatch", "hash"));
|
||||
lock (_gate)
|
||||
{
|
||||
IntoUploads++;
|
||||
var target = Find(sequenceId);
|
||||
if (target.Removed.Any(r => r.Sha256 == sha256)) return null;
|
||||
var index = target.Frames.FindIndex(f => f.Sha256 == sha256);
|
||||
if (index >= 0) return new CompleteUploadResponse(target.Frames[index].Id, target.Id, index);
|
||||
if (Sequences.Values.Where(s => !s.Unhashed && s != target).SelectMany(s => s.Frames).Any(f => f.Sha256 == sha256)) CopiedInto++;
|
||||
var frame = target.AddFrame(bytes, Path.GetFileName(path), capturedAt);
|
||||
return new CompleteUploadResponse(frame.Id, target.Id, target.Frames.IndexOf(frame));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Like the server's <c>frames/delete</c>: 404 for an unknown sequence, 403 with <see cref="ForbidFrameDeletes"/>, 409
|
||||
/// <c>frames_conflict</c> when a frame is not there or the frames version moved, 409 <c>would_empty</c>; a dry run answers
|
||||
/// the plan; else the frames go to the trash, each leaving a tombstone, and the frames version moves.
|
||||
/// </summary>
|
||||
public Task<FrameDeleteResultDto> DeleteFramesAsync(Guid sequenceId, FrameDeleteRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
if (OnceBeforeDelete is { } before)
|
||||
{
|
||||
OnceBeforeDelete = null;
|
||||
before();
|
||||
}
|
||||
lock (_gate)
|
||||
{
|
||||
var sequence = Find(sequenceId);
|
||||
if (ForbidFrameDeletes) throw new ChronosApiException(HttpStatusCode.Forbidden, new ApiError("forbidden", "delete"));
|
||||
var wanted = request.FrameIds.ToHashSet();
|
||||
var frames = sequence.Frames.Where(f => wanted.Contains(f.Id)).ToList();
|
||||
if (frames.Count != wanted.Count || (request.ExpectedFramesVersion is { } expected && expected != sequence.FramesVersion))
|
||||
throw new ChronosApiException(HttpStatusCode.Conflict, new ApiError("frames_conflict", "changed"));
|
||||
if (frames.Count == sequence.Frames.Count) throw new ChronosApiException(HttpStatusCode.Conflict, new ApiError("would_empty", "empty"));
|
||||
var bytes = frames.Sum(f => (long)f.Bytes.Length);
|
||||
if (request.DryRun)
|
||||
return Task.FromResult(new FrameDeleteResultDto(sequenceId, frames.Count, bytes, sequence.FramesVersion, sequence.Frames.Count, null, true));
|
||||
var at = DateTimeOffset.UtcNow;
|
||||
foreach (var frame in frames)
|
||||
{
|
||||
sequence.Frames.Remove(frame);
|
||||
Trashed.Add((sequenceId, frame));
|
||||
if (!sequence.Removed.Any(r => r.Sha256 == frame.Sha256)) sequence.Removed.Add(new RemovedPhotoDto(frame.Sha256, null, "deleted", at));
|
||||
}
|
||||
sequence.FramesVersion++;
|
||||
sequence.UpdatedAt = at;
|
||||
return Task.FromResult(new FrameDeleteResultDto(sequenceId, frames.Count, bytes, sequence.FramesVersion, sequence.Frames.Count,
|
||||
$"trash/photos-{Guid.NewGuid():N}", false));
|
||||
}
|
||||
}
|
||||
|
||||
public FakeSequence Add(string? title, IEnumerable<byte[]> originals, string? settingsJson = null,
|
||||
SequenceStatus status = SequenceStatus.ProxiesBuilt | SequenceStatus.ThumbsBuilt)
|
||||
{
|
||||
@@ -68,6 +218,12 @@ internal sealed class FakeSyncRemote : ISyncRemote
|
||||
return sequence;
|
||||
}
|
||||
|
||||
/// <summary>Someone deletes the sequence on the server: every later call for it answers 404.</summary>
|
||||
public void Drop(Guid id)
|
||||
{
|
||||
lock (_gate) Sequences.Remove(id);
|
||||
}
|
||||
|
||||
/// <summary>Someone else edits the settings on the server.</summary>
|
||||
public void EditSettings(Guid id, string settingsJson)
|
||||
{
|
||||
@@ -357,7 +513,10 @@ internal sealed class FakeSequence
|
||||
}
|
||||
|
||||
public SequenceDto Dto() => new(Id, Title, Title ?? "Untitled", "2026/" + Id.ToString("N"), "Canon", CameraModel, null, FirstCapture, null, Frames.Count, 10,
|
||||
Frames.Sum(f => (long)f.Bytes.Length), (long)Status, Renders.Count, null, Tags, Rating, Notes, SettingsVersion, FramesVersion, 0, null, UpdatedAt);
|
||||
Frames.Sum(f => (long)f.Bytes.Length), (long)Status, Renders.Count, null, Tags, Rating, Notes, SettingsVersion, FramesVersion, 0, OwnerUserId, UpdatedAt);
|
||||
|
||||
/// <summary>The server account that owns the sequence (none by default).</summary>
|
||||
public Guid? OwnerUserId { get; set; }
|
||||
|
||||
public SequenceManifest Manifest(Func<Guid, bool> proxySent) => new(Id, FramesVersion, SettingsVersion, SettingsJson, Tags,
|
||||
[.. Frames.Select((f, i) => new ManifestFrame(f.Id, i, f.FileName, f.Bytes.Length, Unhashed ? string.Empty : f.Sha256,
|
||||
|
||||
Reference in new issue
Block a user