Owner review 2, R7 asks for mass conversion after import on the web panel and the desktop, with a choice at every run between Replace and New sequence. This is the engine and the server API both UIs build on. - Core/Conversion: ConvertOptions (JPEG 4:4:4 at quality 1-100, default 95; TIFF-16 sRGB Deflate with the predictor; linear DNG, the archival choice; converted names keep the stem and take the target's extension in the old one's case). FrameConverter decodes at the native tier only, encodes with the in-house writers, writes Exif from the sidecar (capture time to the tick, offset, exposure) and the original (camera, serial, lens, modes, GPS), hashes the bytes as made, writes atomically and verifies: read back, the metadata again, a decode by the server's decoders. ExifWriter now writes the exposure program, metering mode, flash and white balance. ConvertPlanner: convertible frames, skips with reasons, size estimates per pixel, the plan's space and fingerprint. - FrameTrash.ReplaceAsync swaps the files of frames in place: same ids and indices, the old files and their XMP companions into one Replaced trash entry (records, replacement hashes, a copy of analysis.json), tombstones aliasing old content to new, proxies of the swapped frames dropped, under .chronos/replace.pending.json, which the next writer undoes or completes. Restoring a Replaced entry swaps back while every converted frame is unchanged (the converted files become a Replaced entry in turn), else the originals come back as a new sequence. - MassConverter and JobKind.Convert (19): the dry run (busy, offline, unreadable, space against ArchiveServiceOptions.FreeSpace), stage A converting into staging/convert-<job>/ in parallel with each source re-hashed and each staged file appended to journal/convert-<job>.staged, stage B per sequence all or nothing under the lease at the planned frames version: Replace, or a new sequence through the ingest with the source's tags, rating, notes and project. Resumes keep verified staged files; a sequence changed, busy or with a failed photo is left as it was and fails the job at the end, the journal kept for a retry. - POST /api/v1/archive/convert: dry run with the plan, its space line and fingerprint, or a 202 convert job; formats the server cannot read at full resolution are skipped as convert_on_pc; validated, audited (archive_converted), sequenceChanged converted/created. The server measures the space volume for the check. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> (cherry picked from commit 809210d924cfc3f597ecdea26106fb3ff62d3762) (cherry picked from commit e13d4c0bb1bd9441933b6f3e5b6220e865a7228c)
188 lines
9.2 KiB
C#
188 lines
9.2 KiB
C#
using System.Security.Cryptography;
|
|
using Chronos.Core.Archiving;
|
|
using Chronos.Core.Codecs.Jpeg;
|
|
using Chronos.Core.Imaging;
|
|
using Chronos.Core.Metadata;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Data.Archiving;
|
|
using Chronos.Data.Jobs;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Data.Tests.Archiving;
|
|
|
|
/// <summary>A temporary archive root with its SQLite index and the archive service.</summary>
|
|
internal sealed class ArchiveFixture : IAsyncDisposable
|
|
{
|
|
public static readonly DateTime Start = new(2026, 9, 28, 18, 0, 0);
|
|
|
|
private readonly TestContext _context;
|
|
private readonly List<IAsyncDisposable> _owned = [];
|
|
|
|
private ArchiveFixture(TestContext context, ChronosDatabase db, ArchiveService service, Archive archive)
|
|
{
|
|
_context = context;
|
|
Db = db;
|
|
Service = service;
|
|
Archive = archive;
|
|
_owned.Add(db);
|
|
}
|
|
|
|
public ChronosDatabase Db { get; private set; }
|
|
public ArchiveService Service { get; private set; }
|
|
public Archive Archive { get; private set; }
|
|
public ArchivePaths Paths => new(Archive.RootPath);
|
|
public CancellationToken Ct => _context.Cancellation;
|
|
|
|
/// <param name="database">The index to use (a PostgreSQL schema); a temporary SQLite file when null.</param>
|
|
/// <param name="configure">Changes to the archive service's options (decoders, free space).</param>
|
|
public static async Task<ArchiveFixture> CreateAsync(TestContext context, string? root = null, DatabaseOptions? database = null,
|
|
Func<ArchiveServiceOptions, ArchiveServiceOptions>? configure = null)
|
|
{
|
|
var db = await ChronosDatabase.OpenAsync(database ?? DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("db"), "chronos.db")), context.Cancellation);
|
|
var options = new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) };
|
|
var service = new ArchiveService(db, configure?.Invoke(options) ?? options);
|
|
var archive = await service.AdoptAsync(root ?? context.CreateTempDirectory("archive"), "Main", context.Cancellation);
|
|
return new ArchiveFixture(context, db, service, archive);
|
|
}
|
|
|
|
/// <summary>Switches to a brand-new empty index of the same archive (the "lost database" case).</summary>
|
|
public async Task ReopenWithEmptyIndexAsync()
|
|
{
|
|
var db = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(_context.CreateTempDirectory("db2"), "chronos.db")), Ct);
|
|
_owned.Add(db);
|
|
Db = db;
|
|
Service = new ArchiveService(db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) });
|
|
Archive = await Service.OpenAsync(Archive.RootPath, Ct);
|
|
}
|
|
|
|
public Task<ScanSummary> ScanAsync(ScanMode mode = ScanMode.Incremental) =>
|
|
Service.Indexer.ScanAsync(Archive, mode, null, _ => ValueTask.CompletedTask, Ct);
|
|
|
|
/// <summary>
|
|
/// Another process writing the same archive, such as a desktop sharing a server's archive folder: its own index
|
|
/// (scanned) and archive service; disposed with the fixture.
|
|
/// </summary>
|
|
public async Task<(ArchiveService Service, Archive Archive)> OtherWriterAsync()
|
|
{
|
|
var db = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(_context.CreateTempDirectory("other"), "chronos.db")), Ct);
|
|
_owned.Add(db);
|
|
var service = new ArchiveService(db, new ArchiveServiceOptions { Owner = "other", LeaseWait = TimeSpan.FromSeconds(20) });
|
|
var archive = await service.OpenAsync(Archive.RootPath, Ct);
|
|
await service.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, Ct);
|
|
return (service, archive);
|
|
}
|
|
|
|
/// <summary>A job engine with the archive handlers, started; disposed with the fixture.</summary>
|
|
public async Task<JobEngine> StartEngineAsync()
|
|
{
|
|
var engine = new JobEngine(Db, Service.CreateJobHandlers(),
|
|
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
|
|
_owned.Insert(0, engine);
|
|
await engine.StartAsync(Ct);
|
|
return engine;
|
|
}
|
|
|
|
public async Task<Job> WaitForAsync(Guid jobId, JobState state, int seconds = 30)
|
|
{
|
|
var until = DateTime.UtcNow.AddSeconds(seconds);
|
|
Job? job = null;
|
|
while (DateTime.UtcNow < until)
|
|
{
|
|
job = await Db.Jobs.GetAsync(jobId, Ct);
|
|
if (job?.State == state) return job;
|
|
if (job?.State is JobState.Failed && state != JobState.Failed) Assert.Fail($"Job failed: {job.Error}\n{job.LogTail}");
|
|
await Task.Delay(20, Ct);
|
|
}
|
|
Assert.Fail($"Job {jobId} did not reach {state} (now {job?.State}): {job?.Error}\n{job?.LogTail}");
|
|
return null!;
|
|
}
|
|
|
|
/// <summary>Queues a job directly with a checkpoint, as a crashed earlier attempt would have left it.</summary>
|
|
public async Task<Job> EnqueueWithCheckpointAsync(JobEngine engine, Chronos.Data.Archiving.Jobs.ArchiveJobRequest request, string checkpoint, Guid? id = null)
|
|
{
|
|
var job = new Job
|
|
{
|
|
Id = id ?? Guid.CreateVersion7(),
|
|
Kind = request.Kind,
|
|
TargetIds = request.Targets,
|
|
ParametersJson = request.ParametersJson,
|
|
State = JobState.Queued,
|
|
Checkpoint = checkpoint,
|
|
Created = DateTimeOffset.UtcNow,
|
|
};
|
|
await Db.Jobs.EnqueueAsync(job, Ct);
|
|
engine.Wake();
|
|
return job;
|
|
}
|
|
|
|
/// <summary>Stages a verified upload-like file for ingest (<paramref name="captured"/> null: a photo that records no time): returns its path, size and SHA-256.</summary>
|
|
public (string Path, long Size, string Sha256) Stage(DateTime? captured, int seed, string? make = "Canon", string? model = "Canon EOS R5", string? lens = null)
|
|
{
|
|
var staging = Path.Combine(Paths.StagingDirectory, "uploads");
|
|
Directory.CreateDirectory(staging);
|
|
var bytes = Photos.Jpeg(captured, seed, make, model, lens);
|
|
var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part");
|
|
File.WriteAllBytes(path, bytes);
|
|
return (path, bytes.Length, Convert.ToHexStringLower(SHA256.HashData(bytes)));
|
|
}
|
|
|
|
/// <summary>Every file under <paramref name="folder"/> with the hash of its content, in path order: what a test compares to tell that nothing there changed.</summary>
|
|
public static List<string> FilesOf(string folder) =>
|
|
[
|
|
.. Directory.EnumerateFiles(folder, "*", SearchOption.AllDirectories).Order(StringComparer.Ordinal)
|
|
.Select(p => $"{Path.GetRelativePath(folder, p).Replace('\\', '/')} {Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(p)))}"),
|
|
];
|
|
|
|
public async ValueTask DisposeAsync()
|
|
{
|
|
foreach (var owned in _owned) await owned.DisposeAsync();
|
|
}
|
|
}
|
|
|
|
/// <summary>Small real JPEGs with Exif made by the in-house encoder; each seed gives distinct bytes.</summary>
|
|
internal static class Photos
|
|
{
|
|
public static byte[] Jpeg(DateTime? captured, int seed, string? make = "Canon", string? model = "Canon EOS R5", string? lens = null)
|
|
{
|
|
using var image = new ImageBuffer(48, 32);
|
|
var random = new Random(seed);
|
|
for (var c = 0; c < 3; c++)
|
|
{
|
|
var plane = image.Plane(c);
|
|
var level = (float)random.NextDouble();
|
|
for (var i = 0; i < plane.Length; i++) plane[i] = level * (0.5f + 0.5f * (i % 48) / 48f);
|
|
}
|
|
var exif = ExifWriter.BuildJpegApp1(new FrameMetadata { Make = make, Model = model, LensModel = lens, BodySerial = "0431", CapturedAt = captured, Iso = 100 });
|
|
return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] });
|
|
}
|
|
|
|
public static List<string> WriteSeries(string directory, DateTime start, int count, int firstSeed = 1, string prefix = "IMG_", int firstNumber = 1,
|
|
string make = "Canon", string model = "Canon EOS R5")
|
|
{
|
|
Directory.CreateDirectory(directory);
|
|
var paths = new List<string>(count);
|
|
for (var i = 0; i < count; i++)
|
|
{
|
|
var path = Path.Combine(directory, $"{prefix}{firstNumber + i:0000}.JPG");
|
|
File.WriteAllBytes(path, Jpeg(start.AddSeconds(10 * i), firstSeed + i, make, model));
|
|
paths.Add(path);
|
|
}
|
|
return paths;
|
|
}
|
|
|
|
/// <summary>The index as comparable text: sequences and frames ordered by folder and index.</summary>
|
|
public static async Task<string> DumpIndexAsync(ChronosDatabase db, Guid archiveId)
|
|
{
|
|
var lines = new List<string>();
|
|
foreach (var location in await db.Sequences.ListLocationsAsync(archiveId))
|
|
{
|
|
var s = (await db.Sequences.GetAsync(location.Id))!;
|
|
lines.Add($"S {s.Id} {s.Folder} '{s.Title}' {s.CameraModel} {s.FirstCapture:O} {s.FrameCount} {s.TotalBytes} {s.Status} {s.VerifiedAt:O} " +
|
|
$"{string.Join('|', s.Tags)} {s.SettingsVersion}/{s.FramesVersion} {s.MedianInterval}");
|
|
foreach (var f in await db.Frames.ListBySequenceAsync(s.Id))
|
|
lines.Add($" F {f.Id} {f.Index} {f.RelativePath} {f.OriginalFileName} {f.Size} {f.Sha256} {f.CapturedAt:O} {f.Width}x{f.Height} {f.Format} {f.Flags}");
|
|
}
|
|
return string.Join('\n', lines);
|
|
}
|
|
}
|