CI / Build and test (push) Failing after 3m1s
CI / PostgreSQL 16 integration (push) Skipped
CI / PostgreSQL 17 integration (push) Skipped
CI / Docker image (push) Skipped
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 10s
Release / Build, test and publish (push) Successful in 15m31s
Release / Server image (push) Successful in 1m13s
Convert photos already in the archive to JPEG, TIFF-16 or Linear DNG, from the panel's Library and the desktop's Archive and Library. Each run asks Replace (same frames, originals to the trash, recoverable) or New sequence. A dry run lists what converts, what is skipped and why, size before and after, and free space; the job converts exactly the confirmed photos, resumes, and commits all or nothing per sequence. A conversion never changes what Chronos reads from a photo. Replaced originals are erased only on the archive's own word that their conversion is gone. Desktop TIFFs that WIC showed black or as noise now go to the managed reader; the Edit picker no longer spins a core. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
188 lines
9.2 KiB
C#
188 lines
9.2 KiB
C#
using System.Security.Cryptography;
|
|
using Chronos.Core.Archiving;
|
|
using Chronos.Core.Codecs.Jpeg;
|
|
using Chronos.Core.Imaging;
|
|
using Chronos.Core.Metadata;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Data.Archiving;
|
|
using Chronos.Data.Jobs;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Data.Tests.Archiving;
|
|
|
|
/// <summary>A temporary archive root with its SQLite index and the archive service.</summary>
|
|
internal sealed class ArchiveFixture : IAsyncDisposable
|
|
{
|
|
public static readonly DateTime Start = new(2026, 9, 28, 18, 0, 0);
|
|
|
|
private readonly TestContext _context;
|
|
private readonly List<IAsyncDisposable> _owned = [];
|
|
|
|
private ArchiveFixture(TestContext context, ChronosDatabase db, ArchiveService service, Archive archive)
|
|
{
|
|
_context = context;
|
|
Db = db;
|
|
Service = service;
|
|
Archive = archive;
|
|
_owned.Add(db);
|
|
}
|
|
|
|
public ChronosDatabase Db { get; private set; }
|
|
public ArchiveService Service { get; private set; }
|
|
public Archive Archive { get; private set; }
|
|
public ArchivePaths Paths => new(Archive.RootPath);
|
|
public CancellationToken Ct => _context.Cancellation;
|
|
|
|
/// <param name="database">The index to use (a PostgreSQL schema); a temporary SQLite file when null.</param>
|
|
/// <param name="configure">Changes to the archive service's options (decoders, free space).</param>
|
|
public static async Task<ArchiveFixture> CreateAsync(TestContext context, string? root = null, DatabaseOptions? database = null,
|
|
Func<ArchiveServiceOptions, ArchiveServiceOptions>? configure = null)
|
|
{
|
|
var db = await ChronosDatabase.OpenAsync(database ?? DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("db"), "chronos.db")), context.Cancellation);
|
|
var options = new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) };
|
|
var service = new ArchiveService(db, configure?.Invoke(options) ?? options);
|
|
var archive = await service.AdoptAsync(root ?? context.CreateTempDirectory("archive"), "Main", context.Cancellation);
|
|
return new ArchiveFixture(context, db, service, archive);
|
|
}
|
|
|
|
/// <summary>Switches to a brand-new empty index of the same archive (the "lost database" case).</summary>
|
|
public async Task ReopenWithEmptyIndexAsync()
|
|
{
|
|
var db = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(_context.CreateTempDirectory("db2"), "chronos.db")), Ct);
|
|
_owned.Add(db);
|
|
Db = db;
|
|
Service = new ArchiveService(db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) });
|
|
Archive = await Service.OpenAsync(Archive.RootPath, Ct);
|
|
}
|
|
|
|
public Task<ScanSummary> ScanAsync(ScanMode mode = ScanMode.Incremental) =>
|
|
Service.Indexer.ScanAsync(Archive, mode, null, _ => ValueTask.CompletedTask, Ct);
|
|
|
|
/// <summary>
|
|
/// Another process writing the same archive, such as a desktop sharing a server's archive folder: its own index
|
|
/// (scanned) and archive service; disposed with the fixture.
|
|
/// </summary>
|
|
public async Task<(ArchiveService Service, Archive Archive)> OtherWriterAsync()
|
|
{
|
|
var db = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(_context.CreateTempDirectory("other"), "chronos.db")), Ct);
|
|
_owned.Add(db);
|
|
var service = new ArchiveService(db, new ArchiveServiceOptions { Owner = "other", LeaseWait = TimeSpan.FromSeconds(20) });
|
|
var archive = await service.OpenAsync(Archive.RootPath, Ct);
|
|
await service.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, Ct);
|
|
return (service, archive);
|
|
}
|
|
|
|
/// <summary>A job engine with the archive handlers, started; disposed with the fixture.</summary>
|
|
public async Task<JobEngine> StartEngineAsync()
|
|
{
|
|
var engine = new JobEngine(Db, Service.CreateJobHandlers(),
|
|
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
|
|
_owned.Insert(0, engine);
|
|
await engine.StartAsync(Ct);
|
|
return engine;
|
|
}
|
|
|
|
public async Task<Job> WaitForAsync(Guid jobId, JobState state, int seconds = 30)
|
|
{
|
|
var until = DateTime.UtcNow.AddSeconds(seconds);
|
|
Job? job = null;
|
|
while (DateTime.UtcNow < until)
|
|
{
|
|
job = await Db.Jobs.GetAsync(jobId, Ct);
|
|
if (job?.State == state) return job;
|
|
if (job?.State is JobState.Failed && state != JobState.Failed) Assert.Fail($"Job failed: {job.Error}\n{job.LogTail}");
|
|
await Task.Delay(20, Ct);
|
|
}
|
|
Assert.Fail($"Job {jobId} did not reach {state} (now {job?.State}): {job?.Error}\n{job?.LogTail}");
|
|
return null!;
|
|
}
|
|
|
|
/// <summary>Queues a job directly with a checkpoint, as a crashed earlier attempt would have left it.</summary>
|
|
public async Task<Job> EnqueueWithCheckpointAsync(JobEngine engine, Chronos.Data.Archiving.Jobs.ArchiveJobRequest request, string checkpoint, Guid? id = null)
|
|
{
|
|
var job = new Job
|
|
{
|
|
Id = id ?? Guid.CreateVersion7(),
|
|
Kind = request.Kind,
|
|
TargetIds = request.Targets,
|
|
ParametersJson = request.ParametersJson,
|
|
State = JobState.Queued,
|
|
Checkpoint = checkpoint,
|
|
Created = DateTimeOffset.UtcNow,
|
|
};
|
|
await Db.Jobs.EnqueueAsync(job, Ct);
|
|
engine.Wake();
|
|
return job;
|
|
}
|
|
|
|
/// <summary>Stages a verified upload-like file for ingest (<paramref name="captured"/> null: a photo that records no time): returns its path, size and SHA-256.</summary>
|
|
public (string Path, long Size, string Sha256) Stage(DateTime? captured, int seed, string? make = "Canon", string? model = "Canon EOS R5", string? lens = null)
|
|
{
|
|
var staging = Path.Combine(Paths.StagingDirectory, "uploads");
|
|
Directory.CreateDirectory(staging);
|
|
var bytes = Photos.Jpeg(captured, seed, make, model, lens);
|
|
var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part");
|
|
File.WriteAllBytes(path, bytes);
|
|
return (path, bytes.Length, Convert.ToHexStringLower(SHA256.HashData(bytes)));
|
|
}
|
|
|
|
/// <summary>Every file under <paramref name="folder"/> with the hash of its content, in path order: what a test compares to tell that nothing there changed.</summary>
|
|
public static List<string> FilesOf(string folder) =>
|
|
[
|
|
.. Directory.EnumerateFiles(folder, "*", SearchOption.AllDirectories).Order(StringComparer.Ordinal)
|
|
.Select(p => $"{Path.GetRelativePath(folder, p).Replace('\\', '/')} {Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(p)))}"),
|
|
];
|
|
|
|
public async ValueTask DisposeAsync()
|
|
{
|
|
foreach (var owned in _owned) await owned.DisposeAsync();
|
|
}
|
|
}
|
|
|
|
/// <summary>Small real JPEGs with Exif made by the in-house encoder; each seed gives distinct bytes.</summary>
|
|
internal static class Photos
|
|
{
|
|
public static byte[] Jpeg(DateTime? captured, int seed, string? make = "Canon", string? model = "Canon EOS R5", string? lens = null)
|
|
{
|
|
using var image = new ImageBuffer(48, 32);
|
|
var random = new Random(seed);
|
|
for (var c = 0; c < 3; c++)
|
|
{
|
|
var plane = image.Plane(c);
|
|
var level = (float)random.NextDouble();
|
|
for (var i = 0; i < plane.Length; i++) plane[i] = level * (0.5f + 0.5f * (i % 48) / 48f);
|
|
}
|
|
var exif = ExifWriter.BuildJpegApp1(new FrameMetadata { Make = make, Model = model, LensModel = lens, BodySerial = "0431", CapturedAt = captured, Iso = 100 });
|
|
return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] });
|
|
}
|
|
|
|
public static List<string> WriteSeries(string directory, DateTime start, int count, int firstSeed = 1, string prefix = "IMG_", int firstNumber = 1,
|
|
string make = "Canon", string model = "Canon EOS R5")
|
|
{
|
|
Directory.CreateDirectory(directory);
|
|
var paths = new List<string>(count);
|
|
for (var i = 0; i < count; i++)
|
|
{
|
|
var path = Path.Combine(directory, $"{prefix}{firstNumber + i:0000}.JPG");
|
|
File.WriteAllBytes(path, Jpeg(start.AddSeconds(10 * i), firstSeed + i, make, model));
|
|
paths.Add(path);
|
|
}
|
|
return paths;
|
|
}
|
|
|
|
/// <summary>The index as comparable text: sequences and frames ordered by folder and index.</summary>
|
|
public static async Task<string> DumpIndexAsync(ChronosDatabase db, Guid archiveId)
|
|
{
|
|
var lines = new List<string>();
|
|
foreach (var location in await db.Sequences.ListLocationsAsync(archiveId))
|
|
{
|
|
var s = (await db.Sequences.GetAsync(location.Id))!;
|
|
lines.Add($"S {s.Id} {s.Folder} '{s.Title}' {s.CameraModel} {s.FirstCapture:O} {s.FrameCount} {s.TotalBytes} {s.Status} {s.VerifiedAt:O} " +
|
|
$"{string.Join('|', s.Tags)} {s.SettingsVersion}/{s.FramesVersion} {s.MedianInterval}");
|
|
foreach (var f in await db.Frames.ListBySequenceAsync(s.Id))
|
|
lines.Add($" F {f.Id} {f.Index} {f.RelativePath} {f.OriginalFileName} {f.Size} {f.Sha256} {f.CapturedAt:O} {f.Width}x{f.Height} {f.Format} {f.Flags}");
|
|
}
|
|
return string.Join('\n', lines);
|
|
}
|
|
}
|