diff --git a/src/Chronos.Data/Archiving/ArchiveService.cs b/src/Chronos.Data/Archiving/ArchiveService.cs index adfdc8a3..77b63291 100644 --- a/src/Chronos.Data/Archiving/ArchiveService.cs +++ b/src/Chronos.Data/Archiving/ArchiveService.cs @@ -47,6 +47,7 @@ public sealed class ArchiveService : IJobHandlerProvider FrameTrash = new FrameTrash(db, _writer, Indexer, Ingest, Trash); Converter = new MassConverter(db, _writer, Ingest, FrameTrash, Trash, this); Projects = new ProjectService(db, _writer); + Videos = new Chronos.Data.Videos.VideoLibrary(db, _writer); var shared = Housekeeping.CleanupSteps.Shared(Trash); CleanupSteps = [.. shared, .. _writer.Options.CleanupSteps.Where(s => !shared.Any(o => o.Name == s.Name))]; } @@ -70,9 +71,9 @@ public sealed class ArchiveService : IJobHandlerProvider /// /// The videos library of this index (owner review 1, R9): every render listed, deleted into the trash, restored and copied - /// into a videos folder. The groundwork stub until the library lands ( false). + /// into a videos folder (, among ). /// - public Chronos.Data.Videos.IVideoLibrary Videos { get; } = Chronos.Data.Videos.NotImplementedVideoLibrary.Instance; + public Chronos.Data.Videos.VideoLibrary Videos { get; } /// Converts sequences to JPEG, TIFF-16 or linear DNG, replacing their frames or into new sequences (owner review 2, R7). public MassConverter Converter { get; } @@ -117,6 +118,7 @@ public sealed class ArchiveService : IJobHandlerProvider new EmptyTrashJobHandler(_db, Trash), new MoveJobHandler(_db, Renamer, _writer.Options.SequenceChanged), new ConvertJobHandler(_db, Converter, _writer.Options.SequenceChanged), + new Chronos.Data.Videos.VideoCopyJobHandler(_db, Videos), ]; /// Makes a new, empty archive and indexes it. @@ -172,6 +174,15 @@ public sealed class ArchiveService : IJobHandlerProvider // The journal stays; the next open tries again. The archive itself is usable meanwhile (a server starts all the same). Log.Warning("archive", $"Resuming an interrupted rename in {paths.Root} failed", ex); } + try + { + // A delete of videos into the trash a crash interrupted completes, or is undone (owner review 1, R9). + await Videos.RecoverAsync(archive, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or LeaseUnavailableException or DatabaseException) + { + Log.Warning("archive", $"Settling an interrupted delete of videos in {paths.Root} failed", ex); + } return archive; } @@ -515,10 +526,13 @@ public sealed class ArchiveService : IJobHandlerProvider (await RestoreEntryAsync(archiveId, entryName, null, cancellationToken).ConfigureAwait(false)).Sequence; /// - /// Brings a trash entry back: a sequence into its folder (), or photos into + /// Brings a trash entry back: a sequence into its folder (), photos into /// their sequence (, which queues on what the - /// frames need again). + /// frames need again), or a video into its sequence's renders (, owner + /// review 1, R9: is the video's). /// + /// A video entry whose file no longer matches its record, or whose sequence + /// is not in the index; nothing moved. /// The entry is not in the trash. /// A sequence entry whose sequence is in the index already. /// No file of a photo entry matches its record; nothing moved. @@ -528,6 +542,13 @@ public sealed class ArchiveService : IJobHandlerProvider CancellationToken cancellationToken = default) { var archive = await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false); + if (Chronos.Data.Videos.VideoTrashNames.IsEntryName(entryName)) + { + var render = await Videos.RestoreAsync(archive.Id, entryName, cancellationToken).ConfigureAwait(false); + var owner = await _db.Sequences.GetAsync(render.SequenceId, CancellationToken.None).ConfigureAwait(false) + ?? throw new InvalidOperationException($"The sequence of render {render.Id} is not in the index."); + return new TrashRestoreResult(TrashKind.Videos, owner, 0, [], false); + } if (!Archiving.FrameTrash.IsPhotoEntry(entryName)) return new TrashRestoreResult(TrashKind.Sequence, await Trash.RestoreAsync(archive, entryName, cancellationToken).ConfigureAwait(false), 0, [], false); var photos = await FrameTrash.RestoreAsync(archive, entryName, followUps, cancellationToken).ConfigureAwait(false); diff --git a/src/Chronos.Data/Archiving/SequenceTrash.Empty.cs b/src/Chronos.Data/Archiving/SequenceTrash.Empty.cs index 7c4f66ec..2476a483 100644 --- a/src/Chronos.Data/Archiving/SequenceTrash.Empty.cs +++ b/src/Chronos.Data/Archiving/SequenceTrash.Empty.cs @@ -286,8 +286,8 @@ public sealed partial class SequenceTrash CancellationToken cancellationToken) { var (folder, recordPath) = Location(paths, entry); - var now = FrameTrash.IsPhotoEntry(entry.Name) - ? FrameTrash.ReadListed(paths, entry.ArchiveId, entry.Name) + var now = FrameTrash.IsPhotoEntry(entry.Name) ? FrameTrash.ReadListed(paths, entry.ArchiveId, entry.Name) + : Videos.VideoTrashNames.IsEntryName(entry.Name) ? Videos.VideoTrash.ReadListed(paths, entry.ArchiveId, entry.Name) : ReadRecord(recordPath) is { } record ? Entry(entry.ArchiveId, Path.GetFileName(entry.Name), record) : null; if (now is null || !Directory.Exists(folder)) return (null, null); var (files, bytes) = Measure(folder, cancellationToken); @@ -313,10 +313,11 @@ public sealed partial class SequenceTrash return Survey(archive, paths, entry.Name, record, where, cancellationToken).Deleted ? entry with { ConversionDeleted = true } : entry; } - /// The folder and record of an entry of either kind. + /// The folder and record of an entry of any kind (a sequence, photos, or a video: owner review 1, R9). private static (string Folder, string Record) Location(ArchivePaths paths, TrashEntry entry) { if (FrameTrash.IsPhotoEntry(entry.Name)) return FrameTrash.Location(paths, entry.Name); + if (Videos.VideoTrashNames.IsEntryName(entry.Name)) return Videos.VideoTrash.Location(paths, entry.Name); var folder = Path.Combine(TrashDirectory(paths), Path.GetFileName(entry.Name)); return (folder, folder + RecordExtension); } diff --git a/src/Chronos.Data/Archiving/SequenceTrash.cs b/src/Chronos.Data/Archiving/SequenceTrash.cs index 7ac59ae1..d80a0d6f 100644 --- a/src/Chronos.Data/Archiving/SequenceTrash.cs +++ b/src/Chronos.Data/Archiving/SequenceTrash.cs @@ -213,8 +213,9 @@ public sealed partial class SequenceTrash } /// - /// The entries of an archive's trash, newest first: deleted sequences (trash/*.json) and deleted photos - /// (trash/photos/*.json); records whose folder is gone are left out. + /// The entries of an archive's trash, newest first: deleted sequences (trash/*.json), deleted photos + /// (trash/photos/*.json) and deleted videos (trash/videos/*.json, owner review 1, R9); records whose folder + /// is gone are left out. /// public async Task> ListAsync(Archive archive, CancellationToken cancellationToken = default) { @@ -232,6 +233,7 @@ public sealed partial class SequenceTrash entries.Add(Entry(archive.Id, name, record)); } entries.AddRange(FrameTrash.List(archive, cancellationToken)); + entries.AddRange(Chronos.Data.Videos.VideoTrash.List(archive, cancellationToken)); return entries.OrderByDescending(e => e.DeletedAt).ThenBy(e => e.Name, StringComparer.Ordinal).ToList(); }, cancellationToken).ConfigureAwait(false); } diff --git a/src/Chronos.Data/ChronosDatabase.cs b/src/Chronos.Data/ChronosDatabase.cs index bb095929..a9e23456 100644 --- a/src/Chronos.Data/ChronosDatabase.cs +++ b/src/Chronos.Data/ChronosDatabase.cs @@ -25,6 +25,7 @@ public abstract class ChronosRepositories Schedules = new ScheduleRepository(access); FrameRemovals = new FrameRemovalRepository(access); Projects = new ProjectRepository(access); + RenderCopies = new RenderCopyRepository(access); } public ArchiveRepository Archives { get; } @@ -49,6 +50,9 @@ public abstract class ChronosRepositories /// Projects of sequences and their versions (owner review 2, R8). public ProjectRepository Projects { get; } + + /// Verified copies of renders in videos folders (owner review 1, R9). + public RenderCopyRepository RenderCopies { get; } } /// diff --git a/src/Chronos.Data/Jobs/JobPriorityQueue.cs b/src/Chronos.Data/Jobs/JobPriorityQueue.cs index 71b6d396..b3de4984 100644 --- a/src/Chronos.Data/Jobs/JobPriorityQueue.cs +++ b/src/Chronos.Data/Jobs/JobPriorityQueue.cs @@ -24,7 +24,8 @@ public sealed record JobPriorityQueue /// /// The levels a kind has unless the user changed it: renders, imports and downloads first, and the copy of a project's /// media into the workspace, which Edit waits for (owner review 2, R8), a workspace's local render and the move of the - /// workspace folder, which Edit is held for (phase 5c); a local render's sending home at the normal level; the long + /// workspace folder, which Edit is held for (phase 5c), and the copy of videos into the videos folder, a transfer the user + /// waits for like a download (owner review 1, R9); a local render's sending home at the normal level; the long /// passes that resume anywhere, and the housekeeping of the schedules (backups, clean-ups), last. /// public static IReadOnlyDictionary DefaultLevels { get; } = new Dictionary @@ -36,6 +37,7 @@ public sealed record JobPriorityQueue [JobKind.WorkspaceRender] = JobKindPriority.High, [JobKind.WorkspaceSend] = JobKindPriority.Normal, [JobKind.WorkspaceMove] = JobKindPriority.High, + [JobKind.VideoCopy] = JobKindPriority.High, [JobKind.Verify] = JobKindPriority.Low, [JobKind.Proxies] = JobKindPriority.Low, [JobKind.Backup] = JobKindPriority.Low, diff --git a/src/Chronos.Data/Processing/Jobs/RenderJobHandler.cs b/src/Chronos.Data/Processing/Jobs/RenderJobHandler.cs index a7543966..2c460b5e 100644 --- a/src/Chronos.Data/Processing/Jobs/RenderJobHandler.cs +++ b/src/Chronos.Data/Processing/Jobs/RenderJobHandler.cs @@ -59,7 +59,11 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt var (projectId, projectVersionId) = await ProvenanceAsync(parameters, sequence, cancellation).ConfigureAwait(false); using var names = new OutputNames(); - var all = Targets(export, parameters.PresetName); + // A still saves one frame through the job's preset alone: a batch is a video's. + var all = parameters.IsStill ? [new ExportTarget(ExportResolver.Resolve(export), parameters.PresetName)] : Targets(export, parameters.PresetName); + var still = parameters.IsStill + ? SequenceRender.StillIndex(frames, parameters.StillFrame, parameters.StillPhoto) ?? throw new InvalidOperationException(Loc.L("Export.Render.NoFrames")) + : -1; var resume = RenderCheckpoint.Parse(context.Checkpoint); var done = resume.Done.ToList(); // A crash after a render row was written but before the checkpoint: that entry is finished. @@ -83,29 +87,46 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt await using (var render = new SequenceRender(frames, project, SequenceAnalyser.ReadReport(folder, frames), options, originals: !parameters.UseProxies, companions)) { - outcomes = await render.RenderAllAsync( - targets, - (target, setup) => names.Reserve(taken => ExportNaming.OutputPath(project.Export, folder.Exports, SequenceRender.NameContext(sequence, target, setup), taken)), - (index, path) => context.CheckpointAsync(new RenderCheckpoint(done, index, path, sequence.Folder).ToJson(), context.Done, context.Total), - async (index, target, result) => + async ValueTask Record(int index, ExportTarget target, RenderResult result) + { + // The output is complete: record it even if the job is being cancelled, so it is never orphaned. A file is + // hashed first (owner review 1, R9: copies and restores are verified against it); an image sequence's folder is not. + string? sha = null; + if (File.Exists(result.OutputPath)) { - // The output is complete: record it even if the job is being cancelled, so it is never orphaned. - var entity = result.ToEntity(sequence.Id, target.PresetName, settingsJson, options.ProducedBy) with - { - Id = RenderId(context.Job.Id, index), - OutputPath = Stored(paths, result.OutputPath), - ProjectId = projectId, - ProjectVersionId = projectVersionId, - }; - await db.Renders.InsertAsync(entity, CancellationToken.None).ConfigureAwait(false); - done.Add(index); - await context.CheckpointAsync(new RenderCheckpoint(done, null, null).ToJson(), context.Done, context.Total).ConfigureAwait(false); - context.Log(Loc.L("Export.Log.Rendered", target.PresetName, Path.GetFileName(result.OutputPath))); - foreach (var remark in SequenceRender.Remarks(result)) context.Log(remark); - await options.NotifyAsync(sequence.Id, ProcessingOptions.RenderedChange, CancellationToken.None).ConfigureAwait(false); - }, - context.ReportAsync, - cancellation).ConfigureAwait(false); + await context.ReportAsync(context.Done, context.Total, Loc.L("Export.Job.Verifying")).ConfigureAwait(false); + sha = await ContentHash.Sha256Async(result.OutputPath, cancellationToken: CancellationToken.None).ConfigureAwait(false); + } + var entity = result.ToEntity(sequence.Id, target.PresetName, settingsJson, options.ProducedBy) with + { + Id = RenderId(context.Job.Id, index), + OutputPath = Stored(paths, result.OutputPath), + ProjectId = projectId, + ProjectVersionId = projectVersionId, + Sha256 = sha, + Kind = parameters.IsStill ? RenderKind.Still : null, + }; + if (parameters.IsStill) entity = entity with { Fps = 0, DurationSeconds = 0 }; + await db.Renders.InsertAsync(entity, CancellationToken.None).ConfigureAwait(false); + done.Add(index); + await context.CheckpointAsync(new RenderCheckpoint(done, null, null).ToJson(), context.Done, context.Total).ConfigureAwait(false); + context.Log(parameters.IsStill + ? Loc.L("Export.Log.StillSaved", (still + 1).ToString(Loc.Culture), Path.GetFileName(result.OutputPath)) + : Loc.L("Export.Log.Rendered", target.PresetName, Path.GetFileName(result.OutputPath))); + foreach (var remark in SequenceRender.Remarks(result)) context.Log(remark); + await options.NotifyAsync(sequence.Id, ProcessingOptions.RenderedChange, CancellationToken.None).ConfigureAwait(false); + } + + if (parameters.IsStill) + outcomes = [await StillAsync(context, render, targets[0], still, sequence, project.Export, folder, names, done, Record).ConfigureAwait(false)]; + else + outcomes = await render.RenderAllAsync( + targets, + (target, setup) => names.Reserve(taken => ExportNaming.OutputPath(project.Export, folder.Exports, SequenceRender.NameContext(sequence, target, setup), taken)), + (index, path) => context.CheckpointAsync(new RenderCheckpoint(done, index, path, sequence.Folder).ToJson(), context.Done, context.Total), + Record, + context.ReportAsync, + cancellation).ConfigureAwait(false); if (render.StandIns.Count > 0) context.Log(Loc.L("Export.Log.StandIns", string.Join(", ", render.StandIns.Select(i => (i + 1).ToString(Loc.Culture))))); // Recorded without the job's token: what decoding found holds even if the job is cancelled meanwhile, and a @@ -128,6 +149,34 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt throw new InvalidOperationException(failed.Count == 1 ? failed[0].Error : Loc.L("Errors.Render.Failed", failed.Count, outcomes.Count)); } + /// + /// The still of a still job: frame saved through the job's preset under the export's name and + /// the frame number (), checkpointed and recorded like a video's output. + /// + private static async Task StillAsync(JobContext context, SequenceRender render, (int Index, ExportTarget Target) entry, int frame, + Sequence sequence, ExportSettings export, SequencePaths folder, OutputNames names, List done, + Func record) + { + var (index, target) = entry; + try + { + var result = await render.RenderStillAsync( + target, + frame, + setup => names.Reserve(taken => SequenceRender.StillPath(export, folder.Exports, SequenceRender.NameContext(sequence, target, setup), frame, taken)), + path => context.CheckpointAsync(new RenderCheckpoint(done, index, path, sequence.Folder).ToJson(), context.Done, context.Total), + context.ReportAsync, + context.Cancellation).ConfigureAwait(false); + await record(index, target, result).ConfigureAwait(false); + return new TargetOutcome(target, result, null); + } + catch (Exception ex) when (!(ex is OperationCanceledException && context.Cancellation.IsCancellationRequested)) + { + Core.Diagnostics.Log.Warning("processing", $"The still of frame {frame + 1} of {sequence.DisplayTitle} failed", ex); + return new TargetOutcome(target, null, SequenceRender.Reason(ex)); + } + } + /// /// The settings of the project version a job names without embedding them (owner review 2, R8); null when it names none. /// diff --git a/src/Chronos.Data/Processing/ProcessingJobs.cs b/src/Chronos.Data/Processing/ProcessingJobs.cs index 91752662..e0a099ae 100644 --- a/src/Chronos.Data/Processing/ProcessingJobs.cs +++ b/src/Chronos.Data/Processing/ProcessingJobs.cs @@ -153,7 +153,9 @@ public static class ProcessingJobs { Width = width, Height = height, - RequiredSettingsFormat = int.Max(parameters.RequiredSettingsFormat, RequiredFormat(settings, resolved, sourceWidth, sourceHeight)), + // The job's own export (playback included) is what renders, not the export the settings carry. + RequiredSettingsFormat = int.Max(parameters.RequiredSettingsFormat, + RequiredFormat(settings with { Export = ReadExport(parameters.PresetJson) }, resolved, sourceWidth, sourceHeight)), }; } @@ -166,8 +168,8 @@ public static class ProcessingJobs { ArgumentNullException.ThrowIfNull(settings); ArgumentNullException.ThrowIfNull(export); - return BatchPresets(export).Select(id => ExportResolver.Resolve(BatchExport(export, id))).Prepend(ExportResolver.Resolve(export)) - .Max(resolved => RequiredFormat(settings, resolved, sourceWidth, sourceHeight)); + return BatchPresets(export).Select(id => BatchExport(export, id)).Prepend(export) + .Max(each => RequiredFormat(settings with { Export = each }, ExportResolver.Resolve(each), sourceWidth, sourceHeight)); } /// @@ -210,9 +212,12 @@ public static class ProcessingJobs .Where(id => !id.Equals(main, StringComparison.OrdinalIgnoreCase)); } - /// A batch preset rendered with its own values, named and placed like the export it belongs to. + /// + /// A batch preset rendered with its own values, named, placed and played like the export it belongs to: its playback + /// (owner review 1, R8: direction, loops, fades, hold) lives in the export, so every batch preset inherits it. + /// internal static ExportSettings BatchExport(ExportSettings export, string presetId) => - new() { Preset = new Auto().WithManual(presetId), NameTemplate = export.NameTemplate, Folder = export.Folder }; + new() { Preset = new Auto().WithManual(presetId), NameTemplate = export.NameTemplate, Folder = export.Folder, Playback = export.Playback }; /// The settings rendered, for the output size and the format (else read from ); unknown, the preset's size. private static RenderJobParameters Parameters(ExportSettings export, int sourceWidth, int sourceHeight, string? settingsJson, bool useProxies, @@ -222,7 +227,10 @@ public static class ProcessingJobs settings ??= ReadSettings(settingsJson); var (width, height) = (settings is null ? null : Size(settings, resolved, sourceWidth, sourceHeight)) ?? (resolved.Width > 0 ? resolved.Width : sourceWidth, resolved.Height > 0 ? resolved.Height : sourceHeight); - var format = settings is null ? ProjectSettings.BaseVersion : RequiredFormat(settings, resolved, sourceWidth, sourceHeight); + // The format of what renders: the settings with this export in place of their own (its playback raises it to 3). + var format = settings is null + ? new ProjectSettings { Export = export }.RequiredFormat + : RequiredFormat(settings with { Export = export }, resolved, sourceWidth, sourceHeight); return new RenderJobParameters(PresetName(resolved), JsonSerializer.Serialize(export, ProcessingJson.Default.ExportSettings), CodecName(resolved.Codec), width, height, useProxies, settingsJson, RequiredSettingsFormat: format); } diff --git a/src/Chronos.Data/Processing/SequenceRender.cs b/src/Chronos.Data/Processing/SequenceRender.cs index 705cee18..791be545 100644 --- a/src/Chronos.Data/Processing/SequenceRender.cs +++ b/src/Chronos.Data/Processing/SequenceRender.cs @@ -134,6 +134,99 @@ internal sealed class SequenceRender : IAsyncDisposable return outcomes; } + /// + /// Saves frame as one image ("Save frame", owner review 1, R8): the sequence prepared whole + /// as for a video (so its exposure, colour and stabilisation corrections are the ones the video gives that frame), then + /// that frame alone rendered at the export's output size with every correction and the look, without interpolation, + /// motion blur or playback, through the image encoder of 's codec (PNG, JPEG or TIFF 16-bit). It + /// is written into a hidden working folder beside the output (.<name>.<random>.tmp, which + /// leftover clean-ups find) and moved to 's path once complete. The result's path is the image's, + /// its duration 0 and its frame count 1. + /// + /// The output path of the still from its setup. + /// Runs with that path before anything is written (checkpoints). + /// The frame is not one of the sequence's, or no encoder writes the codec. + public async Task RenderStillAsync(ExportTarget target, int frameIndex, Func name, Func starting, + Func progress, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(target); + ArgumentNullException.ThrowIfNull(name); + if ((uint)frameIndex >= (uint)Input.Frames.Count) throw new RenderException("Export.Render.NoFrames", $"Frame {frameIndex} is not in the sequence."); + // That frame alone, one output period long, whatever the timing would have made of it. + var project = Input.Project with + { + ExcludedFrames = [.. Enumerable.Range(0, Input.Frames.Count).Where(i => i != frameIndex)], + Timing = Input.Project.Timing with { Mode = Input.Project.Timing.Mode.WithManual(TimingMode.Constant), FitToSeconds = null }, + Motion = Input.Project.Motion with { MotionBlur = Input.Project.Motion.MotionBlur.WithManual(false) }, + }; + var still = Input with { Project = project }; + var pipeline = Pipeline(target.Export); + var setup = pipeline.Setup(still, new RenderTarget(target.Export, ExportNaming.FallbackName), _options); + if (!setup.Encoder.IsReady || setup.Plan.Frames.Count == 0) throw new RenderException("Export.Render.NoFrames", Reason(setup)); + var output = name(setup); + await starting(output).ConfigureAwait(false); + var chain = new ProgressChain(progress); + var whole = pipeline.Setup(Input, new RenderTarget(target.Export, ExportNaming.FallbackName), _options); + var preparing = whole.NeedsPreparation ? whole.Plan.IncludedFrames.Count : 0; + var total = preparing + 1L; + // The corrections of the whole sequence, indexed by source frame: the frame gets what the video gives it. + var corrections = await RenderPreparation.PrepareAsync(Input, _options, Stage(chain, 0, preparing, total, Loc.L("Export.Job.Preparing")), cancellationToken) + .ConfigureAwait(false); + var work = Path.Combine(Path.GetDirectoryName(output)!, "." + Path.GetFileName(output) + "." + Guid.NewGuid().ToString("N")[..8] + ".tmp"); + Directory.CreateDirectory(work); + try + { + var result = await pipeline.RenderAsync(still, corrections, new RenderTarget(target.Export, Path.Combine(work, "still")), _options, + Stage(chain, preparing, 1, total, target.PresetName), cancellationToken).ConfigureAwait(false); + await chain.FlushAsync().ConfigureAwait(false); + var image = Directory.EnumerateFiles(result.OutputPath).Order(StringComparer.Ordinal).FirstOrDefault() + ?? throw new RenderException("Export.Render.NoFrames", "The still was not written."); + File.Move(image, output, overwrite: false); + return result with { OutputPath = output, Bytes = new FileInfo(output).Length, DurationSeconds = 0, FrameCount = 1 }; + } + finally + { + try + { + Directory.Delete(work, recursive: true); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Core.Diagnostics.Log.Warning("processing", $"The still's working folder {work} could not be removed", ex); + } + } + } + + /// + /// The path of a still of frame (0-based) in : the export's name + /// template expanded, then " - " and the 1-based frame number in four digits, with the codec's extension + /// ("2026-09-28 Harbour sunset - PNG - 0124.png"); " (2)" and so on when taken. + /// + public static string StillPath(ExportSettings export, string folder, ExportNameContext context, int frameIndex, Func? taken = null) + { + ArgumentNullException.ThrowIfNull(export); + var name = ExportNaming.Expand(export.NameTemplate, context).Name + " - " + (frameIndex + 1).ToString("0000", System.Globalization.CultureInfo.InvariantCulture); + return ExportNaming.Unique(string.IsNullOrWhiteSpace(export.Folder) ? folder : export.Folder, name, Core.Video.VideoCodecInfo.FileExtension(context.Codec), taken); + } + + /// + /// The frame a still job saves (, ): + /// the frame holding the photo when its SHA-256 is given and found (the frames may have moved since), else the index; + /// null when neither names a frame of the sequence. + /// + public static int? StillIndex(SequenceFrames frames, int? frameIndex, string? photoSha256) + { + ArgumentNullException.ThrowIfNull(frames); + if (photoSha256 is { Length: > 0 } sha) + { + for (var i = 0; i < frames.Frames.Count; i++) + { + if (string.Equals(frames.Frames[i].Sha256, sha, StringComparison.OrdinalIgnoreCase)) return i; + } + } + return frameIndex is { } index && (uint)index < (uint)frames.Frames.Count ? index : null; + } + /// Values of the output name tokens: the shoot date and title of the sequence, the codec and size the encoder will use. public static ExportNameContext NameContext(Sequence sequence, ExportTarget target, RenderSetup setup) => new() { diff --git a/src/Chronos.Data/Repositories/RenderCopyRepository.cs b/src/Chronos.Data/Repositories/RenderCopyRepository.cs new file mode 100644 index 00000000..b949db81 --- /dev/null +++ b/src/Chronos.Data/Repositories/RenderCopyRepository.cs @@ -0,0 +1,56 @@ +using Chronos.Core.Model; + +namespace Chronos.Data.Repositories; + +/// +/// The verified copies of renders in videos folders (owner review 1, R9; render_copies). There is no link to +/// renders: the copy of a server render downloaded to a PC is recorded too, and a copy outlives its render. +/// +public sealed class RenderCopyRepository +{ + private const string Columns = "id, render_id, path, sha256, bytes, copied_at, copied_by"; + + private readonly DbAccess _db; + + internal RenderCopyRepository(DbAccess db) => _db = db; + + public Task InsertAsync(RenderCopy copy, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(copy); + return _db.ExecuteAsync($"INSERT INTO render_copies ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7)", + [copy.Id, copy.RenderId, copy.Path, copy.Sha256, copy.Bytes, copy.CopiedAt, copy.CopiedBy], cancellationToken); + } + + /// The copies of a render, oldest first. + public Task> ListByRenderAsync(Guid renderId, CancellationToken cancellationToken = default) => + _db.ListAsync($"SELECT {Columns} FROM render_copies WHERE render_id = $1 ORDER BY copied_at, id", [renderId], Map, cancellationToken); + + /// The copies of each of , oldest first; a render without copies is left out. + public async Task>> ListByRendersAsync(IReadOnlyCollection renderIds, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(renderIds); + var copies = new Dictionary>(); + // In chunks, so the statement stays within every dialect's parameter limit. + foreach (var chunk in renderIds.Distinct().Chunk(200)) + { + var p = new SqlParameters(); + var list = string.Join(", ", chunk.Select(id => p.Add(id))); + var rows = await _db.ListAsync($"SELECT {Columns} FROM render_copies WHERE render_id IN ({list}) ORDER BY copied_at, id", p.ToArray(), Map, + cancellationToken).ConfigureAwait(false); + foreach (var group in rows.GroupBy(c => c.RenderId)) copies[group.Key] = [.. group]; + } + return copies; + } + + private static RenderCopy Map(DbRow r) => new() + { + Id = r.GetGuid(0), + RenderId = r.GetGuid(1), + Path = r.GetString(2), + Sha256 = r.GetStringOrNull(3), + Bytes = r.GetInt64(4), + CopiedAt = r.GetDateTimeOffset(5), + CopiedBy = r.GetStringOrNull(6), + }; +} diff --git a/src/Chronos.Data/Repositories/RenderRepository.cs b/src/Chronos.Data/Repositories/RenderRepository.cs index 416c734d..802698c4 100644 --- a/src/Chronos.Data/Repositories/RenderRepository.cs +++ b/src/Chronos.Data/Repositories/RenderRepository.cs @@ -1,4 +1,6 @@ using Chronos.Core.Model; +using Chronos.Data.Videos; +using Chronos.Protocol; namespace Chronos.Data.Repositories; @@ -7,7 +9,7 @@ public sealed class RenderRepository { private const string Columns = "id, sequence_id, preset_name, settings_json, output_path, codec, width, height, fps, duration_seconds, bytes, created, produced_by, project_id, " + - "project_version_id"; + "project_version_id, sha256, kind"; private const string RecountSql = "UPDATE sequences SET rendered_count = (SELECT count(*) FROM renders WHERE sequence_id = $1) WHERE id = $1"; @@ -20,11 +22,11 @@ public sealed class RenderRepository _db.InTransactionAsync(async session => { await LockSequenceAsync(session, render.SequenceId, cancellationToken).ConfigureAwait(false); - await session.ExecuteAsync($"INSERT INTO renders ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)", + await session.ExecuteAsync($"INSERT INTO renders ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)", [ render.Id, render.SequenceId, render.PresetName, DbParam.Json(render.SettingsJson), render.OutputPath, render.Codec, render.Width, render.Height, render.Fps, render.DurationSeconds, render.Bytes, render.Created, render.ProducedBy, render.ProjectId, - render.ProjectVersionId, + render.ProjectVersionId, render.Sha256, render.Kind, ], cancellationToken).ConfigureAwait(false); return await session.ExecuteAsync(RecountSql, [render.SequenceId], cancellationToken).ConfigureAwait(false); }, cancellationToken); @@ -36,6 +38,80 @@ public sealed class RenderRepository public Task> ListBySequenceAsync(Guid sequenceId, CancellationToken cancellationToken = default) => _db.ListAsync($"SELECT {Columns} FROM renders WHERE sequence_id = $1 ORDER BY created DESC, id DESC", [sequenceId], Map, cancellationToken); + /// + /// One page of every render of the index matching (the videos library, owner review 1, R9), in + /// its order (: newest first unless asked), with the number of renders matching on every page. + /// Text matches the sequence's search text, the preset name or the output path (any case). + /// + /// The cursor is not one of this listing in this order. + public Task<(IReadOnlyList Items, string? Next, long Total)> ListPageAsync(VideoQuery query, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(query); + var p = new SqlParameters(); + var where = Filter(query, p); + var count = new SqlParameters(); + var countWhere = Filter(query, count); + var sort = VideoSorts.All.FirstOrDefault(s => string.Equals(s, query.Sort, StringComparison.OrdinalIgnoreCase)) ?? VideoSorts.Newest; + var scope = "videos:" + sort; + var limit = Math.Clamp(query.Limit, 1, VideoQuery.MaxLimit); + long offset = 0; + string order; + switch (sort) + { + case VideoSorts.Title: + // By title, then newest: paged by position (a title is no key a cursor keeps well). + if (query.Cursor is not null && Cursor.Decode(scope, query.Cursor).Value is long skip) offset = skip; + order = "ORDER BY lower(coalesce(s.title, '')) ASC, r.created DESC, r.id DESC"; + break; + default: + { + var (column, descending) = sort switch + { + VideoSorts.Oldest => ("r.created", false), + VideoSorts.Largest => ("r.bytes", true), + _ => ("r.created", true), + }; + if (query.Cursor is not null) + { + var (value, id) = Cursor.Decode(scope, query.Cursor); + where.Add(Cursor.After(column, "r.id", descending, value, id, p.Add)); + } + var direction = descending ? "DESC" : "ASC"; + order = $"ORDER BY {column} {direction}, r.id {direction}"; + break; + } + } + var columns = string.Join(", ", Columns.Split(", ").Select(c => "r." + c)); + var sql = $"SELECT {columns} FROM renders r JOIN sequences s ON s.id = r.sequence_id{Where(where)} {order} LIMIT {p.Add(limit + 1)}" + + (offset > 0 ? $" OFFSET {p.Add(offset)}" : string.Empty); + var countSql = $"SELECT count(*) FROM renders r JOIN sequences s ON s.id = r.sequence_id{Where(countWhere)}"; + return _db.RunAsync(async session => + { + var rows = await session.QueryListAsync(sql, p.ToArray(), Map, cancellationToken).ConfigureAwait(false); + var total = await session.ScalarAsync(countSql, count.ToArray(), cancellationToken).ConfigureAwait(false); + string? next = null; + if (rows.Count > limit) + { + rows.RemoveAt(limit); + var last = rows[^1]; + next = sort switch + { + VideoSorts.Title => Cursor.Encode(scope, offset + limit, last.Id), + VideoSorts.Largest => Cursor.Encode(scope, last.Bytes, last.Id), + _ => Cursor.Encode(scope, last.Created, last.Id), + }; + } + return ((IReadOnlyList)rows, next, total); + }, cancellationToken); + } + + /// Records the SHA-256 of a render hashed later than it was recorded (owner review 1, R9: renders of before migration 0010). + public Task SetSha256Async(Guid id, string sha256, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrEmpty(sha256); + return _db.ExecuteAsync("UPDATE renders SET sha256 = $2 WHERE id = $1", [id, sha256.ToLowerInvariant()], cancellationToken); + } + public Task DeleteAsync(Guid id, CancellationToken cancellationToken = default) => _db.InTransactionAsync(async session => { @@ -93,6 +169,43 @@ public sealed class RenderRepository if (_db.Sql.IsPostgres) await SequenceRepository.LockAsync(session, sequenceId, cancellationToken).ConfigureAwait(false); } + /// The conditions of a videos query on renders r JOIN sequences s. + private static List Filter(VideoQuery query, SqlParameters p) + { + var where = new List(); + if (query.SequenceId is { } sequence) where.Add($"r.sequence_id = {p.Add(sequence)}"); + if (!string.IsNullOrWhiteSpace(query.Codec)) where.Add($"lower(r.codec) = {p.Add(query.Codec.Trim().ToLowerInvariant())}"); + if (query.Year is { } year and > 0 and < 10000) + { + var from = new DateTimeOffset(year, 1, 1, 0, 0, 0, TimeSpan.Zero); + where.Add($"r.created >= {p.Add(from)} AND r.created < {p.Add(from.AddYears(1))}"); + } + // A sequence's codec names end in "sequence" (pngsequence, tiff16sequence…). + const string Sequence = "(r.kind IS NULL AND lower(coalesce(r.codec, '')) LIKE '%sequence')"; + switch (query.Kind?.Trim().ToLowerInvariant()) + { + case null or "": + break; + case RenderKind.Video: + where.Add($"(r.kind IS NULL AND NOT {Sequence})"); + break; + case RenderKind.Sequence: + where.Add(Sequence); + break; + case var kind: + where.Add($"r.kind = {p.Add(kind)}"); + break; + } + if (!string.IsNullOrWhiteSpace(query.Text)) + { + var pattern = p.Add(SqlDialect.ContainsPattern(query.Text.Trim().ToLowerInvariant())); + where.Add($"(s.search_text LIKE {pattern} ESCAPE '\\' OR lower(r.preset_name) LIKE {pattern} ESCAPE '\\' OR lower(r.output_path) LIKE {pattern} ESCAPE '\\')"); + } + return where; + } + + private static string Where(List conditions) => conditions.Count == 0 ? string.Empty : " WHERE " + string.Join(" AND ", conditions); + private static Render Map(DbRow r) => new() { Id = r.GetGuid(0), @@ -110,5 +223,7 @@ public sealed class RenderRepository ProducedBy = r.GetStringOrNull(12), ProjectId = r.GetGuidOrNull(13), ProjectVersionId = r.GetGuidOrNull(14), + Sha256 = r.GetStringOrNull(15), + Kind = r.GetStringOrNull(16), }; } diff --git a/src/Chronos.Data/Videos/IVideoLibrary.cs b/src/Chronos.Data/Videos/IVideoLibrary.cs index 09a601bc..8dda41d1 100644 --- a/src/Chronos.Data/Videos/IVideoLibrary.cs +++ b/src/Chronos.Data/Videos/IVideoLibrary.cs @@ -127,7 +127,12 @@ public sealed record VideoDeleteResult(IReadOnlyList Deleted, IReadOn /// The videos folder (absolute), checked again when the job runs. /// On the desktop: the renders are the signed-in server's, downloaded into the folder. /// The user (server) or PC (desktop) recorded as . -public sealed record VideoCopyParameters(IReadOnlyList RenderIds, string Folder, bool FromServer = false, string? RequestedBy = null) +/// With : the server the renders are downloaded from (its address as the +/// desktop's connections name it, ). +/// With : the server's sequence of each render, in the order of +/// ; its manifest gives the SHA-256 a download is verified against. +public sealed record VideoCopyParameters(IReadOnlyList RenderIds, string Folder, bool FromServer = false, string? RequestedBy = null, + string? Server = null, IReadOnlyList? SequenceIds = null) { public string ToJson() => JsonSerializer.Serialize(this, VideoJson.Default.VideoCopyParameters); diff --git a/src/Chronos.Data/Videos/VideoCopyJobHandler.cs b/src/Chronos.Data/Videos/VideoCopyJobHandler.cs new file mode 100644 index 00000000..38d127c8 --- /dev/null +++ b/src/Chronos.Data/Videos/VideoCopyJobHandler.cs @@ -0,0 +1,242 @@ +using System.Globalization; +using Chronos.Core.Archiving; +using Chronos.Core.Diagnostics; +using Chronos.Core.Localization; +using Chronos.Core.Model; +using Chronos.Data.Jobs; +using Chronos.Data.Sync; +using Chronos.Protocol; + +namespace Chronos.Data.Videos; + +/// +/// (owner review 1, R9, decision 16): copies renders into the videos folder, on the desktop +/// and on the server. The folder is checked again first, and the free space for every render still to copy, so a copy that +/// cannot fit fails before it writes anything. Each render is written as .incoming-<id>.part in the folder and +/// flushed, hashed and compared with the render's SHA-256 (the render hashed first when it never was, and remembered; an +/// image sequence's folder file by file against its own files), then renamed without overwriting to its own name or a free +/// " (2)" one, and recorded in render_copies. A copy that does not match is deleted and that render fails; the +/// render itself is never touched. On the desktop a server's render () is +/// downloaded into the part file and checked against the SHA-256 its server's manifest gives. Checkpoint: +/// done:<n>, the renders done; leftovers of an interrupted copy are removed before it starts again. +/// +internal sealed class VideoCopyJobHandler(ChronosDatabase db, VideoLibrary library) : IJobHandler +{ + private const string Prefix = "done:"; + + public JobKind Kind => JobKind.VideoCopy; + + public async Task RunAsync(JobContext context) + { + var ct = context.Cancellation; + var parameters = VideoCopyParameters.Read(context.Job.ParametersJson) ?? throw new InvalidDataException(Loc.L("Errors.Render.Parameters")); + if (await library.FolderProblemAsync(parameters.Folder, ct).ConfigureAwait(false) is { } problem) throw VideoLibrary.Refusal(problem); + var folder = parameters.Folder; + Directory.CreateDirectory(folder); + var ids = parameters.RenderIds; + var start = context.Checkpoint is { } checkpoint && checkpoint.StartsWith(Prefix, StringComparison.Ordinal) + && int.TryParse(checkpoint.AsSpan(Prefix.Length), NumberStyles.None, CultureInfo.InvariantCulture, out var done) + ? int.Min(done, ids.Count) + : 0; + foreach (var id in ids.Skip(start)) Remove(VideosFolder.PartPath(folder, id)); + await context.ReportAsync(start, ids.Count, Loc.L("Videos.Job.Copying")).ConfigureAwait(false); + + using var remote = parameters.FromServer ? await ConnectAsync(parameters, ct).ConfigureAwait(false) : null; + var sources = new List(ids.Count); + for (var i = 0; i < ids.Count; i++) + sources.Add(i < start ? null : await SourceAsync(context, parameters, remote, i, ct).ConfigureAwait(false)); + // Every render still to copy must fit before anything is written. + var needed = sources.Where(s => s is not null).Sum(s => s!.Bytes); + if (VideosFolder.FreeBytes(folder, library.FreeSpace) is { } free && free < needed) + throw new VideoLibraryException(VideoErrors.NoSpace, Loc.L("Errors.Videos.NoSpace", Megabytes(needed - free))); + + var failed = 0; + for (var i = start; i < ids.Count; i++) + { + ct.ThrowIfCancellationRequested(); + if (sources[i] is not { } source) + { + failed++; + } + else if (!await CopyAsync(context, parameters, remote, source, ct).ConfigureAwait(false)) + { + failed++; + } + await context.CheckpointAsync(Prefix + (i + 1).ToString(CultureInfo.InvariantCulture), i + 1, ids.Count).ConfigureAwait(false); + } + if (failed > 0) throw new InvalidOperationException(Loc.L("Errors.Render.Failed", failed, ids.Count - start)); + } + + /// What one render's copy reads: its id, name, size, and the SHA-256 it is verified against (null: an image sequence). + /// The render's file or folder on this host; null for a server's render. + /// The server's sequence of a server's render. + private sealed record Source(Guid RenderId, string Name, string? Path, bool IsFolder, long Bytes, string? Sha256, Guid? Sequence); + + /// The render at as it is now; null (logged) when it is gone or its file is missing. + private async Task SourceAsync(JobContext context, VideoCopyParameters parameters, ISyncRemote? remote, int index, CancellationToken ct) + { + var id = parameters.RenderIds[index]; + if (remote is not null) + { + var sequence = parameters.SequenceIds is { } sequences && index < sequences.Count ? sequences[index] : (Guid?)null; + var render = sequence is { } s ? (await remote.GetManifestAsync(s, ct).ConfigureAwait(false)).Renders.FirstOrDefault(r => r.Id == id) : null; + if (render is null) + { + context.Log(Loc.L("Videos.Log.Missing", id.ToString("D"))); + return null; + } + // Named by the server as it downloads. + return new Source(id, id.ToString("N"), null, false, render.Bytes, render.Sha256, sequence); + } + if (await library.GetAsync(id, ct).ConfigureAwait(false) is not { } item || item.Missing) + { + context.Log(Loc.L("Videos.Log.Missing", (await library.GetAsync(id, ct).ConfigureAwait(false))?.FileName ?? id.ToString("D"))); + return null; + } + var folder = Directory.Exists(item.FullPath); + string? sha = null; + if (!folder) + { + sha = item.Render.Sha256; + if (sha is null) + { + // A render of before migration 0010: hashed now, and remembered. + await context.ReportAsync(context.Done, context.Total, Loc.L("Videos.Job.Verifying")).ConfigureAwait(false); + sha = await ContentHash.Sha256Async(item.FullPath, cancellationToken: ct).ConfigureAwait(false); + await db.Renders.SetSha256Async(id, sha, CancellationToken.None).ConfigureAwait(false); + } + } + var bytes = folder ? new DirectoryInfo(item.FullPath).EnumerateFiles("*", SearchOption.AllDirectories).Sum(f => f.Length) : new FileInfo(item.FullPath).Length; + return new Source(id, item.FileName, item.FullPath, folder, bytes, sha, null); + } + + /// Copies one render, verified, under a free name, and records the copy; false (logged) when the copy did not match. + private async Task CopyAsync(JobContext context, VideoCopyParameters parameters, ISyncRemote? remote, Source source, CancellationToken ct) + { + var folder = parameters.Folder; + var part = VideosFolder.PartPath(folder, source.RenderId); + Remove(part); + string? sha; + try + { + if (remote is not null) + { + await using (var file = new FileStream(part, FileMode.Create, FileAccess.ReadWrite, FileShare.None)) + { + var given = await remote.DownloadRenderAsync(source.RenderId, file, ct).ConfigureAwait(false); + file.Flush(flushToDisk: true); + source = source with { Name = SyncTransfers.SafeFileName(given) ?? source.Name }; + } + sha = await Verified(part, source.Sha256, context, ct).ConfigureAwait(false); + } + else if (source.IsFolder) + { + // An image sequence: each file copied and compared with its own. + foreach (var file in Directory.EnumerateFiles(source.Path!, "*", SearchOption.AllDirectories)) + { + var target = Path.Combine(part, Path.GetRelativePath(source.Path!, file)); + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + await VideoTrash.CopyFileAsync(file, target, ct).ConfigureAwait(false); + var expected = await ContentHash.Sha256Async(file, cancellationToken: ct).ConfigureAwait(false); + if (await Verified(target, expected, context, ct).ConfigureAwait(false) is null) + { + Mismatch(context, source, part); + return false; + } + } + Directory.CreateDirectory(part); + sha = null; + await Commit(context, parameters, source, part, sha).ConfigureAwait(false); + return true; + } + else + { + await VideoTrash.CopyFileAsync(source.Path!, part, ct).ConfigureAwait(false); + sha = await Verified(part, source.Sha256, context, ct).ConfigureAwait(false); + } + } + catch + { + Remove(part); + throw; + } + if (sha is null) + { + Mismatch(context, source, part); + return false; + } + await Commit(context, parameters, source, part, sha).ConfigureAwait(false); + return true; + } + + /// The copy takes its name (never overwriting: another free name when one appeared meanwhile) and is recorded. + private async Task Commit(JobContext context, VideoCopyParameters parameters, Source source, string part, string? sha) + { + string target; + for (var attempt = 0; ; attempt++) + { + target = VideosFolder.FreeName(parameters.Folder, source.Name, source.IsFolder); + try + { + if (source.IsFolder) Directory.Move(part, target); + else File.Move(part, target, overwrite: false); + break; + } + catch (IOException) when (attempt < 5 && (File.Exists(target) || Directory.Exists(target))) + { + // Taken between the look and the move: the next free name. + } + } + var bytes = source.IsFolder ? new DirectoryInfo(target).EnumerateFiles("*", SearchOption.AllDirectories).Sum(f => f.Length) : new FileInfo(target).Length; + await db.RenderCopies.InsertAsync(new RenderCopy + { + Id = Guid.CreateVersion7(), + RenderId = source.RenderId, + Path = target, + Sha256 = sha, + Bytes = bytes, + CopiedAt = DateTimeOffset.UtcNow, + CopiedBy = parameters.RequestedBy, + }, CancellationToken.None).ConfigureAwait(false); + context.Log(Loc.L("Videos.Log.Copied", source.Name, target)); + Log.Info("videos", $"Render {source.RenderId} copied to {target} ({bytes} bytes, verified)"); + } + + /// The SHA-256 of the copy at when it matches (any, when none is known); null otherwise. + private static async Task Verified(string path, string? expected, JobContext context, CancellationToken ct) + { + await context.ReportAsync(context.Done, context.Total, Loc.L("Videos.Job.Verifying")).ConfigureAwait(false); + var sha = await ContentHash.Sha256Async(path, cancellationToken: ct).ConfigureAwait(false); + return expected is null || string.Equals(sha, expected, StringComparison.OrdinalIgnoreCase) ? sha : null; + } + + private static void Mismatch(JobContext context, Source source, string part) + { + Remove(part); + context.Log(Loc.L("Videos.Log.Mismatch", source.Name)); + Log.Warning("videos", $"The copy of render {source.RenderId} did not match it; the copy was removed"); + } + + private async Task ConnectAsync(VideoCopyParameters parameters, CancellationToken ct) + { + if (library.Servers is not { } servers || parameters.Server is not { Length: > 0 } server + || await servers.ConnectAsync(server, ct).ConfigureAwait(false) is not { } remote) + throw new InvalidOperationException(Loc.L("Errors.Sync.NotLinked")); + return remote; + } + + private static string Megabytes(long bytes) => Housekeeping.CleanupSteps.Megabytes(bytes); + + private static void Remove(string path) + { + try + { + if (Directory.Exists(path)) Directory.Delete(path, recursive: true); + else if (File.Exists(path)) File.Delete(path); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warning("videos", $"{path} could not be removed", ex); + } + } +} diff --git a/src/Chronos.Data/Videos/VideoLibrary.cs b/src/Chronos.Data/Videos/VideoLibrary.cs new file mode 100644 index 00000000..6163a258 --- /dev/null +++ b/src/Chronos.Data/Videos/VideoLibrary.cs @@ -0,0 +1,216 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using Chronos.Core.Localization; +using Chronos.Core.Model; +using Chronos.Data.Archiving; +using Chronos.Data.Archiving.Jobs; +using Chronos.Data.Sync; +using Chronos.Protocol; + +namespace Chronos.Data.Videos; + +/// +/// The videos library of one index (owner review 1, R9): every render of its archives in one list, newest first unless +/// asked, each with its sequence, its project, whether its file is there and its copies; deleted into its archive's trash +/// after a dry run whose fingerprint the delete gives back (decision 17), restored by id (decision 18), and copied +/// hash-verified into a videos folder by a job (decision 16). The renders themselves are +/// never changed, and copies in a videos folder are never touched. +/// +public sealed class VideoLibrary : IVideoLibrary +{ + private readonly ChronosDatabase _db; + private readonly SequenceWriter _writer; + + internal VideoLibrary(ChronosDatabase db, SequenceWriter writer) + { + _db = db; + _writer = writer; + Trash = new VideoTrash(db, writer); + } + + public bool IsAvailable => true; + + /// + /// Folders a videos folder may not lie in besides the archive roots (decision 15): the desktop's workspace. Asked each + /// time a folder is checked. + /// + public Func>? ProtectedFolders { get; set; } + + /// The desktop's signed-in servers: a copy of a server's renders () downloads through them. + public ISyncConnector? Servers { get; set; } + + /// How the host measures free space under a folder; null: the folder's drive. + public Func? FreeSpace => _writer.Options.FreeSpace; + + internal VideoTrash Trash { get; } + + public async Task ListAsync(VideoQuery query, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(query); + var (renders, next, total) = await _db.Renders.ListPageAsync(query, cancellationToken).ConfigureAwait(false); + return new VideoPage(await ItemsAsync(renders, cancellationToken).ConfigureAwait(false), next, total); + } + + public async Task GetAsync(Guid renderId, CancellationToken cancellationToken = default) => + await _db.Renders.GetAsync(renderId, cancellationToken).ConfigureAwait(false) is { } render + ? (await ItemsAsync([render], cancellationToken).ConfigureAwait(false)).FirstOrDefault() + : null; + + public async Task PlanDeleteAsync(IReadOnlyList renderIds, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(renderIds); + var busy = await BusyAsync(cancellationToken).ConfigureAwait(false); + var items = new List(); + var blocked = new List(); + foreach (var id in renderIds.Distinct()) + { + if (await GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } item) blocked.Add(new VideoBlocked(id, VideoBlockReasons.NotFound)); + else if (busy.Contains(id)) blocked.Add(new VideoBlocked(id, VideoBlockReasons.Busy, item)); + else items.Add(item); + } + return new VideoDeletePlan(items, blocked, Fingerprint(items, blocked)); + } + + public async Task DeleteAsync(IReadOnlyList renderIds, string fingerprint, string? deletedBy, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(renderIds); + var plan = await PlanDeleteAsync(renderIds, cancellationToken).ConfigureAwait(false); + if (!string.Equals(plan.Fingerprint, fingerprint, StringComparison.Ordinal)) + throw new VideoLibraryException(VideoErrors.PlanChanged, Loc.L("Errors.Videos.Fingerprint")); + var deleted = new List(); + var entries = new List(); + var archives = new Dictionary(); + foreach (var item in plan.Items) + { + cancellationToken.ThrowIfCancellationRequested(); + var archive = await ArchiveOfAsync(archives, item.Render.SequenceId, cancellationToken).ConfigureAwait(false); + if (archive is null) + { + // The sequence left the index since the plan: so did its renders. + continue; + } + if (await Trash.DeleteAsync(archive, item.Render, deletedBy, cancellationToken).ConfigureAwait(false) is { } entry) entries.Add(entry.Name); + deleted.Add(item); + } + return new VideoDeleteResult(deleted, plan.Blocked, entries); + } + + public async Task RestoreAsync(Guid archiveId, string entryName, CancellationToken cancellationToken = default) + { + var archive = await _db.Archives.GetAsync(archiveId, cancellationToken).ConfigureAwait(false) + ?? throw new ArgumentException($"Unknown archive {archiveId}.", nameof(archiveId)); + return await Trash.RestoreAsync(archive, entryName, cancellationToken).ConfigureAwait(false); + } + + /// Settles the deletes into the trash a crash interrupted in (). + public Task RecoverAsync(Archive archive, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(archive); + return Trash.RecoverAsync(archive, cancellationToken); + } + + public string? FolderProblem(string? folder) + { + if (string.IsNullOrWhiteSpace(folder)) return VideoErrors.FolderUnset; + // Asked from the UI thread and the server's endpoints alike: the roots are read off any synchronisation context. + var roots = Task.Run(() => _db.Archives.ListAsync()).GetAwaiter().GetResult().Select(a => a.RootPath); + return VideosFolder.Problem(folder, [.. roots, .. ProtectedFolders?.Invoke() ?? []]); + } + + /// without blocking: what the copy job checks again when it runs. + internal async Task FolderProblemAsync(string? folder, CancellationToken cancellationToken) + { + var roots = (await _db.Archives.ListAsync(cancellationToken).ConfigureAwait(false)).Select(a => a.RootPath); + return VideosFolder.Problem(folder, [.. roots, .. ProtectedFolders?.Invoke() ?? []]); + } + + public ArchiveJobRequest CopyJob(VideoCopyParameters parameters) + { + ArgumentNullException.ThrowIfNull(parameters); + if (FolderProblem(parameters.Folder) is { } problem) throw Refusal(problem); + if (parameters.RenderIds.Count == 0) throw new ArgumentException("No render to copy.", nameof(parameters)); + var ids = parameters.RenderIds.Distinct().ToList(); + var folder = VideosFolder.Normalize(parameters.Folder); + return new ArchiveJobRequest(JobKind.VideoCopy, ids, Loc.L("Videos.Target.Copy", ids.Count.ToString(Loc.Culture)), + (parameters with { RenderIds = ids, Folder = folder }).ToJson(), JobPriority.User); + } + + /// The exception of a videos folder that cannot be used, with its localised message. + internal static VideoLibraryException Refusal(string code) => new(code, Loc.L(code == VideoErrors.FolderUnset ? "Errors.Videos.FolderUnset" : "Errors.Videos.FolderInsideArchive")); + + /// + /// The fingerprint of a delete plan: SHA-256 of each render's id, path, size, creation and whether its file is there, and + /// of each render left where it is with why, in id order. Another render, a render moved, grown, gone or busy since gives + /// another fingerprint. + /// + internal static string Fingerprint(IEnumerable items, IEnumerable blocked) + { + var text = new StringBuilder(); + foreach (var item in items.OrderBy(i => i.Render.Id)) + { + var r = item.Render; + text.Append(CultureInfo.InvariantCulture, $"{r.Id:N}|{r.OutputPath}|{r.Bytes}|{r.Created.UtcTicks}|{(item.Missing ? "missing" : "ok")}\n"); + } + foreach (var block in blocked.OrderBy(b => b.RenderId)) text.Append(CultureInfo.InvariantCulture, $"{block.RenderId:N}|blocked|{block.Reason}\n"); + return Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(text.ToString()))); + } + + /// The renders a running or paused job holds: those a copy works on (its targets are render ids). + private async Task> BusyAsync(CancellationToken cancellationToken) + { + var busy = new HashSet(); + foreach (var hold in await _db.Jobs.ListHoldsAsync(null, cancellationToken).ConfigureAwait(false)) + { + if (hold.Kind == JobKind.VideoCopy) busy.UnionWith(hold.TargetIds); + } + return busy; + } + + /// The library items of : each with its sequence, project, file state and copies, in their order. + private async Task> ItemsAsync(IReadOnlyList renders, CancellationToken cancellationToken) + { + if (renders.Count == 0) return []; + var copies = await _db.RenderCopies.ListByRendersAsync([.. renders.Select(r => r.Id)], cancellationToken).ConfigureAwait(false); + var sequences = new Dictionary(); + var archives = new Dictionary(); + var projects = new Dictionary(); + var versions = new Dictionary(); + var items = new List(renders.Count); + foreach (var render in renders) + { + if (!sequences.TryGetValue(render.SequenceId, out var sequence)) + sequences[render.SequenceId] = sequence = await _db.Sequences.GetAsync(render.SequenceId, cancellationToken).ConfigureAwait(false); + if (sequence is null) continue; + if (!archives.TryGetValue(sequence.ArchiveId, out var archive)) + archives[sequence.ArchiveId] = archive = await _db.Archives.GetAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false); + string? projectName = null; + if (render.ProjectId is { } projectId && !projects.TryGetValue(projectId, out projectName)) + projects[projectId] = projectName = (await _db.Projects.GetAsync(projectId, cancellationToken).ConfigureAwait(false))?.Name; + int? number = null; + if (render.ProjectVersionId is { } versionId && !versions.TryGetValue(versionId, out number)) + versions[versionId] = number = (await _db.Projects.GetVersionAsync(versionId, cancellationToken).ConfigureAwait(false))?.Number; + var full = archive is null || Path.IsPathRooted(render.OutputPath) ? render.OutputPath : SyncService.RenderFile(archive, render); + items.Add(new VideoItem + { + Render = render, + SequenceTitle = sequence.DisplayTitle, + SequenceFirstCapture = sequence.FirstCapture, + ProjectName = projectName, + ProjectVersionNumber = projectName is null ? null : number, + FullPath = full, + Missing = !File.Exists(full) && !Directory.Exists(full), + Copies = copies.TryGetValue(render.Id, out var made) ? made : [], + }); + } + return items; + } + + private async Task ArchiveOfAsync(Dictionary known, Guid sequenceId, CancellationToken cancellationToken) + { + if (await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) is not { } sequence) return null; + if (!known.TryGetValue(sequence.ArchiveId, out var archive)) + known[sequence.ArchiveId] = archive = await _db.Archives.GetAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false); + return archive; + } +} diff --git a/src/Chronos.Data/Videos/VideoTrash.cs b/src/Chronos.Data/Videos/VideoTrash.cs new file mode 100644 index 00000000..3e97583c --- /dev/null +++ b/src/Chronos.Data/Videos/VideoTrash.cs @@ -0,0 +1,440 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using Chronos.Core.Archiving; +using Chronos.Core.Diagnostics; +using Chronos.Core.IO; +using Chronos.Core.Localization; +using Chronos.Core.Model; +using Chronos.Data.Archiving; + +namespace Chronos.Data.Videos; + +/// One file of a trashed image sequence, as it was when it went: its path inside the folder, size and SHA-256. +internal sealed record VideoTrashFile(string Path, long Size, string Sha256); + +/// +/// What the trash keeps next to a deleted render (decision 17): the render's row, where its file was, the SHA-256 it is +/// restored against (each file's, for an image sequence's folder), its sequence, and who deleted it when. +/// +/// The file (or folder) name inside the entry's folder: the render's own name. +/// The absolute path the render was at. +/// The sequence's archive-relative folder at deletion: a restore follows a Reorganize since. +internal sealed record VideoTrashRecord(Render Render, string Name, string OriginalPath, Guid ArchiveId, Guid SequenceId, string SequenceTitle, + string SequenceFolder, string? Sha256, IReadOnlyList? Files, int FileCount, long Bytes, DateTimeOffset DeletedAt, string? DeletedBy); + +/// +/// The videos area of an archive's trash (owner review 1, R9, decisions 17 and 18): .chronos/trash/videos/, which +/// holds a folder per deleted render (<render id>-<UTC ticks>, the entry name without its +/// ) with the render's file or folder inside, and its record next to it. A delete runs +/// under the sequence's lease: the record is written first under a pending name (the journal, never listed), then the file +/// moves (renamed on its volume; copied, verified and deleted from another one), then the row goes and the record takes its +/// name. A delete a crash interrupted is settled by : completed when the file reached the +/// trash, undone otherwise. Records live under trash/videos/, so an older Chronos never reads one as a sequence's. +/// +internal sealed class VideoTrash(ChronosDatabase db, SequenceWriter writer) +{ + private const string RecordExtension = ".json"; + + /// The suffix of a record whose delete is under way: the journal a crash leaves. + internal const string PendingSuffix = ".pending"; + + private const string PartSuffix = ".part"; + + /// Tests: move as if the trash were on another volume (copy, verify, delete). + internal bool ForceCopy { get; set; } + + /// Tests: stop a delete right after its file moved, as a crash would (the journal stays). + internal bool CrashAfterMove { get; set; } + + /// The videos area of an archive's trash. + public static string Directory(ArchivePaths archive) => Path.Combine(SequenceTrash.TrashDirectory(archive), VideoTrashNames.FolderName); + + /// The entry's folder and record on disk. + internal static (string Folder, string Record) Location(ArchivePaths paths, string name) + { + var folder = Path.Combine(Directory(paths), DiskName(name)); + return (folder, folder + RecordExtension); + } + + /// The videos entries of an archive's trash (records whose folder is gone, or whose delete is under way, left out). + internal static IReadOnlyList List(Archive archive, CancellationToken cancellationToken) + { + var folder = Directory(new ArchivePaths(archive.RootPath)); + if (!System.IO.Directory.Exists(folder)) return []; + var entries = new List(); + foreach (var path in System.IO.Directory.EnumerateFiles(folder, "*" + RecordExtension)) + { + cancellationToken.ThrowIfCancellationRequested(); + if (!path.EndsWith(RecordExtension, StringComparison.OrdinalIgnoreCase)) continue; + var disk = Path.GetFileNameWithoutExtension(path); + if (!System.IO.Directory.Exists(Path.Combine(folder, disk)) || ReadRecord(path) is not { } record) continue; + entries.Add(Entry(archive.Id, VideoTrashNames.Prefix + disk, record)); + } + return entries; + } + + /// A videos entry as the trash lists it, read from its record now; null when it is not one or cannot be read. + internal static TrashEntry? ReadListed(ArchivePaths paths, Guid archiveId, string name) => + VideoTrashNames.IsEntryName(name) && ReadRecord(Location(paths, name).Record) is { } record ? Entry(archiveId, name, record) : null; + + /// + /// Moves 's file into the trash with its record, and removes its row (its sequence's render count + /// recounted); a render whose file is missing loses only its row. Returns the entry made; null when only the row went + /// (or the render was gone already). + /// + /// The archive is offline or read-only. + /// Another writer holds the sequence. + internal async Task DeleteAsync(Archive archive, Render render, string? deletedBy, CancellationToken cancellationToken) + { + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + var sequence = await db.Sequences.GetAsync(render.SequenceId, cancellationToken).ConfigureAwait(false); + await using (sequence is null ? null : await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false)) + { + if (await db.Renders.GetAsync(render.Id, cancellationToken).ConfigureAwait(false) is not { } row) return null; + var source = Sync.SyncService.RenderFile(archive, row); + var isFile = File.Exists(source); + if (!isFile && !System.IO.Directory.Exists(source)) + { + await db.Renders.DeleteAsync(row.Id, CancellationToken.None).ConfigureAwait(false); + Log.Info("videos", $"Render {row.Id} ({row.OutputPath}) removed from the index: its file is missing"); + return null; + } + string? sha = null; + IReadOnlyList? files = null; + long bytes; + if (isFile) + { + // Hashed now when it never was (a render of before migration 0010), and remembered. + sha = row.Sha256 ?? await ContentHash.Sha256Async(source, cancellationToken: cancellationToken).ConfigureAwait(false); + if (row.Sha256 is null) await db.Renders.SetSha256Async(row.Id, sha, CancellationToken.None).ConfigureAwait(false); + bytes = new FileInfo(source).Length; + } + else + { + files = await HashFolderAsync(source, cancellationToken).ConfigureAwait(false); + bytes = files.Sum(f => f.Size); + } + var now = DateTimeOffset.UtcNow; + var name = VideoTrashNames.EntryName(row.Id, now); + var (entryFolder, recordPath) = Location(paths, name); + var fileName = Path.GetFileName(source.TrimEnd('/', '\\')); + var record = new VideoTrashRecord(row with { Sha256 = sha ?? row.Sha256 }, fileName, source, archive.Id, row.SequenceId, + sequence?.DisplayTitle ?? fileName, sequence?.Folder ?? string.Empty, sha, files, files?.Count ?? 1, bytes, now, deletedBy); + using (await LeaseFile.AcquireAsync(paths.LocksDirectory, SequenceTrash.EntryLease(name), writer.Options.Owner, writer.Options.LeaseWait, + writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false)) + { + cancellationToken.ThrowIfCancellationRequested(); + System.IO.Directory.CreateDirectory(entryFolder); + // The journal first: a crash from here on is settled by RecoverAsync. + AtomicFile.WriteAllText(recordPath + PendingSuffix, JsonSerializer.Serialize(record, VideoTrashJson.Default.VideoTrashRecord)); + var target = Path.Combine(entryFolder, fileName); + try + { + await MoveAsync(source, target, record, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + // A file is open elsewhere, the copy did not verify, or the move was cancelled: the render stays where it was. + Log.Warning("videos", $"{source} could not move to the trash; it stays where it is", ex); + Undo(target, entryFolder, recordPath); + throw; + } + if (CrashAfterMove) throw new IOException("Simulated crash after the move."); + // The file has moved: its row goes too, to the end, whatever the caller cancels meanwhile. + await db.Renders.DeleteAsync(row.Id, CancellationToken.None).ConfigureAwait(false); + Publish(recordPath); + } + Log.Info("videos", $"{source} moved to the trash ({name}, {bytes} bytes)"); + return Entry(archive.Id, name, record); + } + } + + /// + /// Brings a videos entry back: its file re-hashed against its record (refused on a mismatch), its sequence in the index + /// (refused otherwise: restore the sequence first), then moved back where it was (into the sequence's folder as it is now + /// after a Reorganize; a free name when its own is taken) and its row inserted again with the same render id. + /// + /// The entry is not in the trash. + /// or ; nothing moved. + /// The index holds a render of that id already; nothing moved. + internal async Task RestoreAsync(Archive archive, string name, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(name); + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + if (!VideoTrashNames.IsEntryName(name)) throw new DirectoryNotFoundException($"{name} is not in the trash of {archive.Name}."); + var (entryFolder, recordPath) = Location(paths, name); + if (!System.IO.Directory.Exists(entryFolder) || ReadRecord(recordPath) is not { } record) + throw new DirectoryNotFoundException($"{name} is not in the trash of {archive.Name}."); + using (await LeaseFile.AcquireAsync(paths.LocksDirectory, SequenceTrash.EntryLease(name), writer.Options.Owner, writer.Options.LeaseWait, + writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false)) + { + var trashed = Path.Combine(entryFolder, record.Name); + if (!File.Exists(trashed) && !System.IO.Directory.Exists(trashed)) throw new DirectoryNotFoundException($"{name} is not in the trash of {archive.Name}."); + if (await db.Renders.GetAsync(record.Render.Id, cancellationToken).ConfigureAwait(false) is not null) + throw new InvalidOperationException(Loc.L("Server.Error.RenderExists")); + var sequence = await db.Sequences.GetAsync(record.SequenceId, cancellationToken).ConfigureAwait(false) + ?? throw new VideoLibraryException(VideoErrors.SequenceGone, Loc.L("Errors.Videos.SequenceGone", record.SequenceTitle)); + if (!await MatchesAsync(trashed, record, cancellationToken).ConfigureAwait(false)) + throw new VideoLibraryException(VideoErrors.HashMismatch, Loc.L("Errors.Videos.TrashMismatch")); + Render restored; + await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false)) + { + var target = Destination(paths, record, sequence); + System.IO.Directory.CreateDirectory(Path.GetDirectoryName(target)!); + cancellationToken.ThrowIfCancellationRequested(); + await MoveAsync(trashed, target, record, cancellationToken).ConfigureAwait(false); + restored = record.Render with { SequenceId = sequence.Id, OutputPath = Stored(paths, target), Sha256 = record.Sha256 }; + try + { + await db.Renders.InsertAsync(restored, CancellationToken.None).ConfigureAwait(false); + } + catch (Exception ex) + { + Log.Warning("videos", $"{target} could not be indexed again; it goes back into the trash", ex); + await MoveAsync(target, trashed, record, CancellationToken.None).ConfigureAwait(false); + throw; + } + } + File.Delete(recordPath); + RemoveEmpty(entryFolder); + Log.Info("videos", $"{name} restored to {restored.OutputPath}"); + return restored; + } + } + + /// + /// Settles the deletes a crash interrupted in 's trash (their pending records): a delete whose + /// file reached the trash is completed (the row goes, the record takes its name), any other is undone (what was copied + /// goes, the render stays where it is). Each runs under its sequence's lease when the index still holds the sequence. + /// + internal async Task RecoverAsync(Archive archive, CancellationToken cancellationToken) + { + var paths = new ArchivePaths(archive.RootPath); + var folder = Directory(paths); + if (!System.IO.Directory.Exists(folder) || ArchiveVolume.Probe(paths) != ArchiveAvailability.Online) return 0; + var settled = 0; + foreach (var pending in System.IO.Directory.EnumerateFiles(folder, "*" + RecordExtension + PendingSuffix).ToList()) + { + cancellationToken.ThrowIfCancellationRequested(); + var recordPath = pending[..^PendingSuffix.Length]; + var entryFolder = recordPath[..^RecordExtension.Length]; + if (ReadRecord(pending) is not { } record) + { + Log.Warning("videos", $"The pending trash record {pending} cannot be read; it stays"); + continue; + } + var sequence = await db.Sequences.GetAsync(record.SequenceId, cancellationToken).ConfigureAwait(false); + await using (sequence is null ? null : await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false)) + { + if (!File.Exists(pending)) continue; + var target = Path.Combine(entryFolder, record.Name); + if (File.Exists(target) || System.IO.Directory.Exists(target)) + { + // The file reached the trash (a copy only takes its name once verified): the delete completes. A source still + // there is the original a copy left behind, unless something new took its name since. + if ((File.Exists(record.OriginalPath) || System.IO.Directory.Exists(record.OriginalPath)) + && await MatchesAsync(record.OriginalPath, record, cancellationToken).ConfigureAwait(false)) + DeletePath(record.OriginalPath); + if (await db.Renders.GetAsync(record.Render.Id, cancellationToken).ConfigureAwait(false) is not null) + await db.Renders.DeleteAsync(record.Render.Id, CancellationToken.None).ConfigureAwait(false); + Publish(recordPath); + Log.Info("videos", $"An interrupted delete of {record.OriginalPath} completed"); + } + else + { + // The file never reached the trash: what was copied goes, and the render stays as it is. + Undo(target, entryFolder, recordPath); + Log.Info("videos", $"An interrupted delete of {record.OriginalPath} was undone"); + } + settled++; + } + } + return settled; + } + + /// + /// The SHA-256 of every file of an image sequence's folder (paths inside it, forward slashes, in order); the folder's + /// own render records no hash (decision 14). + /// + internal static async Task> HashFolderAsync(string folder, CancellationToken cancellationToken) + { + var files = new List(); + foreach (var file in System.IO.Directory.EnumerateFiles(folder, "*", SearchOption.AllDirectories).Order(StringComparer.Ordinal)) + { + var sha = await ContentHash.Sha256Async(file, cancellationToken: cancellationToken).ConfigureAwait(false); + files.Add(new VideoTrashFile(Path.GetRelativePath(folder, file).Replace('\\', '/'), new FileInfo(file).Length, sha)); + } + return files; + } + + /// Whether the file or folder at holds what says, re-hashed now. + private static async Task MatchesAsync(string path, VideoTrashRecord record, CancellationToken cancellationToken) + { + if (record.Files is { } expected) + { + if (!System.IO.Directory.Exists(path)) return false; + var actual = await HashFolderAsync(path, cancellationToken).ConfigureAwait(false); + return actual.Count == expected.Count && actual.Zip(expected).All(p => + p.First.Path == p.Second.Path && p.First.Size == p.Second.Size && string.Equals(p.First.Sha256, p.Second.Sha256, StringComparison.OrdinalIgnoreCase)); + } + if (!File.Exists(path)) return false; + if (record.Sha256 is null) return true; + var sha = await ContentHash.Sha256Async(path, cancellationToken: cancellationToken).ConfigureAwait(false); + return string.Equals(sha, record.Sha256, StringComparison.OrdinalIgnoreCase); + } + + /// + /// Moves a render's file or folder from to : renamed on one volume; copied + /// under a part name, verified against , renamed and the source deleted across volumes. + /// + private async Task MoveAsync(string from, string to, VideoTrashRecord record, CancellationToken cancellationToken) + { + var folder = System.IO.Directory.Exists(from); + if (!ForceCopy && SameVolume(from, to)) + { + try + { + if (folder) System.IO.Directory.Move(from, to); + else File.Move(from, to, overwrite: false); + return; + } + catch (IOException) when (folder && !System.IO.Directory.Exists(to) && System.IO.Directory.Exists(from)) + { + // Another mount on the same root (Linux): copied instead. + } + } + var part = to + PartSuffix; + DeletePath(part); + if (folder) + { + foreach (var file in System.IO.Directory.EnumerateFiles(from, "*", SearchOption.AllDirectories)) + { + var target = Path.Combine(part, Path.GetRelativePath(from, file)); + System.IO.Directory.CreateDirectory(Path.GetDirectoryName(target)!); + await CopyFileAsync(file, target, cancellationToken).ConfigureAwait(false); + } + System.IO.Directory.CreateDirectory(part); + } + else + { + await CopyFileAsync(from, part, cancellationToken).ConfigureAwait(false); + } + if (!await MatchesAsync(part, record, cancellationToken).ConfigureAwait(false)) + { + DeletePath(part); + throw new IOException(Loc.L("Errors.Videos.TrashMismatch")); + } + if (folder) System.IO.Directory.Move(part, to); + else File.Move(part, to, overwrite: false); + DeletePath(from); + } + + /// Copies a file, flushed to the disk. + internal static async Task CopyFileAsync(string source, string target, CancellationToken cancellationToken) + { + await using var from = new FileStream(source, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 20, FileOptions.Asynchronous | FileOptions.SequentialScan); + await using var to = new FileStream(target, FileMode.Create, FileAccess.Write, FileShare.None, 1 << 20, FileOptions.Asynchronous); + await from.CopyToAsync(to, cancellationToken).ConfigureAwait(false); + to.Flush(flushToDisk: true); + } + + /// Where a restored render goes: where it was (in its sequence's folder as the index has it now), or a free name next to it. + private static string Destination(ArchivePaths paths, VideoTrashRecord record, Sequence sequence) + { + var stored = record.Render.OutputPath; + string full; + if (Path.IsPathRooted(stored)) full = stored; + else + { + var from = record.SequenceFolder; + var rebased = from.Length > 0 && from != sequence.Folder && stored.StartsWith(from + "/", StringComparison.Ordinal) + ? (sequence.Folder.Length == 0 ? stored[(from.Length + 1)..] : sequence.Folder + stored[from.Length..]) + : stored; + full = paths.FromRelative(rebased); + } + return File.Exists(full) || System.IO.Directory.Exists(full) + ? VideosFolder.FreeName(Path.GetDirectoryName(full)!, Path.GetFileName(full), record.Files is not null) + : full; + } + + /// Archive-relative for outputs inside the archive (as renders are recorded), absolute otherwise. + internal static string Stored(ArchivePaths paths, string output) + { + var relative = paths.ToRelative(output); + return relative.StartsWith("..", StringComparison.Ordinal) || Path.IsPathRooted(relative) ? output : relative; + } + + private static bool SameVolume(string a, string b) => + string.Equals(Path.GetPathRoot(Path.GetFullPath(a)), Path.GetPathRoot(Path.GetFullPath(b)), StringComparison.OrdinalIgnoreCase); + + /// Removes what a delete that did not complete left in the trash: the part copy, the entry's folder and its journal. + private static void Undo(string target, string entryFolder, string recordPath) + { + try + { + DeletePath(target + PartSuffix); + RemoveEmpty(entryFolder); + File.Delete(recordPath + PendingSuffix); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warning("videos", $"What an unfinished delete left in {entryFolder} could not be removed; the next start settles it", ex); + } + } + + /// Lists an entry whose delete completed: its record takes its own name. A failure (logged) is settled by the next . + private static void Publish(string recordPath) + { + try + { + File.Move(recordPath + PendingSuffix, recordPath, overwrite: true); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warning("videos", $"Trash record {recordPath} keeps its pending name; the next start lists it", ex); + } + } + + private static void RemoveEmpty(string folder) + { + if (System.IO.Directory.Exists(folder) && !System.IO.Directory.EnumerateFileSystemEntries(folder).Any()) System.IO.Directory.Delete(folder); + } + + private static void DeletePath(string path) + { + if (System.IO.Directory.Exists(path)) SequenceTrash.Erase(path); + else if (File.Exists(path)) File.Delete(path); + } + + /// The folder name of an entry: its name without , never a path. + private static string DiskName(string name) => + Path.GetFileName(name.StartsWith(VideoTrashNames.Prefix, StringComparison.Ordinal) ? name[VideoTrashNames.Prefix.Length..] : name); + + private static TrashEntry Entry(Guid archiveId, string name, VideoTrashRecord record) + { + var stored = record.Render.OutputPath.Replace('\\', '/'); + var cut = stored.LastIndexOf('/'); + return new TrashEntry(archiveId, name, record.SequenceId, record.Name, cut < 0 ? string.Empty : stored[..cut], 0, record.FileCount, record.Bytes, + record.DeletedAt, record.DeletedBy, TrashKind.Videos); + } + + private static VideoTrashRecord? ReadRecord(string path) + { + try + { + return File.Exists(path) ? JsonSerializer.Deserialize(File.ReadAllText(path), VideoTrashJson.Default.VideoTrashRecord) : null; + } + catch (Exception ex) when (ex is JsonException or IOException or UnauthorizedAccessException) + { + Log.Warning("videos", $"Trash record {path} is unreadable", ex); + return null; + } + } +} + +[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + IgnoreReadOnlyProperties = true)] +[JsonSerializable(typeof(VideoTrashRecord))] +internal sealed partial class VideoTrashJson : JsonSerializerContext; diff --git a/src/Chronos.Data/Videos/VideosFolder.cs b/src/Chronos.Data/Videos/VideosFolder.cs new file mode 100644 index 00000000..f2648299 --- /dev/null +++ b/src/Chronos.Data/Videos/VideosFolder.cs @@ -0,0 +1,100 @@ +using System.Globalization; + +namespace Chronos.Data.Videos; + +/// +/// The videos folder (owner review 1, R9, decision 15): a flat folder outside every archive and the workspace that renders +/// are copied into, each under its own name or, when that is taken, "name (2)", "name (3)"… (the export naming rule). A +/// copy is written as .incoming-<render id>.part first and takes its name only once verified. +/// +public static class VideosFolder +{ + /// The prefix of a copy being written; never a name a finished copy takes. + public const string PartPrefix = ".incoming-"; + + /// The suffix of a copy being written. + public const string PartSuffix = ".part"; + + /// + /// Why cannot be the videos folder: when blank, + /// when it is not an absolute path or lies inside (or is) one of + /// (the archive roots, the workspace); null when it can. + /// + public static string? Problem(string? folder, IEnumerable protectedRoots) + { + ArgumentNullException.ThrowIfNull(protectedRoots); + if (string.IsNullOrWhiteSpace(folder)) return VideoErrors.FolderUnset; + string full; + try + { + if (!Path.IsPathFullyQualified(folder)) return VideoErrors.FolderInvalid; + full = Normalize(folder); + } + catch (Exception ex) when (ex is ArgumentException or NotSupportedException or PathTooLongException) + { + return VideoErrors.FolderInvalid; + } + foreach (var root in protectedRoots) + { + if (string.IsNullOrWhiteSpace(root)) continue; + if (IsInside(full, Normalize(root))) return VideoErrors.FolderInvalid; + } + return null; + } + + /// Whether is or lies below it (both full and normalised). + internal static bool IsInside(string path, string root) + { + var comparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; + if (string.Equals(path, root, comparison)) return true; + var prefix = root.EndsWith(Path.DirectorySeparatorChar) ? root : root + Path.DirectorySeparatorChar; + return path.StartsWith(prefix, comparison); + } + + /// The full path without a trailing separator (a drive or file-system root keeps its own). + internal static string Normalize(string path) + { + var full = Path.GetFullPath(path); + var trimmed = full.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + return trimmed.Length == 0 || trimmed.EndsWith(':') ? full : trimmed; + } + + /// + /// The path a copy named takes in : the name itself when free, else + /// "name (2).ext", "name (3).ext"… (a folder's name, an image sequence, has no extension to keep). + /// + /// The copy is an image sequence's folder: its whole name is the stem. + public static string FreeName(string folder, string fileName, bool isFolder = false) + { + ArgumentException.ThrowIfNullOrWhiteSpace(folder); + ArgumentException.ThrowIfNullOrWhiteSpace(fileName); + var name = Path.GetFileName(fileName); + var (stem, extension) = isFolder ? (name, string.Empty) : (Path.GetFileNameWithoutExtension(name), Path.GetExtension(name)); + for (var n = 1; ; n++) + { + var candidate = Path.Combine(folder, (n == 1 ? stem : string.Create(CultureInfo.InvariantCulture, $"{stem} ({n})")) + extension); + if (!File.Exists(candidate) && !Directory.Exists(candidate)) return candidate; + } + } + + /// Where the copy of render is written in until it is verified. + public static string PartPath(string folder, Guid renderId) => Path.Combine(folder, PartPrefix + renderId.ToString("N") + PartSuffix); + + /// + /// The free bytes of the volume of : what says (a host's own measure, + /// tests), else the drive's; null when unknown. + /// + public static long? FreeBytes(string folder, Func? measure = null) + { + if (measure?.Invoke(folder) is { } measured) return measured; + try + { + var root = Path.GetPathRoot(Path.GetFullPath(folder)); + return string.IsNullOrEmpty(root) ? null : new DriveInfo(root).AvailableFreeSpace; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException) + { + return null; + } + } +} diff --git a/src/Chronos.Data/Workspaces/WorkspaceRenderJobHandler.cs b/src/Chronos.Data/Workspaces/WorkspaceRenderJobHandler.cs index 2f9d59ca..f531260a 100644 --- a/src/Chronos.Data/Workspaces/WorkspaceRenderJobHandler.cs +++ b/src/Chronos.Data/Workspaces/WorkspaceRenderJobHandler.cs @@ -62,7 +62,13 @@ internal sealed class WorkspaceRenderJobHandler(ChronosDatabase db, ArchiveServi var exports = project.ExportsDirectory; Directory.CreateDirectory(exports); using var names = new OutputNames(); - var all = RenderJobHandler.Targets(projectSettings.Export, render.PresetName); + // A still saves one frame through the job's preset alone: a batch is a video's. + var all = render.IsStill + ? [new ExportTarget(ExportResolver.Resolve(projectSettings.Export), render.PresetName)] + : RenderJobHandler.Targets(projectSettings.Export, render.PresetName); + var still = render.IsStill + ? SequenceRender.StillIndex(frames, render.StillFrame, render.StillPhoto) ?? throw new InvalidOperationException(Loc.L("Export.Render.NoFrames")) + : -1; var resume = RenderCheckpoint.Parse(context.Checkpoint); var done = resume.Done.ToList(); var ledger = project.ReadRenders(); @@ -90,34 +96,67 @@ internal sealed class WorkspaceRenderJobHandler(ChronosDatabase db, ArchiveServi try { await using var renderer = new SequenceRender(frames, projectSettings, inputs.Analysis, options.Processing, originals: true, companions: null); - outcomes = await renderer.RenderAllAsync( - targets, - (target, setup) => names.Reserve(taken => ExportNaming.OutputPath(projectSettings.Export, exports, SequenceRender.NameContext(sequence, target, setup), - name => taken(name) || ledger.Entries.Any(e => SamePath(project.ExportFile(e.File), name)))), - (index, path) => + // A name the ledger holds is taken too (an output sent home and deleted here keeps its entry). + bool Taken(Func taken, string name) => taken(name) || ledger.Entries.Any(e => SamePath(project.ExportFile(e.File), name)); + ValueTask Starting(int index, string path) + { + writing = path; + return context.CheckpointAsync(new RenderCheckpoint(done, index, path).ToJson(), context.Done, context.Total); + } + async ValueTask Record(int index, ExportTarget target, RenderResult result) + { + // The output is complete: recorded even if the job is being cancelled, so it is never orphaned. + var id = RenderJobHandler.RenderId(context.Job.Id, index); + var entry = await EntryAsync(project, context.Job.Id, id, target, result, versionId, settingsHash).ConfigureAwait(false); + if (render.IsStill) entry = entry with { Fps = 0, DurationSeconds = 0, Kind = RenderKind.Still }; + using (await project.LockAsync(LockWait, CancellationToken.None).ConfigureAwait(false)) { - writing = path; - return context.CheckpointAsync(new RenderCheckpoint(done, index, path).ToJson(), context.Done, context.Total); - }, - async (index, target, result) => + ledger = project.ReadRenders().With(entry); + project.WriteRenders(ledger); + } + done.Add(index); + if (!recorded.Contains(id)) recorded.Add(id); + writing = null; + await context.CheckpointAsync(new RenderCheckpoint(done, null, null).ToJson(), context.Done, context.Total).ConfigureAwait(false); + context.Log(render.IsStill + ? Loc.L("Export.Log.StillSaved", (still + 1).ToString(Loc.Culture), Path.GetFileName(result.OutputPath)) + : Loc.L("Export.Log.Rendered", target.PresetName, Path.GetFileName(result.OutputPath))); + foreach (var remark in SequenceRender.Remarks(result)) context.Log(remark); + } + + if (render.IsStill) + { + var (index, target) = targets[0]; + try { - // The output is complete: recorded even if the job is being cancelled, so it is never orphaned. - var id = RenderJobHandler.RenderId(context.Job.Id, index); - var entry = await EntryAsync(project, context.Job.Id, id, target, result, versionId, settingsHash).ConfigureAwait(false); - using (await project.LockAsync(LockWait, CancellationToken.None).ConfigureAwait(false)) - { - ledger = project.ReadRenders().With(entry); - project.WriteRenders(ledger); - } - done.Add(index); - if (!recorded.Contains(id)) recorded.Add(id); - writing = null; - await context.CheckpointAsync(new RenderCheckpoint(done, null, null).ToJson(), context.Done, context.Total).ConfigureAwait(false); - context.Log(Loc.L("Export.Log.Rendered", target.PresetName, Path.GetFileName(result.OutputPath))); - foreach (var remark in SequenceRender.Remarks(result)) context.Log(remark); - }, - context.ReportAsync, - cancellation).ConfigureAwait(false); + var result = await renderer.RenderStillAsync( + target, + still, + setup => names.Reserve(taken => SequenceRender.StillPath(projectSettings.Export, exports, SequenceRender.NameContext(sequence, target, setup), still, + name => Taken(taken, name))), + path => Starting(index, path), + context.ReportAsync, + cancellation).ConfigureAwait(false); + await Record(index, target, result).ConfigureAwait(false); + outcomes = [new TargetOutcome(target, result, null)]; + } + catch (Exception ex) when (!(ex is OperationCanceledException && cancellation.IsCancellationRequested)) + { + Core.Diagnostics.Log.Warning("workspace", $"The still of frame {still + 1} of {sequence.DisplayTitle} failed", ex); + outcomes = [new TargetOutcome(target, null, SequenceRender.Reason(ex))]; + } + } + else + { + outcomes = await renderer.RenderAllAsync( + targets, + (target, setup) => names.Reserve(taken => ExportNaming.OutputPath(projectSettings.Export, exports, SequenceRender.NameContext(sequence, target, setup), + name => Taken(taken, name))), + Starting, + Record, + context.ReportAsync, + cancellation).ConfigureAwait(false); + } if (renderer.StandIns.Count > 0) context.Log(Loc.L("Export.Log.StandIns", string.Join(", ", renderer.StandIns.Select(i => (i + 1).ToString(Loc.Culture))))); } diff --git a/src/Chronos.Data/Workspaces/WorkspaceRenders.cs b/src/Chronos.Data/Workspaces/WorkspaceRenders.cs index 88d0dc5e..fe37969b 100644 --- a/src/Chronos.Data/Workspaces/WorkspaceRenders.cs +++ b/src/Chronos.Data/Workspaces/WorkspaceRenders.cs @@ -25,6 +25,7 @@ namespace Chronos.Data.Workspaces; /// , or (a later Chronos may write others: read as pending). /// When the home took it; null until then. /// Why the last send failed (one localized sentence); null otherwise. +/// The render's kind (): still for a saved frame, null for a video or an image sequence. public sealed record WorkspaceRenderEntry( Guid RenderId, Guid JobId, @@ -43,7 +44,8 @@ public sealed record WorkspaceRenderEntry( DateTimeOffset Rendered, string State = WorkspaceRenderEntry.Pending, DateTimeOffset? Sent = null, - string? Error = null) + string? Error = null, + string? Kind = null) { /// Written, not at the home yet. public const string Pending = "pending"; diff --git a/src/Chronos.Data/Workspaces/WorkspaceSendJobHandler.cs b/src/Chronos.Data/Workspaces/WorkspaceSendJobHandler.cs index 99c16f80..d3085371 100644 --- a/src/Chronos.Data/Workspaces/WorkspaceSendJobHandler.cs +++ b/src/Chronos.Data/Workspaces/WorkspaceSendJobHandler.cs @@ -286,6 +286,9 @@ internal sealed class WorkspaceSendJobHandler(ChronosDatabase db, ArchiveService ProducedBy = options.Processing.ProducedBy, ProjectId = entry.ProjectId, ProjectVersionId = entry.ProjectVersionId, + // A file's hash is the file's own (an image sequence's ledger hash is of its listing, not a render's). + Sha256 = folder ? null : entry.Sha256.ToLowerInvariant(), + Kind = entry.Kind, }, ct).ConfigureAwait(false); return filed is not null ? true : throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved")); } @@ -329,7 +332,7 @@ internal sealed class WorkspaceSendJobHandler(ChronosDatabase db, ArchiveService { var name = SyncTransfers.SafeFileName(Path.GetFileName(entry.File)) ?? entry.RenderId.ToString("N"); var request = new RenderUploadRequest(entry.RenderId, name, entry.Size, entry.Sha256.ToLowerInvariant(), entry.PresetName, entry.Codec, entry.Width, entry.Height, - entry.Fps, entry.DurationSeconds, options.Processing.ProducedBy, entry.ProjectId, entry.ProjectVersionId); + entry.Fps, entry.DurationSeconds, options.Processing.ProducedBy, entry.ProjectId, entry.ProjectVersionId, entry.Kind); try { if (await remote.UploadRenderAsync(sequenceId, file, request, ct).ConfigureAwait(false) is not { } filed) return false; diff --git a/tests/Chronos.Data.Tests/Videos/VideoCopyTests.cs b/tests/Chronos.Data.Tests/Videos/VideoCopyTests.cs new file mode 100644 index 00000000..e2764b6a --- /dev/null +++ b/tests/Chronos.Data.Tests/Videos/VideoCopyTests.cs @@ -0,0 +1,129 @@ +using System.Security.Cryptography; +using Chronos.Core.Localization; +using Chronos.Core.Model; +using Chronos.Data.Videos; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Videos; + +/// Owner review 1, R9, decision 16: a VideoCopy job copies renders into the videos folder, verified, never overwriting. +[Category("videos")] +public sealed class VideoCopyTests(TestContext context) +{ + [Test] + public async Task ACopyIsVerifiedAndRecordedAndATakenNameGetsANumber() + { + await using var fx = await VideoFixture.CreateAsync(context); + var sequence = await fx.SequenceAsync(); + var render = await fx.RenderAsync(sequence, "sunset.mp4", 5000); + var old = await fx.RenderAsync(sequence, "old.mp4", 3000, hashed: false); + await File.WriteAllTextAsync(Path.Combine(fx.VideosFolder, "sunset.mp4"), "someone else's", fx.Ct); + await fx.StartEngineAsync(); + + var request = fx.Library.CopyJob(new VideoCopyParameters([render.Id, old.Id, render.Id], fx.VideosFolder, RequestedBy: "alberto")); + Assert.Equal((JobKind.VideoCopy, 2), (request.Kind, request.Targets.Count)); + var job = await fx.EnqueueAsync(request); + await fx.Archive.WaitForAsync(job.Id, JobState.Done); + + var copy = Path.Combine(fx.VideosFolder, "sunset (2).mp4"); + Assert.SequenceEqual(await File.ReadAllBytesAsync(fx.FileOf(render), fx.Ct), await File.ReadAllBytesAsync(copy, fx.Ct)); + Assert.Equal("someone else's", await File.ReadAllTextAsync(Path.Combine(fx.VideosFolder, "sunset.mp4"), fx.Ct), "never overwritten"); + Assert.True(File.Exists(Path.Combine(fx.VideosFolder, "old.mp4"))); + Assert.Empty(Directory.EnumerateFileSystemEntries(fx.VideosFolder, ".incoming-*")); + + var copies = await fx.Db.RenderCopies.ListByRenderAsync(render.Id, fx.Ct); + Assert.Equal((copy, render.Sha256, 5000L, "alberto"), (copies.Single().Path, copies.Single().Sha256, copies.Single().Bytes, copies.Single().CopiedBy)); + // A render of before migration 0010 is hashed by its first copy, and remembered. + var hashed = (await fx.Db.Renders.GetAsync(old.Id, fx.Ct))!.Sha256; + Assert.Equal(Convert.ToHexStringLower(SHA256.HashData(await File.ReadAllBytesAsync(fx.FileOf(old), fx.Ct))), hashed); + Assert.Equal(hashed, (await fx.Db.RenderCopies.ListByRenderAsync(old.Id, fx.Ct)).Single().Sha256); + Assert.Equal(1, (await fx.Library.GetAsync(render.Id, fx.Ct))!.Copies.Count); + } + + [Test] + public async Task ACopyThatDoesNotMatchIsRemovedAndTheRenderFails() + { + await using var fx = await VideoFixture.CreateAsync(context); + var sequence = await fx.SequenceAsync(); + var good = await fx.RenderAsync(sequence, "good.mp4", 2000); + var bad = await fx.RenderAsync(sequence, "bad.mp4", 2000); + // The file changed since it was recorded: its copy cannot match the render's hash. + await File.WriteAllBytesAsync(fx.FileOf(bad), VideoFixture.Content("other", 2000), fx.Ct); + await fx.StartEngineAsync(); + + var job = await fx.EnqueueAsync(fx.Library.CopyJob(new VideoCopyParameters([bad.Id, good.Id], fx.VideosFolder))); + var failed = await fx.Archive.WaitForAsync(job.Id, JobState.Failed); + Assert.Equal(Loc.L("Errors.Render.Failed", 1, 2), failed.Error); + Assert.SequenceEqual(["good.mp4"], Directory.GetFileSystemEntries(fx.VideosFolder).Select(Path.GetFileName)); + Assert.Empty(await fx.Db.RenderCopies.ListByRenderAsync(bad.Id, fx.Ct)); + Assert.True(File.Exists(fx.FileOf(bad)), "the render itself is never touched"); + } + + [Test] + public async Task ACopyThatCannotFitFailsBeforeWritingAnything() + { + await using var fx = await VideoFixture.CreateAsync(context, freeSpace: 6000); + var sequence = await fx.SequenceAsync(); + var first = await fx.RenderAsync(sequence, "first.mp4", 4000); + var second = await fx.RenderAsync(sequence, "second.mp4", 4000); + await fx.StartEngineAsync(); + + var job = await fx.EnqueueAsync(fx.Library.CopyJob(new VideoCopyParameters([first.Id, second.Id], fx.VideosFolder))); + var failed = await fx.Archive.WaitForAsync(job.Id, JobState.Failed); + Assert.Contains(Loc.L("Errors.Videos.NoSpace", Chronos.Data.Housekeeping.CleanupSteps.Megabytes(2000)), failed.Error!); + Assert.Empty(Directory.EnumerateFileSystemEntries(fx.VideosFolder)); + Assert.Empty(await fx.Db.RenderCopies.ListByRenderAsync(first.Id, fx.Ct)); + } + + [Test] + public async Task AVideosFolderInsideTheArchiveIsRefused() + { + await using var fx = await VideoFixture.CreateAsync(context); + var sequence = await fx.SequenceAsync(); + var render = await fx.RenderAsync(sequence, "sunset.mp4", 1000); + var inside = Path.Combine(fx.Archive.Paths.Root, "Videos"); + + Assert.Equal(VideoErrors.FolderInvalid, fx.Library.FolderProblem(inside)); + Assert.Equal(VideoErrors.FolderInvalid, fx.Library.FolderProblem(fx.Archive.Paths.Root)); + Assert.Equal(VideoErrors.FolderUnset, fx.Library.FolderProblem(" ")); + Assert.Equal(VideoErrors.FolderInvalid, fx.Library.FolderProblem("relative/videos")); + Assert.Null(fx.Library.FolderProblem(fx.VideosFolder)); + var refused = Assert.Throws(() => fx.Library.CopyJob(new VideoCopyParameters([render.Id], inside))); + Assert.Equal((VideoErrors.FolderInvalid, Loc.L("Errors.Videos.FolderInsideArchive")), (refused.Code, refused.Message)); + + // A workspace the host protects is refused too, and so is a folder that became one after the job was queued. + var workspace = context.CreateTempDirectory("workspace"); + fx.Library.ProtectedFolders = () => [workspace]; + Assert.Equal(VideoErrors.FolderInvalid, fx.Library.FolderProblem(Path.Combine(workspace, "out"))); + fx.Library.ProtectedFolders = null; + var request = fx.Library.CopyJob(new VideoCopyParameters([render.Id], Path.Combine(workspace, "out"))); + fx.Library.ProtectedFolders = () => [workspace]; + await fx.StartEngineAsync(); + var failed = await fx.Archive.WaitForAsync((await fx.EnqueueAsync(request)).Id, JobState.Failed); + Assert.Equal(Loc.L("Errors.Videos.FolderInsideArchive"), failed.Error); + Assert.False(Directory.Exists(Path.Combine(workspace, "out"))); + } + + [Test] + public async Task AnImageSequenceIsCopiedAndVerifiedFileByFile() + { + await using var fx = await VideoFixture.CreateAsync(context); + var sequence = await fx.SequenceAsync(); + var pngs = await fx.SequenceRenderAsync(sequence, "clip-PNG", 5); + Directory.CreateDirectory(Path.Combine(fx.VideosFolder, "clip-PNG")); + await fx.StartEngineAsync(); + + var job = await fx.EnqueueAsync(fx.Library.CopyJob(new VideoCopyParameters([pngs.Id], fx.VideosFolder))); + await fx.Archive.WaitForAsync(job.Id, JobState.Done); + + var copy = Path.Combine(fx.VideosFolder, "clip-PNG (2)"); + var sources = Directory.GetFiles(fx.FileOf(pngs)).Order(StringComparer.Ordinal).ToList(); + var copied = Directory.GetFiles(copy).Order(StringComparer.Ordinal).ToList(); + Assert.SequenceEqual(sources.Select(Path.GetFileName), copied.Select(Path.GetFileName)); + for (var i = 0; i < sources.Count; i++) + Assert.SequenceEqual(await File.ReadAllBytesAsync(sources[i], fx.Ct), await File.ReadAllBytesAsync(copied[i], fx.Ct)); + var record = (await fx.Db.RenderCopies.ListByRenderAsync(pngs.Id, fx.Ct)).Single(); + Assert.Equal((copy, (string?)null, pngs.Bytes), (record.Path, record.Sha256, record.Bytes)); + Assert.Null((await fx.Db.Renders.GetAsync(pngs.Id, fx.Ct))!.Sha256, "a folder records no hash of its own"); + } +} diff --git a/tests/Chronos.Data.Tests/Videos/VideoFixture.cs b/tests/Chronos.Data.Tests/Videos/VideoFixture.cs new file mode 100644 index 00000000..56507837 --- /dev/null +++ b/tests/Chronos.Data.Tests/Videos/VideoFixture.cs @@ -0,0 +1,138 @@ +using System.Security.Cryptography; +using Chronos.Core.Codecs.Jpeg; +using Chronos.Core.Diagnostics.Synthetic; +using Chronos.Core.Metadata; +using Chronos.Core.Model; +using Chronos.Data.Archiving; +using Chronos.Data.Tests.Archiving; +using Chronos.Data.Videos; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Videos; + +/// A temporary archive with small synthetic sequences and renders recorded on files written by hand. +internal sealed class VideoFixture : IAsyncDisposable +{ + private VideoFixture(TestContext context, ArchiveFixture archive) + { + Archive = archive; + VideosFolder = context.CreateTempDirectory("videos"); + } + + public ArchiveFixture Archive { get; } + public ChronosDatabase Db => Archive.Db; + public VideoLibrary Library => Archive.Service.Videos; + public CancellationToken Ct => Archive.Ct; + + /// A folder outside the archive that copies go to. + public string VideosFolder { get; } + + /// How much space the videos folder has (bytes); null: the drive's. + public static async Task CreateAsync(TestContext context, long? freeSpace = null) + { + string? videos = null; + var archive = await ArchiveFixture.CreateAsync(context, configure: o => o with + { + FreeSpace = path => freeSpace is { } free && videos is not null && path.StartsWith(videos, StringComparison.OrdinalIgnoreCase) ? free : null, + }); + var fixture = new VideoFixture(context, archive); + videos = fixture.VideosFolder; + return fixture; + } + + /// A sequence of four synthetic frames in , indexed. + public async Task SequenceAsync(string folder = "Harbour", int seed = 1) + { + var synthetic = SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 4, Seed = seed, Start = new DateTime(2024, 6, 21, 19, 0, 0).AddDays(seed) }); + var directory = Path.Combine(Archive.Paths.Root, folder); + Directory.CreateDirectory(directory); + for (var i = 0; i < synthetic.Count; i++) + { + using var image = synthetic.Render(i); + var exif = ExifWriter.BuildJpegApp1(synthetic.Metadata[i]); + File.WriteAllBytes(Path.Combine(directory, $"IMG_{i + 1:0000}.JPG"), JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 90, App1Segments = [exif[4..]] })); + } + await Archive.ScanAsync(Chronos.Data.Archiving.ScanMode.Full); + var location = (await Db.Sequences.ListLocationsAsync(Archive.Archive.Id, Ct)).Single(l => l.Folder == folder); + return (await Db.Sequences.GetAsync(location.Id, Ct))!; + } + + /// A render of recorded on a file of written into its exports. + public async Task RenderAsync(Sequence sequence, string name, int bytes, DateTimeOffset? created = null, string codec = "H264", string? kind = null, + bool hashed = true) + { + var exports = Archive.Paths.Sequence(sequence.Folder).Exports; + Directory.CreateDirectory(exports); + var content = Content(name, bytes); + await File.WriteAllBytesAsync(Path.Combine(exports, name), content, Ct); + var render = new Render + { + Id = Guid.CreateVersion7(), + SequenceId = sequence.Id, + PresetName = "Test preset", + OutputPath = $"{sequence.Folder}/exports/{name}", + Codec = codec, + Width = 96, + Height = 64, + Fps = kind == RenderKind.Still ? 0 : 24, + DurationSeconds = kind == RenderKind.Still ? 0 : 2, + Bytes = bytes, + Created = created ?? DateTimeOffset.UtcNow, + ProducedBy = "test-pc", + Sha256 = hashed ? Convert.ToHexStringLower(SHA256.HashData(content)) : null, + Kind = kind, + }; + await Db.Renders.InsertAsync(render, Ct); + return render; + } + + /// An image-sequence render: a folder of PNG-named files. + public async Task SequenceRenderAsync(Sequence sequence, string name, int files) + { + var folder = Path.Combine(Archive.Paths.Sequence(sequence.Folder).Exports, name); + Directory.CreateDirectory(folder); + long total = 0; + for (var i = 0; i < files; i++) + { + var content = Content(name + i, 300 + i); + total += content.Length; + await File.WriteAllBytesAsync(Path.Combine(folder, $"{name}_{i + 1:000000}.png"), content, Ct); + } + var render = new Render + { + Id = Guid.CreateVersion7(), + SequenceId = sequence.Id, + PresetName = "PNG", + OutputPath = $"{sequence.Folder}/exports/{name}", + Codec = "PngSequence", + Width = 96, + Height = 64, + Fps = 24, + Bytes = total, + Created = DateTimeOffset.UtcNow, + }; + await Db.Renders.InsertAsync(render, Ct); + return render; + } + + public string FileOf(Render render) => Archive.Paths.FromRelative(render.OutputPath); + + private Chronos.Data.Jobs.JobEngine? _engine; + + /// Starts the archive's job engine (its handlers include the videos copy). + public async Task StartEngineAsync() => _engine = await Archive.StartEngineAsync(); + + public Task EnqueueAsync(Chronos.Data.Archiving.Jobs.ArchiveJobRequest request) => + (_engine ?? throw new InvalidOperationException("Start the engine first.")).EnqueueAsync(request, cancellationToken: Ct); + + /// Deterministic bytes, different for each name. + public static byte[] Content(string name, int bytes) + { + var content = new byte[bytes]; + new Random(StringComparer.Ordinal.GetHashCode(name) & 0x7FFFFFFF).NextBytes(content); + content[0] = (byte)name.Length; + return content; + } + + public ValueTask DisposeAsync() => Archive.DisposeAsync(); +} diff --git a/tests/Chronos.Data.Tests/Videos/VideoLibraryTests.cs b/tests/Chronos.Data.Tests/Videos/VideoLibraryTests.cs new file mode 100644 index 00000000..6868eefe --- /dev/null +++ b/tests/Chronos.Data.Tests/Videos/VideoLibraryTests.cs @@ -0,0 +1,211 @@ +using Chronos.Core.Model; +using Chronos.Data.Videos; +using Chronos.Protocol; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Videos; + +/// Owner review 1, R9: the videos library lists every render, deletes into the trash after a dry run and restores by id. +[Category("videos")] +public sealed class VideoLibraryTests(TestContext context) +{ + private static readonly DateTimeOffset Day = new(2026, 10, 1, 12, 0, 0, TimeSpan.Zero); + + [Test] + public async Task TheListPagesNewestFirstAndFilters() + { + await using var fx = await VideoFixture.CreateAsync(context); + var harbour = await fx.SequenceAsync("Harbour", 1); + var roof = await fx.SequenceAsync("Roof", 2); + var renders = new List(); + for (var i = 0; i < 5; i++) renders.Add(await fx.RenderAsync(harbour, $"harbour-{i}.mp4", 1000 + (i * 100), Day.AddHours(i))); + var still = await fx.RenderAsync(roof, "roof - 0003.png", 5000, Day.AddDays(-400), codec: "PngSequence", kind: RenderKind.Still); + var gif = await fx.RenderAsync(roof, "roof.gif", 700, Day.AddMinutes(30), codec: "Gif"); + var pngs = await fx.SequenceRenderAsync(roof, "roof-PNG", 3); + + // Newest first, two at a time, with the total. + var seen = new List(); + string? cursor = null; + do + { + var page = await fx.Library.ListAsync(new VideoQuery { Limit = 3, Cursor = cursor }, fx.Ct); + Assert.Equal(8L, page.Total); + Assert.True(page.Items.Count <= 3); + seen.AddRange(page.Items.Select(i => i.Render.Id)); + cursor = page.Next; + } + while (cursor is not null); + var all = renders.Append(still).Append(gif).Append(pngs).OrderByDescending(r => r.Created).Select(r => r.Id).ToList(); + Assert.SequenceEqual(all, seen); + + async Task> Ids(VideoQuery query) => [.. (await fx.Library.ListAsync(query, fx.Ct)).Items.Select(i => i.Render.Id)]; + Assert.SequenceEqual([still.Id], await Ids(new VideoQuery { Kind = RenderKind.Still })); + Assert.SequenceEqual([pngs.Id], await Ids(new VideoQuery { Kind = RenderKind.Sequence })); + Assert.Equal(6, (await Ids(new VideoQuery { Kind = RenderKind.Video })).Count); + Assert.SequenceEqual([gif.Id], await Ids(new VideoQuery { Codec = "gif" })); + Assert.SequenceEqual([still.Id], await Ids(new VideoQuery { Year = Day.AddDays(-400).Year })); + Assert.Equal(3, (await Ids(new VideoQuery { SequenceId = roof.Id })).Count); + Assert.SequenceEqual([renders[3].Id], await Ids(new VideoQuery { Text = "harbour-3" })); + Assert.SequenceEqual([still.Id, renders[4].Id], (await Ids(new VideoQuery { Sort = VideoSorts.Largest })).Take(2)); + Assert.SequenceEqual([still.Id], (await Ids(new VideoQuery { Sort = VideoSorts.Oldest })).Take(1)); + + // The title sort pages too. + var titled = new List(); + cursor = null; + do + { + var page = await fx.Library.ListAsync(new VideoQuery { Sort = VideoSorts.Title, Limit = 3, Cursor = cursor }, fx.Ct); + titled.AddRange(page.Items.Select(i => i.Render.Id)); + cursor = page.Next; + } + while (cursor is not null); + Assert.Equal(8, titled.Distinct().Count()); + + var item = (await fx.Library.GetAsync(still.Id, fx.Ct))!; + Assert.Equal((RenderKind.Still, false, fx.FileOf(still), "roof - 0003.png"), (item.Kind, item.Missing, item.FullPath, item.FileName)); + File.Delete(fx.FileOf(gif)); + Assert.True((await fx.Library.GetAsync(gif.Id, fx.Ct))!.Missing); + } + + [Test] + public async Task TheDryRunFingerprintChangesWhenTheVideosDo() + { + await using var fx = await VideoFixture.CreateAsync(context); + var sequence = await fx.SequenceAsync(); + var first = await fx.RenderAsync(sequence, "a.mp4", 1200); + var later = Guid.CreateVersion7(); + + var plan = await fx.Library.PlanDeleteAsync([first.Id, later], fx.Ct); + Assert.SequenceEqual([first.Id], plan.Items.Select(i => i.Render.Id)); + Assert.Equal((later, VideoBlockReasons.NotFound), (plan.Blocked.Single().RenderId, plan.Blocked.Single().Reason)); + Assert.Equal(plan.Fingerprint, (await fx.Library.PlanDeleteAsync([later, first.Id], fx.Ct)).Fingerprint, "the order asked does not matter"); + + // A render added since: the plan given back no longer holds, and nothing is deleted. + await fx.Db.Renders.InsertAsync(new Render { Id = later, SequenceId = sequence.Id, PresetName = "p", OutputPath = "Harbour/exports/b.mp4", Created = DateTimeOffset.UtcNow }, fx.Ct); + var replanned = await fx.Library.PlanDeleteAsync([first.Id, later], fx.Ct); + Assert.NotEqual(plan.Fingerprint, replanned.Fingerprint); + var refused = await Assert.ThrowsAsync(() => fx.Library.DeleteAsync([first.Id, later], plan.Fingerprint, "me", fx.Ct)); + Assert.Equal(VideoErrors.PlanChanged, refused.Code); + Assert.True(File.Exists(fx.FileOf(first))); + Assert.NotNull(await fx.Db.Renders.GetAsync(first.Id, fx.Ct)); + } + + [Test] + public async Task ADeleteMovesTheFileAndTheRowIntoTheTrashAndARestoreBringsBothBack() + { + await using var fx = await VideoFixture.CreateAsync(context); + var sequence = await fx.SequenceAsync(); + var kept = await fx.RenderAsync(sequence, "kept.mp4", 900); + var render = await fx.RenderAsync(sequence, "sunset.mp4", 4096, hashed: false); + Assert.Equal(2, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.RenderedCount); + + var plan = await fx.Library.PlanDeleteAsync([render.Id], fx.Ct); + Assert.Equal(4096L, plan.Bytes); + var result = await fx.Library.DeleteAsync([render.Id], plan.Fingerprint, "alberto", fx.Ct); + var name = result.TrashEntries.Single(); + Assert.True(VideoTrashNames.IsEntryName(name)); + Assert.False(File.Exists(fx.FileOf(render))); + Assert.Null(await fx.Db.Renders.GetAsync(render.Id, fx.Ct)); + Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.RenderedCount); + Assert.NotNull(await fx.Db.Renders.GetAsync(kept.Id, fx.Ct)); + // The record is published (no journal left) and the trash lists the entry. + var (folder, record) = VideoTrash.Location(fx.Archive.Paths, name); + Assert.True(File.Exists(record)); + Assert.False(File.Exists(record + VideoTrash.PendingSuffix)); + Assert.True(File.Exists(Path.Combine(folder, "sunset.mp4"))); + var entry = (await fx.Archive.Service.Trash.ListAsync(fx.Archive.Archive, fx.Ct)).Single(e => e.Name == name); + Assert.Equal((Chronos.Data.Archiving.TrashKind.Videos, 4096L, "alberto"), (entry.Kind, entry.Bytes, entry.DeletedBy)); + + var restored = await fx.Library.RestoreAsync(fx.Archive.Archive.Id, name, fx.Ct); + Assert.Equal(render.Id, restored.Id); + Assert.Equal(render.OutputPath, restored.OutputPath); + Assert.NotNull(restored.Sha256); + Assert.True(File.Exists(fx.FileOf(render))); + Assert.Equal(restored.Sha256, (await fx.Db.Renders.GetAsync(render.Id, fx.Ct))!.Sha256); + Assert.Equal(2, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.RenderedCount); + Assert.False(File.Exists(record)); + Assert.False(Directory.Exists(folder)); + } + + [Test] + public async Task ATamperedTrashedVideoIsNotRestored() + { + await using var fx = await VideoFixture.CreateAsync(context); + var sequence = await fx.SequenceAsync(); + var render = await fx.RenderAsync(sequence, "sunset.mp4", 2048); + var plan = await fx.Library.PlanDeleteAsync([render.Id], fx.Ct); + var name = (await fx.Library.DeleteAsync([render.Id], plan.Fingerprint, null, fx.Ct)).TrashEntries.Single(); + var trashed = Path.Combine(VideoTrash.Location(fx.Archive.Paths, name).Folder, "sunset.mp4"); + var bytes = await File.ReadAllBytesAsync(trashed, fx.Ct); + bytes[100] ^= 0xFF; + await File.WriteAllBytesAsync(trashed, bytes, fx.Ct); + + var refused = await Assert.ThrowsAsync(() => fx.Library.RestoreAsync(fx.Archive.Archive.Id, name, fx.Ct)); + Assert.Equal(VideoErrors.HashMismatch, refused.Code); + Assert.True(File.Exists(trashed), "it stays in the trash"); + Assert.False(File.Exists(fx.FileOf(render))); + Assert.Null(await fx.Db.Renders.GetAsync(render.Id, fx.Ct)); + } + + [Test] + public async Task ADeleteACrashInterruptedIsCompletedWhenTheArchiveOpens() + { + await using var fx = await VideoFixture.CreateAsync(context); + var sequence = await fx.SequenceAsync(); + var render = await fx.RenderAsync(sequence, "sunset.mp4", 3000); + fx.Library.Trash.CrashAfterMove = true; + fx.Library.Trash.ForceCopy = true; + var plan = await fx.Library.PlanDeleteAsync([render.Id], fx.Ct); + await Assert.ThrowsAsync(() => fx.Library.DeleteAsync([render.Id], plan.Fingerprint, null, fx.Ct)); + fx.Library.Trash.CrashAfterMove = false; + + // The journal is there, the row still is, and the trash does not list a delete under way. + var trash = VideoTrash.Directory(fx.Archive.Paths); + var pending = Directory.GetFiles(trash, "*" + VideoTrash.PendingSuffix).Single(); + Assert.NotNull(await fx.Db.Renders.GetAsync(render.Id, fx.Ct)); + Assert.Empty((await fx.Archive.Service.Trash.ListAsync(fx.Archive.Archive, fx.Ct)).Where(e => e.Kind == Chronos.Data.Archiving.TrashKind.Videos)); + + Assert.Equal(1, await fx.Library.RecoverAsync(fx.Archive.Archive, fx.Ct)); + Assert.False(File.Exists(pending)); + Assert.Null(await fx.Db.Renders.GetAsync(render.Id, fx.Ct)); + Assert.False(File.Exists(fx.FileOf(render))); + Assert.Equal(0, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.RenderedCount); + var entry = (await fx.Archive.Service.Trash.ListAsync(fx.Archive.Archive, fx.Ct)).Single(e => e.Kind == Chronos.Data.Archiving.TrashKind.Videos); + Assert.Equal(render.Id, (await fx.Library.RestoreAsync(fx.Archive.Archive.Id, entry.Name, fx.Ct)).Id); + } + + [Test] + public async Task AVideoWhoseFileIsMissingLosesOnlyItsRow() + { + await using var fx = await VideoFixture.CreateAsync(context); + var sequence = await fx.SequenceAsync(); + var render = await fx.RenderAsync(sequence, "gone.mp4", 1500); + File.Delete(fx.FileOf(render)); + + var plan = await fx.Library.PlanDeleteAsync([render.Id], fx.Ct); + Assert.True(plan.Items.Single().Missing); + Assert.Equal(0L, plan.Bytes); + var result = await fx.Library.DeleteAsync([render.Id], plan.Fingerprint, null, fx.Ct); + Assert.SequenceEqual([render.Id], result.Deleted.Select(d => d.Render.Id)); + Assert.Empty(result.TrashEntries); + Assert.Null(await fx.Db.Renders.GetAsync(render.Id, fx.Ct)); + Assert.False(Directory.Exists(VideoTrash.Directory(fx.Archive.Paths)) && Directory.EnumerateFileSystemEntries(VideoTrash.Directory(fx.Archive.Paths)).Any()); + } + + [Test] + public async Task AnImageSequenceFolderGoesToTheTrashAndBackFileByFile() + { + await using var fx = await VideoFixture.CreateAsync(context); + var sequence = await fx.SequenceAsync(); + var pngs = await fx.SequenceRenderAsync(sequence, "clip-PNG", 4); + var plan = await fx.Library.PlanDeleteAsync([pngs.Id], fx.Ct); + var name = (await fx.Library.DeleteAsync([pngs.Id], plan.Fingerprint, null, fx.Ct)).TrashEntries.Single(); + Assert.False(Directory.Exists(fx.FileOf(pngs))); + var entry = (await fx.Archive.Service.Trash.ListAsync(fx.Archive.Archive, fx.Ct)).Single(e => e.Name == name); + Assert.Equal(4, entry.Files); + + var restored = await fx.Library.RestoreAsync(fx.Archive.Archive.Id, name, fx.Ct); + Assert.Equal((pngs.Id, (string?)null), (restored.Id, restored.Sha256)); + Assert.Equal(4, Directory.GetFiles(fx.FileOf(pngs)).Length); + } +} diff --git a/tests/Chronos.Data.Tests/Videos/VideoRenderTests.cs b/tests/Chronos.Data.Tests/Videos/VideoRenderTests.cs new file mode 100644 index 00000000..7335a21b --- /dev/null +++ b/tests/Chronos.Data.Tests/Videos/VideoRenderTests.cs @@ -0,0 +1,136 @@ +using System.Security.Cryptography; +using Chronos.Core.Diagnostics.Synthetic; +using Chronos.Core.Export; +using Chronos.Core.Imaging; +using Chronos.Core.Imaging.Decoders; +using Chronos.Core.Model; +using Chronos.Core.Project; +using Chronos.Core.Render; +using Chronos.Core.Video; +using Chronos.Data.Processing; +using Chronos.Data.Tests.Processing; +using Chronos.Testing; + +namespace Chronos.Data.Tests.Videos; + +/// Owner review 1, phase 5: renders record their hash and kind, a still job saves one frame, playback routes as format 3. +[Category("videos")] +public sealed class VideoRenderTests(TestContext context) +{ + private const int Frames = 8; + + private static readonly ExportSettings Mjpeg = new() + { + Preset = new Auto().WithManual("mjpeg-avi"), + Overrides = new ExportOverrides { Width = 96, Height = 64 }, + NameTemplate = "clip-{codec}", + }; + + private static readonly PlaybackSettings Boomerang = new() { Direction = PlaybackDirection.Boomerang, Loops = 2 }; + + private static SyntheticSequence Scene() => SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = Frames, StartEv = 0.5, EndEv = -0.5 }); + + [Test] + public void ABatchPresetKeepsThePlaybackOfItsExport() + { + var export = Mjpeg with { Playback = Boomerang, Batch = ["gif-preview"] }; + Assert.Equal(Boomerang, ProcessingJobs.BatchExport(export, "gif-preview").Playback); + } + + [Test] + public void APlaybackJobNeedsFormatThreeAndANodeOfFormatTwoNeverGetsIt() + { + var sequence = new Sequence { Id = Guid.CreateVersion7(), ArchiveId = Guid.CreateVersion7(), Folder = "Synthetic", Title = "Roof" }; + static Chronos.Protocol.RenderJobParameters Read(Chronos.Data.Archiving.Jobs.ArchiveJobRequest request) => ProcessingJobs.ReadRender(request.ParametersJson).Parameters; + + Assert.Equal(1, Read(ProcessingJobs.Render(sequence, Mjpeg, 96, 64, new ProjectSettings())).RequiredSettingsFormat); + Assert.Equal(ProjectSettings.PlaybackVersion, Read(ProcessingJobs.Render(sequence, Mjpeg with { Playback = Boomerang }, 96, 64, new ProjectSettings())).RequiredSettingsFormat); + Assert.Equal(ProjectSettings.PlaybackVersion, Read(ProcessingJobs.Render(sequence, Mjpeg with { Playback = Boomerang }, 96, 64)).RequiredSettingsFormat, + "without settings too"); + // Only a batch preset plays back: the job of that preset needs format 3, the plain one does not. + var split = ProcessingJobs.Split(Read(ProcessingJobs.Render(sequence, Mjpeg with { Batch = ["gif-preview"] }, 96, 64, new ProjectSettings())), 96, 64, new ProjectSettings()); + Assert.SequenceEqual([1, 1], split.Select(p => p.RequiredSettingsFormat)); + // Routed with settings whose own export plays back, the job's export (which does not) is what counts, and the other way round. + var plain = Read(ProcessingJobs.Render(sequence, Mjpeg, 96, 64, new ProjectSettings())); + var playing = new ProjectSettings { Export = Mjpeg with { Playback = Boomerang } }; + Assert.Equal(1, ProcessingJobs.Routed(plain, playing, 96, 64).RequiredSettingsFormat); + var posted = Read(ProcessingJobs.Render(sequence, Mjpeg with { Playback = Boomerang }, 96, 64, new ProjectSettings())) with { RequiredSettingsFormat = 1 }; + Assert.Equal(ProjectSettings.PlaybackVersion, ProcessingJobs.Routed(posted, new ProjectSettings(), 96, 64).RequiredSettingsFormat); + Assert.Equal(ProjectSettings.PlaybackVersion, ProcessingJobs.RequiredFormat(new ProjectSettings(), Mjpeg with { Batch = ["gif-preview"], Playback = Boomerang }, 96, 64)); + // A still needs format 3 whatever it renders. + var still = Read(ProcessingJobs.Still(sequence, Mjpeg, VideoCodec.PngSequence, 2, null, 96, 64, new ProjectSettings())); + Assert.Equal((RenderRules.StillFormat, (int?)2, true), (still.RequiredSettingsFormat, still.StillFrame, still.IsStill)); + } + + [Test] + public async Task AVideoRecordsItsHashAndNoKind() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(Scene()); + await fx.StartEngineAsync(); + + await fx.WaitForAsync((await fx.EnqueueAsync(ProcessingJobs.Render(sequence, Mjpeg with { Batch = ["png-sequence"] }, 96, 64))).Id, JobState.Done); + + var renders = (await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct)).OrderBy(r => r.OutputPath, StringComparer.Ordinal).ToList(); + var avi = Path.Combine(fx.Folder(sequence).Exports, "clip-MJPEG.avi"); + Assert.Equal(Convert.ToHexStringLower(SHA256.HashData(await File.ReadAllBytesAsync(avi, fx.Ct))), renders[0].Sha256); + Assert.Null(renders[0].Kind); + Assert.Equal(((string?)null, (string?)null), (renders[1].Sha256, renders[1].Kind)); + Assert.Equal(RenderKind.Sequence, RenderKind.Of(renders[1])); + } + + [Test] + public async Task AStillJobSavesOneFrameAsAnImageAndRecordsIt() + { + await using var fx = await ProcessingFixture.CreateAsync(context); + var sequence = await fx.AddAsync(Scene()); + await fx.StartEngineAsync(); + + var request = ProcessingJobs.Still(sequence, Mjpeg with { NameTemplate = "still-{codec}" }, VideoCodec.PngSequence, 2, null, 96, 64); + await fx.WaitForAsync((await fx.EnqueueAsync(request)).Id, JobState.Done); + + var exports = fx.Folder(sequence).Exports; + var image = Path.Combine(exports, "still-PNG - 0003.png"); + Assert.True(File.Exists(image), string.Join(", ", Directory.EnumerateFileSystemEntries(exports))); + Assert.Equal(1, Directory.EnumerateFileSystemEntries(exports).Count()); + using (var decoded = DecoderFactory.CreateServer().Decode(new DecodeRequest(image))) + Assert.Equal((96, 64), (Assert.NotNull(decoded).Pixels.Width, decoded.Pixels.Height)); + var render = (await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct)).Single(); + Assert.Equal(("Synthetic/exports/still-PNG - 0003.png", RenderKind.Still, 0.0, 0.0), (render.OutputPath, render.Kind, render.Fps, render.DurationSeconds)); + Assert.Equal(Convert.ToHexStringLower(SHA256.HashData(await File.ReadAllBytesAsync(image, fx.Ct))), render.Sha256); + + // The same frame again: another name, never over the first. + await fx.WaitForAsync((await fx.EnqueueAsync(request)).Id, JobState.Done); + Assert.True(File.Exists(Path.Combine(exports, "still-PNG - 0003 (2).png"))); + // A frame outside the sequence fails the job and writes nothing. + var outside = ProcessingJobs.Still(sequence, Mjpeg, VideoCodec.JpegSequence, Frames + 5, null, 96, 64); + await fx.WaitForAsync((await fx.EnqueueAsync(outside)).Id, JobState.Failed); + Assert.Equal(2, Directory.EnumerateFileSystemEntries(exports).Count()); + } + + [Test] + public void AStillIsFoundByItsPhotoBeforeItsIndex() + { + var sequence = new Sequence { Id = Guid.CreateVersion7(), ArchiveId = Guid.CreateVersion7(), Folder = "S", Title = "S" }; + var frames = Enumerable.Range(0, 4).Select(i => new Frame + { + Id = Guid.CreateVersion7(), + SequenceId = sequence.Id, + Index = i, + RelativePath = $"S/{i}.jpg", + FileName = $"{i}.jpg", + Sha256 = new string((char)('a' + i), 64), + }).ToList(); + var set = new SequenceFrames(sequence, frames, frames.Select(f => (string?)f.RelativePath).ToList()); + Assert.Equal(3, SequenceRender.StillIndex(set, 1, new string('d', 64))); + Assert.Equal(3, SequenceRender.StillIndex(set, 1, new string('D', 64)), "any case"); + Assert.Equal(1, SequenceRender.StillIndex(set, 1, new string('z', 64)), "a photo no longer there: the index"); + Assert.Null(SequenceRender.StillIndex(set, 9, null)); + // Named by the export's template and the frame number from 1; a taken name gets " (2)". + var folder = context.CreateTempDirectory("still"); + var name = new ExportNameContext { Date = new DateTime(2026, 9, 28), Title = "Harbour", PresetName = "PNG", Codec = VideoCodec.PngSequence }; + var export = new ExportSettings { NameTemplate = "{title} - {codec}" }; + Assert.Equal(Path.Combine(folder, "Harbour - PNG - 0042.png"), SequenceRender.StillPath(export, folder, name, 41)); + Assert.Equal(Path.Combine(folder, "Harbour - PNG - 0042 (2).png"), SequenceRender.StillPath(export, folder, name, 41, p => p.EndsWith("0042.png", StringComparison.Ordinal))); + } +} diff --git a/tests/Chronos.Server.Tests/Jobs/PlaybackRoutingTests.cs b/tests/Chronos.Server.Tests/Jobs/PlaybackRoutingTests.cs new file mode 100644 index 00000000..af66eaa9 --- /dev/null +++ b/tests/Chronos.Server.Tests/Jobs/PlaybackRoutingTests.cs @@ -0,0 +1,39 @@ +using Chronos.Core.Model; +using Chronos.Core.Project; +using Chronos.Core.Video; +using Chronos.Data.Processing; +using Chronos.Protocol; +using Chronos.Server.Jobs; +using Chronos.Testing; + +namespace Chronos.Server.Tests.Jobs; + +/// Owner review 1, phase 5: a job that plays back, or saves a still, goes only to a node that reads settings format 3. +[Category("server")] +public sealed class PlaybackRoutingTests +{ + private static readonly ExportSettings Mjpeg = new() + { + Preset = new Auto().WithManual("mjpeg-avi"), + Overrides = new ExportOverrides { Width = 96, Height = 64 }, + }; + + [Test] + public void APlaybackJobIsNeverGivenToAFormatTwoNode() + { + var sequence = new Sequence { Id = Guid.CreateVersion7(), ArchiveId = Guid.CreateVersion7(), Folder = "Roof", Title = "Roof" }; + var playing = ProcessingJobs.ReadRender(ProcessingJobs.Render(sequence, Mjpeg with { Playback = new PlaybackSettings { Loops = 3 } }, 96, 64, + new ProjectSettings()).ParametersJson).Parameters; + var plain = ProcessingJobs.ReadRender(ProcessingJobs.Render(sequence, Mjpeg, 96, 64, new ProjectSettings()).ParametersJson).Parameters; + Assert.Equal((3, 1), (playing.RequiredSettingsFormat, plain.RequiredSettingsFormat)); + + RegisterNodeRequest Node(int format) => new("node", [new NodeEncoder("mjpeg", "managed", false, false)], 0, 0, format); + Assert.False(RenderNodeService.Renders(Node(2), playing), "a format-2 node would render it forward once"); + Assert.True(RenderNodeService.Renders(Node(3), playing)); + Assert.True(RenderNodeService.Renders(Node(2), plain)); + + // A still needs format 3 too (and an image codec no node encodes). + var still = ProcessingJobs.ReadRender(ProcessingJobs.Still(sequence, Mjpeg, VideoCodec.JpegSequence, 0, null, 96, 64, new ProjectSettings()).ParametersJson).Parameters; + Assert.False(RenderNodeService.Renders(Node(2) with { Encoders = [new NodeEncoder(still.Codec, "managed", false, false)] }, still)); + } +}