merge: phase 5 parcel P2

This commit is contained in:
Alby96 committed 2026-11-05 02:29:30 +01:00
commit 51f60fbbdf
23 files changed
+2123 -72

No files matched your search

+25 -4
View File
@@ -47,6 +47,7 @@ public sealed class ArchiveService : IJobHandlerProvider
FrameTrash = new FrameTrash(db, _writer, Indexer, Ingest, Trash);
Converter = new MassConverter(db, _writer, Ingest, FrameTrash, Trash, this);
Projects = new ProjectService(db, _writer);
Videos = new Chronos.Data.Videos.VideoLibrary(db, _writer);
var shared = Housekeeping.CleanupSteps.Shared(Trash);
CleanupSteps = [.. shared, .. _writer.Options.CleanupSteps.Where(s => !shared.Any(o => o.Name == s.Name))];
}
@@ -70,9 +71,9 @@ public sealed class ArchiveService : IJobHandlerProvider
/// <summary>
/// The videos library of this index (owner review 1, R9): every render listed, deleted into the trash, restored and copied
/// into a videos folder. The groundwork stub until the library lands (<see cref="Chronos.Data.Videos.IVideoLibrary.IsAvailable"/> false).
/// into a videos folder (<see cref="JobKind.VideoCopy"/>, among <see cref="CreateJobHandlers"/>).
/// </summary>
public Chronos.Data.Videos.IVideoLibrary Videos { get; } = Chronos.Data.Videos.NotImplementedVideoLibrary.Instance;
public Chronos.Data.Videos.VideoLibrary Videos { get; }
/// <summary>Converts sequences to JPEG, TIFF-16 or linear DNG, replacing their frames or into new sequences (owner review 2, R7).</summary>
public MassConverter Converter { get; }
@@ -117,6 +118,7 @@ public sealed class ArchiveService : IJobHandlerProvider
new EmptyTrashJobHandler(_db, Trash),
new MoveJobHandler(_db, Renamer, _writer.Options.SequenceChanged),
new ConvertJobHandler(_db, Converter, _writer.Options.SequenceChanged),
new Chronos.Data.Videos.VideoCopyJobHandler(_db, Videos),
];
/// <summary>Makes <paramref name="root"/> a new, empty archive and indexes it.</summary>
@@ -172,6 +174,15 @@ public sealed class ArchiveService : IJobHandlerProvider
// The journal stays; the next open tries again. The archive itself is usable meanwhile (a server starts all the same).
Log.Warning("archive", $"Resuming an interrupted rename in {paths.Root} failed", ex);
}
try
{
// A delete of videos into the trash a crash interrupted completes, or is undone (owner review 1, R9).
await Videos.RecoverAsync(archive, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or LeaseUnavailableException or DatabaseException)
{
Log.Warning("archive", $"Settling an interrupted delete of videos in {paths.Root} failed", ex);
}
return archive;
}
@@ -515,10 +526,13 @@ public sealed class ArchiveService : IJobHandlerProvider
(await RestoreEntryAsync(archiveId, entryName, null, cancellationToken).ConfigureAwait(false)).Sequence;
/// <summary>
/// Brings a trash entry back: a sequence into its folder (<see cref="SequenceTrash.RestoreAsync"/>), or photos into
/// Brings a trash entry back: a sequence into its folder (<see cref="SequenceTrash.RestoreAsync"/>), photos into
/// their sequence (<see cref="Archiving.FrameTrash.RestoreAsync"/>, which queues on <paramref name="followUps"/> what the
/// frames need again).
/// frames need again), or a video into its sequence's renders (<see cref="Chronos.Data.Videos.VideoLibrary.RestoreAsync"/>, owner
/// review 1, R9: <see cref="TrashRestoreResult.Sequence"/> is the video's).
/// </summary>
/// <exception cref="Chronos.Data.Videos.VideoLibraryException">A video entry whose file no longer matches its record, or whose sequence
/// is not in the index; nothing moved.</exception>
/// <exception cref="DirectoryNotFoundException">The entry is not in the trash.</exception>
/// <exception cref="SequenceInLibraryException">A sequence entry whose sequence is in the index already.</exception>
/// <exception cref="TrashMismatchException">No file of a photo entry matches its record; nothing moved.</exception>
@@ -528,6 +542,13 @@ public sealed class ArchiveService : IJobHandlerProvider
CancellationToken cancellationToken = default)
{
var archive = await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false);
if (Chronos.Data.Videos.VideoTrashNames.IsEntryName(entryName))
{
var render = await Videos.RestoreAsync(archive.Id, entryName, cancellationToken).ConfigureAwait(false);
var owner = await _db.Sequences.GetAsync(render.SequenceId, CancellationToken.None).ConfigureAwait(false)
?? throw new InvalidOperationException($"The sequence of render {render.Id} is not in the index.");
return new TrashRestoreResult(TrashKind.Videos, owner, 0, [], false);
}
if (!Archiving.FrameTrash.IsPhotoEntry(entryName))
return new TrashRestoreResult(TrashKind.Sequence, await Trash.RestoreAsync(archive, entryName, cancellationToken).ConfigureAwait(false), 0, [], false);
var photos = await FrameTrash.RestoreAsync(archive, entryName, followUps, cancellationToken).ConfigureAwait(false);
@@ -286,8 +286,8 @@ public sealed partial class SequenceTrash
CancellationToken cancellationToken)
{
var (folder, recordPath) = Location(paths, entry);
var now = FrameTrash.IsPhotoEntry(entry.Name)
? FrameTrash.ReadListed(paths, entry.ArchiveId, entry.Name)
var now = FrameTrash.IsPhotoEntry(entry.Name) ? FrameTrash.ReadListed(paths, entry.ArchiveId, entry.Name)
: Videos.VideoTrashNames.IsEntryName(entry.Name) ? Videos.VideoTrash.ReadListed(paths, entry.ArchiveId, entry.Name)
: ReadRecord(recordPath) is { } record ? Entry(entry.ArchiveId, Path.GetFileName(entry.Name), record) : null;
if (now is null || !Directory.Exists(folder)) return (null, null);
var (files, bytes) = Measure(folder, cancellationToken);
@@ -313,10 +313,11 @@ public sealed partial class SequenceTrash
return Survey(archive, paths, entry.Name, record, where, cancellationToken).Deleted ? entry with { ConversionDeleted = true } : entry;
}
/// <summary>The folder and record of an entry of either kind.</summary>
/// <summary>The folder and record of an entry of any kind (a sequence, photos, or a video: owner review 1, R9).</summary>
private static (string Folder, string Record) Location(ArchivePaths paths, TrashEntry entry)
{
if (FrameTrash.IsPhotoEntry(entry.Name)) return FrameTrash.Location(paths, entry.Name);
if (Videos.VideoTrashNames.IsEntryName(entry.Name)) return Videos.VideoTrash.Location(paths, entry.Name);
var folder = Path.Combine(TrashDirectory(paths), Path.GetFileName(entry.Name));
return (folder, folder + RecordExtension);
}
+4 -2
View File
@@ -213,8 +213,9 @@ public sealed partial class SequenceTrash
}
/// <summary>
/// The entries of an archive's trash, newest first: deleted sequences (<c>trash/*.json</c>) and deleted photos
/// (<c>trash/photos/*.json</c>); records whose folder is gone are left out.
/// The entries of an archive's trash, newest first: deleted sequences (<c>trash/*.json</c>), deleted photos
/// (<c>trash/photos/*.json</c>) and deleted videos (<c>trash/videos/*.json</c>, owner review 1, R9); records whose folder
/// is gone are left out.
/// </summary>
public async Task<IReadOnlyList<TrashEntry>> ListAsync(Archive archive, CancellationToken cancellationToken = default)
{
@@ -232,6 +233,7 @@ public sealed partial class SequenceTrash
entries.Add(Entry(archive.Id, name, record));
}
entries.AddRange(FrameTrash.List(archive, cancellationToken));
entries.AddRange(Chronos.Data.Videos.VideoTrash.List(archive, cancellationToken));
return entries.OrderByDescending(e => e.DeletedAt).ThenBy(e => e.Name, StringComparer.Ordinal).ToList();
}, cancellationToken).ConfigureAwait(false);
}
+4
View File
@@ -25,6 +25,7 @@ public abstract class ChronosRepositories
Schedules = new ScheduleRepository(access);
FrameRemovals = new FrameRemovalRepository(access);
Projects = new ProjectRepository(access);
RenderCopies = new RenderCopyRepository(access);
}
public ArchiveRepository Archives { get; }
@@ -49,6 +50,9 @@ public abstract class ChronosRepositories
/// <summary>Projects of sequences and their versions (owner review 2, R8).</summary>
public ProjectRepository Projects { get; }
/// <summary>Verified copies of renders in videos folders (owner review 1, R9).</summary>
public RenderCopyRepository RenderCopies { get; }
}
/// <summary>
+3 -1
View File
@@ -24,7 +24,8 @@ public sealed record JobPriorityQueue
/// <summary>
/// The levels a kind has unless the user changed it: renders, imports and downloads first, and the copy of a project's
/// media into the workspace, which Edit waits for (owner review 2, R8), a workspace's local render and the move of the
/// workspace folder, which Edit is held for (phase 5c); a local render's sending home at the normal level; the long
/// workspace folder, which Edit is held for (phase 5c), and the copy of videos into the videos folder, a transfer the user
/// waits for like a download (owner review 1, R9); a local render's sending home at the normal level; the long
/// passes that resume anywhere, and the housekeeping of the schedules (backups, clean-ups), last.
/// </summary>
public static IReadOnlyDictionary<JobKind, JobKindPriority> DefaultLevels { get; } = new Dictionary<JobKind, JobKindPriority>
@@ -36,6 +37,7 @@ public sealed record JobPriorityQueue
[JobKind.WorkspaceRender] = JobKindPriority.High,
[JobKind.WorkspaceSend] = JobKindPriority.Normal,
[JobKind.WorkspaceMove] = JobKindPriority.High,
[JobKind.VideoCopy] = JobKindPriority.High,
[JobKind.Verify] = JobKindPriority.Low,
[JobKind.Proxies] = JobKindPriority.Low,
[JobKind.Backup] = JobKindPriority.Low,
@@ -59,7 +59,11 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt
var (projectId, projectVersionId) = await ProvenanceAsync(parameters, sequence, cancellation).ConfigureAwait(false);
using var names = new OutputNames();
var all = Targets(export, parameters.PresetName);
// A still saves one frame through the job's preset alone: a batch is a video's.
var all = parameters.IsStill ? [new ExportTarget(ExportResolver.Resolve(export), parameters.PresetName)] : Targets(export, parameters.PresetName);
var still = parameters.IsStill
? SequenceRender.StillIndex(frames, parameters.StillFrame, parameters.StillPhoto) ?? throw new InvalidOperationException(Loc.L("Export.Render.NoFrames"))
: -1;
var resume = RenderCheckpoint.Parse(context.Checkpoint);
var done = resume.Done.ToList();
// A crash after a render row was written but before the checkpoint: that entry is finished.
@@ -83,29 +87,46 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt
await using (var render = new SequenceRender(frames, project, SequenceAnalyser.ReadReport(folder, frames), options, originals: !parameters.UseProxies,
companions))
{
outcomes = await render.RenderAllAsync(
targets,
(target, setup) => names.Reserve(taken => ExportNaming.OutputPath(project.Export, folder.Exports, SequenceRender.NameContext(sequence, target, setup), taken)),
(index, path) => context.CheckpointAsync(new RenderCheckpoint(done, index, path, sequence.Folder).ToJson(), context.Done, context.Total),
async (index, target, result) =>
async ValueTask Record(int index, ExportTarget target, RenderResult result)
{
// The output is complete: record it even if the job is being cancelled, so it is never orphaned. A file is
// hashed first (owner review 1, R9: copies and restores are verified against it); an image sequence's folder is not.
string? sha = null;
if (File.Exists(result.OutputPath))
{
// The output is complete: record it even if the job is being cancelled, so it is never orphaned.
var entity = result.ToEntity(sequence.Id, target.PresetName, settingsJson, options.ProducedBy) with
{
Id = RenderId(context.Job.Id, index),
OutputPath = Stored(paths, result.OutputPath),
ProjectId = projectId,
ProjectVersionId = projectVersionId,
};
await db.Renders.InsertAsync(entity, CancellationToken.None).ConfigureAwait(false);
done.Add(index);
await context.CheckpointAsync(new RenderCheckpoint(done, null, null).ToJson(), context.Done, context.Total).ConfigureAwait(false);
context.Log(Loc.L("Export.Log.Rendered", target.PresetName, Path.GetFileName(result.OutputPath)));
foreach (var remark in SequenceRender.Remarks(result)) context.Log(remark);
await options.NotifyAsync(sequence.Id, ProcessingOptions.RenderedChange, CancellationToken.None).ConfigureAwait(false);
},
context.ReportAsync,
cancellation).ConfigureAwait(false);
await context.ReportAsync(context.Done, context.Total, Loc.L("Export.Job.Verifying")).ConfigureAwait(false);
sha = await ContentHash.Sha256Async(result.OutputPath, cancellationToken: CancellationToken.None).ConfigureAwait(false);
}
var entity = result.ToEntity(sequence.Id, target.PresetName, settingsJson, options.ProducedBy) with
{
Id = RenderId(context.Job.Id, index),
OutputPath = Stored(paths, result.OutputPath),
ProjectId = projectId,
ProjectVersionId = projectVersionId,
Sha256 = sha,
Kind = parameters.IsStill ? RenderKind.Still : null,
};
if (parameters.IsStill) entity = entity with { Fps = 0, DurationSeconds = 0 };
await db.Renders.InsertAsync(entity, CancellationToken.None).ConfigureAwait(false);
done.Add(index);
await context.CheckpointAsync(new RenderCheckpoint(done, null, null).ToJson(), context.Done, context.Total).ConfigureAwait(false);
context.Log(parameters.IsStill
? Loc.L("Export.Log.StillSaved", (still + 1).ToString(Loc.Culture), Path.GetFileName(result.OutputPath))
: Loc.L("Export.Log.Rendered", target.PresetName, Path.GetFileName(result.OutputPath)));
foreach (var remark in SequenceRender.Remarks(result)) context.Log(remark);
await options.NotifyAsync(sequence.Id, ProcessingOptions.RenderedChange, CancellationToken.None).ConfigureAwait(false);
}
if (parameters.IsStill)
outcomes = [await StillAsync(context, render, targets[0], still, sequence, project.Export, folder, names, done, Record).ConfigureAwait(false)];
else
outcomes = await render.RenderAllAsync(
targets,
(target, setup) => names.Reserve(taken => ExportNaming.OutputPath(project.Export, folder.Exports, SequenceRender.NameContext(sequence, target, setup), taken)),
(index, path) => context.CheckpointAsync(new RenderCheckpoint(done, index, path, sequence.Folder).ToJson(), context.Done, context.Total),
Record,
context.ReportAsync,
cancellation).ConfigureAwait(false);
if (render.StandIns.Count > 0)
context.Log(Loc.L("Export.Log.StandIns", string.Join(", ", render.StandIns.Select(i => (i + 1).ToString(Loc.Culture)))));
// Recorded without the job's token: what decoding found holds even if the job is cancelled meanwhile, and a
@@ -128,6 +149,34 @@ internal sealed class RenderJobHandler(ChronosDatabase db, ProcessingOptions opt
throw new InvalidOperationException(failed.Count == 1 ? failed[0].Error : Loc.L("Errors.Render.Failed", failed.Count, outcomes.Count));
}
/// <summary>
/// The still of a still job: frame <paramref name="frame"/> saved through the job's preset under the export's name and
/// the frame number (<see cref="SequenceRender.StillPath"/>), checkpointed and recorded like a video's output.
/// </summary>
private static async Task<TargetOutcome> StillAsync(JobContext context, SequenceRender render, (int Index, ExportTarget Target) entry, int frame,
Sequence sequence, ExportSettings export, SequencePaths folder, OutputNames names, List<int> done,
Func<int, ExportTarget, RenderResult, ValueTask> record)
{
var (index, target) = entry;
try
{
var result = await render.RenderStillAsync(
target,
frame,
setup => names.Reserve(taken => SequenceRender.StillPath(export, folder.Exports, SequenceRender.NameContext(sequence, target, setup), frame, taken)),
path => context.CheckpointAsync(new RenderCheckpoint(done, index, path, sequence.Folder).ToJson(), context.Done, context.Total),
context.ReportAsync,
context.Cancellation).ConfigureAwait(false);
await record(index, target, result).ConfigureAwait(false);
return new TargetOutcome(target, result, null);
}
catch (Exception ex) when (!(ex is OperationCanceledException && context.Cancellation.IsCancellationRequested))
{
Core.Diagnostics.Log.Warning("processing", $"The still of frame {frame + 1} of {sequence.DisplayTitle} failed", ex);
return new TargetOutcome(target, null, SequenceRender.Reason(ex));
}
}
/// <summary>
/// The settings of the project version a job names without embedding them (owner review 2, R8); null when it names none.
/// </summary>
+14 -6
View File
@@ -153,7 +153,9 @@ public static class ProcessingJobs
{
Width = width,
Height = height,
RequiredSettingsFormat = int.Max(parameters.RequiredSettingsFormat, RequiredFormat(settings, resolved, sourceWidth, sourceHeight)),
// The job's own export (playback included) is what renders, not the export the settings carry.
RequiredSettingsFormat = int.Max(parameters.RequiredSettingsFormat,
RequiredFormat(settings with { Export = ReadExport(parameters.PresetJson) }, resolved, sourceWidth, sourceHeight)),
};
}
@@ -166,8 +168,8 @@ public static class ProcessingJobs
{
ArgumentNullException.ThrowIfNull(settings);
ArgumentNullException.ThrowIfNull(export);
return BatchPresets(export).Select(id => ExportResolver.Resolve(BatchExport(export, id))).Prepend(ExportResolver.Resolve(export))
.Max(resolved => RequiredFormat(settings, resolved, sourceWidth, sourceHeight));
return BatchPresets(export).Select(id => BatchExport(export, id)).Prepend(export)
.Max(each => RequiredFormat(settings with { Export = each }, ExportResolver.Resolve(each), sourceWidth, sourceHeight));
}
/// <summary>
@@ -210,9 +212,12 @@ public static class ProcessingJobs
.Where(id => !id.Equals(main, StringComparison.OrdinalIgnoreCase));
}
/// <summary>A batch preset rendered with its own values, named and placed like the export it belongs to.</summary>
/// <summary>
/// A batch preset rendered with its own values, named, placed and played like the export it belongs to: its playback
/// (owner review 1, R8: direction, loops, fades, hold) lives in the export, so every batch preset inherits it.
/// </summary>
internal static ExportSettings BatchExport(ExportSettings export, string presetId) =>
new() { Preset = new Auto<string>().WithManual(presetId), NameTemplate = export.NameTemplate, Folder = export.Folder };
new() { Preset = new Auto<string>().WithManual(presetId), NameTemplate = export.NameTemplate, Folder = export.Folder, Playback = export.Playback };
/// <param name="settings">The settings rendered, for the output size and the format (else read from <paramref name="settingsJson"/>); unknown, the preset's size.</param>
private static RenderJobParameters Parameters(ExportSettings export, int sourceWidth, int sourceHeight, string? settingsJson, bool useProxies,
@@ -222,7 +227,10 @@ public static class ProcessingJobs
settings ??= ReadSettings(settingsJson);
var (width, height) = (settings is null ? null : Size(settings, resolved, sourceWidth, sourceHeight))
?? (resolved.Width > 0 ? resolved.Width : sourceWidth, resolved.Height > 0 ? resolved.Height : sourceHeight);
var format = settings is null ? ProjectSettings.BaseVersion : RequiredFormat(settings, resolved, sourceWidth, sourceHeight);
// The format of what renders: the settings with this export in place of their own (its playback raises it to 3).
var format = settings is null
? new ProjectSettings { Export = export }.RequiredFormat
: RequiredFormat(settings with { Export = export }, resolved, sourceWidth, sourceHeight);
return new RenderJobParameters(PresetName(resolved), JsonSerializer.Serialize(export, ProcessingJson.Default.ExportSettings), CodecName(resolved.Codec),
width, height, useProxies, settingsJson, RequiredSettingsFormat: format);
}
@@ -134,6 +134,99 @@ internal sealed class SequenceRender : IAsyncDisposable
return outcomes;
}
/// <summary>
/// Saves frame <paramref name="frameIndex"/> as one image ("Save frame", owner review 1, R8): the sequence prepared whole
/// as for a video (so its exposure, colour and stabilisation corrections are the ones the video gives that frame), then
/// that frame alone rendered at the export's output size with every correction and the look, without interpolation,
/// motion blur or playback, through the image encoder of <paramref name="target"/>'s codec (PNG, JPEG or TIFF 16-bit). It
/// is written into a hidden working folder beside the output (<c>.&lt;name&gt;.&lt;random&gt;.tmp</c>, which
/// leftover clean-ups find) and moved to <paramref name="name"/>'s path once complete. The result's path is the image's,
/// its duration 0 and its frame count 1.
/// </summary>
/// <param name="name">The output path of the still from its setup.</param>
/// <param name="starting">Runs with that path before anything is written (checkpoints).</param>
/// <exception cref="RenderException">The frame is not one of the sequence's, or no encoder writes the codec.</exception>
public async Task<RenderResult> RenderStillAsync(ExportTarget target, int frameIndex, Func<RenderSetup, string> name, Func<string, ValueTask> starting,
Func<long, long, string?, ValueTask> progress, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(target);
ArgumentNullException.ThrowIfNull(name);
if ((uint)frameIndex >= (uint)Input.Frames.Count) throw new RenderException("Export.Render.NoFrames", $"Frame {frameIndex} is not in the sequence.");
// That frame alone, one output period long, whatever the timing would have made of it.
var project = Input.Project with
{
ExcludedFrames = [.. Enumerable.Range(0, Input.Frames.Count).Where(i => i != frameIndex)],
Timing = Input.Project.Timing with { Mode = Input.Project.Timing.Mode.WithManual(TimingMode.Constant), FitToSeconds = null },
Motion = Input.Project.Motion with { MotionBlur = Input.Project.Motion.MotionBlur.WithManual(false) },
};
var still = Input with { Project = project };
var pipeline = Pipeline(target.Export);
var setup = pipeline.Setup(still, new RenderTarget(target.Export, ExportNaming.FallbackName), _options);
if (!setup.Encoder.IsReady || setup.Plan.Frames.Count == 0) throw new RenderException("Export.Render.NoFrames", Reason(setup));
var output = name(setup);
await starting(output).ConfigureAwait(false);
var chain = new ProgressChain(progress);
var whole = pipeline.Setup(Input, new RenderTarget(target.Export, ExportNaming.FallbackName), _options);
var preparing = whole.NeedsPreparation ? whole.Plan.IncludedFrames.Count : 0;
var total = preparing + 1L;
// The corrections of the whole sequence, indexed by source frame: the frame gets what the video gives it.
var corrections = await RenderPreparation.PrepareAsync(Input, _options, Stage(chain, 0, preparing, total, Loc.L("Export.Job.Preparing")), cancellationToken)
.ConfigureAwait(false);
var work = Path.Combine(Path.GetDirectoryName(output)!, "." + Path.GetFileName(output) + "." + Guid.NewGuid().ToString("N")[..8] + ".tmp");
Directory.CreateDirectory(work);
try
{
var result = await pipeline.RenderAsync(still, corrections, new RenderTarget(target.Export, Path.Combine(work, "still")), _options,
Stage(chain, preparing, 1, total, target.PresetName), cancellationToken).ConfigureAwait(false);
await chain.FlushAsync().ConfigureAwait(false);
var image = Directory.EnumerateFiles(result.OutputPath).Order(StringComparer.Ordinal).FirstOrDefault()
?? throw new RenderException("Export.Render.NoFrames", "The still was not written.");
File.Move(image, output, overwrite: false);
return result with { OutputPath = output, Bytes = new FileInfo(output).Length, DurationSeconds = 0, FrameCount = 1 };
}
finally
{
try
{
Directory.Delete(work, recursive: true);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Core.Diagnostics.Log.Warning("processing", $"The still's working folder {work} could not be removed", ex);
}
}
}
/// <summary>
/// The path of a still of frame <paramref name="frameIndex"/> (0-based) in <paramref name="folder"/>: the export's name
/// template expanded, then " - " and the 1-based frame number in four digits, with the codec's extension
/// ("2026-09-28 Harbour sunset - PNG - 0124.png"); " (2)" and so on when taken.
/// </summary>
public static string StillPath(ExportSettings export, string folder, ExportNameContext context, int frameIndex, Func<string, bool>? taken = null)
{
ArgumentNullException.ThrowIfNull(export);
var name = ExportNaming.Expand(export.NameTemplate, context).Name + " - " + (frameIndex + 1).ToString("0000", System.Globalization.CultureInfo.InvariantCulture);
return ExportNaming.Unique(string.IsNullOrWhiteSpace(export.Folder) ? folder : export.Folder, name, Core.Video.VideoCodecInfo.FileExtension(context.Codec), taken);
}
/// <summary>
/// The frame a still job saves (<see cref="Protocol.RenderJobParameters.StillFrame"/>, <see cref="Protocol.RenderJobParameters.StillPhoto"/>):
/// the frame holding the photo when its SHA-256 is given and found (the frames may have moved since), else the index;
/// null when neither names a frame of the sequence.
/// </summary>
public static int? StillIndex(SequenceFrames frames, int? frameIndex, string? photoSha256)
{
ArgumentNullException.ThrowIfNull(frames);
if (photoSha256 is { Length: > 0 } sha)
{
for (var i = 0; i < frames.Frames.Count; i++)
{
if (string.Equals(frames.Frames[i].Sha256, sha, StringComparison.OrdinalIgnoreCase)) return i;
}
}
return frameIndex is { } index && (uint)index < (uint)frames.Frames.Count ? index : null;
}
/// <summary>Values of the output name tokens: the shoot date and title of the sequence, the codec and size the encoder will use.</summary>
public static ExportNameContext NameContext(Sequence sequence, ExportTarget target, RenderSetup setup) => new()
{
@@ -0,0 +1,56 @@
using Chronos.Core.Model;
namespace Chronos.Data.Repositories;
/// <summary>
/// The verified copies of renders in videos folders (owner review 1, R9; <c>render_copies</c>). There is no link to
/// <c>renders</c>: the copy of a server render downloaded to a PC is recorded too, and a copy outlives its render.
/// </summary>
public sealed class RenderCopyRepository
{
private const string Columns = "id, render_id, path, sha256, bytes, copied_at, copied_by";
private readonly DbAccess _db;
internal RenderCopyRepository(DbAccess db) => _db = db;
public Task<int> InsertAsync(RenderCopy copy, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(copy);
return _db.ExecuteAsync($"INSERT INTO render_copies ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7)",
[copy.Id, copy.RenderId, copy.Path, copy.Sha256, copy.Bytes, copy.CopiedAt, copy.CopiedBy], cancellationToken);
}
/// <summary>The copies of a render, oldest first.</summary>
public Task<IReadOnlyList<RenderCopy>> ListByRenderAsync(Guid renderId, CancellationToken cancellationToken = default) =>
_db.ListAsync($"SELECT {Columns} FROM render_copies WHERE render_id = $1 ORDER BY copied_at, id", [renderId], Map, cancellationToken);
/// <summary>The copies of each of <paramref name="renderIds"/>, oldest first; a render without copies is left out.</summary>
public async Task<IReadOnlyDictionary<Guid, IReadOnlyList<RenderCopy>>> ListByRendersAsync(IReadOnlyCollection<Guid> renderIds,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(renderIds);
var copies = new Dictionary<Guid, IReadOnlyList<RenderCopy>>();
// In chunks, so the statement stays within every dialect's parameter limit.
foreach (var chunk in renderIds.Distinct().Chunk(200))
{
var p = new SqlParameters();
var list = string.Join(", ", chunk.Select(id => p.Add(id)));
var rows = await _db.ListAsync($"SELECT {Columns} FROM render_copies WHERE render_id IN ({list}) ORDER BY copied_at, id", p.ToArray(), Map,
cancellationToken).ConfigureAwait(false);
foreach (var group in rows.GroupBy(c => c.RenderId)) copies[group.Key] = [.. group];
}
return copies;
}
private static RenderCopy Map(DbRow r) => new()
{
Id = r.GetGuid(0),
RenderId = r.GetGuid(1),
Path = r.GetString(2),
Sha256 = r.GetStringOrNull(3),
Bytes = r.GetInt64(4),
CopiedAt = r.GetDateTimeOffset(5),
CopiedBy = r.GetStringOrNull(6),
};
}
@@ -1,4 +1,6 @@
using Chronos.Core.Model;
using Chronos.Data.Videos;
using Chronos.Protocol;
namespace Chronos.Data.Repositories;
@@ -7,7 +9,7 @@ public sealed class RenderRepository
{
private const string Columns =
"id, sequence_id, preset_name, settings_json, output_path, codec, width, height, fps, duration_seconds, bytes, created, produced_by, project_id, " +
"project_version_id";
"project_version_id, sha256, kind";
private const string RecountSql =
"UPDATE sequences SET rendered_count = (SELECT count(*) FROM renders WHERE sequence_id = $1) WHERE id = $1";
@@ -20,11 +22,11 @@ public sealed class RenderRepository
_db.InTransactionAsync(async session =>
{
await LockSequenceAsync(session, render.SequenceId, cancellationToken).ConfigureAwait(false);
await session.ExecuteAsync($"INSERT INTO renders ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)",
await session.ExecuteAsync($"INSERT INTO renders ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)",
[
render.Id, render.SequenceId, render.PresetName, DbParam.Json(render.SettingsJson), render.OutputPath, render.Codec,
render.Width, render.Height, render.Fps, render.DurationSeconds, render.Bytes, render.Created, render.ProducedBy, render.ProjectId,
render.ProjectVersionId,
render.ProjectVersionId, render.Sha256, render.Kind,
], cancellationToken).ConfigureAwait(false);
return await session.ExecuteAsync(RecountSql, [render.SequenceId], cancellationToken).ConfigureAwait(false);
}, cancellationToken);
@@ -36,6 +38,80 @@ public sealed class RenderRepository
public Task<IReadOnlyList<Render>> ListBySequenceAsync(Guid sequenceId, CancellationToken cancellationToken = default) =>
_db.ListAsync($"SELECT {Columns} FROM renders WHERE sequence_id = $1 ORDER BY created DESC, id DESC", [sequenceId], Map, cancellationToken);
/// <summary>
/// One page of every render of the index matching <paramref name="query"/> (the videos library, owner review 1, R9), in
/// its order (<see cref="VideoSorts"/>: newest first unless asked), with the number of renders matching on every page.
/// Text matches the sequence's search text, the preset name or the output path (any case).
/// </summary>
/// <exception cref="FormatException">The cursor is not one of this listing in this order.</exception>
public Task<(IReadOnlyList<Render> Items, string? Next, long Total)> ListPageAsync(VideoQuery query, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(query);
var p = new SqlParameters();
var where = Filter(query, p);
var count = new SqlParameters();
var countWhere = Filter(query, count);
var sort = VideoSorts.All.FirstOrDefault(s => string.Equals(s, query.Sort, StringComparison.OrdinalIgnoreCase)) ?? VideoSorts.Newest;
var scope = "videos:" + sort;
var limit = Math.Clamp(query.Limit, 1, VideoQuery.MaxLimit);
long offset = 0;
string order;
switch (sort)
{
case VideoSorts.Title:
// By title, then newest: paged by position (a title is no key a cursor keeps well).
if (query.Cursor is not null && Cursor.Decode(scope, query.Cursor).Value is long skip) offset = skip;
order = "ORDER BY lower(coalesce(s.title, '')) ASC, r.created DESC, r.id DESC";
break;
default:
{
var (column, descending) = sort switch
{
VideoSorts.Oldest => ("r.created", false),
VideoSorts.Largest => ("r.bytes", true),
_ => ("r.created", true),
};
if (query.Cursor is not null)
{
var (value, id) = Cursor.Decode(scope, query.Cursor);
where.Add(Cursor.After(column, "r.id", descending, value, id, p.Add));
}
var direction = descending ? "DESC" : "ASC";
order = $"ORDER BY {column} {direction}, r.id {direction}";
break;
}
}
var columns = string.Join(", ", Columns.Split(", ").Select(c => "r." + c));
var sql = $"SELECT {columns} FROM renders r JOIN sequences s ON s.id = r.sequence_id{Where(where)} {order} LIMIT {p.Add(limit + 1)}"
+ (offset > 0 ? $" OFFSET {p.Add(offset)}" : string.Empty);
var countSql = $"SELECT count(*) FROM renders r JOIN sequences s ON s.id = r.sequence_id{Where(countWhere)}";
return _db.RunAsync(async session =>
{
var rows = await session.QueryListAsync(sql, p.ToArray(), Map, cancellationToken).ConfigureAwait(false);
var total = await session.ScalarAsync<long>(countSql, count.ToArray(), cancellationToken).ConfigureAwait(false);
string? next = null;
if (rows.Count > limit)
{
rows.RemoveAt(limit);
var last = rows[^1];
next = sort switch
{
VideoSorts.Title => Cursor.Encode(scope, offset + limit, last.Id),
VideoSorts.Largest => Cursor.Encode(scope, last.Bytes, last.Id),
_ => Cursor.Encode(scope, last.Created, last.Id),
};
}
return ((IReadOnlyList<Render>)rows, next, total);
}, cancellationToken);
}
/// <summary>Records the SHA-256 of a render hashed later than it was recorded (owner review 1, R9: renders of before migration 0010).</summary>
public Task<int> SetSha256Async(Guid id, string sha256, CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrEmpty(sha256);
return _db.ExecuteAsync("UPDATE renders SET sha256 = $2 WHERE id = $1", [id, sha256.ToLowerInvariant()], cancellationToken);
}
public Task<bool> DeleteAsync(Guid id, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
@@ -93,6 +169,43 @@ public sealed class RenderRepository
if (_db.Sql.IsPostgres) await SequenceRepository.LockAsync(session, sequenceId, cancellationToken).ConfigureAwait(false);
}
/// <summary>The conditions of a videos query on <c>renders r JOIN sequences s</c>.</summary>
private static List<string> Filter(VideoQuery query, SqlParameters p)
{
var where = new List<string>();
if (query.SequenceId is { } sequence) where.Add($"r.sequence_id = {p.Add(sequence)}");
if (!string.IsNullOrWhiteSpace(query.Codec)) where.Add($"lower(r.codec) = {p.Add(query.Codec.Trim().ToLowerInvariant())}");
if (query.Year is { } year and > 0 and < 10000)
{
var from = new DateTimeOffset(year, 1, 1, 0, 0, 0, TimeSpan.Zero);
where.Add($"r.created >= {p.Add(from)} AND r.created < {p.Add(from.AddYears(1))}");
}
// A sequence's codec names end in "sequence" (pngsequence, tiff16sequence…).
const string Sequence = "(r.kind IS NULL AND lower(coalesce(r.codec, '')) LIKE '%sequence')";
switch (query.Kind?.Trim().ToLowerInvariant())
{
case null or "":
break;
case RenderKind.Video:
where.Add($"(r.kind IS NULL AND NOT {Sequence})");
break;
case RenderKind.Sequence:
where.Add(Sequence);
break;
case var kind:
where.Add($"r.kind = {p.Add(kind)}");
break;
}
if (!string.IsNullOrWhiteSpace(query.Text))
{
var pattern = p.Add(SqlDialect.ContainsPattern(query.Text.Trim().ToLowerInvariant()));
where.Add($"(s.search_text LIKE {pattern} ESCAPE '\\' OR lower(r.preset_name) LIKE {pattern} ESCAPE '\\' OR lower(r.output_path) LIKE {pattern} ESCAPE '\\')");
}
return where;
}
private static string Where(List<string> conditions) => conditions.Count == 0 ? string.Empty : " WHERE " + string.Join(" AND ", conditions);
private static Render Map(DbRow r) => new()
{
Id = r.GetGuid(0),
@@ -110,5 +223,7 @@ public sealed class RenderRepository
ProducedBy = r.GetStringOrNull(12),
ProjectId = r.GetGuidOrNull(13),
ProjectVersionId = r.GetGuidOrNull(14),
Sha256 = r.GetStringOrNull(15),
Kind = r.GetStringOrNull(16),
};
}
+6 -1
View File
@@ -127,7 +127,12 @@ public sealed record VideoDeleteResult(IReadOnlyList<VideoItem> Deleted, IReadOn
/// <param name="Folder">The videos folder (absolute), checked again when the job runs.</param>
/// <param name="FromServer">On the desktop: the renders are the signed-in server's, downloaded into the folder.</param>
/// <param name="RequestedBy">The user (server) or PC (desktop) recorded as <see cref="RenderCopy.CopiedBy"/>.</param>
public sealed record VideoCopyParameters(IReadOnlyList<Guid> RenderIds, string Folder, bool FromServer = false, string? RequestedBy = null)
/// <param name="Server">With <paramref name="FromServer"/>: the server the renders are downloaded from (its address as the
/// desktop's connections name it, <see cref="Sync.ISyncConnector.ConnectAsync"/>).</param>
/// <param name="SequenceIds">With <paramref name="FromServer"/>: the server's sequence of each render, in the order of
/// <paramref name="RenderIds"/>; its manifest gives the SHA-256 a download is verified against.</param>
public sealed record VideoCopyParameters(IReadOnlyList<Guid> RenderIds, string Folder, bool FromServer = false, string? RequestedBy = null,
string? Server = null, IReadOnlyList<Guid>? SequenceIds = null)
{
public string ToJson() => JsonSerializer.Serialize(this, VideoJson.Default.VideoCopyParameters);
@@ -0,0 +1,242 @@
using System.Globalization;
using Chronos.Core.Archiving;
using Chronos.Core.Diagnostics;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Data.Jobs;
using Chronos.Data.Sync;
using Chronos.Protocol;
namespace Chronos.Data.Videos;
/// <summary>
/// <see cref="JobKind.VideoCopy"/> (owner review 1, R9, decision 16): copies renders into the videos folder, on the desktop
/// and on the server. The folder is checked again first, and the free space for every render still to copy, so a copy that
/// cannot fit fails before it writes anything. Each render is written as <c>.incoming-&lt;id&gt;.part</c> in the folder and
/// flushed, hashed and compared with the render's SHA-256 (the render hashed first when it never was, and remembered; an
/// image sequence's folder file by file against its own files), then renamed without overwriting to its own name or a free
/// " (2)" one, and recorded in <c>render_copies</c>. A copy that does not match is deleted and that render fails; the
/// render itself is never touched. On the desktop a server's render (<see cref="VideoCopyParameters.FromServer"/>) is
/// downloaded into the part file and checked against the SHA-256 its server's manifest gives. Checkpoint:
/// <c>done:&lt;n&gt;</c>, the renders done; leftovers of an interrupted copy are removed before it starts again.
/// </summary>
internal sealed class VideoCopyJobHandler(ChronosDatabase db, VideoLibrary library) : IJobHandler
{
private const string Prefix = "done:";
public JobKind Kind => JobKind.VideoCopy;
public async Task RunAsync(JobContext context)
{
var ct = context.Cancellation;
var parameters = VideoCopyParameters.Read(context.Job.ParametersJson) ?? throw new InvalidDataException(Loc.L("Errors.Render.Parameters"));
if (await library.FolderProblemAsync(parameters.Folder, ct).ConfigureAwait(false) is { } problem) throw VideoLibrary.Refusal(problem);
var folder = parameters.Folder;
Directory.CreateDirectory(folder);
var ids = parameters.RenderIds;
var start = context.Checkpoint is { } checkpoint && checkpoint.StartsWith(Prefix, StringComparison.Ordinal)
&& int.TryParse(checkpoint.AsSpan(Prefix.Length), NumberStyles.None, CultureInfo.InvariantCulture, out var done)
? int.Min(done, ids.Count)
: 0;
foreach (var id in ids.Skip(start)) Remove(VideosFolder.PartPath(folder, id));
await context.ReportAsync(start, ids.Count, Loc.L("Videos.Job.Copying")).ConfigureAwait(false);
using var remote = parameters.FromServer ? await ConnectAsync(parameters, ct).ConfigureAwait(false) : null;
var sources = new List<Source?>(ids.Count);
for (var i = 0; i < ids.Count; i++)
sources.Add(i < start ? null : await SourceAsync(context, parameters, remote, i, ct).ConfigureAwait(false));
// Every render still to copy must fit before anything is written.
var needed = sources.Where(s => s is not null).Sum(s => s!.Bytes);
if (VideosFolder.FreeBytes(folder, library.FreeSpace) is { } free && free < needed)
throw new VideoLibraryException(VideoErrors.NoSpace, Loc.L("Errors.Videos.NoSpace", Megabytes(needed - free)));
var failed = 0;
for (var i = start; i < ids.Count; i++)
{
ct.ThrowIfCancellationRequested();
if (sources[i] is not { } source)
{
failed++;
}
else if (!await CopyAsync(context, parameters, remote, source, ct).ConfigureAwait(false))
{
failed++;
}
await context.CheckpointAsync(Prefix + (i + 1).ToString(CultureInfo.InvariantCulture), i + 1, ids.Count).ConfigureAwait(false);
}
if (failed > 0) throw new InvalidOperationException(Loc.L("Errors.Render.Failed", failed, ids.Count - start));
}
/// <summary>What one render's copy reads: its id, name, size, and the SHA-256 it is verified against (null: an image sequence).</summary>
/// <param name="Path">The render's file or folder on this host; null for a server's render.</param>
/// <param name="Sequence">The server's sequence of a server's render.</param>
private sealed record Source(Guid RenderId, string Name, string? Path, bool IsFolder, long Bytes, string? Sha256, Guid? Sequence);
/// <summary>The render at <paramref name="index"/> as it is now; null (logged) when it is gone or its file is missing.</summary>
private async Task<Source?> SourceAsync(JobContext context, VideoCopyParameters parameters, ISyncRemote? remote, int index, CancellationToken ct)
{
var id = parameters.RenderIds[index];
if (remote is not null)
{
var sequence = parameters.SequenceIds is { } sequences && index < sequences.Count ? sequences[index] : (Guid?)null;
var render = sequence is { } s ? (await remote.GetManifestAsync(s, ct).ConfigureAwait(false)).Renders.FirstOrDefault(r => r.Id == id) : null;
if (render is null)
{
context.Log(Loc.L("Videos.Log.Missing", id.ToString("D")));
return null;
}
// Named by the server as it downloads.
return new Source(id, id.ToString("N"), null, false, render.Bytes, render.Sha256, sequence);
}
if (await library.GetAsync(id, ct).ConfigureAwait(false) is not { } item || item.Missing)
{
context.Log(Loc.L("Videos.Log.Missing", (await library.GetAsync(id, ct).ConfigureAwait(false))?.FileName ?? id.ToString("D")));
return null;
}
var folder = Directory.Exists(item.FullPath);
string? sha = null;
if (!folder)
{
sha = item.Render.Sha256;
if (sha is null)
{
// A render of before migration 0010: hashed now, and remembered.
await context.ReportAsync(context.Done, context.Total, Loc.L("Videos.Job.Verifying")).ConfigureAwait(false);
sha = await ContentHash.Sha256Async(item.FullPath, cancellationToken: ct).ConfigureAwait(false);
await db.Renders.SetSha256Async(id, sha, CancellationToken.None).ConfigureAwait(false);
}
}
var bytes = folder ? new DirectoryInfo(item.FullPath).EnumerateFiles("*", SearchOption.AllDirectories).Sum(f => f.Length) : new FileInfo(item.FullPath).Length;
return new Source(id, item.FileName, item.FullPath, folder, bytes, sha, null);
}
/// <summary>Copies one render, verified, under a free name, and records the copy; false (logged) when the copy did not match.</summary>
private async Task<bool> CopyAsync(JobContext context, VideoCopyParameters parameters, ISyncRemote? remote, Source source, CancellationToken ct)
{
var folder = parameters.Folder;
var part = VideosFolder.PartPath(folder, source.RenderId);
Remove(part);
string? sha;
try
{
if (remote is not null)
{
await using (var file = new FileStream(part, FileMode.Create, FileAccess.ReadWrite, FileShare.None))
{
var given = await remote.DownloadRenderAsync(source.RenderId, file, ct).ConfigureAwait(false);
file.Flush(flushToDisk: true);
source = source with { Name = SyncTransfers.SafeFileName(given) ?? source.Name };
}
sha = await Verified(part, source.Sha256, context, ct).ConfigureAwait(false);
}
else if (source.IsFolder)
{
// An image sequence: each file copied and compared with its own.
foreach (var file in Directory.EnumerateFiles(source.Path!, "*", SearchOption.AllDirectories))
{
var target = Path.Combine(part, Path.GetRelativePath(source.Path!, file));
Directory.CreateDirectory(Path.GetDirectoryName(target)!);
await VideoTrash.CopyFileAsync(file, target, ct).ConfigureAwait(false);
var expected = await ContentHash.Sha256Async(file, cancellationToken: ct).ConfigureAwait(false);
if (await Verified(target, expected, context, ct).ConfigureAwait(false) is null)
{
Mismatch(context, source, part);
return false;
}
}
Directory.CreateDirectory(part);
sha = null;
await Commit(context, parameters, source, part, sha).ConfigureAwait(false);
return true;
}
else
{
await VideoTrash.CopyFileAsync(source.Path!, part, ct).ConfigureAwait(false);
sha = await Verified(part, source.Sha256, context, ct).ConfigureAwait(false);
}
}
catch
{
Remove(part);
throw;
}
if (sha is null)
{
Mismatch(context, source, part);
return false;
}
await Commit(context, parameters, source, part, sha).ConfigureAwait(false);
return true;
}
/// <summary>The copy takes its name (never overwriting: another free name when one appeared meanwhile) and is recorded.</summary>
private async Task Commit(JobContext context, VideoCopyParameters parameters, Source source, string part, string? sha)
{
string target;
for (var attempt = 0; ; attempt++)
{
target = VideosFolder.FreeName(parameters.Folder, source.Name, source.IsFolder);
try
{
if (source.IsFolder) Directory.Move(part, target);
else File.Move(part, target, overwrite: false);
break;
}
catch (IOException) when (attempt < 5 && (File.Exists(target) || Directory.Exists(target)))
{
// Taken between the look and the move: the next free name.
}
}
var bytes = source.IsFolder ? new DirectoryInfo(target).EnumerateFiles("*", SearchOption.AllDirectories).Sum(f => f.Length) : new FileInfo(target).Length;
await db.RenderCopies.InsertAsync(new RenderCopy
{
Id = Guid.CreateVersion7(),
RenderId = source.RenderId,
Path = target,
Sha256 = sha,
Bytes = bytes,
CopiedAt = DateTimeOffset.UtcNow,
CopiedBy = parameters.RequestedBy,
}, CancellationToken.None).ConfigureAwait(false);
context.Log(Loc.L("Videos.Log.Copied", source.Name, target));
Log.Info("videos", $"Render {source.RenderId} copied to {target} ({bytes} bytes, verified)");
}
/// <summary>The SHA-256 of the copy at <paramref name="path"/> when it matches <paramref name="expected"/> (any, when none is known); null otherwise.</summary>
private static async Task<string?> Verified(string path, string? expected, JobContext context, CancellationToken ct)
{
await context.ReportAsync(context.Done, context.Total, Loc.L("Videos.Job.Verifying")).ConfigureAwait(false);
var sha = await ContentHash.Sha256Async(path, cancellationToken: ct).ConfigureAwait(false);
return expected is null || string.Equals(sha, expected, StringComparison.OrdinalIgnoreCase) ? sha : null;
}
private static void Mismatch(JobContext context, Source source, string part)
{
Remove(part);
context.Log(Loc.L("Videos.Log.Mismatch", source.Name));
Log.Warning("videos", $"The copy of render {source.RenderId} did not match it; the copy was removed");
}
private async Task<ISyncRemote> ConnectAsync(VideoCopyParameters parameters, CancellationToken ct)
{
if (library.Servers is not { } servers || parameters.Server is not { Length: > 0 } server
|| await servers.ConnectAsync(server, ct).ConfigureAwait(false) is not { } remote)
throw new InvalidOperationException(Loc.L("Errors.Sync.NotLinked"));
return remote;
}
private static string Megabytes(long bytes) => Housekeeping.CleanupSteps.Megabytes(bytes);
private static void Remove(string path)
{
try
{
if (Directory.Exists(path)) Directory.Delete(path, recursive: true);
else if (File.Exists(path)) File.Delete(path);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("videos", $"{path} could not be removed", ex);
}
}
}
+216
View File
@@ -0,0 +1,216 @@
using System.Globalization;
using System.Security.Cryptography;
using System.Text;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Data.Archiving;
using Chronos.Data.Archiving.Jobs;
using Chronos.Data.Sync;
using Chronos.Protocol;
namespace Chronos.Data.Videos;
/// <summary>
/// The videos library of one index (owner review 1, R9): every render of its archives in one list, newest first unless
/// asked, each with its sequence, its project, whether its file is there and its copies; deleted into its archive's trash
/// after a dry run whose fingerprint the delete gives back (decision 17), restored by id (decision 18), and copied
/// hash-verified into a videos folder by a <see cref="JobKind.VideoCopy"/> job (decision 16). The renders themselves are
/// never changed, and copies in a videos folder are never touched.
/// </summary>
public sealed class VideoLibrary : IVideoLibrary
{
private readonly ChronosDatabase _db;
private readonly SequenceWriter _writer;
internal VideoLibrary(ChronosDatabase db, SequenceWriter writer)
{
_db = db;
_writer = writer;
Trash = new VideoTrash(db, writer);
}
public bool IsAvailable => true;
/// <summary>
/// Folders a videos folder may not lie in besides the archive roots (decision 15): the desktop's workspace. Asked each
/// time a folder is checked.
/// </summary>
public Func<IReadOnlyList<string>>? ProtectedFolders { get; set; }
/// <summary>The desktop's signed-in servers: a copy of a server's renders (<see cref="VideoCopyParameters.FromServer"/>) downloads through them.</summary>
public ISyncConnector? Servers { get; set; }
/// <summary>How the host measures free space under a folder; null: the folder's drive.</summary>
public Func<string, long?>? FreeSpace => _writer.Options.FreeSpace;
internal VideoTrash Trash { get; }
public async Task<VideoPage> ListAsync(VideoQuery query, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(query);
var (renders, next, total) = await _db.Renders.ListPageAsync(query, cancellationToken).ConfigureAwait(false);
return new VideoPage(await ItemsAsync(renders, cancellationToken).ConfigureAwait(false), next, total);
}
public async Task<VideoItem?> GetAsync(Guid renderId, CancellationToken cancellationToken = default) =>
await _db.Renders.GetAsync(renderId, cancellationToken).ConfigureAwait(false) is { } render
? (await ItemsAsync([render], cancellationToken).ConfigureAwait(false)).FirstOrDefault()
: null;
public async Task<VideoDeletePlan> PlanDeleteAsync(IReadOnlyList<Guid> renderIds, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(renderIds);
var busy = await BusyAsync(cancellationToken).ConfigureAwait(false);
var items = new List<VideoItem>();
var blocked = new List<VideoBlocked>();
foreach (var id in renderIds.Distinct())
{
if (await GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } item) blocked.Add(new VideoBlocked(id, VideoBlockReasons.NotFound));
else if (busy.Contains(id)) blocked.Add(new VideoBlocked(id, VideoBlockReasons.Busy, item));
else items.Add(item);
}
return new VideoDeletePlan(items, blocked, Fingerprint(items, blocked));
}
public async Task<VideoDeleteResult> DeleteAsync(IReadOnlyList<Guid> renderIds, string fingerprint, string? deletedBy, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(renderIds);
var plan = await PlanDeleteAsync(renderIds, cancellationToken).ConfigureAwait(false);
if (!string.Equals(plan.Fingerprint, fingerprint, StringComparison.Ordinal))
throw new VideoLibraryException(VideoErrors.PlanChanged, Loc.L("Errors.Videos.Fingerprint"));
var deleted = new List<VideoItem>();
var entries = new List<string>();
var archives = new Dictionary<Guid, Archive?>();
foreach (var item in plan.Items)
{
cancellationToken.ThrowIfCancellationRequested();
var archive = await ArchiveOfAsync(archives, item.Render.SequenceId, cancellationToken).ConfigureAwait(false);
if (archive is null)
{
// The sequence left the index since the plan: so did its renders.
continue;
}
if (await Trash.DeleteAsync(archive, item.Render, deletedBy, cancellationToken).ConfigureAwait(false) is { } entry) entries.Add(entry.Name);
deleted.Add(item);
}
return new VideoDeleteResult(deleted, plan.Blocked, entries);
}
public async Task<Render> RestoreAsync(Guid archiveId, string entryName, CancellationToken cancellationToken = default)
{
var archive = await _db.Archives.GetAsync(archiveId, cancellationToken).ConfigureAwait(false)
?? throw new ArgumentException($"Unknown archive {archiveId}.", nameof(archiveId));
return await Trash.RestoreAsync(archive, entryName, cancellationToken).ConfigureAwait(false);
}
/// <summary>Settles the deletes into the trash a crash interrupted in <paramref name="archive"/> (<see cref="VideoTrash.RecoverAsync"/>).</summary>
public Task<int> RecoverAsync(Archive archive, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(archive);
return Trash.RecoverAsync(archive, cancellationToken);
}
public string? FolderProblem(string? folder)
{
if (string.IsNullOrWhiteSpace(folder)) return VideoErrors.FolderUnset;
// Asked from the UI thread and the server's endpoints alike: the roots are read off any synchronisation context.
var roots = Task.Run(() => _db.Archives.ListAsync()).GetAwaiter().GetResult().Select(a => a.RootPath);
return VideosFolder.Problem(folder, [.. roots, .. ProtectedFolders?.Invoke() ?? []]);
}
/// <summary><see cref="FolderProblem"/> without blocking: what the copy job checks again when it runs.</summary>
internal async Task<string?> FolderProblemAsync(string? folder, CancellationToken cancellationToken)
{
var roots = (await _db.Archives.ListAsync(cancellationToken).ConfigureAwait(false)).Select(a => a.RootPath);
return VideosFolder.Problem(folder, [.. roots, .. ProtectedFolders?.Invoke() ?? []]);
}
public ArchiveJobRequest CopyJob(VideoCopyParameters parameters)
{
ArgumentNullException.ThrowIfNull(parameters);
if (FolderProblem(parameters.Folder) is { } problem) throw Refusal(problem);
if (parameters.RenderIds.Count == 0) throw new ArgumentException("No render to copy.", nameof(parameters));
var ids = parameters.RenderIds.Distinct().ToList();
var folder = VideosFolder.Normalize(parameters.Folder);
return new ArchiveJobRequest(JobKind.VideoCopy, ids, Loc.L("Videos.Target.Copy", ids.Count.ToString(Loc.Culture)),
(parameters with { RenderIds = ids, Folder = folder }).ToJson(), JobPriority.User);
}
/// <summary>The exception of a videos folder that cannot be used, with its localised message.</summary>
internal static VideoLibraryException Refusal(string code) => new(code, Loc.L(code == VideoErrors.FolderUnset ? "Errors.Videos.FolderUnset" : "Errors.Videos.FolderInsideArchive"));
/// <summary>
/// The fingerprint of a delete plan: SHA-256 of each render's id, path, size, creation and whether its file is there, and
/// of each render left where it is with why, in id order. Another render, a render moved, grown, gone or busy since gives
/// another fingerprint.
/// </summary>
internal static string Fingerprint(IEnumerable<VideoItem> items, IEnumerable<VideoBlocked> blocked)
{
var text = new StringBuilder();
foreach (var item in items.OrderBy(i => i.Render.Id))
{
var r = item.Render;
text.Append(CultureInfo.InvariantCulture, $"{r.Id:N}|{r.OutputPath}|{r.Bytes}|{r.Created.UtcTicks}|{(item.Missing ? "missing" : "ok")}\n");
}
foreach (var block in blocked.OrderBy(b => b.RenderId)) text.Append(CultureInfo.InvariantCulture, $"{block.RenderId:N}|blocked|{block.Reason}\n");
return Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(text.ToString())));
}
/// <summary>The renders a running or paused job holds: those a copy works on (its targets are render ids).</summary>
private async Task<HashSet<Guid>> BusyAsync(CancellationToken cancellationToken)
{
var busy = new HashSet<Guid>();
foreach (var hold in await _db.Jobs.ListHoldsAsync(null, cancellationToken).ConfigureAwait(false))
{
if (hold.Kind == JobKind.VideoCopy) busy.UnionWith(hold.TargetIds);
}
return busy;
}
/// <summary>The library items of <paramref name="renders"/>: each with its sequence, project, file state and copies, in their order.</summary>
private async Task<IReadOnlyList<VideoItem>> ItemsAsync(IReadOnlyList<Render> renders, CancellationToken cancellationToken)
{
if (renders.Count == 0) return [];
var copies = await _db.RenderCopies.ListByRendersAsync([.. renders.Select(r => r.Id)], cancellationToken).ConfigureAwait(false);
var sequences = new Dictionary<Guid, Sequence?>();
var archives = new Dictionary<Guid, Archive?>();
var projects = new Dictionary<Guid, string?>();
var versions = new Dictionary<Guid, int?>();
var items = new List<VideoItem>(renders.Count);
foreach (var render in renders)
{
if (!sequences.TryGetValue(render.SequenceId, out var sequence))
sequences[render.SequenceId] = sequence = await _db.Sequences.GetAsync(render.SequenceId, cancellationToken).ConfigureAwait(false);
if (sequence is null) continue;
if (!archives.TryGetValue(sequence.ArchiveId, out var archive))
archives[sequence.ArchiveId] = archive = await _db.Archives.GetAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false);
string? projectName = null;
if (render.ProjectId is { } projectId && !projects.TryGetValue(projectId, out projectName))
projects[projectId] = projectName = (await _db.Projects.GetAsync(projectId, cancellationToken).ConfigureAwait(false))?.Name;
int? number = null;
if (render.ProjectVersionId is { } versionId && !versions.TryGetValue(versionId, out number))
versions[versionId] = number = (await _db.Projects.GetVersionAsync(versionId, cancellationToken).ConfigureAwait(false))?.Number;
var full = archive is null || Path.IsPathRooted(render.OutputPath) ? render.OutputPath : SyncService.RenderFile(archive, render);
items.Add(new VideoItem
{
Render = render,
SequenceTitle = sequence.DisplayTitle,
SequenceFirstCapture = sequence.FirstCapture,
ProjectName = projectName,
ProjectVersionNumber = projectName is null ? null : number,
FullPath = full,
Missing = !File.Exists(full) && !Directory.Exists(full),
Copies = copies.TryGetValue(render.Id, out var made) ? made : [],
});
}
return items;
}
private async Task<Archive?> ArchiveOfAsync(Dictionary<Guid, Archive?> known, Guid sequenceId, CancellationToken cancellationToken)
{
if (await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) is not { } sequence) return null;
if (!known.TryGetValue(sequence.ArchiveId, out var archive))
known[sequence.ArchiveId] = archive = await _db.Archives.GetAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false);
return archive;
}
}
+440
View File
@@ -0,0 +1,440 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using Chronos.Core.Archiving;
using Chronos.Core.Diagnostics;
using Chronos.Core.IO;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Data.Archiving;
namespace Chronos.Data.Videos;
/// <summary>One file of a trashed image sequence, as it was when it went: its path inside the folder, size and SHA-256.</summary>
internal sealed record VideoTrashFile(string Path, long Size, string Sha256);
/// <summary>
/// What the trash keeps next to a deleted render (decision 17): the render's row, where its file was, the SHA-256 it is
/// restored against (each file's, for an image sequence's folder), its sequence, and who deleted it when.
/// </summary>
/// <param name="Name">The file (or folder) name inside the entry's folder: the render's own name.</param>
/// <param name="OriginalPath">The absolute path the render was at.</param>
/// <param name="SequenceFolder">The sequence's archive-relative folder at deletion: a restore follows a Reorganize since.</param>
internal sealed record VideoTrashRecord(Render Render, string Name, string OriginalPath, Guid ArchiveId, Guid SequenceId, string SequenceTitle,
string SequenceFolder, string? Sha256, IReadOnlyList<VideoTrashFile>? Files, int FileCount, long Bytes, DateTimeOffset DeletedAt, string? DeletedBy);
/// <summary>
/// The videos area of an archive's trash (owner review 1, R9, decisions 17 and 18): <c>.chronos/trash/videos/</c>, which
/// holds a folder per deleted render (<c>&lt;render id&gt;-&lt;UTC ticks&gt;</c>, the entry name without its
/// <see cref="VideoTrashNames.Prefix"/>) with the render's file or folder inside, and its record next to it. A delete runs
/// under the sequence's lease: the record is written first under a pending name (the journal, never listed), then the file
/// moves (renamed on its volume; copied, verified and deleted from another one), then the row goes and the record takes its
/// name. A delete a crash interrupted is settled by <see cref="RecoverAsync"/>: completed when the file reached the
/// trash, undone otherwise. Records live under <c>trash/videos/</c>, so an older Chronos never reads one as a sequence's.
/// </summary>
internal sealed class VideoTrash(ChronosDatabase db, SequenceWriter writer)
{
private const string RecordExtension = ".json";
/// <summary>The suffix of a record whose delete is under way: the journal a crash leaves.</summary>
internal const string PendingSuffix = ".pending";
private const string PartSuffix = ".part";
/// <summary>Tests: move as if the trash were on another volume (copy, verify, delete).</summary>
internal bool ForceCopy { get; set; }
/// <summary>Tests: stop a delete right after its file moved, as a crash would (the journal stays).</summary>
internal bool CrashAfterMove { get; set; }
/// <summary>The videos area of an archive's trash.</summary>
public static string Directory(ArchivePaths archive) => Path.Combine(SequenceTrash.TrashDirectory(archive), VideoTrashNames.FolderName);
/// <summary>The entry's folder and record on disk.</summary>
internal static (string Folder, string Record) Location(ArchivePaths paths, string name)
{
var folder = Path.Combine(Directory(paths), DiskName(name));
return (folder, folder + RecordExtension);
}
/// <summary>The videos entries of an archive's trash (records whose folder is gone, or whose delete is under way, left out).</summary>
internal static IReadOnlyList<TrashEntry> List(Archive archive, CancellationToken cancellationToken)
{
var folder = Directory(new ArchivePaths(archive.RootPath));
if (!System.IO.Directory.Exists(folder)) return [];
var entries = new List<TrashEntry>();
foreach (var path in System.IO.Directory.EnumerateFiles(folder, "*" + RecordExtension))
{
cancellationToken.ThrowIfCancellationRequested();
if (!path.EndsWith(RecordExtension, StringComparison.OrdinalIgnoreCase)) continue;
var disk = Path.GetFileNameWithoutExtension(path);
if (!System.IO.Directory.Exists(Path.Combine(folder, disk)) || ReadRecord(path) is not { } record) continue;
entries.Add(Entry(archive.Id, VideoTrashNames.Prefix + disk, record));
}
return entries;
}
/// <summary>A videos entry as the trash lists it, read from its record now; null when it is not one or cannot be read.</summary>
internal static TrashEntry? ReadListed(ArchivePaths paths, Guid archiveId, string name) =>
VideoTrashNames.IsEntryName(name) && ReadRecord(Location(paths, name).Record) is { } record ? Entry(archiveId, name, record) : null;
/// <summary>
/// Moves <paramref name="render"/>'s file into the trash with its record, and removes its row (its sequence's render count
/// recounted); a render whose file is missing loses only its row. Returns the entry made; null when only the row went
/// (or the render was gone already).
/// </summary>
/// <exception cref="ArchiveOfflineException">The archive is offline or read-only.</exception>
/// <exception cref="LeaseUnavailableException">Another writer holds the sequence.</exception>
internal async Task<TrashEntry?> DeleteAsync(Archive archive, Render render, string? deletedBy, CancellationToken cancellationToken)
{
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var sequence = await db.Sequences.GetAsync(render.SequenceId, cancellationToken).ConfigureAwait(false);
await using (sequence is null ? null : await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
{
if (await db.Renders.GetAsync(render.Id, cancellationToken).ConfigureAwait(false) is not { } row) return null;
var source = Sync.SyncService.RenderFile(archive, row);
var isFile = File.Exists(source);
if (!isFile && !System.IO.Directory.Exists(source))
{
await db.Renders.DeleteAsync(row.Id, CancellationToken.None).ConfigureAwait(false);
Log.Info("videos", $"Render {row.Id} ({row.OutputPath}) removed from the index: its file is missing");
return null;
}
string? sha = null;
IReadOnlyList<VideoTrashFile>? files = null;
long bytes;
if (isFile)
{
// Hashed now when it never was (a render of before migration 0010), and remembered.
sha = row.Sha256 ?? await ContentHash.Sha256Async(source, cancellationToken: cancellationToken).ConfigureAwait(false);
if (row.Sha256 is null) await db.Renders.SetSha256Async(row.Id, sha, CancellationToken.None).ConfigureAwait(false);
bytes = new FileInfo(source).Length;
}
else
{
files = await HashFolderAsync(source, cancellationToken).ConfigureAwait(false);
bytes = files.Sum(f => f.Size);
}
var now = DateTimeOffset.UtcNow;
var name = VideoTrashNames.EntryName(row.Id, now);
var (entryFolder, recordPath) = Location(paths, name);
var fileName = Path.GetFileName(source.TrimEnd('/', '\\'));
var record = new VideoTrashRecord(row with { Sha256 = sha ?? row.Sha256 }, fileName, source, archive.Id, row.SequenceId,
sequence?.DisplayTitle ?? fileName, sequence?.Folder ?? string.Empty, sha, files, files?.Count ?? 1, bytes, now, deletedBy);
using (await LeaseFile.AcquireAsync(paths.LocksDirectory, SequenceTrash.EntryLease(name), writer.Options.Owner, writer.Options.LeaseWait,
writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false))
{
cancellationToken.ThrowIfCancellationRequested();
System.IO.Directory.CreateDirectory(entryFolder);
// The journal first: a crash from here on is settled by RecoverAsync.
AtomicFile.WriteAllText(recordPath + PendingSuffix, JsonSerializer.Serialize(record, VideoTrashJson.Default.VideoTrashRecord));
var target = Path.Combine(entryFolder, fileName);
try
{
await MoveAsync(source, target, record, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
// A file is open elsewhere, the copy did not verify, or the move was cancelled: the render stays where it was.
Log.Warning("videos", $"{source} could not move to the trash; it stays where it is", ex);
Undo(target, entryFolder, recordPath);
throw;
}
if (CrashAfterMove) throw new IOException("Simulated crash after the move.");
// The file has moved: its row goes too, to the end, whatever the caller cancels meanwhile.
await db.Renders.DeleteAsync(row.Id, CancellationToken.None).ConfigureAwait(false);
Publish(recordPath);
}
Log.Info("videos", $"{source} moved to the trash ({name}, {bytes} bytes)");
return Entry(archive.Id, name, record);
}
}
/// <summary>
/// Brings a videos entry back: its file re-hashed against its record (refused on a mismatch), its sequence in the index
/// (refused otherwise: restore the sequence first), then moved back where it was (into the sequence's folder as it is now
/// after a Reorganize; a free name when its own is taken) and its row inserted again with the same render id.
/// </summary>
/// <exception cref="DirectoryNotFoundException">The entry is not in the trash.</exception>
/// <exception cref="VideoLibraryException"><see cref="VideoErrors.HashMismatch"/> or <see cref="VideoErrors.SequenceGone"/>; nothing moved.</exception>
/// <exception cref="InvalidOperationException">The index holds a render of that id already; nothing moved.</exception>
internal async Task<Render> RestoreAsync(Archive archive, string name, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(name);
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
if (!VideoTrashNames.IsEntryName(name)) throw new DirectoryNotFoundException($"{name} is not in the trash of {archive.Name}.");
var (entryFolder, recordPath) = Location(paths, name);
if (!System.IO.Directory.Exists(entryFolder) || ReadRecord(recordPath) is not { } record)
throw new DirectoryNotFoundException($"{name} is not in the trash of {archive.Name}.");
using (await LeaseFile.AcquireAsync(paths.LocksDirectory, SequenceTrash.EntryLease(name), writer.Options.Owner, writer.Options.LeaseWait,
writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false))
{
var trashed = Path.Combine(entryFolder, record.Name);
if (!File.Exists(trashed) && !System.IO.Directory.Exists(trashed)) throw new DirectoryNotFoundException($"{name} is not in the trash of {archive.Name}.");
if (await db.Renders.GetAsync(record.Render.Id, cancellationToken).ConfigureAwait(false) is not null)
throw new InvalidOperationException(Loc.L("Server.Error.RenderExists"));
var sequence = await db.Sequences.GetAsync(record.SequenceId, cancellationToken).ConfigureAwait(false)
?? throw new VideoLibraryException(VideoErrors.SequenceGone, Loc.L("Errors.Videos.SequenceGone", record.SequenceTitle));
if (!await MatchesAsync(trashed, record, cancellationToken).ConfigureAwait(false))
throw new VideoLibraryException(VideoErrors.HashMismatch, Loc.L("Errors.Videos.TrashMismatch"));
Render restored;
await using (await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
{
var target = Destination(paths, record, sequence);
System.IO.Directory.CreateDirectory(Path.GetDirectoryName(target)!);
cancellationToken.ThrowIfCancellationRequested();
await MoveAsync(trashed, target, record, cancellationToken).ConfigureAwait(false);
restored = record.Render with { SequenceId = sequence.Id, OutputPath = Stored(paths, target), Sha256 = record.Sha256 };
try
{
await db.Renders.InsertAsync(restored, CancellationToken.None).ConfigureAwait(false);
}
catch (Exception ex)
{
Log.Warning("videos", $"{target} could not be indexed again; it goes back into the trash", ex);
await MoveAsync(target, trashed, record, CancellationToken.None).ConfigureAwait(false);
throw;
}
}
File.Delete(recordPath);
RemoveEmpty(entryFolder);
Log.Info("videos", $"{name} restored to {restored.OutputPath}");
return restored;
}
}
/// <summary>
/// Settles the deletes a crash interrupted in <paramref name="archive"/>'s trash (their pending records): a delete whose
/// file reached the trash is completed (the row goes, the record takes its name), any other is undone (what was copied
/// goes, the render stays where it is). Each runs under its sequence's lease when the index still holds the sequence.
/// </summary>
internal async Task<int> RecoverAsync(Archive archive, CancellationToken cancellationToken)
{
var paths = new ArchivePaths(archive.RootPath);
var folder = Directory(paths);
if (!System.IO.Directory.Exists(folder) || ArchiveVolume.Probe(paths) != ArchiveAvailability.Online) return 0;
var settled = 0;
foreach (var pending in System.IO.Directory.EnumerateFiles(folder, "*" + RecordExtension + PendingSuffix).ToList())
{
cancellationToken.ThrowIfCancellationRequested();
var recordPath = pending[..^PendingSuffix.Length];
var entryFolder = recordPath[..^RecordExtension.Length];
if (ReadRecord(pending) is not { } record)
{
Log.Warning("videos", $"The pending trash record {pending} cannot be read; it stays");
continue;
}
var sequence = await db.Sequences.GetAsync(record.SequenceId, cancellationToken).ConfigureAwait(false);
await using (sequence is null ? null : await writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
{
if (!File.Exists(pending)) continue;
var target = Path.Combine(entryFolder, record.Name);
if (File.Exists(target) || System.IO.Directory.Exists(target))
{
// The file reached the trash (a copy only takes its name once verified): the delete completes. A source still
// there is the original a copy left behind, unless something new took its name since.
if ((File.Exists(record.OriginalPath) || System.IO.Directory.Exists(record.OriginalPath))
&& await MatchesAsync(record.OriginalPath, record, cancellationToken).ConfigureAwait(false))
DeletePath(record.OriginalPath);
if (await db.Renders.GetAsync(record.Render.Id, cancellationToken).ConfigureAwait(false) is not null)
await db.Renders.DeleteAsync(record.Render.Id, CancellationToken.None).ConfigureAwait(false);
Publish(recordPath);
Log.Info("videos", $"An interrupted delete of {record.OriginalPath} completed");
}
else
{
// The file never reached the trash: what was copied goes, and the render stays as it is.
Undo(target, entryFolder, recordPath);
Log.Info("videos", $"An interrupted delete of {record.OriginalPath} was undone");
}
settled++;
}
}
return settled;
}
/// <summary>
/// The SHA-256 of every file of an image sequence's folder (paths inside it, forward slashes, in order); the folder's
/// own render records no hash (decision 14).
/// </summary>
internal static async Task<IReadOnlyList<VideoTrashFile>> HashFolderAsync(string folder, CancellationToken cancellationToken)
{
var files = new List<VideoTrashFile>();
foreach (var file in System.IO.Directory.EnumerateFiles(folder, "*", SearchOption.AllDirectories).Order(StringComparer.Ordinal))
{
var sha = await ContentHash.Sha256Async(file, cancellationToken: cancellationToken).ConfigureAwait(false);
files.Add(new VideoTrashFile(Path.GetRelativePath(folder, file).Replace('\\', '/'), new FileInfo(file).Length, sha));
}
return files;
}
/// <summary>Whether the file or folder at <paramref name="path"/> holds what <paramref name="record"/> says, re-hashed now.</summary>
private static async Task<bool> MatchesAsync(string path, VideoTrashRecord record, CancellationToken cancellationToken)
{
if (record.Files is { } expected)
{
if (!System.IO.Directory.Exists(path)) return false;
var actual = await HashFolderAsync(path, cancellationToken).ConfigureAwait(false);
return actual.Count == expected.Count && actual.Zip(expected).All(p =>
p.First.Path == p.Second.Path && p.First.Size == p.Second.Size && string.Equals(p.First.Sha256, p.Second.Sha256, StringComparison.OrdinalIgnoreCase));
}
if (!File.Exists(path)) return false;
if (record.Sha256 is null) return true;
var sha = await ContentHash.Sha256Async(path, cancellationToken: cancellationToken).ConfigureAwait(false);
return string.Equals(sha, record.Sha256, StringComparison.OrdinalIgnoreCase);
}
/// <summary>
/// Moves a render's file or folder from <paramref name="from"/> to <paramref name="to"/>: renamed on one volume; copied
/// under a part name, verified against <paramref name="record"/>, renamed and the source deleted across volumes.
/// </summary>
private async Task MoveAsync(string from, string to, VideoTrashRecord record, CancellationToken cancellationToken)
{
var folder = System.IO.Directory.Exists(from);
if (!ForceCopy && SameVolume(from, to))
{
try
{
if (folder) System.IO.Directory.Move(from, to);
else File.Move(from, to, overwrite: false);
return;
}
catch (IOException) when (folder && !System.IO.Directory.Exists(to) && System.IO.Directory.Exists(from))
{
// Another mount on the same root (Linux): copied instead.
}
}
var part = to + PartSuffix;
DeletePath(part);
if (folder)
{
foreach (var file in System.IO.Directory.EnumerateFiles(from, "*", SearchOption.AllDirectories))
{
var target = Path.Combine(part, Path.GetRelativePath(from, file));
System.IO.Directory.CreateDirectory(Path.GetDirectoryName(target)!);
await CopyFileAsync(file, target, cancellationToken).ConfigureAwait(false);
}
System.IO.Directory.CreateDirectory(part);
}
else
{
await CopyFileAsync(from, part, cancellationToken).ConfigureAwait(false);
}
if (!await MatchesAsync(part, record, cancellationToken).ConfigureAwait(false))
{
DeletePath(part);
throw new IOException(Loc.L("Errors.Videos.TrashMismatch"));
}
if (folder) System.IO.Directory.Move(part, to);
else File.Move(part, to, overwrite: false);
DeletePath(from);
}
/// <summary>Copies a file, flushed to the disk.</summary>
internal static async Task CopyFileAsync(string source, string target, CancellationToken cancellationToken)
{
await using var from = new FileStream(source, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 20, FileOptions.Asynchronous | FileOptions.SequentialScan);
await using var to = new FileStream(target, FileMode.Create, FileAccess.Write, FileShare.None, 1 << 20, FileOptions.Asynchronous);
await from.CopyToAsync(to, cancellationToken).ConfigureAwait(false);
to.Flush(flushToDisk: true);
}
/// <summary>Where a restored render goes: where it was (in its sequence's folder as the index has it now), or a free name next to it.</summary>
private static string Destination(ArchivePaths paths, VideoTrashRecord record, Sequence sequence)
{
var stored = record.Render.OutputPath;
string full;
if (Path.IsPathRooted(stored)) full = stored;
else
{
var from = record.SequenceFolder;
var rebased = from.Length > 0 && from != sequence.Folder && stored.StartsWith(from + "/", StringComparison.Ordinal)
? (sequence.Folder.Length == 0 ? stored[(from.Length + 1)..] : sequence.Folder + stored[from.Length..])
: stored;
full = paths.FromRelative(rebased);
}
return File.Exists(full) || System.IO.Directory.Exists(full)
? VideosFolder.FreeName(Path.GetDirectoryName(full)!, Path.GetFileName(full), record.Files is not null)
: full;
}
/// <summary>Archive-relative for outputs inside the archive (as renders are recorded), absolute otherwise.</summary>
internal static string Stored(ArchivePaths paths, string output)
{
var relative = paths.ToRelative(output);
return relative.StartsWith("..", StringComparison.Ordinal) || Path.IsPathRooted(relative) ? output : relative;
}
private static bool SameVolume(string a, string b) =>
string.Equals(Path.GetPathRoot(Path.GetFullPath(a)), Path.GetPathRoot(Path.GetFullPath(b)), StringComparison.OrdinalIgnoreCase);
/// <summary>Removes what a delete that did not complete left in the trash: the part copy, the entry's folder and its journal.</summary>
private static void Undo(string target, string entryFolder, string recordPath)
{
try
{
DeletePath(target + PartSuffix);
RemoveEmpty(entryFolder);
File.Delete(recordPath + PendingSuffix);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("videos", $"What an unfinished delete left in {entryFolder} could not be removed; the next start settles it", ex);
}
}
/// <summary>Lists an entry whose delete completed: its record takes its own name. A failure (logged) is settled by the next <see cref="RecoverAsync"/>.</summary>
private static void Publish(string recordPath)
{
try
{
File.Move(recordPath + PendingSuffix, recordPath, overwrite: true);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("videos", $"Trash record {recordPath} keeps its pending name; the next start lists it", ex);
}
}
private static void RemoveEmpty(string folder)
{
if (System.IO.Directory.Exists(folder) && !System.IO.Directory.EnumerateFileSystemEntries(folder).Any()) System.IO.Directory.Delete(folder);
}
private static void DeletePath(string path)
{
if (System.IO.Directory.Exists(path)) SequenceTrash.Erase(path);
else if (File.Exists(path)) File.Delete(path);
}
/// <summary>The folder name of an entry: its name without <see cref="VideoTrashNames.Prefix"/>, never a path.</summary>
private static string DiskName(string name) =>
Path.GetFileName(name.StartsWith(VideoTrashNames.Prefix, StringComparison.Ordinal) ? name[VideoTrashNames.Prefix.Length..] : name);
private static TrashEntry Entry(Guid archiveId, string name, VideoTrashRecord record)
{
var stored = record.Render.OutputPath.Replace('\\', '/');
var cut = stored.LastIndexOf('/');
return new TrashEntry(archiveId, name, record.SequenceId, record.Name, cut < 0 ? string.Empty : stored[..cut], 0, record.FileCount, record.Bytes,
record.DeletedAt, record.DeletedBy, TrashKind.Videos);
}
private static VideoTrashRecord? ReadRecord(string path)
{
try
{
return File.Exists(path) ? JsonSerializer.Deserialize(File.ReadAllText(path), VideoTrashJson.Default.VideoTrashRecord) : null;
}
catch (Exception ex) when (ex is JsonException or IOException or UnauthorizedAccessException)
{
Log.Warning("videos", $"Trash record {path} is unreadable", ex);
return null;
}
}
}
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
IgnoreReadOnlyProperties = true)]
[JsonSerializable(typeof(VideoTrashRecord))]
internal sealed partial class VideoTrashJson : JsonSerializerContext;
+100
View File
@@ -0,0 +1,100 @@
using System.Globalization;
namespace Chronos.Data.Videos;
/// <summary>
/// The videos folder (owner review 1, R9, decision 15): a flat folder outside every archive and the workspace that renders
/// are copied into, each under its own name or, when that is taken, "name (2)", "name (3)"… (the export naming rule). A
/// copy is written as <c>.incoming-&lt;render id&gt;.part</c> first and takes its name only once verified.
/// </summary>
public static class VideosFolder
{
/// <summary>The prefix of a copy being written; never a name a finished copy takes.</summary>
public const string PartPrefix = ".incoming-";
/// <summary>The suffix of a copy being written.</summary>
public const string PartSuffix = ".part";
/// <summary>
/// Why <paramref name="folder"/> cannot be the videos folder: <see cref="VideoErrors.FolderUnset"/> when blank,
/// <see cref="VideoErrors.FolderInvalid"/> when it is not an absolute path or lies inside (or is) one of
/// <paramref name="protectedRoots"/> (the archive roots, the workspace); null when it can.
/// </summary>
public static string? Problem(string? folder, IEnumerable<string?> protectedRoots)
{
ArgumentNullException.ThrowIfNull(protectedRoots);
if (string.IsNullOrWhiteSpace(folder)) return VideoErrors.FolderUnset;
string full;
try
{
if (!Path.IsPathFullyQualified(folder)) return VideoErrors.FolderInvalid;
full = Normalize(folder);
}
catch (Exception ex) when (ex is ArgumentException or NotSupportedException or PathTooLongException)
{
return VideoErrors.FolderInvalid;
}
foreach (var root in protectedRoots)
{
if (string.IsNullOrWhiteSpace(root)) continue;
if (IsInside(full, Normalize(root))) return VideoErrors.FolderInvalid;
}
return null;
}
/// <summary>Whether <paramref name="path"/> is <paramref name="root"/> or lies below it (both full and normalised).</summary>
internal static bool IsInside(string path, string root)
{
var comparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
if (string.Equals(path, root, comparison)) return true;
var prefix = root.EndsWith(Path.DirectorySeparatorChar) ? root : root + Path.DirectorySeparatorChar;
return path.StartsWith(prefix, comparison);
}
/// <summary>The full path without a trailing separator (a drive or file-system root keeps its own).</summary>
internal static string Normalize(string path)
{
var full = Path.GetFullPath(path);
var trimmed = full.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
return trimmed.Length == 0 || trimmed.EndsWith(':') ? full : trimmed;
}
/// <summary>
/// The path a copy named <paramref name="fileName"/> takes in <paramref name="folder"/>: the name itself when free, else
/// "name (2).ext", "name (3).ext"… (a folder's name, an image sequence, has no extension to keep).
/// </summary>
/// <param name="isFolder">The copy is an image sequence's folder: its whole name is the stem.</param>
public static string FreeName(string folder, string fileName, bool isFolder = false)
{
ArgumentException.ThrowIfNullOrWhiteSpace(folder);
ArgumentException.ThrowIfNullOrWhiteSpace(fileName);
var name = Path.GetFileName(fileName);
var (stem, extension) = isFolder ? (name, string.Empty) : (Path.GetFileNameWithoutExtension(name), Path.GetExtension(name));
for (var n = 1; ; n++)
{
var candidate = Path.Combine(folder, (n == 1 ? stem : string.Create(CultureInfo.InvariantCulture, $"{stem} ({n})")) + extension);
if (!File.Exists(candidate) && !Directory.Exists(candidate)) return candidate;
}
}
/// <summary>Where the copy of render <paramref name="renderId"/> is written in <paramref name="folder"/> until it is verified.</summary>
public static string PartPath(string folder, Guid renderId) => Path.Combine(folder, PartPrefix + renderId.ToString("N") + PartSuffix);
/// <summary>
/// The free bytes of the volume of <paramref name="folder"/>: what <paramref name="measure"/> says (a host's own measure,
/// tests), else the drive's; null when unknown.
/// </summary>
public static long? FreeBytes(string folder, Func<string, long?>? measure = null)
{
if (measure?.Invoke(folder) is { } measured) return measured;
try
{
var root = Path.GetPathRoot(Path.GetFullPath(folder));
return string.IsNullOrEmpty(root) ? null : new DriveInfo(root).AvailableFreeSpace;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException)
{
return null;
}
}
}
@@ -62,7 +62,13 @@ internal sealed class WorkspaceRenderJobHandler(ChronosDatabase db, ArchiveServi
var exports = project.ExportsDirectory;
Directory.CreateDirectory(exports);
using var names = new OutputNames();
var all = RenderJobHandler.Targets(projectSettings.Export, render.PresetName);
// A still saves one frame through the job's preset alone: a batch is a video's.
var all = render.IsStill
? [new ExportTarget(ExportResolver.Resolve(projectSettings.Export), render.PresetName)]
: RenderJobHandler.Targets(projectSettings.Export, render.PresetName);
var still = render.IsStill
? SequenceRender.StillIndex(frames, render.StillFrame, render.StillPhoto) ?? throw new InvalidOperationException(Loc.L("Export.Render.NoFrames"))
: -1;
var resume = RenderCheckpoint.Parse(context.Checkpoint);
var done = resume.Done.ToList();
var ledger = project.ReadRenders();
@@ -90,34 +96,67 @@ internal sealed class WorkspaceRenderJobHandler(ChronosDatabase db, ArchiveServi
try
{
await using var renderer = new SequenceRender(frames, projectSettings, inputs.Analysis, options.Processing, originals: true, companions: null);
outcomes = await renderer.RenderAllAsync(
targets,
(target, setup) => names.Reserve(taken => ExportNaming.OutputPath(projectSettings.Export, exports, SequenceRender.NameContext(sequence, target, setup),
name => taken(name) || ledger.Entries.Any(e => SamePath(project.ExportFile(e.File), name)))),
(index, path) =>
// A name the ledger holds is taken too (an output sent home and deleted here keeps its entry).
bool Taken(Func<string, bool> taken, string name) => taken(name) || ledger.Entries.Any(e => SamePath(project.ExportFile(e.File), name));
ValueTask Starting(int index, string path)
{
writing = path;
return context.CheckpointAsync(new RenderCheckpoint(done, index, path).ToJson(), context.Done, context.Total);
}
async ValueTask Record(int index, ExportTarget target, RenderResult result)
{
// The output is complete: recorded even if the job is being cancelled, so it is never orphaned.
var id = RenderJobHandler.RenderId(context.Job.Id, index);
var entry = await EntryAsync(project, context.Job.Id, id, target, result, versionId, settingsHash).ConfigureAwait(false);
if (render.IsStill) entry = entry with { Fps = 0, DurationSeconds = 0, Kind = RenderKind.Still };
using (await project.LockAsync(LockWait, CancellationToken.None).ConfigureAwait(false))
{
writing = path;
return context.CheckpointAsync(new RenderCheckpoint(done, index, path).ToJson(), context.Done, context.Total);
},
async (index, target, result) =>
ledger = project.ReadRenders().With(entry);
project.WriteRenders(ledger);
}
done.Add(index);
if (!recorded.Contains(id)) recorded.Add(id);
writing = null;
await context.CheckpointAsync(new RenderCheckpoint(done, null, null).ToJson(), context.Done, context.Total).ConfigureAwait(false);
context.Log(render.IsStill
? Loc.L("Export.Log.StillSaved", (still + 1).ToString(Loc.Culture), Path.GetFileName(result.OutputPath))
: Loc.L("Export.Log.Rendered", target.PresetName, Path.GetFileName(result.OutputPath)));
foreach (var remark in SequenceRender.Remarks(result)) context.Log(remark);
}
if (render.IsStill)
{
var (index, target) = targets[0];
try
{
// The output is complete: recorded even if the job is being cancelled, so it is never orphaned.
var id = RenderJobHandler.RenderId(context.Job.Id, index);
var entry = await EntryAsync(project, context.Job.Id, id, target, result, versionId, settingsHash).ConfigureAwait(false);
using (await project.LockAsync(LockWait, CancellationToken.None).ConfigureAwait(false))
{
ledger = project.ReadRenders().With(entry);
project.WriteRenders(ledger);
}
done.Add(index);
if (!recorded.Contains(id)) recorded.Add(id);
writing = null;
await context.CheckpointAsync(new RenderCheckpoint(done, null, null).ToJson(), context.Done, context.Total).ConfigureAwait(false);
context.Log(Loc.L("Export.Log.Rendered", target.PresetName, Path.GetFileName(result.OutputPath)));
foreach (var remark in SequenceRender.Remarks(result)) context.Log(remark);
},
context.ReportAsync,
cancellation).ConfigureAwait(false);
var result = await renderer.RenderStillAsync(
target,
still,
setup => names.Reserve(taken => SequenceRender.StillPath(projectSettings.Export, exports, SequenceRender.NameContext(sequence, target, setup), still,
name => Taken(taken, name))),
path => Starting(index, path),
context.ReportAsync,
cancellation).ConfigureAwait(false);
await Record(index, target, result).ConfigureAwait(false);
outcomes = [new TargetOutcome(target, result, null)];
}
catch (Exception ex) when (!(ex is OperationCanceledException && cancellation.IsCancellationRequested))
{
Core.Diagnostics.Log.Warning("workspace", $"The still of frame {still + 1} of {sequence.DisplayTitle} failed", ex);
outcomes = [new TargetOutcome(target, null, SequenceRender.Reason(ex))];
}
}
else
{
outcomes = await renderer.RenderAllAsync(
targets,
(target, setup) => names.Reserve(taken => ExportNaming.OutputPath(projectSettings.Export, exports, SequenceRender.NameContext(sequence, target, setup),
name => Taken(taken, name))),
Starting,
Record,
context.ReportAsync,
cancellation).ConfigureAwait(false);
}
if (renderer.StandIns.Count > 0)
context.Log(Loc.L("Export.Log.StandIns", string.Join(", ", renderer.StandIns.Select(i => (i + 1).ToString(Loc.Culture)))));
}
@@ -25,6 +25,7 @@ namespace Chronos.Data.Workspaces;
/// <param name="State"><see cref="Pending"/>, <see cref="SentState"/> or <see cref="Failed"/> (a later Chronos may write others: read as pending).</param>
/// <param name="Sent">When the home took it; null until then.</param>
/// <param name="Error">Why the last send failed (one localized sentence); null otherwise.</param>
/// <param name="Kind">The render's kind (<see cref="RenderKind"/>): <c>still</c> for a saved frame, null for a video or an image sequence.</param>
public sealed record WorkspaceRenderEntry(
Guid RenderId,
Guid JobId,
@@ -43,7 +44,8 @@ public sealed record WorkspaceRenderEntry(
DateTimeOffset Rendered,
string State = WorkspaceRenderEntry.Pending,
DateTimeOffset? Sent = null,
string? Error = null)
string? Error = null,
string? Kind = null)
{
/// <summary>Written, not at the home yet.</summary>
public const string Pending = "pending";
@@ -286,6 +286,9 @@ internal sealed class WorkspaceSendJobHandler(ChronosDatabase db, ArchiveService
ProducedBy = options.Processing.ProducedBy,
ProjectId = entry.ProjectId,
ProjectVersionId = entry.ProjectVersionId,
// A file's hash is the file's own (an image sequence's ledger hash is of its listing, not a render's).
Sha256 = folder ? null : entry.Sha256.ToLowerInvariant(),
Kind = entry.Kind,
}, ct).ConfigureAwait(false);
return filed is not null ? true : throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
}
@@ -329,7 +332,7 @@ internal sealed class WorkspaceSendJobHandler(ChronosDatabase db, ArchiveService
{
var name = SyncTransfers.SafeFileName(Path.GetFileName(entry.File)) ?? entry.RenderId.ToString("N");
var request = new RenderUploadRequest(entry.RenderId, name, entry.Size, entry.Sha256.ToLowerInvariant(), entry.PresetName, entry.Codec, entry.Width, entry.Height,
entry.Fps, entry.DurationSeconds, options.Processing.ProducedBy, entry.ProjectId, entry.ProjectVersionId);
entry.Fps, entry.DurationSeconds, options.Processing.ProducedBy, entry.ProjectId, entry.ProjectVersionId, entry.Kind);
try
{
if (await remote.UploadRenderAsync(sequenceId, file, request, ct).ConfigureAwait(false) is not { } filed) return false;
@@ -0,0 +1,129 @@
using System.Security.Cryptography;
using Chronos.Core.Localization;
using Chronos.Core.Model;
using Chronos.Data.Videos;
using Chronos.Testing;
namespace Chronos.Data.Tests.Videos;
/// <summary>Owner review 1, R9, decision 16: a VideoCopy job copies renders into the videos folder, verified, never overwriting.</summary>
[Category("videos")]
public sealed class VideoCopyTests(TestContext context)
{
[Test]
public async Task ACopyIsVerifiedAndRecordedAndATakenNameGetsANumber()
{
await using var fx = await VideoFixture.CreateAsync(context);
var sequence = await fx.SequenceAsync();
var render = await fx.RenderAsync(sequence, "sunset.mp4", 5000);
var old = await fx.RenderAsync(sequence, "old.mp4", 3000, hashed: false);
await File.WriteAllTextAsync(Path.Combine(fx.VideosFolder, "sunset.mp4"), "someone else's", fx.Ct);
await fx.StartEngineAsync();
var request = fx.Library.CopyJob(new VideoCopyParameters([render.Id, old.Id, render.Id], fx.VideosFolder, RequestedBy: "alberto"));
Assert.Equal((JobKind.VideoCopy, 2), (request.Kind, request.Targets.Count));
var job = await fx.EnqueueAsync(request);
await fx.Archive.WaitForAsync(job.Id, JobState.Done);
var copy = Path.Combine(fx.VideosFolder, "sunset (2).mp4");
Assert.SequenceEqual(await File.ReadAllBytesAsync(fx.FileOf(render), fx.Ct), await File.ReadAllBytesAsync(copy, fx.Ct));
Assert.Equal("someone else's", await File.ReadAllTextAsync(Path.Combine(fx.VideosFolder, "sunset.mp4"), fx.Ct), "never overwritten");
Assert.True(File.Exists(Path.Combine(fx.VideosFolder, "old.mp4")));
Assert.Empty(Directory.EnumerateFileSystemEntries(fx.VideosFolder, ".incoming-*"));
var copies = await fx.Db.RenderCopies.ListByRenderAsync(render.Id, fx.Ct);
Assert.Equal((copy, render.Sha256, 5000L, "alberto"), (copies.Single().Path, copies.Single().Sha256, copies.Single().Bytes, copies.Single().CopiedBy));
// A render of before migration 0010 is hashed by its first copy, and remembered.
var hashed = (await fx.Db.Renders.GetAsync(old.Id, fx.Ct))!.Sha256;
Assert.Equal(Convert.ToHexStringLower(SHA256.HashData(await File.ReadAllBytesAsync(fx.FileOf(old), fx.Ct))), hashed);
Assert.Equal(hashed, (await fx.Db.RenderCopies.ListByRenderAsync(old.Id, fx.Ct)).Single().Sha256);
Assert.Equal(1, (await fx.Library.GetAsync(render.Id, fx.Ct))!.Copies.Count);
}
[Test]
public async Task ACopyThatDoesNotMatchIsRemovedAndTheRenderFails()
{
await using var fx = await VideoFixture.CreateAsync(context);
var sequence = await fx.SequenceAsync();
var good = await fx.RenderAsync(sequence, "good.mp4", 2000);
var bad = await fx.RenderAsync(sequence, "bad.mp4", 2000);
// The file changed since it was recorded: its copy cannot match the render's hash.
await File.WriteAllBytesAsync(fx.FileOf(bad), VideoFixture.Content("other", 2000), fx.Ct);
await fx.StartEngineAsync();
var job = await fx.EnqueueAsync(fx.Library.CopyJob(new VideoCopyParameters([bad.Id, good.Id], fx.VideosFolder)));
var failed = await fx.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Equal(Loc.L("Errors.Render.Failed", 1, 2), failed.Error);
Assert.SequenceEqual(["good.mp4"], Directory.GetFileSystemEntries(fx.VideosFolder).Select(Path.GetFileName));
Assert.Empty(await fx.Db.RenderCopies.ListByRenderAsync(bad.Id, fx.Ct));
Assert.True(File.Exists(fx.FileOf(bad)), "the render itself is never touched");
}
[Test]
public async Task ACopyThatCannotFitFailsBeforeWritingAnything()
{
await using var fx = await VideoFixture.CreateAsync(context, freeSpace: 6000);
var sequence = await fx.SequenceAsync();
var first = await fx.RenderAsync(sequence, "first.mp4", 4000);
var second = await fx.RenderAsync(sequence, "second.mp4", 4000);
await fx.StartEngineAsync();
var job = await fx.EnqueueAsync(fx.Library.CopyJob(new VideoCopyParameters([first.Id, second.Id], fx.VideosFolder)));
var failed = await fx.Archive.WaitForAsync(job.Id, JobState.Failed);
Assert.Contains(Loc.L("Errors.Videos.NoSpace", Chronos.Data.Housekeeping.CleanupSteps.Megabytes(2000)), failed.Error!);
Assert.Empty(Directory.EnumerateFileSystemEntries(fx.VideosFolder));
Assert.Empty(await fx.Db.RenderCopies.ListByRenderAsync(first.Id, fx.Ct));
}
[Test]
public async Task AVideosFolderInsideTheArchiveIsRefused()
{
await using var fx = await VideoFixture.CreateAsync(context);
var sequence = await fx.SequenceAsync();
var render = await fx.RenderAsync(sequence, "sunset.mp4", 1000);
var inside = Path.Combine(fx.Archive.Paths.Root, "Videos");
Assert.Equal(VideoErrors.FolderInvalid, fx.Library.FolderProblem(inside));
Assert.Equal(VideoErrors.FolderInvalid, fx.Library.FolderProblem(fx.Archive.Paths.Root));
Assert.Equal(VideoErrors.FolderUnset, fx.Library.FolderProblem(" "));
Assert.Equal(VideoErrors.FolderInvalid, fx.Library.FolderProblem("relative/videos"));
Assert.Null(fx.Library.FolderProblem(fx.VideosFolder));
var refused = Assert.Throws<VideoLibraryException>(() => fx.Library.CopyJob(new VideoCopyParameters([render.Id], inside)));
Assert.Equal((VideoErrors.FolderInvalid, Loc.L("Errors.Videos.FolderInsideArchive")), (refused.Code, refused.Message));
// A workspace the host protects is refused too, and so is a folder that became one after the job was queued.
var workspace = context.CreateTempDirectory("workspace");
fx.Library.ProtectedFolders = () => [workspace];
Assert.Equal(VideoErrors.FolderInvalid, fx.Library.FolderProblem(Path.Combine(workspace, "out")));
fx.Library.ProtectedFolders = null;
var request = fx.Library.CopyJob(new VideoCopyParameters([render.Id], Path.Combine(workspace, "out")));
fx.Library.ProtectedFolders = () => [workspace];
await fx.StartEngineAsync();
var failed = await fx.Archive.WaitForAsync((await fx.EnqueueAsync(request)).Id, JobState.Failed);
Assert.Equal(Loc.L("Errors.Videos.FolderInsideArchive"), failed.Error);
Assert.False(Directory.Exists(Path.Combine(workspace, "out")));
}
[Test]
public async Task AnImageSequenceIsCopiedAndVerifiedFileByFile()
{
await using var fx = await VideoFixture.CreateAsync(context);
var sequence = await fx.SequenceAsync();
var pngs = await fx.SequenceRenderAsync(sequence, "clip-PNG", 5);
Directory.CreateDirectory(Path.Combine(fx.VideosFolder, "clip-PNG"));
await fx.StartEngineAsync();
var job = await fx.EnqueueAsync(fx.Library.CopyJob(new VideoCopyParameters([pngs.Id], fx.VideosFolder)));
await fx.Archive.WaitForAsync(job.Id, JobState.Done);
var copy = Path.Combine(fx.VideosFolder, "clip-PNG (2)");
var sources = Directory.GetFiles(fx.FileOf(pngs)).Order(StringComparer.Ordinal).ToList();
var copied = Directory.GetFiles(copy).Order(StringComparer.Ordinal).ToList();
Assert.SequenceEqual(sources.Select(Path.GetFileName), copied.Select(Path.GetFileName));
for (var i = 0; i < sources.Count; i++)
Assert.SequenceEqual(await File.ReadAllBytesAsync(sources[i], fx.Ct), await File.ReadAllBytesAsync(copied[i], fx.Ct));
var record = (await fx.Db.RenderCopies.ListByRenderAsync(pngs.Id, fx.Ct)).Single();
Assert.Equal((copy, (string?)null, pngs.Bytes), (record.Path, record.Sha256, record.Bytes));
Assert.Null((await fx.Db.Renders.GetAsync(pngs.Id, fx.Ct))!.Sha256, "a folder records no hash of its own");
}
}
@@ -0,0 +1,138 @@
using System.Security.Cryptography;
using Chronos.Core.Codecs.Jpeg;
using Chronos.Core.Diagnostics.Synthetic;
using Chronos.Core.Metadata;
using Chronos.Core.Model;
using Chronos.Data.Archiving;
using Chronos.Data.Tests.Archiving;
using Chronos.Data.Videos;
using Chronos.Testing;
namespace Chronos.Data.Tests.Videos;
/// <summary>A temporary archive with small synthetic sequences and renders recorded on files written by hand.</summary>
internal sealed class VideoFixture : IAsyncDisposable
{
private VideoFixture(TestContext context, ArchiveFixture archive)
{
Archive = archive;
VideosFolder = context.CreateTempDirectory("videos");
}
public ArchiveFixture Archive { get; }
public ChronosDatabase Db => Archive.Db;
public VideoLibrary Library => Archive.Service.Videos;
public CancellationToken Ct => Archive.Ct;
/// <summary>A folder outside the archive that copies go to.</summary>
public string VideosFolder { get; }
/// <param name="freeSpace">How much space the videos folder has (bytes); null: the drive's.</param>
public static async Task<VideoFixture> CreateAsync(TestContext context, long? freeSpace = null)
{
string? videos = null;
var archive = await ArchiveFixture.CreateAsync(context, configure: o => o with
{
FreeSpace = path => freeSpace is { } free && videos is not null && path.StartsWith(videos, StringComparison.OrdinalIgnoreCase) ? free : null,
});
var fixture = new VideoFixture(context, archive);
videos = fixture.VideosFolder;
return fixture;
}
/// <summary>A sequence of four synthetic frames in <paramref name="folder"/>, indexed.</summary>
public async Task<Sequence> SequenceAsync(string folder = "Harbour", int seed = 1)
{
var synthetic = SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = 4, Seed = seed, Start = new DateTime(2024, 6, 21, 19, 0, 0).AddDays(seed) });
var directory = Path.Combine(Archive.Paths.Root, folder);
Directory.CreateDirectory(directory);
for (var i = 0; i < synthetic.Count; i++)
{
using var image = synthetic.Render(i);
var exif = ExifWriter.BuildJpegApp1(synthetic.Metadata[i]);
File.WriteAllBytes(Path.Combine(directory, $"IMG_{i + 1:0000}.JPG"), JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 90, App1Segments = [exif[4..]] }));
}
await Archive.ScanAsync(Chronos.Data.Archiving.ScanMode.Full);
var location = (await Db.Sequences.ListLocationsAsync(Archive.Archive.Id, Ct)).Single(l => l.Folder == folder);
return (await Db.Sequences.GetAsync(location.Id, Ct))!;
}
/// <summary>A render of <paramref name="sequence"/> recorded on a file of <paramref name="bytes"/> written into its exports.</summary>
public async Task<Render> RenderAsync(Sequence sequence, string name, int bytes, DateTimeOffset? created = null, string codec = "H264", string? kind = null,
bool hashed = true)
{
var exports = Archive.Paths.Sequence(sequence.Folder).Exports;
Directory.CreateDirectory(exports);
var content = Content(name, bytes);
await File.WriteAllBytesAsync(Path.Combine(exports, name), content, Ct);
var render = new Render
{
Id = Guid.CreateVersion7(),
SequenceId = sequence.Id,
PresetName = "Test preset",
OutputPath = $"{sequence.Folder}/exports/{name}",
Codec = codec,
Width = 96,
Height = 64,
Fps = kind == RenderKind.Still ? 0 : 24,
DurationSeconds = kind == RenderKind.Still ? 0 : 2,
Bytes = bytes,
Created = created ?? DateTimeOffset.UtcNow,
ProducedBy = "test-pc",
Sha256 = hashed ? Convert.ToHexStringLower(SHA256.HashData(content)) : null,
Kind = kind,
};
await Db.Renders.InsertAsync(render, Ct);
return render;
}
/// <summary>An image-sequence render: a folder of <paramref name="files"/> PNG-named files.</summary>
public async Task<Render> SequenceRenderAsync(Sequence sequence, string name, int files)
{
var folder = Path.Combine(Archive.Paths.Sequence(sequence.Folder).Exports, name);
Directory.CreateDirectory(folder);
long total = 0;
for (var i = 0; i < files; i++)
{
var content = Content(name + i, 300 + i);
total += content.Length;
await File.WriteAllBytesAsync(Path.Combine(folder, $"{name}_{i + 1:000000}.png"), content, Ct);
}
var render = new Render
{
Id = Guid.CreateVersion7(),
SequenceId = sequence.Id,
PresetName = "PNG",
OutputPath = $"{sequence.Folder}/exports/{name}",
Codec = "PngSequence",
Width = 96,
Height = 64,
Fps = 24,
Bytes = total,
Created = DateTimeOffset.UtcNow,
};
await Db.Renders.InsertAsync(render, Ct);
return render;
}
public string FileOf(Render render) => Archive.Paths.FromRelative(render.OutputPath);
private Chronos.Data.Jobs.JobEngine? _engine;
/// <summary>Starts the archive's job engine (its handlers include the videos copy).</summary>
public async Task StartEngineAsync() => _engine = await Archive.StartEngineAsync();
public Task<Job> EnqueueAsync(Chronos.Data.Archiving.Jobs.ArchiveJobRequest request) =>
(_engine ?? throw new InvalidOperationException("Start the engine first.")).EnqueueAsync(request, cancellationToken: Ct);
/// <summary>Deterministic bytes, different for each name.</summary>
public static byte[] Content(string name, int bytes)
{
var content = new byte[bytes];
new Random(StringComparer.Ordinal.GetHashCode(name) & 0x7FFFFFFF).NextBytes(content);
content[0] = (byte)name.Length;
return content;
}
public ValueTask DisposeAsync() => Archive.DisposeAsync();
}
@@ -0,0 +1,211 @@
using Chronos.Core.Model;
using Chronos.Data.Videos;
using Chronos.Protocol;
using Chronos.Testing;
namespace Chronos.Data.Tests.Videos;
/// <summary>Owner review 1, R9: the videos library lists every render, deletes into the trash after a dry run and restores by id.</summary>
[Category("videos")]
public sealed class VideoLibraryTests(TestContext context)
{
private static readonly DateTimeOffset Day = new(2026, 10, 1, 12, 0, 0, TimeSpan.Zero);
[Test]
public async Task TheListPagesNewestFirstAndFilters()
{
await using var fx = await VideoFixture.CreateAsync(context);
var harbour = await fx.SequenceAsync("Harbour", 1);
var roof = await fx.SequenceAsync("Roof", 2);
var renders = new List<Render>();
for (var i = 0; i < 5; i++) renders.Add(await fx.RenderAsync(harbour, $"harbour-{i}.mp4", 1000 + (i * 100), Day.AddHours(i)));
var still = await fx.RenderAsync(roof, "roof - 0003.png", 5000, Day.AddDays(-400), codec: "PngSequence", kind: RenderKind.Still);
var gif = await fx.RenderAsync(roof, "roof.gif", 700, Day.AddMinutes(30), codec: "Gif");
var pngs = await fx.SequenceRenderAsync(roof, "roof-PNG", 3);
// Newest first, two at a time, with the total.
var seen = new List<Guid>();
string? cursor = null;
do
{
var page = await fx.Library.ListAsync(new VideoQuery { Limit = 3, Cursor = cursor }, fx.Ct);
Assert.Equal(8L, page.Total);
Assert.True(page.Items.Count <= 3);
seen.AddRange(page.Items.Select(i => i.Render.Id));
cursor = page.Next;
}
while (cursor is not null);
var all = renders.Append(still).Append(gif).Append(pngs).OrderByDescending(r => r.Created).Select(r => r.Id).ToList();
Assert.SequenceEqual(all, seen);
async Task<List<Guid>> Ids(VideoQuery query) => [.. (await fx.Library.ListAsync(query, fx.Ct)).Items.Select(i => i.Render.Id)];
Assert.SequenceEqual([still.Id], await Ids(new VideoQuery { Kind = RenderKind.Still }));
Assert.SequenceEqual([pngs.Id], await Ids(new VideoQuery { Kind = RenderKind.Sequence }));
Assert.Equal(6, (await Ids(new VideoQuery { Kind = RenderKind.Video })).Count);
Assert.SequenceEqual([gif.Id], await Ids(new VideoQuery { Codec = "gif" }));
Assert.SequenceEqual([still.Id], await Ids(new VideoQuery { Year = Day.AddDays(-400).Year }));
Assert.Equal(3, (await Ids(new VideoQuery { SequenceId = roof.Id })).Count);
Assert.SequenceEqual([renders[3].Id], await Ids(new VideoQuery { Text = "harbour-3" }));
Assert.SequenceEqual([still.Id, renders[4].Id], (await Ids(new VideoQuery { Sort = VideoSorts.Largest })).Take(2));
Assert.SequenceEqual([still.Id], (await Ids(new VideoQuery { Sort = VideoSorts.Oldest })).Take(1));
// The title sort pages too.
var titled = new List<Guid>();
cursor = null;
do
{
var page = await fx.Library.ListAsync(new VideoQuery { Sort = VideoSorts.Title, Limit = 3, Cursor = cursor }, fx.Ct);
titled.AddRange(page.Items.Select(i => i.Render.Id));
cursor = page.Next;
}
while (cursor is not null);
Assert.Equal(8, titled.Distinct().Count());
var item = (await fx.Library.GetAsync(still.Id, fx.Ct))!;
Assert.Equal((RenderKind.Still, false, fx.FileOf(still), "roof - 0003.png"), (item.Kind, item.Missing, item.FullPath, item.FileName));
File.Delete(fx.FileOf(gif));
Assert.True((await fx.Library.GetAsync(gif.Id, fx.Ct))!.Missing);
}
[Test]
public async Task TheDryRunFingerprintChangesWhenTheVideosDo()
{
await using var fx = await VideoFixture.CreateAsync(context);
var sequence = await fx.SequenceAsync();
var first = await fx.RenderAsync(sequence, "a.mp4", 1200);
var later = Guid.CreateVersion7();
var plan = await fx.Library.PlanDeleteAsync([first.Id, later], fx.Ct);
Assert.SequenceEqual([first.Id], plan.Items.Select(i => i.Render.Id));
Assert.Equal((later, VideoBlockReasons.NotFound), (plan.Blocked.Single().RenderId, plan.Blocked.Single().Reason));
Assert.Equal(plan.Fingerprint, (await fx.Library.PlanDeleteAsync([later, first.Id], fx.Ct)).Fingerprint, "the order asked does not matter");
// A render added since: the plan given back no longer holds, and nothing is deleted.
await fx.Db.Renders.InsertAsync(new Render { Id = later, SequenceId = sequence.Id, PresetName = "p", OutputPath = "Harbour/exports/b.mp4", Created = DateTimeOffset.UtcNow }, fx.Ct);
var replanned = await fx.Library.PlanDeleteAsync([first.Id, later], fx.Ct);
Assert.NotEqual(plan.Fingerprint, replanned.Fingerprint);
var refused = await Assert.ThrowsAsync<VideoLibraryException>(() => fx.Library.DeleteAsync([first.Id, later], plan.Fingerprint, "me", fx.Ct));
Assert.Equal(VideoErrors.PlanChanged, refused.Code);
Assert.True(File.Exists(fx.FileOf(first)));
Assert.NotNull(await fx.Db.Renders.GetAsync(first.Id, fx.Ct));
}
[Test]
public async Task ADeleteMovesTheFileAndTheRowIntoTheTrashAndARestoreBringsBothBack()
{
await using var fx = await VideoFixture.CreateAsync(context);
var sequence = await fx.SequenceAsync();
var kept = await fx.RenderAsync(sequence, "kept.mp4", 900);
var render = await fx.RenderAsync(sequence, "sunset.mp4", 4096, hashed: false);
Assert.Equal(2, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.RenderedCount);
var plan = await fx.Library.PlanDeleteAsync([render.Id], fx.Ct);
Assert.Equal(4096L, plan.Bytes);
var result = await fx.Library.DeleteAsync([render.Id], plan.Fingerprint, "alberto", fx.Ct);
var name = result.TrashEntries.Single();
Assert.True(VideoTrashNames.IsEntryName(name));
Assert.False(File.Exists(fx.FileOf(render)));
Assert.Null(await fx.Db.Renders.GetAsync(render.Id, fx.Ct));
Assert.Equal(1, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.RenderedCount);
Assert.NotNull(await fx.Db.Renders.GetAsync(kept.Id, fx.Ct));
// The record is published (no journal left) and the trash lists the entry.
var (folder, record) = VideoTrash.Location(fx.Archive.Paths, name);
Assert.True(File.Exists(record));
Assert.False(File.Exists(record + VideoTrash.PendingSuffix));
Assert.True(File.Exists(Path.Combine(folder, "sunset.mp4")));
var entry = (await fx.Archive.Service.Trash.ListAsync(fx.Archive.Archive, fx.Ct)).Single(e => e.Name == name);
Assert.Equal((Chronos.Data.Archiving.TrashKind.Videos, 4096L, "alberto"), (entry.Kind, entry.Bytes, entry.DeletedBy));
var restored = await fx.Library.RestoreAsync(fx.Archive.Archive.Id, name, fx.Ct);
Assert.Equal(render.Id, restored.Id);
Assert.Equal(render.OutputPath, restored.OutputPath);
Assert.NotNull(restored.Sha256);
Assert.True(File.Exists(fx.FileOf(render)));
Assert.Equal(restored.Sha256, (await fx.Db.Renders.GetAsync(render.Id, fx.Ct))!.Sha256);
Assert.Equal(2, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.RenderedCount);
Assert.False(File.Exists(record));
Assert.False(Directory.Exists(folder));
}
[Test]
public async Task ATamperedTrashedVideoIsNotRestored()
{
await using var fx = await VideoFixture.CreateAsync(context);
var sequence = await fx.SequenceAsync();
var render = await fx.RenderAsync(sequence, "sunset.mp4", 2048);
var plan = await fx.Library.PlanDeleteAsync([render.Id], fx.Ct);
var name = (await fx.Library.DeleteAsync([render.Id], plan.Fingerprint, null, fx.Ct)).TrashEntries.Single();
var trashed = Path.Combine(VideoTrash.Location(fx.Archive.Paths, name).Folder, "sunset.mp4");
var bytes = await File.ReadAllBytesAsync(trashed, fx.Ct);
bytes[100] ^= 0xFF;
await File.WriteAllBytesAsync(trashed, bytes, fx.Ct);
var refused = await Assert.ThrowsAsync<VideoLibraryException>(() => fx.Library.RestoreAsync(fx.Archive.Archive.Id, name, fx.Ct));
Assert.Equal(VideoErrors.HashMismatch, refused.Code);
Assert.True(File.Exists(trashed), "it stays in the trash");
Assert.False(File.Exists(fx.FileOf(render)));
Assert.Null(await fx.Db.Renders.GetAsync(render.Id, fx.Ct));
}
[Test]
public async Task ADeleteACrashInterruptedIsCompletedWhenTheArchiveOpens()
{
await using var fx = await VideoFixture.CreateAsync(context);
var sequence = await fx.SequenceAsync();
var render = await fx.RenderAsync(sequence, "sunset.mp4", 3000);
fx.Library.Trash.CrashAfterMove = true;
fx.Library.Trash.ForceCopy = true;
var plan = await fx.Library.PlanDeleteAsync([render.Id], fx.Ct);
await Assert.ThrowsAsync<IOException>(() => fx.Library.DeleteAsync([render.Id], plan.Fingerprint, null, fx.Ct));
fx.Library.Trash.CrashAfterMove = false;
// The journal is there, the row still is, and the trash does not list a delete under way.
var trash = VideoTrash.Directory(fx.Archive.Paths);
var pending = Directory.GetFiles(trash, "*" + VideoTrash.PendingSuffix).Single();
Assert.NotNull(await fx.Db.Renders.GetAsync(render.Id, fx.Ct));
Assert.Empty((await fx.Archive.Service.Trash.ListAsync(fx.Archive.Archive, fx.Ct)).Where(e => e.Kind == Chronos.Data.Archiving.TrashKind.Videos));
Assert.Equal(1, await fx.Library.RecoverAsync(fx.Archive.Archive, fx.Ct));
Assert.False(File.Exists(pending));
Assert.Null(await fx.Db.Renders.GetAsync(render.Id, fx.Ct));
Assert.False(File.Exists(fx.FileOf(render)));
Assert.Equal(0, (await fx.Db.Sequences.GetAsync(sequence.Id, fx.Ct))!.RenderedCount);
var entry = (await fx.Archive.Service.Trash.ListAsync(fx.Archive.Archive, fx.Ct)).Single(e => e.Kind == Chronos.Data.Archiving.TrashKind.Videos);
Assert.Equal(render.Id, (await fx.Library.RestoreAsync(fx.Archive.Archive.Id, entry.Name, fx.Ct)).Id);
}
[Test]
public async Task AVideoWhoseFileIsMissingLosesOnlyItsRow()
{
await using var fx = await VideoFixture.CreateAsync(context);
var sequence = await fx.SequenceAsync();
var render = await fx.RenderAsync(sequence, "gone.mp4", 1500);
File.Delete(fx.FileOf(render));
var plan = await fx.Library.PlanDeleteAsync([render.Id], fx.Ct);
Assert.True(plan.Items.Single().Missing);
Assert.Equal(0L, plan.Bytes);
var result = await fx.Library.DeleteAsync([render.Id], plan.Fingerprint, null, fx.Ct);
Assert.SequenceEqual([render.Id], result.Deleted.Select(d => d.Render.Id));
Assert.Empty(result.TrashEntries);
Assert.Null(await fx.Db.Renders.GetAsync(render.Id, fx.Ct));
Assert.False(Directory.Exists(VideoTrash.Directory(fx.Archive.Paths)) && Directory.EnumerateFileSystemEntries(VideoTrash.Directory(fx.Archive.Paths)).Any());
}
[Test]
public async Task AnImageSequenceFolderGoesToTheTrashAndBackFileByFile()
{
await using var fx = await VideoFixture.CreateAsync(context);
var sequence = await fx.SequenceAsync();
var pngs = await fx.SequenceRenderAsync(sequence, "clip-PNG", 4);
var plan = await fx.Library.PlanDeleteAsync([pngs.Id], fx.Ct);
var name = (await fx.Library.DeleteAsync([pngs.Id], plan.Fingerprint, null, fx.Ct)).TrashEntries.Single();
Assert.False(Directory.Exists(fx.FileOf(pngs)));
var entry = (await fx.Archive.Service.Trash.ListAsync(fx.Archive.Archive, fx.Ct)).Single(e => e.Name == name);
Assert.Equal(4, entry.Files);
var restored = await fx.Library.RestoreAsync(fx.Archive.Archive.Id, name, fx.Ct);
Assert.Equal((pngs.Id, (string?)null), (restored.Id, restored.Sha256));
Assert.Equal(4, Directory.GetFiles(fx.FileOf(pngs)).Length);
}
}
@@ -0,0 +1,136 @@
using System.Security.Cryptography;
using Chronos.Core.Diagnostics.Synthetic;
using Chronos.Core.Export;
using Chronos.Core.Imaging;
using Chronos.Core.Imaging.Decoders;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Core.Render;
using Chronos.Core.Video;
using Chronos.Data.Processing;
using Chronos.Data.Tests.Processing;
using Chronos.Testing;
namespace Chronos.Data.Tests.Videos;
/// <summary>Owner review 1, phase 5: renders record their hash and kind, a still job saves one frame, playback routes as format 3.</summary>
[Category("videos")]
public sealed class VideoRenderTests(TestContext context)
{
private const int Frames = 8;
private static readonly ExportSettings Mjpeg = new()
{
Preset = new Auto<string>().WithManual("mjpeg-avi"),
Overrides = new ExportOverrides { Width = 96, Height = 64 },
NameTemplate = "clip-{codec}",
};
private static readonly PlaybackSettings Boomerang = new() { Direction = PlaybackDirection.Boomerang, Loops = 2 };
private static SyntheticSequence Scene() => SyntheticSequence.Create(new SyntheticSequenceOptions { FrameCount = Frames, StartEv = 0.5, EndEv = -0.5 });
[Test]
public void ABatchPresetKeepsThePlaybackOfItsExport()
{
var export = Mjpeg with { Playback = Boomerang, Batch = ["gif-preview"] };
Assert.Equal(Boomerang, ProcessingJobs.BatchExport(export, "gif-preview").Playback);
}
[Test]
public void APlaybackJobNeedsFormatThreeAndANodeOfFormatTwoNeverGetsIt()
{
var sequence = new Sequence { Id = Guid.CreateVersion7(), ArchiveId = Guid.CreateVersion7(), Folder = "Synthetic", Title = "Roof" };
static Chronos.Protocol.RenderJobParameters Read(Chronos.Data.Archiving.Jobs.ArchiveJobRequest request) => ProcessingJobs.ReadRender(request.ParametersJson).Parameters;
Assert.Equal(1, Read(ProcessingJobs.Render(sequence, Mjpeg, 96, 64, new ProjectSettings())).RequiredSettingsFormat);
Assert.Equal(ProjectSettings.PlaybackVersion, Read(ProcessingJobs.Render(sequence, Mjpeg with { Playback = Boomerang }, 96, 64, new ProjectSettings())).RequiredSettingsFormat);
Assert.Equal(ProjectSettings.PlaybackVersion, Read(ProcessingJobs.Render(sequence, Mjpeg with { Playback = Boomerang }, 96, 64)).RequiredSettingsFormat,
"without settings too");
// Only a batch preset plays back: the job of that preset needs format 3, the plain one does not.
var split = ProcessingJobs.Split(Read(ProcessingJobs.Render(sequence, Mjpeg with { Batch = ["gif-preview"] }, 96, 64, new ProjectSettings())), 96, 64, new ProjectSettings());
Assert.SequenceEqual([1, 1], split.Select(p => p.RequiredSettingsFormat));
// Routed with settings whose own export plays back, the job's export (which does not) is what counts, and the other way round.
var plain = Read(ProcessingJobs.Render(sequence, Mjpeg, 96, 64, new ProjectSettings()));
var playing = new ProjectSettings { Export = Mjpeg with { Playback = Boomerang } };
Assert.Equal(1, ProcessingJobs.Routed(plain, playing, 96, 64).RequiredSettingsFormat);
var posted = Read(ProcessingJobs.Render(sequence, Mjpeg with { Playback = Boomerang }, 96, 64, new ProjectSettings())) with { RequiredSettingsFormat = 1 };
Assert.Equal(ProjectSettings.PlaybackVersion, ProcessingJobs.Routed(posted, new ProjectSettings(), 96, 64).RequiredSettingsFormat);
Assert.Equal(ProjectSettings.PlaybackVersion, ProcessingJobs.RequiredFormat(new ProjectSettings(), Mjpeg with { Batch = ["gif-preview"], Playback = Boomerang }, 96, 64));
// A still needs format 3 whatever it renders.
var still = Read(ProcessingJobs.Still(sequence, Mjpeg, VideoCodec.PngSequence, 2, null, 96, 64, new ProjectSettings()));
Assert.Equal((RenderRules.StillFormat, (int?)2, true), (still.RequiredSettingsFormat, still.StillFrame, still.IsStill));
}
[Test]
public async Task AVideoRecordsItsHashAndNoKind()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var sequence = await fx.AddAsync(Scene());
await fx.StartEngineAsync();
await fx.WaitForAsync((await fx.EnqueueAsync(ProcessingJobs.Render(sequence, Mjpeg with { Batch = ["png-sequence"] }, 96, 64))).Id, JobState.Done);
var renders = (await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct)).OrderBy(r => r.OutputPath, StringComparer.Ordinal).ToList();
var avi = Path.Combine(fx.Folder(sequence).Exports, "clip-MJPEG.avi");
Assert.Equal(Convert.ToHexStringLower(SHA256.HashData(await File.ReadAllBytesAsync(avi, fx.Ct))), renders[0].Sha256);
Assert.Null(renders[0].Kind);
Assert.Equal(((string?)null, (string?)null), (renders[1].Sha256, renders[1].Kind));
Assert.Equal(RenderKind.Sequence, RenderKind.Of(renders[1]));
}
[Test]
public async Task AStillJobSavesOneFrameAsAnImageAndRecordsIt()
{
await using var fx = await ProcessingFixture.CreateAsync(context);
var sequence = await fx.AddAsync(Scene());
await fx.StartEngineAsync();
var request = ProcessingJobs.Still(sequence, Mjpeg with { NameTemplate = "still-{codec}" }, VideoCodec.PngSequence, 2, null, 96, 64);
await fx.WaitForAsync((await fx.EnqueueAsync(request)).Id, JobState.Done);
var exports = fx.Folder(sequence).Exports;
var image = Path.Combine(exports, "still-PNG - 0003.png");
Assert.True(File.Exists(image), string.Join(", ", Directory.EnumerateFileSystemEntries(exports)));
Assert.Equal(1, Directory.EnumerateFileSystemEntries(exports).Count());
using (var decoded = DecoderFactory.CreateServer().Decode(new DecodeRequest(image)))
Assert.Equal((96, 64), (Assert.NotNull(decoded).Pixels.Width, decoded.Pixels.Height));
var render = (await fx.Db.Renders.ListBySequenceAsync(sequence.Id, fx.Ct)).Single();
Assert.Equal(("Synthetic/exports/still-PNG - 0003.png", RenderKind.Still, 0.0, 0.0), (render.OutputPath, render.Kind, render.Fps, render.DurationSeconds));
Assert.Equal(Convert.ToHexStringLower(SHA256.HashData(await File.ReadAllBytesAsync(image, fx.Ct))), render.Sha256);
// The same frame again: another name, never over the first.
await fx.WaitForAsync((await fx.EnqueueAsync(request)).Id, JobState.Done);
Assert.True(File.Exists(Path.Combine(exports, "still-PNG - 0003 (2).png")));
// A frame outside the sequence fails the job and writes nothing.
var outside = ProcessingJobs.Still(sequence, Mjpeg, VideoCodec.JpegSequence, Frames + 5, null, 96, 64);
await fx.WaitForAsync((await fx.EnqueueAsync(outside)).Id, JobState.Failed);
Assert.Equal(2, Directory.EnumerateFileSystemEntries(exports).Count());
}
[Test]
public void AStillIsFoundByItsPhotoBeforeItsIndex()
{
var sequence = new Sequence { Id = Guid.CreateVersion7(), ArchiveId = Guid.CreateVersion7(), Folder = "S", Title = "S" };
var frames = Enumerable.Range(0, 4).Select(i => new Frame
{
Id = Guid.CreateVersion7(),
SequenceId = sequence.Id,
Index = i,
RelativePath = $"S/{i}.jpg",
FileName = $"{i}.jpg",
Sha256 = new string((char)('a' + i), 64),
}).ToList();
var set = new SequenceFrames(sequence, frames, frames.Select(f => (string?)f.RelativePath).ToList());
Assert.Equal(3, SequenceRender.StillIndex(set, 1, new string('d', 64)));
Assert.Equal(3, SequenceRender.StillIndex(set, 1, new string('D', 64)), "any case");
Assert.Equal(1, SequenceRender.StillIndex(set, 1, new string('z', 64)), "a photo no longer there: the index");
Assert.Null(SequenceRender.StillIndex(set, 9, null));
// Named by the export's template and the frame number from 1; a taken name gets " (2)".
var folder = context.CreateTempDirectory("still");
var name = new ExportNameContext { Date = new DateTime(2026, 9, 28), Title = "Harbour", PresetName = "PNG", Codec = VideoCodec.PngSequence };
var export = new ExportSettings { NameTemplate = "{title} - {codec}" };
Assert.Equal(Path.Combine(folder, "Harbour - PNG - 0042.png"), SequenceRender.StillPath(export, folder, name, 41));
Assert.Equal(Path.Combine(folder, "Harbour - PNG - 0042 (2).png"), SequenceRender.StillPath(export, folder, name, 41, p => p.EndsWith("0042.png", StringComparison.Ordinal)));
}
}
@@ -0,0 +1,39 @@
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Core.Video;
using Chronos.Data.Processing;
using Chronos.Protocol;
using Chronos.Server.Jobs;
using Chronos.Testing;
namespace Chronos.Server.Tests.Jobs;
/// <summary>Owner review 1, phase 5: a job that plays back, or saves a still, goes only to a node that reads settings format 3.</summary>
[Category("server")]
public sealed class PlaybackRoutingTests
{
private static readonly ExportSettings Mjpeg = new()
{
Preset = new Auto<string>().WithManual("mjpeg-avi"),
Overrides = new ExportOverrides { Width = 96, Height = 64 },
};
[Test]
public void APlaybackJobIsNeverGivenToAFormatTwoNode()
{
var sequence = new Sequence { Id = Guid.CreateVersion7(), ArchiveId = Guid.CreateVersion7(), Folder = "Roof", Title = "Roof" };
var playing = ProcessingJobs.ReadRender(ProcessingJobs.Render(sequence, Mjpeg with { Playback = new PlaybackSettings { Loops = 3 } }, 96, 64,
new ProjectSettings()).ParametersJson).Parameters;
var plain = ProcessingJobs.ReadRender(ProcessingJobs.Render(sequence, Mjpeg, 96, 64, new ProjectSettings()).ParametersJson).Parameters;
Assert.Equal((3, 1), (playing.RequiredSettingsFormat, plain.RequiredSettingsFormat));
RegisterNodeRequest Node(int format) => new("node", [new NodeEncoder("mjpeg", "managed", false, false)], 0, 0, format);
Assert.False(RenderNodeService.Renders(Node(2), playing), "a format-2 node would render it forward once");
Assert.True(RenderNodeService.Renders(Node(3), playing));
Assert.True(RenderNodeService.Renders(Node(2), plain));
// A still needs format 3 too (and an image codec no node encodes).
var still = ProcessingJobs.ReadRender(ProcessingJobs.Still(sequence, Mjpeg, VideoCodec.JpegSequence, 0, null, 96, 64, new ProjectSettings()).ParametersJson).Parameters;
Assert.False(RenderNodeService.Renders(Node(2) with { Encoders = [new NodeEncoder(still.Codec, "managed", false, false)] }, still));
}
}