Mass passes need to say what they do now (photo x of y, sequence n of m) without slowing down. This adds the contract and the engine side; handlers and displays follow in their own parcels. - Core: JobDetail (step, item, item x of y, part n of m with unit), JobStep and JobPartUnit appended-only and stored by kebab name; Job.Detail. - Protocol: JobDetailDto, appended as JobDto.Detail, JobProgressEvent.Detail and NodeProgress.Detail, so older peers ignore it and newer ones fall back. - Data: JobContext keeps a detail across reports (ReportAsync with a detail, ReportItemAsync without allocating, CheckpointAsync with a detail) and publishes at most once per ProgressInterval (500 ms on the engine's TimeProvider), at once for the first report, a new step or a completed count; a dropped report costs no allocation or lock, and the persist that falls due publishes what it saves. Persist stays at most every 5 s. JobWire maps steps, units and details; an unknown name reads as none. - Migration 0011 adds jobs.progress_detail (nullable JSON text); an unreadable cell reads as no detail. Lists return it. - Server: ProgressGate (500 ms per job, step-change and completion bypass) shared by the engine host and render nodes; node details are stored and broadcast, an older node without one still works. Refs #65 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
212 lines
11 KiB
C#
212 lines
11 KiB
C#
using Chronos.Core.Model;
|
|
using Chronos.Data.Migrations;
|
|
using Chronos.Data.Tests.Infrastructure;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Data.Tests.Migrations;
|
|
|
|
public abstract class MigrationTests(TestContext context) : RepositoryTest(context)
|
|
{
|
|
private static readonly string[] Tables =
|
|
[
|
|
"archives", "sequences", "frames", "renders", "jobs", "media_cards", "import_history", "users", "tokens",
|
|
"render_nodes", "settings", "tags", "sequence_tags", "audit_log", "uploads", "schedules", "frame_removals", "projects", "project_versions",
|
|
"render_copies",
|
|
];
|
|
|
|
[Test]
|
|
public void EmbeddedScriptsAreFoundForBothDialects()
|
|
{
|
|
foreach (var dialect in new[] { Dialect.Sqlite, Dialect.Postgres })
|
|
{
|
|
var migrations = MigrationRunner.Load(dialect);
|
|
Assert.NotEmpty(migrations);
|
|
Assert.Equal(1, migrations[0].Version);
|
|
Assert.Equal("initial", migrations[0].Name);
|
|
Assert.Contains("CREATE TABLE frames", migrations[0].Sql);
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public async Task CreatesEveryTableAndIsIdempotent()
|
|
{
|
|
var chronos = await OpenChronosAsync();
|
|
var latest = MigrationRunner.Load(Dialect)[^1].Version;
|
|
Assert.Equal(latest, chronos.SchemaVersion);
|
|
Assert.Equal(latest, await MigrationRunner.MigrateAsync(chronos.Database, Ct));
|
|
|
|
await using var session = await chronos.Database.OpenAsync(Ct);
|
|
foreach (var table in Tables) Assert.Equal(0L, await session.ScalarAsync<long>($"SELECT count(*) FROM {table}", [], Ct));
|
|
var recorded = await session.QueryListAsync("SELECT version, name, applied_at FROM schema_migrations ORDER BY version", [], r => r, Ct);
|
|
Assert.Equal(MigrationRunner.Load(Dialect).Count, recorded.Count);
|
|
Assert.Equal("initial", recorded[0].GetString(1));
|
|
Assert.True(recorded[0].GetDateTimeOffset(2) > DateTimeOffset.UtcNow.AddMinutes(-5));
|
|
}
|
|
|
|
[Test]
|
|
public async Task ProjectsKeepOneMainAndOneLiveNamePerSequence()
|
|
{
|
|
var chronos = await OpenChronosAsync();
|
|
var archive = TestData.Archive("Main");
|
|
await chronos.Archives.UpsertAsync(archive, Ct);
|
|
var sequence = TestData.Sequence(archive.Id);
|
|
await chronos.Sequences.UpsertAsync(sequence, Ct);
|
|
var now = DateTimeOffset.UtcNow;
|
|
SequenceProject Project(string name, bool main) => new() { Id = Guid.CreateVersion7(), SequenceId = sequence.Id, Name = name, IsMain = main, CreatedAt = now, UpdatedAt = now };
|
|
|
|
var main = Project("Main", true);
|
|
await chronos.Projects.InsertAsync(main, Ct);
|
|
await Assert.ThrowsAsync<DatabaseException>(() => chronos.Projects.InsertAsync(Project("Other main", true), Ct));
|
|
var vertical = Project("Vertical", false);
|
|
await chronos.Projects.InsertAsync(vertical, Ct);
|
|
await Assert.ThrowsAsync<DatabaseException>(() => chronos.Projects.InsertAsync(Project("Vertical", false), Ct));
|
|
// A deleted project gives its name back.
|
|
await chronos.Projects.UpdateAsync(vertical with { DeletedAt = now }, Ct);
|
|
await chronos.Projects.InsertAsync(Project("Vertical", false), Ct);
|
|
|
|
// Versions: one number each, settings kept byte for byte, and they go with the sequence.
|
|
const string settings = "{\n \"version\": 1,\n \"exposure\": {}\n}";
|
|
var version = new ProjectVersion
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
ProjectId = main.Id,
|
|
Number = 1,
|
|
Kind = ProjectVersionKind.Imported,
|
|
SettingsJson = settings,
|
|
ContentHash = new string('0', 64),
|
|
FramesVersion = 3,
|
|
CreatedAt = now,
|
|
StoredAt = now,
|
|
};
|
|
await chronos.Projects.InsertVersionAsync(version, Ct);
|
|
await Assert.ThrowsAsync<DatabaseException>(() => chronos.Projects.InsertVersionAsync(version with { Id = Guid.CreateVersion7() }, Ct));
|
|
var stored = Assert.NotNull(await chronos.Projects.GetVersionAsync(version.Id, Ct));
|
|
Assert.Equal(settings, stored.SettingsJson);
|
|
Assert.Equal((ProjectVersionKind.Imported, 3), (stored.Kind, stored.FramesVersion));
|
|
Assert.True(await chronos.Projects.AdvanceHeadAsync(main.Id, 0, version.Id, 1, now, Ct));
|
|
Assert.False(await chronos.Projects.AdvanceHeadAsync(main.Id, 0, version.Id, 1, now, Ct), "the head moved already");
|
|
|
|
var render = new Render
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
SequenceId = sequence.Id,
|
|
PresetName = "MJPEG",
|
|
OutputPath = "x/exports/a.avi",
|
|
Created = now,
|
|
ProjectId = main.Id,
|
|
ProjectVersionId = version.Id,
|
|
};
|
|
await chronos.Renders.InsertAsync(render, Ct);
|
|
Assert.Equal((main.Id, version.Id), ((await chronos.Renders.GetAsync(render.Id, Ct))!.ProjectId, (await chronos.Renders.GetAsync(render.Id, Ct))!.ProjectVersionId));
|
|
|
|
await chronos.Sequences.DeleteAsync(sequence.Id, Ct);
|
|
Assert.Empty(await chronos.Projects.ListBySequenceAsync(sequence.Id, includeDeleted: true, Ct));
|
|
Assert.Null(await chronos.Projects.GetVersionAsync(version.Id, Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task RendersKeepTheirHashAndKindAndCopiesOutliveThem()
|
|
{
|
|
var chronos = await OpenChronosAsync();
|
|
var archive = TestData.Archive("Main");
|
|
await chronos.Archives.UpsertAsync(archive, Ct);
|
|
var sequence = TestData.Sequence(archive.Id);
|
|
await chronos.Sequences.UpsertAsync(sequence, Ct);
|
|
var now = DateTimeOffset.UtcNow;
|
|
var render = new Render { Id = Guid.CreateVersion7(), SequenceId = sequence.Id, PresetName = "PNG", OutputPath = "x/exports/a.png", Created = now };
|
|
await chronos.Renders.InsertAsync(render, Ct);
|
|
|
|
await using var session = await chronos.Database.OpenAsync(Ct);
|
|
// Rows recorded before migration 0010 have neither: null, told apart by the codec.
|
|
Assert.Equal(1L, await session.ScalarAsync<long>("SELECT count(*) FROM renders WHERE sha256 IS NULL AND kind IS NULL", [], Ct));
|
|
var hash = new string('a', 64);
|
|
await session.ExecuteAsync("UPDATE renders SET sha256 = $1, kind = $2 WHERE id = $3", [hash, "still", render.Id], Ct);
|
|
Assert.Equal(hash, await session.ScalarAsync<string>("SELECT sha256 FROM renders WHERE id = $1", [render.Id], Ct));
|
|
Assert.Equal("still", await session.ScalarAsync<string>("SELECT kind FROM renders WHERE id = $1", [render.Id], Ct));
|
|
|
|
// A copy names its render without a foreign key: it is kept when the render goes, and may name one never stored here.
|
|
foreach (var renderId in new[] { render.Id, Guid.CreateVersion7() })
|
|
{
|
|
await session.ExecuteAsync(
|
|
"INSERT INTO render_copies (id, render_id, path, sha256, bytes, copied_at, copied_by) VALUES ($1, $2, $3, $4, $5, $6, $7)",
|
|
[Guid.CreateVersion7(), renderId, $"C:/Videos/{renderId:N}.png", hash, 1234L, now, "admin"], Ct);
|
|
}
|
|
await chronos.Renders.DeleteAsync(render.Id, Ct);
|
|
Assert.Equal(2L, await session.ScalarAsync<long>("SELECT count(*) FROM render_copies", [], Ct));
|
|
Assert.Equal(1234L, await session.ScalarAsync<long>("SELECT bytes FROM render_copies WHERE render_id = $1", [render.Id], Ct));
|
|
// Bytes default to 0 and the hash and author may be null.
|
|
await session.ExecuteAsync("INSERT INTO render_copies (id, render_id, path, copied_at) VALUES ($1, $2, $3, $4)", [Guid.CreateVersion7(), render.Id, "C:/Videos/b", now], Ct);
|
|
Assert.Equal(0L, await session.ScalarAsync<long>("SELECT bytes FROM render_copies WHERE path = $1", ["C:/Videos/b"], Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheProgressDetailColumnJoinsAnIndexOfTheVideosRelease()
|
|
{
|
|
var database = await OpenDatabaseAsync();
|
|
var all = MigrationRunner.Load(Dialect);
|
|
Assert.Equal(10, await MigrationRunner.MigrateAsync(database, [.. all.Where(m => m.Version <= 10)], Ct));
|
|
var id = Guid.CreateVersion7();
|
|
await using (var session = await database.OpenAsync(Ct))
|
|
await session.ExecuteAsync("INSERT INTO jobs (id, kind, state, priority, created) VALUES ($1, $2, $3, $4, $5)",
|
|
[id, (int)JobKind.Verify, (int)JobState.Paused, (int)JobPriority.Normal, DateTimeOffset.UtcNow], Ct);
|
|
|
|
Assert.Equal(11, all.Single(m => m.Name == "job_progress_detail").Version);
|
|
Assert.Equal(all[^1].Version, await MigrationRunner.MigrateAsync(database, Ct));
|
|
await using var after = await database.OpenAsync(Ct);
|
|
// No backfill: a job of 1.0 has no detail.
|
|
Assert.Equal(1L, await after.ScalarAsync<long>("SELECT count(*) FROM jobs WHERE id = $1 AND progress_detail IS NULL", [id], Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AppliesOnlyPendingMigrations()
|
|
{
|
|
var chronos = await OpenChronosAsync();
|
|
var next = MigrationRunner.Load(Dialect).Append(new Migration(9000, "extra", "CREATE TABLE extra (id integer PRIMARY KEY)")).ToList();
|
|
Assert.Equal(9000, await MigrationRunner.MigrateAsync(chronos.Database, next, Ct));
|
|
Assert.Equal(9000, await MigrationRunner.MigrateAsync(chronos.Database, next, Ct));
|
|
await using var session = await chronos.Database.OpenAsync(Ct);
|
|
Assert.Equal(1L, await session.ScalarAsync<long>("SELECT count(*) FROM schema_migrations WHERE version = $1", [9000], Ct));
|
|
Assert.Equal(0L, await session.ScalarAsync<long>("SELECT count(*) FROM extra", [], Ct));
|
|
}
|
|
|
|
[Test]
|
|
public async Task RefusesADatabaseFromANewerBuild()
|
|
{
|
|
var chronos = await OpenChronosAsync();
|
|
await using (var session = await chronos.Database.OpenAsync(Ct))
|
|
await session.ExecuteAsync("INSERT INTO schema_migrations (version, name, applied_at) VALUES ($1, $2, $3)", [9999, "future", DateTimeOffset.UtcNow], Ct);
|
|
var error = await Assert.ThrowsAsync<SchemaTooNewException>(() => MigrationRunner.MigrateAsync(chronos.Database, Ct));
|
|
Assert.Equal(9999, error.DatabaseVersion);
|
|
Assert.Equal(MigrationRunner.Load(Dialect)[^1].Version, error.SupportedVersion);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AFailingMigrationLeavesNothingBehind()
|
|
{
|
|
var database = await OpenDatabaseAsync();
|
|
List<Migration> broken =
|
|
[
|
|
new(1, "good", "CREATE TABLE good_one (id integer PRIMARY KEY)"),
|
|
new(2, "bad", "CREATE TABLE good_two (id integer PRIMARY KEY); THIS IS NOT SQL"),
|
|
];
|
|
await Assert.ThrowsAsync<DatabaseException>(() => MigrationRunner.MigrateAsync(database, broken, Ct));
|
|
await using var session = await database.OpenAsync(Ct);
|
|
var leftovers = await session.ScalarAsync<long>(ForDialect(
|
|
"SELECT count(*) FROM sqlite_master WHERE name IN ('good_one', 'good_two', 'schema_migrations')",
|
|
"SELECT count(*) FROM information_schema.tables WHERE table_schema = current_schema() AND table_name IN ('good_one', 'good_two', 'schema_migrations')"),
|
|
[], Ct);
|
|
Assert.Equal(0L, leftovers);
|
|
}
|
|
}
|
|
|
|
public sealed class SqliteMigrationTests(TestContext context) : MigrationTests(context)
|
|
{
|
|
protected override Backend Backend => Backend.SqliteMemory;
|
|
}
|
|
|
|
[RequiresEnvironment(PgTestServer.Variable)]
|
|
public sealed class PostgresMigrationTests(TestContext context) : MigrationTests(context)
|
|
{
|
|
protected override Backend Backend => Backend.Postgres;
|
|
}
|