diff --git a/src/Titano.Data/Postgres/PgAuthentication.cs b/src/Titano.Data/Postgres/PgAuthentication.cs index f3933958..96f3299f 100644 --- a/src/Titano.Data/Postgres/PgAuthentication.cs +++ b/src/Titano.Data/Postgres/PgAuthentication.cs @@ -26,6 +26,9 @@ internal sealed class ScramSha256 public const string Mechanism = "SCRAM-SHA-256"; + /// The server proved knowledge of the password with a valid final signature. + public bool IsServerVerified { get; private set; } + public byte[] ClientFirstMessage() { _clientFirstBare = $"n={_username},r={_clientNonce}"; @@ -71,6 +74,7 @@ internal sealed class ScramSha256 if (_expectedServerSignature is null || value is null || !CryptographicOperations.FixedTimeEquals(Convert.FromBase64String(value), _expectedServerSignature)) throw new AuthenticationException("The server's SCRAM signature is invalid."); + IsServerVerified = true; } /// diff --git a/src/Titano.Data/Postgres/PgConnector.cs b/src/Titano.Data/Postgres/PgConnector.cs index a06d5292..74edffc4 100644 --- a/src/Titano.Data/Postgres/PgConnector.cs +++ b/src/Titano.Data/Postgres/PgConnector.cs @@ -4,6 +4,7 @@ using System.Net.Security; using System.Net.Sockets; using System.Runtime.CompilerServices; using System.Runtime.ExceptionServices; +using System.Security.Authentication; using System.Security.Cryptography.X509Certificates; using System.Text; using Titano.Core.Diagnostics; @@ -423,6 +424,9 @@ internal sealed class PgConnector : IAsyncDisposable switch (code) { case 0: + // A server that started SCRAM must finish it: only its final signature proves it knows the password. + if (scram is { IsServerVerified: false }) + throw new AuthenticationException("The server skipped the final SCRAM message."); return scram; case 3: WritePassword(RequirePassword()); @@ -551,6 +555,12 @@ internal sealed class PgConnector : IAsyncDisposable _ioAbort.Dispose(); _ioAbort = new CancellationTokenSource(); } + // A huge row (a large bytea) grew the buffer; do not keep it for the life of a pooled connection. + if (_read.Length > 1 << 20 && _readPos == _readEnd) + { + _read = new byte[16 * 1024]; + _readPos = _readEnd = 0; + } _active = null; MarkIdle(); } diff --git a/src/Titano.Data/Postgres/PgListener.cs b/src/Titano.Data/Postgres/PgListener.cs index dd5455c8..b6bcedf8 100644 --- a/src/Titano.Data/Postgres/PgListener.cs +++ b/src/Titano.Data/Postgres/PgListener.cs @@ -13,7 +13,9 @@ public sealed class PgListener : IAsyncDisposable private readonly PgConnector _connector; private readonly SemaphoreSlim _commandLock = new(1, 1); private readonly CancellationTokenSource _stop = new(); + private readonly Lock _gate = new(); private TaskCompletionSource? _pending; + private bool _closed; internal PgListener(PgConnector connector) { @@ -48,39 +50,59 @@ public sealed class PgListener : IAsyncDisposable _stop.Dispose(); } + /// + /// Sends one command and waits for its ReadyForQuery. The command lock is held until the server has answered, + /// even when the caller stops waiting, so a late answer can never be taken for the next command's. + /// private async Task CommandAsync(string sql, CancellationToken cancellationToken) { ObjectDisposedException.ThrowIf(_stop.IsCancellationRequested, this); await _commandLock.WaitAsync(cancellationToken).ConfigureAwait(false); + var pending = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); try { - if (Completion.IsCompleted) throw new DatabaseConnectionException("The listener connection is closed.", Completion.Exception?.InnerException); - var pending = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - Volatile.Write(ref _pending, pending); - await _connector.SendQueryAsync(sql, cancellationToken).ConfigureAwait(false); - if (await pending.Task.WaitAsync(cancellationToken).ConfigureAwait(false) is { } error) throw error; + lock (_gate) + { + if (_closed) throw new DatabaseConnectionException("The listener connection is closed.", Completion.Exception?.InnerException); + _pending = pending; + } + // The listener's own token: a write torn by the caller's token would corrupt the protocol stream. + await _connector.SendQueryAsync(sql, _stop.Token).ConfigureAwait(false); } - finally + catch { _commandLock.Release(); + throw; } + _ = pending.Task.ContinueWith(static (_, state) => ((SemaphoreSlim)state!).Release(), _commandLock, + CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); + if (await pending.Task.WaitAsync(cancellationToken).ConfigureAwait(false) is { } error) throw error; } private async Task RunAsync() { try { - await _connector.RunListenerAsync(error => Volatile.Read(ref _pending)?.TrySetResult(error), _stop.Token).ConfigureAwait(false); + await _connector.RunListenerAsync(OnCommandCompleted, _stop.Token).ConfigureAwait(false); } catch (Exception) when (_stop.IsCancellationRequested) { } finally { - Volatile.Read(ref _pending)?.TrySetException(new DatabaseConnectionException("The listener connection is closed.")); + lock (_gate) + { + _closed = true; + _pending?.TrySetException(new DatabaseConnectionException("The listener connection is closed.")); + } } } + private void OnCommandCompleted(PostgresException? error) + { + lock (_gate) _pending?.TrySetResult(error); + } + private void Raise(PgNotification notification) { try diff --git a/src/Titano.Data/Postgres/PgPool.cs b/src/Titano.Data/Postgres/PgPool.cs index 8f2beb39..c9f8e9f4 100644 --- a/src/Titano.Data/Postgres/PgPool.cs +++ b/src/Titano.Data/Postgres/PgPool.cs @@ -38,9 +38,19 @@ internal sealed class PgPool : IAsyncDisposable { while (TakeIdle() is { } connector) { - if (!connector.IsBroken - && (connector.IdleTime < _settings.HealthCheckInterval || await connector.PingAsync(cancellationToken).ConfigureAwait(false))) - return connector; + bool healthy; + try + { + healthy = !connector.IsBroken + && (connector.IdleTime < _settings.HealthCheckInterval || await connector.PingAsync(cancellationToken).ConfigureAwait(false)); + } + catch + { + // The caller gave up during the health check; the connection's state is unknown. + await connector.DisposeAsync().ConfigureAwait(false); + throw; + } + if (healthy) return connector; await connector.DisposeAsync().ConfigureAwait(false); } return await OpenAsync(cancellationToken).ConfigureAwait(false); diff --git a/src/Titano.Data/Postgres/PostgresDatabase.cs b/src/Titano.Data/Postgres/PostgresDatabase.cs index 81fc0417..84bf644d 100644 --- a/src/Titano.Data/Postgres/PostgresDatabase.cs +++ b/src/Titano.Data/Postgres/PostgresDatabase.cs @@ -112,28 +112,32 @@ internal sealed class PostgresSession(PgPool pool, PgConnector connector) : IDbS _disposed = true; if (_connector is not { } connection) return; _connector = null; - if (_transaction is not null && !connection.IsBroken) + try { - try - { - await connection.SimpleQueryAsync("ROLLBACK", CancellationToken.None).ConfigureAwait(false); - } - catch (DatabaseException ex) - { - Log.Warning(LogCategory, "Rollback on session close failed; the connection is discarded.", ex); - } + if (_transaction is not null) await RollBackQuietlyAsync(connection).ConfigureAwait(false); + } + finally + { + _transaction = null; + pool.Return(connection); } - _transaction = null; - pool.Return(connection); } - /// Ends if it is still active. COMMIT of an aborted transaction is reported, not silently rolled back. - internal async Task EndTransactionAsync(PostgresTransaction transaction, bool commit, CancellationToken cancellationToken) + /// + /// Ends if it is still active. COMMIT of an aborted transaction is reported, not + /// silently rolled back. A rollback while disposing never throws: a broken connection's transaction dies with + /// it, and on a busy one the session stays in the transaction until it is disposed (the pool then discards it). + /// + internal async Task EndTransactionAsync(PostgresTransaction transaction, bool commit, bool disposing, CancellationToken cancellationToken) { if (_transaction != transaction) return; try { - if (!commit) + if (disposing) + { + if (_connector is { } connection) await RollBackQuietlyAsync(connection).ConfigureAwait(false); + } + else if (!commit) { await Connector.SimpleQueryAsync("ROLLBACK", cancellationToken).ConfigureAwait(false); } @@ -153,6 +157,19 @@ internal sealed class PostgresSession(PgPool pool, PgConnector connector) : IDbS } } + private static async Task RollBackQuietlyAsync(PgConnector connection) + { + if (connection.IsBroken || connection.IsBusy) return; + try + { + await connection.SimpleQueryAsync("ROLLBACK", CancellationToken.None).ConfigureAwait(false); + } + catch (DatabaseException ex) + { + Log.Warning(LogCategory, "Rollback on dispose failed; the connection is discarded.", ex); + } + } + private static bool IsConnectionFailure(Exception ex) => ex is DatabaseConnectionException or PostgresException { IsConnectionFailure: true }; @@ -196,21 +213,21 @@ internal sealed class PostgresTransaction(PostgresSession session) : IDbTransact public async Task CommitAsync(CancellationToken cancellationToken = default) { ObjectDisposedException.ThrowIf(_completed, this); + await session.EndTransactionAsync(this, commit: true, disposing: false, cancellationToken).ConfigureAwait(false); _completed = true; - await session.EndTransactionAsync(this, commit: true, cancellationToken).ConfigureAwait(false); } public async Task RollbackAsync(CancellationToken cancellationToken = default) { ObjectDisposedException.ThrowIf(_completed, this); + await session.EndTransactionAsync(this, commit: false, disposing: false, cancellationToken).ConfigureAwait(false); _completed = true; - await session.EndTransactionAsync(this, commit: false, cancellationToken).ConfigureAwait(false); } public async ValueTask DisposeAsync() { if (_completed) return; _completed = true; - await session.EndTransactionAsync(this, commit: false, CancellationToken.None).ConfigureAwait(false); + await session.EndTransactionAsync(this, commit: false, disposing: true, CancellationToken.None).ConfigureAwait(false); } } diff --git a/src/Titano.Data/Postgres/PostgresSettings.cs b/src/Titano.Data/Postgres/PostgresSettings.cs index 76e003d6..00f1601c 100644 --- a/src/Titano.Data/Postgres/PostgresSettings.cs +++ b/src/Titano.Data/Postgres/PostgresSettings.cs @@ -68,8 +68,8 @@ public sealed record PostgresSettings cs.ThrowOnUnknownKeys(); if (settings.MaxPoolSize < 1 || settings.MinPoolSize < 0 || settings.MinPoolSize > settings.MaxPoolSize) throw new FormatException("Pool sizes must satisfy 0 ≤ Minimum Pool Size ≤ Maximum Pool Size, Maximum ≥ 1."); - if (settings.StatementCacheSize < 0 || settings.ConnectTimeout <= TimeSpan.Zero) - throw new FormatException("'Statement Cache Size' must be ≥ 0 and 'Timeout' > 0."); + if (settings.StatementCacheSize < 0 || settings.ConnectTimeout <= TimeSpan.Zero || settings.IdleLifetime <= TimeSpan.Zero) + throw new FormatException("'Statement Cache Size' must be ≥ 0; 'Timeout' and 'Connection Idle Lifetime' must be > 0."); return settings; } diff --git a/src/Titano.Data/Repositories/JobRepository.cs b/src/Titano.Data/Repositories/JobRepository.cs index 228a8159..b3bd9857 100644 --- a/src/Titano.Data/Repositories/JobRepository.cs +++ b/src/Titano.Data/Repositories/JobRepository.cs @@ -43,7 +43,7 @@ public sealed class JobRepository var claim = $"UPDATE jobs SET state = {(int)JobState.Running}, node_id = $1, started = $2, finished = NULL, error = NULL, attempts = attempts + 1"; DbParam[] args = [workerId, DateTimeOffset.UtcNow]; if (_db.Sql.IsPostgres) - return _db.SingleAsync($"{claim} WHERE id = ({pick} FOR UPDATE SKIP LOCKED) RETURNING {Columns}", args, Map, cancellationToken); + return _db.SingleAsync($"{claim} WHERE id = ({pick}{_db.Sql.ForUpdateSkipLocked}) RETURNING {Columns}", args, Map, cancellationToken); return _db.InTransactionAsync(async session => { diff --git a/src/Titano.Data/Repositories/RenderRepository.cs b/src/Titano.Data/Repositories/RenderRepository.cs index 98448d28..f7d52f45 100644 --- a/src/Titano.Data/Repositories/RenderRepository.cs +++ b/src/Titano.Data/Repositories/RenderRepository.cs @@ -18,6 +18,7 @@ public sealed class RenderRepository public Task InsertAsync(Render render, CancellationToken cancellationToken = default) => _db.InTransactionAsync(async session => { + await LockSequenceAsync(session, render.SequenceId, cancellationToken).ConfigureAwait(false); await session.ExecuteAsync($"INSERT INTO renders ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)", [ render.Id, render.SequenceId, render.PresetName, DbParam.Json(render.SettingsJson), render.OutputPath, render.Codec, @@ -38,11 +39,21 @@ public sealed class RenderRepository { var sequenceId = await session.ScalarAsync("SELECT sequence_id FROM renders WHERE id = $1", [id], cancellationToken).ConfigureAwait(false); if (sequenceId is null) return false; - await session.ExecuteAsync("DELETE FROM renders WHERE id = $1", [id], cancellationToken).ConfigureAwait(false); + await LockSequenceAsync(session, sequenceId.Value, cancellationToken).ConfigureAwait(false); + if (await session.ExecuteAsync("DELETE FROM renders WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) == 0) return false; await session.ExecuteAsync(RecountSql, [sequenceId], cancellationToken).ConfigureAwait(false); return true; }, cancellationToken); + /// + /// Serialises render changes per sequence on PostgreSQL, so the recount (a new statement, hence a new snapshot) + /// sees every committed render; SQLite's write lock already does this. + /// + private async Task LockSequenceAsync(IDbSession session, Guid sequenceId, CancellationToken cancellationToken) + { + if (_db.Sql.IsPostgres) await SequenceRepository.LockAsync(session, sequenceId, cancellationToken).ConfigureAwait(false); + } + private static Render Map(DbRow r) => new() { Id = r.GetGuid(0), diff --git a/src/Titano.Data/Repositories/SequenceRepository.cs b/src/Titano.Data/Repositories/SequenceRepository.cs index 422ad364..4912cec5 100644 --- a/src/Titano.Data/Repositories/SequenceRepository.cs +++ b/src/Titano.Data/Repositories/SequenceRepository.cs @@ -87,13 +87,14 @@ public sealed class SequenceRepository "s.frame_count, s.median_interval, s.total_bytes, s.status, s.rendered_count, s.verified_at, s.hero_frame_id, s.rating, s.notes, " + "s.settings_json, s.analysis_json, s.origin, s.sync_state, s.settings_version, s.frames_version, s.owner_user_id, s.warning_count, s.updated_at"; + /// rendered_count is owned by once the row exists. private static readonly string UpsertSql = SqlDialect.Upsert("sequences", [ "id", "archive_id", "title", "title_key", "search_text", "folder", "camera_make", "camera_model", "camera_serial", "lens", "first_capture", "last_capture", "frame_count", "median_interval", "total_bytes", "status", "rendered_count", "verified_at", "hero_frame_id", "rating", "notes", "settings_json", "analysis_json", "origin", "sync_state", "settings_version", "frames_version", "owner_user_id", "warning_count", "updated_at", - ], 1); + ], 1, ["rendered_count"]); /// Ordinal of the tag list after , and of the sort key a listing appends after it. private const int TagsOrdinal = 28; @@ -112,7 +113,11 @@ public sealed class SequenceRepository $"SELECT {Columns}, (SELECT {_db.Sql.StringAgg("t.name")} FROM sequence_tags st JOIN tags t ON t.id = st.tag_id " + $"WHERE st.sequence_id = s.id) AS tags{extraColumns} FROM sequences s"; - /// Inserts or replaces the sequence row and its tag set. + /// + /// Inserts the sequence, or replaces every stored field and the tag set with the record's (last writer wins); + /// only of an existing row is kept, as renders maintain it. For concurrent + /// partial changes use , or . + /// public Task UpsertAsync(Sequence sequence, CancellationToken cancellationToken = default) => _db.InTransactionAsync(async session => { @@ -132,8 +137,7 @@ public sealed class SequenceRepository public async Task SetTagsAsync(Guid id, IEnumerable tags, CancellationToken cancellationToken = default) => await _db.InTransactionAsync(async session => { - if (await session.ScalarAsync("SELECT id FROM sequences WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) is null) - return false; + if (!await LockAsync(session, id, cancellationToken).ConfigureAwait(false)) return false; await ReplaceTagsAsync(session, id, tags, cancellationToken).ConfigureAwait(false); await session.ExecuteAsync("UPDATE sequences SET settings_version = settings_version + 1, updated_at = $2 WHERE id = $1", [id, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false); @@ -149,7 +153,8 @@ public sealed class SequenceRepository public Task RefreshStatsAsync(Guid id, CancellationToken cancellationToken = default) => _db.InTransactionAsync(async session => { - var current = await session.QuerySingleOrDefaultAsync($"{_select} WHERE s.id = $1", [id], Map, cancellationToken).ConfigureAwait(false); + var current = await session.QuerySingleOrDefaultAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken) + .ConfigureAwait(false); if (current is null) return null; var stats = await session.QuerySingleAsync( "SELECT count(*), COALESCE(SUM(size), 0), MIN(captured_at), MAX(captured_at) FROM frames WHERE sequence_id = $1", @@ -312,16 +317,37 @@ public sealed class SequenceRepository ], cancellationToken); } - private static async Task ReplaceTagsAsync(IDbSession session, Guid sequenceId, IEnumerable tags, CancellationToken cancellationToken) + /// + /// Row-locks a sequence for a read-modify-write (PostgreSQL; SQLite's write lock already serialises writers). + /// False when it does not exist. + /// + internal static async Task LockAsync(IDbSession session, Guid id, CancellationToken cancellationToken) => + await session.ScalarAsync($"SELECT id FROM sequences WHERE id = $1{SqlDialect.For(session.Dialect).ForUpdate}", [id], cancellationToken) + .ConfigureAwait(false) is not null; + + /// Replaces the tag links of a sequence whose row the caller has locked (or just upserted). + private async Task ReplaceTagsAsync(IDbSession session, Guid sequenceId, IEnumerable tags, CancellationToken cancellationToken) { await session.ExecuteAsync("DELETE FROM sequence_tags WHERE sequence_id = $1", [sequenceId], cancellationToken).ConfigureAwait(false); foreach (var tag in TagRepository.NormalizeSet(tags)) { - var key = TagRepository.Key(tag); - await session.ExecuteAsync("INSERT INTO tags (id, name, name_key) VALUES ($1, $2, $3) ON CONFLICT (name_key) DO NOTHING", - [Guid.CreateVersion7(), tag, key], cancellationToken).ConfigureAwait(false); - await session.ExecuteAsync("INSERT INTO sequence_tags (sequence_id, tag_id) SELECT $1, id FROM tags WHERE name_key = $2", - [sequenceId, key], cancellationToken).ConfigureAwait(false); + DbParam[] args = [Guid.CreateVersion7(), tag, TagRepository.Key(tag)]; + Guid tagId; + if (_db.Sql.IsPostgres) + { + // The no-op update locks an existing tag, so a concurrent DeleteUnusedAsync cannot remove it before the link exists. + tagId = (await session.ScalarAsync( + "INSERT INTO tags (id, name, name_key) VALUES ($1, $2, $3) ON CONFLICT (name_key) DO UPDATE SET name_key = excluded.name_key RETURNING id", + args, cancellationToken).ConfigureAwait(false))!.Value; + } + else + { + await session.ExecuteAsync("INSERT INTO tags (id, name, name_key) VALUES ($1, $2, $3) ON CONFLICT (name_key) DO NOTHING", + args, cancellationToken).ConfigureAwait(false); + tagId = (await session.ScalarAsync("SELECT id FROM tags WHERE name_key = $1", [args[2]], cancellationToken).ConfigureAwait(false))!.Value; + } + await session.ExecuteAsync("INSERT INTO sequence_tags (sequence_id, tag_id) VALUES ($1, $2)", [sequenceId, tagId], cancellationToken) + .ConfigureAwait(false); } } diff --git a/src/Titano.Data/Repositories/TagRepository.cs b/src/Titano.Data/Repositories/TagRepository.cs index 07de96ce..bb8a616a 100644 --- a/src/Titano.Data/Repositories/TagRepository.cs +++ b/src/Titano.Data/Repositories/TagRepository.cs @@ -36,9 +36,27 @@ public sealed class TagRepository public async Task DeleteAsync(Guid id, CancellationToken cancellationToken = default) => await _db.ExecuteAsync("DELETE FROM tags WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) > 0; - /// Deletes tags no sequence uses any more; returns how many. + /// + /// Deletes tags no sequence uses any more; returns how many. Candidates are locked first (skipping tags a + /// concurrent tagger holds) and re-checked by a later statement, whose fresh snapshot sees every link committed + /// before the lock, so a tag being attached at the same moment survives. + /// public Task DeleteUnusedAsync(CancellationToken cancellationToken = default) => - _db.ExecuteAsync("DELETE FROM tags WHERE NOT EXISTS (SELECT 1 FROM sequence_tags st WHERE st.tag_id = tags.id)", [], cancellationToken); + _db.InTransactionAsync(async session => + { + const string unused = "NOT EXISTS (SELECT 1 FROM sequence_tags st WHERE st.tag_id = tags.id)"; + var candidates = await session.QueryListAsync($"SELECT id FROM tags WHERE {unused}{_db.Sql.ForUpdateSkipLocked}", [], + r => r.GetGuid(0), cancellationToken).ConfigureAwait(false); + var deleted = 0; + foreach (var chunk in candidates.Chunk(500)) + { + var p = new SqlParameters(); + var ids = string.Join(", ", chunk.Select(id => p.Add(id))); + deleted += await session.ExecuteAsync($"DELETE FROM tags WHERE id IN ({ids}) AND {unused}", p.ToArray(), cancellationToken) + .ConfigureAwait(false); + } + return deleted; + }, cancellationToken); /// Trims, collapses inner whitespace and validates a tag name. public static string Normalize(string name) diff --git a/src/Titano.Data/SqlDialect.cs b/src/Titano.Data/SqlDialect.cs index 1581a9ad..a847dbec 100644 --- a/src/Titano.Data/SqlDialect.cs +++ b/src/Titano.Data/SqlDialect.cs @@ -27,6 +27,9 @@ public sealed class SqlDialect /// Row lock for read-modify-write inside a transaction; empty on SQLite, whose BEGIN IMMEDIATE already holds the write lock. public string ForUpdate => IsPostgres ? " FOR UPDATE" : string.Empty; + /// Like , but rows another transaction has locked are skipped instead of awaited. + public string ForUpdateSkipLocked => IsPostgres ? " FOR UPDATE SKIP LOCKED" : string.Empty; + /// Calendar year of a timestamp column (SQLite stores ISO-8601 text). public string Year(string expression) => IsPostgres ? $"CAST(EXTRACT(YEAR FROM {expression}) AS integer)" @@ -44,21 +47,19 @@ public sealed class SqlDialect /// /// INSERT … VALUES ($1…$n) ON CONFLICT (keys) DO UPDATE SET …; the first - /// columns form the conflict target. The syntax is shared by SQLite ≥ 3.24 and PostgreSQL ≥ 9.5. + /// columns form the conflict target and columns keep their stored value on + /// conflict. The syntax is shared by SQLite ≥ 3.24 and PostgreSQL ≥ 9.5. /// - public static string Upsert(string table, IReadOnlyList columns, int keyCount) + public static string Upsert(string table, IReadOnlyList columns, int keyCount, IReadOnlyCollection? insertOnly = null) { ArgumentOutOfRangeException.ThrowIfLessThan(keyCount, 1); ArgumentOutOfRangeException.ThrowIfGreaterThan(keyCount, columns.Count); var sb = new StringBuilder("INSERT INTO ").Append(table).Append(" (").AppendJoin(", ", columns).Append(") VALUES ("); for (var i = 1; i <= columns.Count; i++) sb.Append(i > 1 ? ", $" : "$").Append(i); - sb.Append(") ON CONFLICT ("); - for (var i = 0; i < keyCount; i++) sb.Append(i > 0 ? ", " : string.Empty).Append(columns[i]); - if (keyCount == columns.Count) return sb.Append(") DO NOTHING").ToString(); - sb.Append(") DO UPDATE SET "); - for (var i = keyCount; i < columns.Count; i++) - sb.Append(i > keyCount ? ", " : string.Empty).Append(columns[i]).Append(" = excluded.").Append(columns[i]); - return sb.ToString(); + sb.Append(") ON CONFLICT (").AppendJoin(", ", columns.Take(keyCount)).Append(')'); + var updated = columns.Skip(keyCount).Where(c => insertOnly is null || !insertOnly.Contains(c)).ToList(); + if (updated.Count == 0) return sb.Append(" DO NOTHING").ToString(); + return sb.Append(" DO UPDATE SET ").AppendJoin(", ", updated.Select(c => $"{c} = excluded.{c}")).ToString(); } /// Double-quotes an identifier (both dialects). diff --git a/src/Titano.Data/Sqlite/SqliteDatabase.cs b/src/Titano.Data/Sqlite/SqliteDatabase.cs index 2387a4d3..50a5a09e 100644 --- a/src/Titano.Data/Sqlite/SqliteDatabase.cs +++ b/src/Titano.Data/Sqlite/SqliteDatabase.cs @@ -42,6 +42,7 @@ public sealed class SqliteDatabase : IDatabase private readonly int _maxConnections; private readonly SemaphoreSlim _slots; private readonly SemaphoreSlim _writeLock = new(1, 1); + private readonly SemaphoreSlim _restoreGate = new(1, 1); private readonly Stack _idle = new(); private readonly Lock _gate = new(); private bool _disposed; @@ -126,9 +127,17 @@ public sealed class SqliteDatabase : IDatabase await Task.Delay(2, cancellationToken).ConfigureAwait(false); } var temp = target + ".tmp"; - await using (var session = (SqliteSession)await OpenAsync(cancellationToken).ConfigureAwait(false)) - await Task.Run(() => session.VacuumInto(temp, cancellationToken), cancellationToken).ConfigureAwait(false); - File.Move(temp, target); + try + { + await using (var session = (SqliteSession)await OpenAsync(cancellationToken).ConfigureAwait(false)) + await Task.Run(() => session.VacuumInto(temp, cancellationToken), cancellationToken).ConfigureAwait(false); + File.Move(temp, target); + } + catch + { + File.Delete(temp); + throw; + } var pattern = new Regex("^" + Regex.Escape(stem) + @"-\d{8}-\d{6}-\d{3}\.db$", RegexOptions.CultureInvariant); var backups = Directory.EnumerateFiles(directory) @@ -142,7 +151,8 @@ public sealed class SqliteDatabase : IDatabase /// /// Replaces the database file with a backup after checking it with PRAGMA quick_check. Waits until - /// every session of this instance is returned; no other process may have the file open. + /// every session of this instance is returned (a caller must not hold one meanwhile); no other process may + /// have the file open. /// public async Task RestoreAsync(string backupPath, CancellationToken cancellationToken = default) { @@ -150,7 +160,8 @@ public sealed class SqliteDatabase : IDatabase if (!File.Exists(backupPath)) throw new FileNotFoundException("Backup file not found.", backupPath); Validate(backupPath); - await _writeLock.WaitAsync(cancellationToken).ConfigureAwait(false); + // Holding every pool slot excludes all sessions, and with them the write lock, which only sessions take. + await _restoreGate.WaitAsync(cancellationToken).ConfigureAwait(false); var taken = 0; try { @@ -170,7 +181,7 @@ public sealed class SqliteDatabase : IDatabase finally { if (taken > 0) _slots.Release(taken); - _writeLock.Release(); + _restoreGate.Release(); } } diff --git a/tests/Titano.Data.Tests/Core/ValueConversionTests.cs b/tests/Titano.Data.Tests/Core/ValueConversionTests.cs index d480a998..94f98d15 100644 --- a/tests/Titano.Data.Tests/Core/ValueConversionTests.cs +++ b/tests/Titano.Data.Tests/Core/ValueConversionTests.cs @@ -81,6 +81,9 @@ public sealed class ValueConversionTests "INSERT INTO t (a, b, c) VALUES ($1, $2, $3) ON CONFLICT (a) DO UPDATE SET b = excluded.b, c = excluded.c", SqlDialect.Upsert("t", ["a", "b", "c"], 1)); Assert.Equal("INSERT INTO t (a, b) VALUES ($1, $2) ON CONFLICT (a, b) DO NOTHING", SqlDialect.Upsert("t", ["a", "b"], 2)); + Assert.Equal( + "INSERT INTO t (a, b, c) VALUES ($1, $2, $3) ON CONFLICT (a) DO UPDATE SET c = excluded.c", + SqlDialect.Upsert("t", ["a", "b", "c"], 1, ["b"])); Assert.Equal("%50\\%\\_off\\\\%", SqlDialect.ContainsPattern("50%_off\\")); Assert.Equal("\"we\"\"ird\"", SqlDialect.QuoteIdentifier("we\"ird")); } diff --git a/tests/Titano.Data.Tests/Postgres/PgProtocolUnitTests.cs b/tests/Titano.Data.Tests/Postgres/PgProtocolUnitTests.cs index 51e76d87..1f037797 100644 --- a/tests/Titano.Data.Tests/Postgres/PgProtocolUnitTests.cs +++ b/tests/Titano.Data.Tests/Postgres/PgProtocolUnitTests.cs @@ -18,8 +18,11 @@ public sealed class PgProtocolUnitTests Assert.Equal( "c=biws,r=rOprNGfwEbeRWgbNEkqO%hvYDpWUa2RaTCAfuxFIlj)hNlF$k0,p=dHzbZapWIk4jUhN+Ute9ytag9zjfMHgsqmmiz7AndVQ=", Encoding.UTF8.GetString(final)); - scram.VerifyServerFinal("v=6rriTRBi23WpRR/wtup+mMhUZUn/dB5nLTJRsjl95G4="u8); + Assert.False(scram.IsServerVerified); Assert.Throws(() => scram.VerifyServerFinal("v=AAAA"u8)); + Assert.False(scram.IsServerVerified); + scram.VerifyServerFinal("v=6rriTRBi23WpRR/wtup+mMhUZUn/dB5nLTJRsjl95G4="u8); + Assert.True(scram.IsServerVerified); } [Test] diff --git a/tests/Titano.Data.Tests/Postgres/PgWireTests.cs b/tests/Titano.Data.Tests/Postgres/PgWireTests.cs index b9f15d97..132c549b 100644 --- a/tests/Titano.Data.Tests/Postgres/PgWireTests.cs +++ b/tests/Titano.Data.Tests/Postgres/PgWireTests.cs @@ -271,6 +271,24 @@ public sealed class PgWireTests(TestContext context) : DatabaseTest(context) Assert.Equal(1, await session.ScalarAsync("SELECT 1", [], Ct)); } + [Test] + public async Task DisposingABusySessionInATransactionFreesItsPoolSlot() + { + var database = await OpenAsync("Maximum Pool Size=1;Timeout=3"); + var session = await database.OpenAsync(Ct); + await session.ExecuteScriptAsync("CREATE TABLE t (id integer)", Ct); + var tx = await session.BeginAsync(Ct); + await session.ExecuteAsync("INSERT INTO t VALUES ($1)", [1], Ct); + var rows = session.QueryAsync("SELECT generate_series(1, 1000)", [], Ct).GetAsyncEnumerator(Ct); + Assert.True(await rows.MoveNextAsync()); + await tx.DisposeAsync(); + Assert.True(session.InTransaction, "a busy session cannot roll back yet"); + await session.DisposeAsync(); + + await using var next = await database.OpenAsync(Ct); + Assert.Equal(0L, await next.ScalarAsync("SELECT count(*) FROM t", [], Ct)); + } + [Test] public async Task NoticesAndScriptsDoNotDisturbTheSession() { diff --git a/tests/Titano.Data.Tests/Repositories/CatalogRepositoryTests.cs b/tests/Titano.Data.Tests/Repositories/CatalogRepositoryTests.cs index 6c0e1277..62270019 100644 --- a/tests/Titano.Data.Tests/Repositories/CatalogRepositoryTests.cs +++ b/tests/Titano.Data.Tests/Repositories/CatalogRepositoryTests.cs @@ -304,6 +304,8 @@ public abstract class CatalogRepositoryTests(TestContext context) : RepositoryTe await db.Renders.InsertAsync(first, Ct); await db.Renders.InsertAsync(second, Ct); Assert.Equal(2, (await db.Sequences.GetAsync(sequence.Id, Ct))!.RenderedCount); + await db.Sequences.UpsertAsync(sequence with { Title = "Renamed from a stale copy" }, Ct); + Assert.Equal(2, (await db.Sequences.GetAsync(sequence.Id, Ct))!.RenderedCount); var list = await db.Renders.ListBySequenceAsync(sequence.Id, Ct); Assert.SequenceEqual([second.Id, first.Id], list.Select(r => r.Id)); Assert.Equal(second, list[0]); diff --git a/tests/Titano.Data.Tests/Sqlite/SqliteBindingTests.cs b/tests/Titano.Data.Tests/Sqlite/SqliteBindingTests.cs index 65942b0b..f0ddbc6b 100644 --- a/tests/Titano.Data.Tests/Sqlite/SqliteBindingTests.cs +++ b/tests/Titano.Data.Tests/Sqlite/SqliteBindingTests.cs @@ -117,6 +117,29 @@ public sealed class SqliteBindingTests(TestContext context) : DatabaseTest(conte Assert.Equal("original", await session.ScalarAsync("SELECT v FROM b WHERE id = 1", [], Ct)); } + [Test] + public async Task RestoreWaitsForSessionsThatStillWrite() + { + var path = Path.Combine(Context.CreateTempDirectory(), "titano.db"); + var database = (SqliteDatabase)Own(DatabaseFactory.Open(DatabaseOptions.Sqlite(path))); + await using (var setup = await database.OpenAsync(Ct)) + await setup.ExecuteScriptAsync("CREATE TABLE b (id INTEGER PRIMARY KEY)", Ct); + var backup = await database.BackupAsync(Context.CreateTempDirectory(), cancellationToken: Ct); + + var session = await database.OpenAsync(Ct); + var restore = database.RestoreAsync(backup, Ct); + await Task.Delay(100, Ct); + Assert.False(restore.IsCompleted, "restore must wait for the open session"); + // The session can still take the write lock while the restore waits for its pool slot. + await session.ExecuteAsync("INSERT INTO b VALUES ($1)", [1], Ct); + await using (var tx = await session.BeginAsync(Ct)) await tx.CommitAsync(Ct); + await session.DisposeAsync(); + await restore.WaitAsync(TimeSpan.FromSeconds(10), Ct); + + await using var check = await database.OpenAsync(Ct); + Assert.Equal(0L, await check.ScalarAsync("SELECT count(*) FROM b", [], Ct)); + } + [Test] public async Task RestoreRefusesAFileThatIsNotADatabase() {