fix(data): close concurrency and cleanup gaps found in review
- SQLite restore holds every pool slot instead of the write lock first, removing a lock-order deadlock with sessions that begin to write. - PostgreSQL sessions always return their connection on dispose, a rollback while disposing never masks the original exception, and a busy session keeps its transaction until it is discarded. - The pool disposes a connection whose health check the caller cancels. - Listener commands keep their lock until the server answers and never tear a write with the caller's token; closing and pending no longer race. - Tag replacement locks the sequence and the tag row; unused-tag cleanup locks candidates and re-checks them with a fresh snapshot; render counts and stats refreshes lock the sequence row on PostgreSQL. - Sequence upserts keep rendered_count, which renders maintain. - AuthenticationOk after an unfinished SCRAM exchange is rejected. - Oversized read buffers are released; failed backups leave no .tmp; a zero idle lifetime is rejected. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
17 files changed
+240
-61
No files matched your search
@@ -26,6 +26,9 @@ internal sealed class ScramSha256
|
||||
|
||||
public const string Mechanism = "SCRAM-SHA-256";
|
||||
|
||||
/// <summary>The server proved knowledge of the password with a valid final signature.</summary>
|
||||
public bool IsServerVerified { get; private set; }
|
||||
|
||||
public byte[] ClientFirstMessage()
|
||||
{
|
||||
_clientFirstBare = $"n={_username},r={_clientNonce}";
|
||||
@@ -71,6 +74,7 @@ internal sealed class ScramSha256
|
||||
if (_expectedServerSignature is null || value is null
|
||||
|| !CryptographicOperations.FixedTimeEquals(Convert.FromBase64String(value), _expectedServerSignature))
|
||||
throw new AuthenticationException("The server's SCRAM signature is invalid.");
|
||||
IsServerVerified = true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -4,6 +4,7 @@ using System.Net.Security;
|
||||
using System.Net.Sockets;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.ExceptionServices;
|
||||
using System.Security.Authentication;
|
||||
using System.Security.Cryptography.X509Certificates;
|
||||
using System.Text;
|
||||
using Titano.Core.Diagnostics;
|
||||
@@ -423,6 +424,9 @@ internal sealed class PgConnector : IAsyncDisposable
|
||||
switch (code)
|
||||
{
|
||||
case 0:
|
||||
// A server that started SCRAM must finish it: only its final signature proves it knows the password.
|
||||
if (scram is { IsServerVerified: false })
|
||||
throw new AuthenticationException("The server skipped the final SCRAM message.");
|
||||
return scram;
|
||||
case 3:
|
||||
WritePassword(RequirePassword());
|
||||
@@ -551,6 +555,12 @@ internal sealed class PgConnector : IAsyncDisposable
|
||||
_ioAbort.Dispose();
|
||||
_ioAbort = new CancellationTokenSource();
|
||||
}
|
||||
// A huge row (a large bytea) grew the buffer; do not keep it for the life of a pooled connection.
|
||||
if (_read.Length > 1 << 20 && _readPos == _readEnd)
|
||||
{
|
||||
_read = new byte[16 * 1024];
|
||||
_readPos = _readEnd = 0;
|
||||
}
|
||||
_active = null;
|
||||
MarkIdle();
|
||||
}
|
||||
|
||||
@@ -13,7 +13,9 @@ public sealed class PgListener : IAsyncDisposable
|
||||
private readonly PgConnector _connector;
|
||||
private readonly SemaphoreSlim _commandLock = new(1, 1);
|
||||
private readonly CancellationTokenSource _stop = new();
|
||||
private readonly Lock _gate = new();
|
||||
private TaskCompletionSource<PostgresException?>? _pending;
|
||||
private bool _closed;
|
||||
|
||||
internal PgListener(PgConnector connector)
|
||||
{
|
||||
@@ -48,39 +50,59 @@ public sealed class PgListener : IAsyncDisposable
|
||||
_stop.Dispose();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sends one command and waits for its ReadyForQuery. The command lock is held until the server has answered,
|
||||
/// even when the caller stops waiting, so a late answer can never be taken for the next command's.
|
||||
/// </summary>
|
||||
private async Task CommandAsync(string sql, CancellationToken cancellationToken)
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(_stop.IsCancellationRequested, this);
|
||||
await _commandLock.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
var pending = new TaskCompletionSource<PostgresException?>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
try
|
||||
{
|
||||
if (Completion.IsCompleted) throw new DatabaseConnectionException("The listener connection is closed.", Completion.Exception?.InnerException);
|
||||
var pending = new TaskCompletionSource<PostgresException?>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
Volatile.Write(ref _pending, pending);
|
||||
await _connector.SendQueryAsync(sql, cancellationToken).ConfigureAwait(false);
|
||||
if (await pending.Task.WaitAsync(cancellationToken).ConfigureAwait(false) is { } error) throw error;
|
||||
lock (_gate)
|
||||
{
|
||||
if (_closed) throw new DatabaseConnectionException("The listener connection is closed.", Completion.Exception?.InnerException);
|
||||
_pending = pending;
|
||||
}
|
||||
// The listener's own token: a write torn by the caller's token would corrupt the protocol stream.
|
||||
await _connector.SendQueryAsync(sql, _stop.Token).ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
catch
|
||||
{
|
||||
_commandLock.Release();
|
||||
throw;
|
||||
}
|
||||
_ = pending.Task.ContinueWith(static (_, state) => ((SemaphoreSlim)state!).Release(), _commandLock,
|
||||
CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
|
||||
if (await pending.Task.WaitAsync(cancellationToken).ConfigureAwait(false) is { } error) throw error;
|
||||
}
|
||||
|
||||
private async Task RunAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
await _connector.RunListenerAsync(error => Volatile.Read(ref _pending)?.TrySetResult(error), _stop.Token).ConfigureAwait(false);
|
||||
await _connector.RunListenerAsync(OnCommandCompleted, _stop.Token).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception) when (_stop.IsCancellationRequested)
|
||||
{
|
||||
}
|
||||
finally
|
||||
{
|
||||
Volatile.Read(ref _pending)?.TrySetException(new DatabaseConnectionException("The listener connection is closed."));
|
||||
lock (_gate)
|
||||
{
|
||||
_closed = true;
|
||||
_pending?.TrySetException(new DatabaseConnectionException("The listener connection is closed."));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void OnCommandCompleted(PostgresException? error)
|
||||
{
|
||||
lock (_gate) _pending?.TrySetResult(error);
|
||||
}
|
||||
|
||||
private void Raise(PgNotification notification)
|
||||
{
|
||||
try
|
||||
|
||||
@@ -38,9 +38,19 @@ internal sealed class PgPool : IAsyncDisposable
|
||||
{
|
||||
while (TakeIdle() is { } connector)
|
||||
{
|
||||
if (!connector.IsBroken
|
||||
&& (connector.IdleTime < _settings.HealthCheckInterval || await connector.PingAsync(cancellationToken).ConfigureAwait(false)))
|
||||
return connector;
|
||||
bool healthy;
|
||||
try
|
||||
{
|
||||
healthy = !connector.IsBroken
|
||||
&& (connector.IdleTime < _settings.HealthCheckInterval || await connector.PingAsync(cancellationToken).ConfigureAwait(false));
|
||||
}
|
||||
catch
|
||||
{
|
||||
// The caller gave up during the health check; the connection's state is unknown.
|
||||
await connector.DisposeAsync().ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
if (healthy) return connector;
|
||||
await connector.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
return await OpenAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
@@ -112,28 +112,32 @@ internal sealed class PostgresSession(PgPool pool, PgConnector connector) : IDbS
|
||||
_disposed = true;
|
||||
if (_connector is not { } connection) return;
|
||||
_connector = null;
|
||||
if (_transaction is not null && !connection.IsBroken)
|
||||
try
|
||||
{
|
||||
try
|
||||
{
|
||||
await connection.SimpleQueryAsync("ROLLBACK", CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
catch (DatabaseException ex)
|
||||
{
|
||||
Log.Warning(LogCategory, "Rollback on session close failed; the connection is discarded.", ex);
|
||||
}
|
||||
if (_transaction is not null) await RollBackQuietlyAsync(connection).ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_transaction = null;
|
||||
pool.Return(connection);
|
||||
}
|
||||
_transaction = null;
|
||||
pool.Return(connection);
|
||||
}
|
||||
|
||||
/// <summary>Ends <paramref name="transaction"/> if it is still active. COMMIT of an aborted transaction is reported, not silently rolled back.</summary>
|
||||
internal async Task EndTransactionAsync(PostgresTransaction transaction, bool commit, CancellationToken cancellationToken)
|
||||
/// <summary>
|
||||
/// Ends <paramref name="transaction"/> if it is still active. COMMIT of an aborted transaction is reported, not
|
||||
/// silently rolled back. A rollback while disposing never throws: a broken connection's transaction dies with
|
||||
/// it, and on a busy one the session stays in the transaction until it is disposed (the pool then discards it).
|
||||
/// </summary>
|
||||
internal async Task EndTransactionAsync(PostgresTransaction transaction, bool commit, bool disposing, CancellationToken cancellationToken)
|
||||
{
|
||||
if (_transaction != transaction) return;
|
||||
try
|
||||
{
|
||||
if (!commit)
|
||||
if (disposing)
|
||||
{
|
||||
if (_connector is { } connection) await RollBackQuietlyAsync(connection).ConfigureAwait(false);
|
||||
}
|
||||
else if (!commit)
|
||||
{
|
||||
await Connector.SimpleQueryAsync("ROLLBACK", cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
@@ -153,6 +157,19 @@ internal sealed class PostgresSession(PgPool pool, PgConnector connector) : IDbS
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task RollBackQuietlyAsync(PgConnector connection)
|
||||
{
|
||||
if (connection.IsBroken || connection.IsBusy) return;
|
||||
try
|
||||
{
|
||||
await connection.SimpleQueryAsync("ROLLBACK", CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
catch (DatabaseException ex)
|
||||
{
|
||||
Log.Warning(LogCategory, "Rollback on dispose failed; the connection is discarded.", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsConnectionFailure(Exception ex) =>
|
||||
ex is DatabaseConnectionException or PostgresException { IsConnectionFailure: true };
|
||||
|
||||
@@ -196,21 +213,21 @@ internal sealed class PostgresTransaction(PostgresSession session) : IDbTransact
|
||||
public async Task CommitAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(_completed, this);
|
||||
await session.EndTransactionAsync(this, commit: true, disposing: false, cancellationToken).ConfigureAwait(false);
|
||||
_completed = true;
|
||||
await session.EndTransactionAsync(this, commit: true, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task RollbackAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(_completed, this);
|
||||
await session.EndTransactionAsync(this, commit: false, disposing: false, cancellationToken).ConfigureAwait(false);
|
||||
_completed = true;
|
||||
await session.EndTransactionAsync(this, commit: false, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_completed) return;
|
||||
_completed = true;
|
||||
await session.EndTransactionAsync(this, commit: false, CancellationToken.None).ConfigureAwait(false);
|
||||
await session.EndTransactionAsync(this, commit: false, disposing: true, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -68,8 +68,8 @@ public sealed record PostgresSettings
|
||||
cs.ThrowOnUnknownKeys();
|
||||
if (settings.MaxPoolSize < 1 || settings.MinPoolSize < 0 || settings.MinPoolSize > settings.MaxPoolSize)
|
||||
throw new FormatException("Pool sizes must satisfy 0 ≤ Minimum Pool Size ≤ Maximum Pool Size, Maximum ≥ 1.");
|
||||
if (settings.StatementCacheSize < 0 || settings.ConnectTimeout <= TimeSpan.Zero)
|
||||
throw new FormatException("'Statement Cache Size' must be ≥ 0 and 'Timeout' > 0.");
|
||||
if (settings.StatementCacheSize < 0 || settings.ConnectTimeout <= TimeSpan.Zero || settings.IdleLifetime <= TimeSpan.Zero)
|
||||
throw new FormatException("'Statement Cache Size' must be ≥ 0; 'Timeout' and 'Connection Idle Lifetime' must be > 0.");
|
||||
return settings;
|
||||
}
|
||||
|
||||
|
||||
@@ -43,7 +43,7 @@ public sealed class JobRepository
|
||||
var claim = $"UPDATE jobs SET state = {(int)JobState.Running}, node_id = $1, started = $2, finished = NULL, error = NULL, attempts = attempts + 1";
|
||||
DbParam[] args = [workerId, DateTimeOffset.UtcNow];
|
||||
if (_db.Sql.IsPostgres)
|
||||
return _db.SingleAsync($"{claim} WHERE id = ({pick} FOR UPDATE SKIP LOCKED) RETURNING {Columns}", args, Map, cancellationToken);
|
||||
return _db.SingleAsync($"{claim} WHERE id = ({pick}{_db.Sql.ForUpdateSkipLocked}) RETURNING {Columns}", args, Map, cancellationToken);
|
||||
|
||||
return _db.InTransactionAsync(async session =>
|
||||
{
|
||||
|
||||
@@ -18,6 +18,7 @@ public sealed class RenderRepository
|
||||
public Task InsertAsync(Render render, CancellationToken cancellationToken = default) =>
|
||||
_db.InTransactionAsync(async session =>
|
||||
{
|
||||
await LockSequenceAsync(session, render.SequenceId, cancellationToken).ConfigureAwait(false);
|
||||
await session.ExecuteAsync($"INSERT INTO renders ({Columns}) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)",
|
||||
[
|
||||
render.Id, render.SequenceId, render.PresetName, DbParam.Json(render.SettingsJson), render.OutputPath, render.Codec,
|
||||
@@ -38,11 +39,21 @@ public sealed class RenderRepository
|
||||
{
|
||||
var sequenceId = await session.ScalarAsync<Guid?>("SELECT sequence_id FROM renders WHERE id = $1", [id], cancellationToken).ConfigureAwait(false);
|
||||
if (sequenceId is null) return false;
|
||||
await session.ExecuteAsync("DELETE FROM renders WHERE id = $1", [id], cancellationToken).ConfigureAwait(false);
|
||||
await LockSequenceAsync(session, sequenceId.Value, cancellationToken).ConfigureAwait(false);
|
||||
if (await session.ExecuteAsync("DELETE FROM renders WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) == 0) return false;
|
||||
await session.ExecuteAsync(RecountSql, [sequenceId], cancellationToken).ConfigureAwait(false);
|
||||
return true;
|
||||
}, cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Serialises render changes per sequence on PostgreSQL, so the recount (a new statement, hence a new snapshot)
|
||||
/// sees every committed render; SQLite's write lock already does this.
|
||||
/// </summary>
|
||||
private async Task LockSequenceAsync(IDbSession session, Guid sequenceId, CancellationToken cancellationToken)
|
||||
{
|
||||
if (_db.Sql.IsPostgres) await SequenceRepository.LockAsync(session, sequenceId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static Render Map(DbRow r) => new()
|
||||
{
|
||||
Id = r.GetGuid(0),
|
||||
|
||||
@@ -87,13 +87,14 @@ public sealed class SequenceRepository
|
||||
"s.frame_count, s.median_interval, s.total_bytes, s.status, s.rendered_count, s.verified_at, s.hero_frame_id, s.rating, s.notes, " +
|
||||
"s.settings_json, s.analysis_json, s.origin, s.sync_state, s.settings_version, s.frames_version, s.owner_user_id, s.warning_count, s.updated_at";
|
||||
|
||||
/// <summary>rendered_count is owned by <see cref="RenderRepository"/> once the row exists.</summary>
|
||||
private static readonly string UpsertSql = SqlDialect.Upsert("sequences",
|
||||
[
|
||||
"id", "archive_id", "title", "title_key", "search_text", "folder", "camera_make", "camera_model", "camera_serial", "lens",
|
||||
"first_capture", "last_capture", "frame_count", "median_interval", "total_bytes", "status", "rendered_count", "verified_at",
|
||||
"hero_frame_id", "rating", "notes", "settings_json", "analysis_json", "origin", "sync_state", "settings_version",
|
||||
"frames_version", "owner_user_id", "warning_count", "updated_at",
|
||||
], 1);
|
||||
], 1, ["rendered_count"]);
|
||||
|
||||
/// <summary>Ordinal of the tag list after <see cref="Columns"/>, and of the sort key a listing appends after it.</summary>
|
||||
private const int TagsOrdinal = 28;
|
||||
@@ -112,7 +113,11 @@ public sealed class SequenceRepository
|
||||
$"SELECT {Columns}, (SELECT {_db.Sql.StringAgg("t.name")} FROM sequence_tags st JOIN tags t ON t.id = st.tag_id " +
|
||||
$"WHERE st.sequence_id = s.id) AS tags{extraColumns} FROM sequences s";
|
||||
|
||||
/// <summary>Inserts or replaces the sequence row and its tag set.</summary>
|
||||
/// <summary>
|
||||
/// Inserts the sequence, or replaces every stored field and the tag set with the record's (last writer wins);
|
||||
/// only <see cref="Sequence.RenderedCount"/> of an existing row is kept, as renders maintain it. For concurrent
|
||||
/// partial changes use <see cref="SetTagsAsync"/>, <see cref="UpdateStatusAsync"/> or <see cref="RefreshStatsAsync"/>.
|
||||
/// </summary>
|
||||
public Task UpsertAsync(Sequence sequence, CancellationToken cancellationToken = default) =>
|
||||
_db.InTransactionAsync(async session =>
|
||||
{
|
||||
@@ -132,8 +137,7 @@ public sealed class SequenceRepository
|
||||
public async Task<bool> SetTagsAsync(Guid id, IEnumerable<string> tags, CancellationToken cancellationToken = default) =>
|
||||
await _db.InTransactionAsync(async session =>
|
||||
{
|
||||
if (await session.ScalarAsync<Guid?>("SELECT id FROM sequences WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) is null)
|
||||
return false;
|
||||
if (!await LockAsync(session, id, cancellationToken).ConfigureAwait(false)) return false;
|
||||
await ReplaceTagsAsync(session, id, tags, cancellationToken).ConfigureAwait(false);
|
||||
await session.ExecuteAsync("UPDATE sequences SET settings_version = settings_version + 1, updated_at = $2 WHERE id = $1",
|
||||
[id, DateTimeOffset.UtcNow], cancellationToken).ConfigureAwait(false);
|
||||
@@ -149,7 +153,8 @@ public sealed class SequenceRepository
|
||||
public Task<Sequence?> RefreshStatsAsync(Guid id, CancellationToken cancellationToken = default) =>
|
||||
_db.InTransactionAsync(async session =>
|
||||
{
|
||||
var current = await session.QuerySingleOrDefaultAsync($"{_select} WHERE s.id = $1", [id], Map, cancellationToken).ConfigureAwait(false);
|
||||
var current = await session.QuerySingleOrDefaultAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
if (current is null) return null;
|
||||
var stats = await session.QuerySingleAsync(
|
||||
"SELECT count(*), COALESCE(SUM(size), 0), MIN(captured_at), MAX(captured_at) FROM frames WHERE sequence_id = $1",
|
||||
@@ -312,16 +317,37 @@ public sealed class SequenceRepository
|
||||
], cancellationToken);
|
||||
}
|
||||
|
||||
private static async Task ReplaceTagsAsync(IDbSession session, Guid sequenceId, IEnumerable<string> tags, CancellationToken cancellationToken)
|
||||
/// <summary>
|
||||
/// Row-locks a sequence for a read-modify-write (PostgreSQL; SQLite's write lock already serialises writers).
|
||||
/// False when it does not exist.
|
||||
/// </summary>
|
||||
internal static async Task<bool> LockAsync(IDbSession session, Guid id, CancellationToken cancellationToken) =>
|
||||
await session.ScalarAsync<Guid?>($"SELECT id FROM sequences WHERE id = $1{SqlDialect.For(session.Dialect).ForUpdate}", [id], cancellationToken)
|
||||
.ConfigureAwait(false) is not null;
|
||||
|
||||
/// <summary>Replaces the tag links of a sequence whose row the caller has locked (or just upserted).</summary>
|
||||
private async Task ReplaceTagsAsync(IDbSession session, Guid sequenceId, IEnumerable<string> tags, CancellationToken cancellationToken)
|
||||
{
|
||||
await session.ExecuteAsync("DELETE FROM sequence_tags WHERE sequence_id = $1", [sequenceId], cancellationToken).ConfigureAwait(false);
|
||||
foreach (var tag in TagRepository.NormalizeSet(tags))
|
||||
{
|
||||
var key = TagRepository.Key(tag);
|
||||
await session.ExecuteAsync("INSERT INTO tags (id, name, name_key) VALUES ($1, $2, $3) ON CONFLICT (name_key) DO NOTHING",
|
||||
[Guid.CreateVersion7(), tag, key], cancellationToken).ConfigureAwait(false);
|
||||
await session.ExecuteAsync("INSERT INTO sequence_tags (sequence_id, tag_id) SELECT $1, id FROM tags WHERE name_key = $2",
|
||||
[sequenceId, key], cancellationToken).ConfigureAwait(false);
|
||||
DbParam[] args = [Guid.CreateVersion7(), tag, TagRepository.Key(tag)];
|
||||
Guid tagId;
|
||||
if (_db.Sql.IsPostgres)
|
||||
{
|
||||
// The no-op update locks an existing tag, so a concurrent DeleteUnusedAsync cannot remove it before the link exists.
|
||||
tagId = (await session.ScalarAsync<Guid?>(
|
||||
"INSERT INTO tags (id, name, name_key) VALUES ($1, $2, $3) ON CONFLICT (name_key) DO UPDATE SET name_key = excluded.name_key RETURNING id",
|
||||
args, cancellationToken).ConfigureAwait(false))!.Value;
|
||||
}
|
||||
else
|
||||
{
|
||||
await session.ExecuteAsync("INSERT INTO tags (id, name, name_key) VALUES ($1, $2, $3) ON CONFLICT (name_key) DO NOTHING",
|
||||
args, cancellationToken).ConfigureAwait(false);
|
||||
tagId = (await session.ScalarAsync<Guid?>("SELECT id FROM tags WHERE name_key = $1", [args[2]], cancellationToken).ConfigureAwait(false))!.Value;
|
||||
}
|
||||
await session.ExecuteAsync("INSERT INTO sequence_tags (sequence_id, tag_id) VALUES ($1, $2)", [sequenceId, tagId], cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -36,9 +36,27 @@ public sealed class TagRepository
|
||||
public async Task<bool> DeleteAsync(Guid id, CancellationToken cancellationToken = default) =>
|
||||
await _db.ExecuteAsync("DELETE FROM tags WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) > 0;
|
||||
|
||||
/// <summary>Deletes tags no sequence uses any more; returns how many.</summary>
|
||||
/// <summary>
|
||||
/// Deletes tags no sequence uses any more; returns how many. Candidates are locked first (skipping tags a
|
||||
/// concurrent tagger holds) and re-checked by a later statement, whose fresh snapshot sees every link committed
|
||||
/// before the lock, so a tag being attached at the same moment survives.
|
||||
/// </summary>
|
||||
public Task<int> DeleteUnusedAsync(CancellationToken cancellationToken = default) =>
|
||||
_db.ExecuteAsync("DELETE FROM tags WHERE NOT EXISTS (SELECT 1 FROM sequence_tags st WHERE st.tag_id = tags.id)", [], cancellationToken);
|
||||
_db.InTransactionAsync(async session =>
|
||||
{
|
||||
const string unused = "NOT EXISTS (SELECT 1 FROM sequence_tags st WHERE st.tag_id = tags.id)";
|
||||
var candidates = await session.QueryListAsync($"SELECT id FROM tags WHERE {unused}{_db.Sql.ForUpdateSkipLocked}", [],
|
||||
r => r.GetGuid(0), cancellationToken).ConfigureAwait(false);
|
||||
var deleted = 0;
|
||||
foreach (var chunk in candidates.Chunk(500))
|
||||
{
|
||||
var p = new SqlParameters();
|
||||
var ids = string.Join(", ", chunk.Select(id => p.Add(id)));
|
||||
deleted += await session.ExecuteAsync($"DELETE FROM tags WHERE id IN ({ids}) AND {unused}", p.ToArray(), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
return deleted;
|
||||
}, cancellationToken);
|
||||
|
||||
/// <summary>Trims, collapses inner whitespace and validates a tag name.</summary>
|
||||
public static string Normalize(string name)
|
||||
|
||||
@@ -27,6 +27,9 @@ public sealed class SqlDialect
|
||||
/// <summary>Row lock for read-modify-write inside a transaction; empty on SQLite, whose <c>BEGIN IMMEDIATE</c> already holds the write lock.</summary>
|
||||
public string ForUpdate => IsPostgres ? " FOR UPDATE" : string.Empty;
|
||||
|
||||
/// <summary>Like <see cref="ForUpdate"/>, but rows another transaction has locked are skipped instead of awaited.</summary>
|
||||
public string ForUpdateSkipLocked => IsPostgres ? " FOR UPDATE SKIP LOCKED" : string.Empty;
|
||||
|
||||
/// <summary>Calendar year of a timestamp column (SQLite stores ISO-8601 text).</summary>
|
||||
public string Year(string expression) => IsPostgres
|
||||
? $"CAST(EXTRACT(YEAR FROM {expression}) AS integer)"
|
||||
@@ -44,21 +47,19 @@ public sealed class SqlDialect
|
||||
|
||||
/// <summary>
|
||||
/// <c>INSERT … VALUES ($1…$n) ON CONFLICT (keys) DO UPDATE SET …</c>; the first <paramref name="keyCount"/>
|
||||
/// columns form the conflict target. The syntax is shared by SQLite ≥ 3.24 and PostgreSQL ≥ 9.5.
|
||||
/// columns form the conflict target and <paramref name="insertOnly"/> columns keep their stored value on
|
||||
/// conflict. The syntax is shared by SQLite ≥ 3.24 and PostgreSQL ≥ 9.5.
|
||||
/// </summary>
|
||||
public static string Upsert(string table, IReadOnlyList<string> columns, int keyCount)
|
||||
public static string Upsert(string table, IReadOnlyList<string> columns, int keyCount, IReadOnlyCollection<string>? insertOnly = null)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfLessThan(keyCount, 1);
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThan(keyCount, columns.Count);
|
||||
var sb = new StringBuilder("INSERT INTO ").Append(table).Append(" (").AppendJoin(", ", columns).Append(") VALUES (");
|
||||
for (var i = 1; i <= columns.Count; i++) sb.Append(i > 1 ? ", $" : "$").Append(i);
|
||||
sb.Append(") ON CONFLICT (");
|
||||
for (var i = 0; i < keyCount; i++) sb.Append(i > 0 ? ", " : string.Empty).Append(columns[i]);
|
||||
if (keyCount == columns.Count) return sb.Append(") DO NOTHING").ToString();
|
||||
sb.Append(") DO UPDATE SET ");
|
||||
for (var i = keyCount; i < columns.Count; i++)
|
||||
sb.Append(i > keyCount ? ", " : string.Empty).Append(columns[i]).Append(" = excluded.").Append(columns[i]);
|
||||
return sb.ToString();
|
||||
sb.Append(") ON CONFLICT (").AppendJoin(", ", columns.Take(keyCount)).Append(')');
|
||||
var updated = columns.Skip(keyCount).Where(c => insertOnly is null || !insertOnly.Contains(c)).ToList();
|
||||
if (updated.Count == 0) return sb.Append(" DO NOTHING").ToString();
|
||||
return sb.Append(" DO UPDATE SET ").AppendJoin(", ", updated.Select(c => $"{c} = excluded.{c}")).ToString();
|
||||
}
|
||||
|
||||
/// <summary>Double-quotes an identifier (both dialects).</summary>
|
||||
|
||||
@@ -42,6 +42,7 @@ public sealed class SqliteDatabase : IDatabase
|
||||
private readonly int _maxConnections;
|
||||
private readonly SemaphoreSlim _slots;
|
||||
private readonly SemaphoreSlim _writeLock = new(1, 1);
|
||||
private readonly SemaphoreSlim _restoreGate = new(1, 1);
|
||||
private readonly Stack<SqliteConnection> _idle = new();
|
||||
private readonly Lock _gate = new();
|
||||
private bool _disposed;
|
||||
@@ -126,9 +127,17 @@ public sealed class SqliteDatabase : IDatabase
|
||||
await Task.Delay(2, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
var temp = target + ".tmp";
|
||||
await using (var session = (SqliteSession)await OpenAsync(cancellationToken).ConfigureAwait(false))
|
||||
await Task.Run(() => session.VacuumInto(temp, cancellationToken), cancellationToken).ConfigureAwait(false);
|
||||
File.Move(temp, target);
|
||||
try
|
||||
{
|
||||
await using (var session = (SqliteSession)await OpenAsync(cancellationToken).ConfigureAwait(false))
|
||||
await Task.Run(() => session.VacuumInto(temp, cancellationToken), cancellationToken).ConfigureAwait(false);
|
||||
File.Move(temp, target);
|
||||
}
|
||||
catch
|
||||
{
|
||||
File.Delete(temp);
|
||||
throw;
|
||||
}
|
||||
|
||||
var pattern = new Regex("^" + Regex.Escape(stem) + @"-\d{8}-\d{6}-\d{3}\.db$", RegexOptions.CultureInvariant);
|
||||
var backups = Directory.EnumerateFiles(directory)
|
||||
@@ -142,7 +151,8 @@ public sealed class SqliteDatabase : IDatabase
|
||||
|
||||
/// <summary>
|
||||
/// Replaces the database file with a backup after checking it with <c>PRAGMA quick_check</c>. Waits until
|
||||
/// every session of this instance is returned; no other process may have the file open.
|
||||
/// every session of this instance is returned (a caller must not hold one meanwhile); no other process may
|
||||
/// have the file open.
|
||||
/// </summary>
|
||||
public async Task RestoreAsync(string backupPath, CancellationToken cancellationToken = default)
|
||||
{
|
||||
@@ -150,7 +160,8 @@ public sealed class SqliteDatabase : IDatabase
|
||||
if (!File.Exists(backupPath)) throw new FileNotFoundException("Backup file not found.", backupPath);
|
||||
Validate(backupPath);
|
||||
|
||||
await _writeLock.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
// Holding every pool slot excludes all sessions, and with them the write lock, which only sessions take.
|
||||
await _restoreGate.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
var taken = 0;
|
||||
try
|
||||
{
|
||||
@@ -170,7 +181,7 @@ public sealed class SqliteDatabase : IDatabase
|
||||
finally
|
||||
{
|
||||
if (taken > 0) _slots.Release(taken);
|
||||
_writeLock.Release();
|
||||
_restoreGate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -81,6 +81,9 @@ public sealed class ValueConversionTests
|
||||
"INSERT INTO t (a, b, c) VALUES ($1, $2, $3) ON CONFLICT (a) DO UPDATE SET b = excluded.b, c = excluded.c",
|
||||
SqlDialect.Upsert("t", ["a", "b", "c"], 1));
|
||||
Assert.Equal("INSERT INTO t (a, b) VALUES ($1, $2) ON CONFLICT (a, b) DO NOTHING", SqlDialect.Upsert("t", ["a", "b"], 2));
|
||||
Assert.Equal(
|
||||
"INSERT INTO t (a, b, c) VALUES ($1, $2, $3) ON CONFLICT (a) DO UPDATE SET c = excluded.c",
|
||||
SqlDialect.Upsert("t", ["a", "b", "c"], 1, ["b"]));
|
||||
Assert.Equal("%50\\%\\_off\\\\%", SqlDialect.ContainsPattern("50%_off\\"));
|
||||
Assert.Equal("\"we\"\"ird\"", SqlDialect.QuoteIdentifier("we\"ird"));
|
||||
}
|
||||
|
||||
@@ -18,8 +18,11 @@ public sealed class PgProtocolUnitTests
|
||||
Assert.Equal(
|
||||
"c=biws,r=rOprNGfwEbeRWgbNEkqO%hvYDpWUa2RaTCAfuxFIlj)hNlF$k0,p=dHzbZapWIk4jUhN+Ute9ytag9zjfMHgsqmmiz7AndVQ=",
|
||||
Encoding.UTF8.GetString(final));
|
||||
scram.VerifyServerFinal("v=6rriTRBi23WpRR/wtup+mMhUZUn/dB5nLTJRsjl95G4="u8);
|
||||
Assert.False(scram.IsServerVerified);
|
||||
Assert.Throws<System.Security.Authentication.AuthenticationException>(() => scram.VerifyServerFinal("v=AAAA"u8));
|
||||
Assert.False(scram.IsServerVerified);
|
||||
scram.VerifyServerFinal("v=6rriTRBi23WpRR/wtup+mMhUZUn/dB5nLTJRsjl95G4="u8);
|
||||
Assert.True(scram.IsServerVerified);
|
||||
}
|
||||
|
||||
[Test]
|
||||
|
||||
@@ -271,6 +271,24 @@ public sealed class PgWireTests(TestContext context) : DatabaseTest(context)
|
||||
Assert.Equal(1, await session.ScalarAsync<int>("SELECT 1", [], Ct));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task DisposingABusySessionInATransactionFreesItsPoolSlot()
|
||||
{
|
||||
var database = await OpenAsync("Maximum Pool Size=1;Timeout=3");
|
||||
var session = await database.OpenAsync(Ct);
|
||||
await session.ExecuteScriptAsync("CREATE TABLE t (id integer)", Ct);
|
||||
var tx = await session.BeginAsync(Ct);
|
||||
await session.ExecuteAsync("INSERT INTO t VALUES ($1)", [1], Ct);
|
||||
var rows = session.QueryAsync("SELECT generate_series(1, 1000)", [], Ct).GetAsyncEnumerator(Ct);
|
||||
Assert.True(await rows.MoveNextAsync());
|
||||
await tx.DisposeAsync();
|
||||
Assert.True(session.InTransaction, "a busy session cannot roll back yet");
|
||||
await session.DisposeAsync();
|
||||
|
||||
await using var next = await database.OpenAsync(Ct);
|
||||
Assert.Equal(0L, await next.ScalarAsync<long>("SELECT count(*) FROM t", [], Ct));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task NoticesAndScriptsDoNotDisturbTheSession()
|
||||
{
|
||||
|
||||
@@ -304,6 +304,8 @@ public abstract class CatalogRepositoryTests(TestContext context) : RepositoryTe
|
||||
await db.Renders.InsertAsync(first, Ct);
|
||||
await db.Renders.InsertAsync(second, Ct);
|
||||
Assert.Equal(2, (await db.Sequences.GetAsync(sequence.Id, Ct))!.RenderedCount);
|
||||
await db.Sequences.UpsertAsync(sequence with { Title = "Renamed from a stale copy" }, Ct);
|
||||
Assert.Equal(2, (await db.Sequences.GetAsync(sequence.Id, Ct))!.RenderedCount);
|
||||
var list = await db.Renders.ListBySequenceAsync(sequence.Id, Ct);
|
||||
Assert.SequenceEqual([second.Id, first.Id], list.Select(r => r.Id));
|
||||
Assert.Equal(second, list[0]);
|
||||
|
||||
@@ -117,6 +117,29 @@ public sealed class SqliteBindingTests(TestContext context) : DatabaseTest(conte
|
||||
Assert.Equal("original", await session.ScalarAsync<string>("SELECT v FROM b WHERE id = 1", [], Ct));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task RestoreWaitsForSessionsThatStillWrite()
|
||||
{
|
||||
var path = Path.Combine(Context.CreateTempDirectory(), "titano.db");
|
||||
var database = (SqliteDatabase)Own(DatabaseFactory.Open(DatabaseOptions.Sqlite(path)));
|
||||
await using (var setup = await database.OpenAsync(Ct))
|
||||
await setup.ExecuteScriptAsync("CREATE TABLE b (id INTEGER PRIMARY KEY)", Ct);
|
||||
var backup = await database.BackupAsync(Context.CreateTempDirectory(), cancellationToken: Ct);
|
||||
|
||||
var session = await database.OpenAsync(Ct);
|
||||
var restore = database.RestoreAsync(backup, Ct);
|
||||
await Task.Delay(100, Ct);
|
||||
Assert.False(restore.IsCompleted, "restore must wait for the open session");
|
||||
// The session can still take the write lock while the restore waits for its pool slot.
|
||||
await session.ExecuteAsync("INSERT INTO b VALUES ($1)", [1], Ct);
|
||||
await using (var tx = await session.BeginAsync(Ct)) await tx.CommitAsync(Ct);
|
||||
await session.DisposeAsync();
|
||||
await restore.WaitAsync(TimeSpan.FromSeconds(10), Ct);
|
||||
|
||||
await using var check = await database.OpenAsync(Ct);
|
||||
Assert.Equal(0L, await check.ScalarAsync<long>("SELECT count(*) FROM b", [], Ct));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task RestoreRefusesAFileThatIsNotADatabase()
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user