A transaction left open by mistake now fails other writers with a transient SQLITE_BUSY after 60 s instead of hanging them. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
286 lines
11 KiB
C#
286 lines
11 KiB
C#
using System.Globalization;
|
|
using System.Text.RegularExpressions;
|
|
using Titano.Core.Diagnostics;
|
|
using Titano.Core.IO;
|
|
using static Titano.Data.Sqlite.SqliteNative;
|
|
|
|
namespace Titano.Data.Sqlite;
|
|
|
|
/// <summary>Parsed SQLite connection string: <c>Data Source=<path>|:memory:;Pool Size=4</c>.</summary>
|
|
internal sealed record SqliteSettings(string DataSource, int PoolSize)
|
|
{
|
|
public bool IsMemory => DataSource == ":memory:";
|
|
|
|
public static SqliteSettings Parse(string connectionString)
|
|
{
|
|
var cs = new ConnectionString(connectionString);
|
|
var source = cs.Get("Data Source", "DataSource", "Filename")
|
|
?? throw new FormatException("A SQLite connection string needs 'Data Source'.");
|
|
var pool = cs.GetInt(4, "Pool Size", "Max Pool Size");
|
|
cs.ThrowOnUnknownKeys();
|
|
if (pool < 1) throw new FormatException("'Pool Size' must be at least 1.");
|
|
return new SqliteSettings(source, pool);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// SQLite through the system library (BRIEF §6.2). Up to <c>Pool Size</c> connections serve concurrent
|
|
/// readers (WAL); writes are serialised in-process by one async write lock, so writers wait in a queue instead
|
|
/// of spinning on SQLITE_BUSY. An in-memory database is a shared-cache URI that lives while the pool holds a
|
|
/// connection; its table locks (SQLITE_LOCKED) are retried by the sessions.
|
|
/// </summary>
|
|
public sealed class SqliteDatabase : IDatabase
|
|
{
|
|
/// <summary>3.31.0, the oldest library accepted: it has UPSERT, NULLS LAST and <c>VACUUM INTO</c>.</summary>
|
|
public const int MinimumVersionNumber = 3_031_000;
|
|
|
|
internal const int BusyTimeoutMs = 5000;
|
|
private const string LogCategory = "Data.Sqlite";
|
|
private static readonly TimeSpan WriteLockTimeout = TimeSpan.FromSeconds(60);
|
|
|
|
private readonly string _filename;
|
|
private readonly bool _memory;
|
|
private readonly int _maxConnections;
|
|
private readonly SemaphoreSlim _slots;
|
|
private readonly SemaphoreSlim _writeLock = new(1, 1);
|
|
private readonly SemaphoreSlim _restoreGate = new(1, 1);
|
|
private readonly Stack<SqliteConnection> _idle = new();
|
|
private readonly Lock _gate = new();
|
|
private bool _disposed;
|
|
|
|
private SqliteDatabase(SqliteSettings settings)
|
|
{
|
|
_memory = settings.IsMemory;
|
|
if (_memory)
|
|
{
|
|
_filename = $"file:titano-{Guid.NewGuid():N}?mode=memory&cache=shared";
|
|
}
|
|
else
|
|
{
|
|
Path = System.IO.Path.GetFullPath(settings.DataSource);
|
|
var directory = System.IO.Path.GetDirectoryName(Path);
|
|
if (!string.IsNullOrEmpty(directory)) Directory.CreateDirectory(directory);
|
|
_filename = Path;
|
|
}
|
|
_maxConnections = settings.PoolSize;
|
|
_slots = new SemaphoreSlim(_maxConnections, _maxConnections);
|
|
}
|
|
|
|
public Dialect Dialect => Dialect.Sqlite;
|
|
|
|
/// <summary>Full path of the database file; null for an in-memory database.</summary>
|
|
public string? Path { get; }
|
|
|
|
/// <summary>Version string of the loaded SQLite library, e.g. "3.45.1".</summary>
|
|
public static string LibraryVersion
|
|
{
|
|
get
|
|
{
|
|
EnsureLibrary();
|
|
unsafe
|
|
{
|
|
return Utf8(sqlite3_libversion()) ?? "?";
|
|
}
|
|
}
|
|
}
|
|
|
|
internal static SqliteDatabase Open(SqliteSettings settings)
|
|
{
|
|
EnsureLibrary();
|
|
var database = new SqliteDatabase(settings);
|
|
// The first connection validates the path, switches the file to WAL and keeps a memory database alive.
|
|
database._idle.Push(database.CreateConnection());
|
|
return database;
|
|
}
|
|
|
|
public async Task<IDbSession> OpenAsync(CancellationToken cancellationToken)
|
|
{
|
|
ObjectDisposedException.ThrowIf(_disposed, this);
|
|
await _slots.WaitAsync(cancellationToken).ConfigureAwait(false);
|
|
try
|
|
{
|
|
SqliteConnection? connection;
|
|
lock (_gate) _idle.TryPop(out connection);
|
|
return new SqliteSession(this, connection ?? CreateConnection());
|
|
}
|
|
catch
|
|
{
|
|
_slots.Release();
|
|
throw;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Writes a consistent copy with <c>VACUUM INTO</c> to <paramref name="directory"/> as
|
|
/// <c><name>-yyyyMMdd-HHmmss-fff.db</c> and deletes all but the <paramref name="keep"/> newest copies.
|
|
/// Readers and writers continue meanwhile. Returns the new file.
|
|
/// </summary>
|
|
public async Task<string> BackupAsync(string directory, int keep = 3, CancellationToken cancellationToken = default)
|
|
{
|
|
ArgumentOutOfRangeException.ThrowIfLessThan(keep, 1);
|
|
Directory.CreateDirectory(directory);
|
|
var stem = BackupStem;
|
|
string target;
|
|
while (true)
|
|
{
|
|
target = System.IO.Path.Combine(directory, string.Create(CultureInfo.InvariantCulture, $"{stem}-{DateTime.UtcNow:yyyyMMdd-HHmmss-fff}.db"));
|
|
if (!File.Exists(target)) break;
|
|
await Task.Delay(2, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
var temp = target + ".tmp";
|
|
try
|
|
{
|
|
await using (var session = (SqliteSession)await OpenAsync(cancellationToken).ConfigureAwait(false))
|
|
await Task.Run(() => session.VacuumInto(temp, cancellationToken), cancellationToken).ConfigureAwait(false);
|
|
File.Move(temp, target);
|
|
}
|
|
catch
|
|
{
|
|
File.Delete(temp);
|
|
throw;
|
|
}
|
|
|
|
var pattern = new Regex("^" + Regex.Escape(stem) + @"-\d{8}-\d{6}-\d{3}\.db$", RegexOptions.CultureInvariant);
|
|
var backups = Directory.EnumerateFiles(directory)
|
|
.Where(f => pattern.IsMatch(System.IO.Path.GetFileName(f)))
|
|
.OrderByDescending(f => System.IO.Path.GetFileName(f), StringComparer.Ordinal)
|
|
.ToList();
|
|
foreach (var old in backups.Skip(keep)) File.Delete(old);
|
|
Log.Info(LogCategory, $"Backup written to {target}");
|
|
return target;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Replaces the database file with a backup after checking it with <c>PRAGMA quick_check</c>. Waits until
|
|
/// every session of this instance is returned (a caller must not hold one meanwhile); no other process may
|
|
/// have the file open.
|
|
/// </summary>
|
|
public async Task RestoreAsync(string backupPath, CancellationToken cancellationToken = default)
|
|
{
|
|
if (Path is null) throw new NotSupportedException("An in-memory database cannot be restored.");
|
|
if (!File.Exists(backupPath)) throw new FileNotFoundException("Backup file not found.", backupPath);
|
|
Validate(backupPath);
|
|
|
|
// Holding every pool slot excludes all sessions, and with them the write lock, which only sessions take.
|
|
await _restoreGate.WaitAsync(cancellationToken).ConfigureAwait(false);
|
|
var taken = 0;
|
|
try
|
|
{
|
|
for (; taken < _maxConnections; taken++) await _slots.WaitAsync(cancellationToken).ConfigureAwait(false);
|
|
lock (_gate)
|
|
{
|
|
while (_idle.TryPop(out var connection)) connection.Dispose();
|
|
}
|
|
var temp = AtomicFile.TempPathFor(Path);
|
|
File.Copy(backupPath, temp);
|
|
// A WAL left behind belongs to the old file and would be replayed onto the restored one.
|
|
File.Delete(Path + "-wal");
|
|
File.Delete(Path + "-shm");
|
|
AtomicFile.Commit(temp, Path);
|
|
Log.Info(LogCategory, $"Database restored from {backupPath}");
|
|
}
|
|
finally
|
|
{
|
|
if (taken > 0) _slots.Release(taken);
|
|
_restoreGate.Release();
|
|
}
|
|
}
|
|
|
|
public ValueTask DisposeAsync()
|
|
{
|
|
lock (_gate)
|
|
{
|
|
if (_disposed) return ValueTask.CompletedTask;
|
|
_disposed = true;
|
|
while (_idle.TryPop(out var connection)) connection.Dispose();
|
|
}
|
|
return ValueTask.CompletedTask;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Queues for the write lock. The wait is bounded so a transaction left open by mistake surfaces as a transient
|
|
/// SQLITE_BUSY instead of hanging every writer; the bound is generous because bulk imports hold the lock for long.
|
|
/// </summary>
|
|
internal async Task AcquireWriteAsync(CancellationToken cancellationToken)
|
|
{
|
|
if (!await _writeLock.WaitAsync(WriteLockTimeout, cancellationToken).ConfigureAwait(false))
|
|
throw new SqliteException($"database is locked: no write lock within {WriteLockTimeout.TotalSeconds} s", SQLITE_BUSY);
|
|
}
|
|
|
|
internal void ReleaseWrite() => _writeLock.Release();
|
|
|
|
/// <summary>Takes back a session's connection; one left inside a transaction is rolled back or dropped.</summary>
|
|
internal void Return(SqliteConnection connection)
|
|
{
|
|
try
|
|
{
|
|
if (!connection.IsAutoCommit)
|
|
{
|
|
try
|
|
{
|
|
connection.Exec("ROLLBACK");
|
|
}
|
|
catch (SqliteException ex)
|
|
{
|
|
Log.Warning(LogCategory, "Dropping a connection whose transaction could not be rolled back.", ex);
|
|
connection.Dispose();
|
|
return;
|
|
}
|
|
}
|
|
lock (_gate)
|
|
{
|
|
if (_disposed) connection.Dispose();
|
|
else _idle.Push(connection);
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
_slots.Release();
|
|
}
|
|
}
|
|
|
|
private string BackupStem => Path is null ? "titano" : System.IO.Path.GetFileNameWithoutExtension(Path);
|
|
|
|
private SqliteConnection CreateConnection() => SqliteConnection.Open(_filename, _memory, BusyTimeoutMs);
|
|
|
|
private static void EnsureLibrary()
|
|
{
|
|
EnsureResolver();
|
|
int version;
|
|
try
|
|
{
|
|
version = sqlite3_libversion_number();
|
|
}
|
|
catch (DllNotFoundException ex)
|
|
{
|
|
throw new NotSupportedException("The system SQLite library was not found (winsqlite3.dll, libsqlite3.so.0 or libsqlite3.dylib).", ex);
|
|
}
|
|
if (version < MinimumVersionNumber)
|
|
throw new NotSupportedException($"SQLite {version / 1_000_000}.{version / 1000 % 1000}.{version % 1000} is too old; 3.31 or newer is required.");
|
|
}
|
|
|
|
private static void Validate(string backupPath)
|
|
{
|
|
try
|
|
{
|
|
using var connection = SqliteConnection.OpenReadOnly(backupPath);
|
|
var statement = connection.Rent("PRAGMA quick_check");
|
|
try
|
|
{
|
|
var rc = statement.Step();
|
|
if (rc != SQLITE_ROW) throw connection.Error(rc);
|
|
var result = statement.ReadRow(statement.ReadColumns()).GetStringOrNull(0);
|
|
if (result != "ok") throw new InvalidDataException($"The backup failed its integrity check: {result}");
|
|
}
|
|
finally
|
|
{
|
|
connection.Return(statement);
|
|
}
|
|
}
|
|
catch (SqliteException ex)
|
|
{
|
|
throw new InvalidDataException("The file is not a readable SQLite database.", ex);
|
|
}
|
|
}
|
|
}
|