From 738c43ba0a31087bb8614f9b97e7ac233897a88f Mon Sep 17 00:00:00 2001 From: Alberto Balbo Date: Wed, 11 Nov 2026 13:11:34 +0100 Subject: [PATCH] feat: data layer with SQLite and PostgreSQL providers, migrations and encrypted company registry (M1) Every later milestone stores its data through this layer, so it has to behave the same on both engines and be safe by construction: - Khaemhat.Data: explicit entity maps, typed queries over a safe expression subset, SqlFragment where every interpolated value becomes a parameter, unit of work with transactions, repositories with audit columns and optimistic concurrency on row_version, retries on transient errors. - SQLite provider (STRICT tables, WAL, exact decimals as integers scaled by 10^6, Unicode case-insensitive LIKE, serialised writes) and PostgreSQL 16+ provider (numeric(19,6), timestamptz, jsonb, advisory locks, database creation and server info). Databases are configured from postgres:// and sqlite:// URLs whose password never appears in messages. - C# migrations with system and company catalogues, schema_version with checksums, one transaction per migration under a lock, refusal of unknown applied versions; first system tables sys_companies and sys_settings. - CompanyRegistry stores each company's database URL encrypted with the instance key (AES-256-GCM, HKDF-derived key, purpose as associated data), so a copied row cannot be read for another company. - tools/Khaemhat.DataModelTool, scripts/gen-datamodel.sh and scripts/new-migration.sh: the current schema section of docs/DATA-MODEL.md is generated from the migrations and a test fails when it is stale. - C# 14 binds array.Contains to MemoryExtensions over spans; the query translator unwraps the span conversion. - Tests run every data scenario on SQLite and PostgreSQL (PostgreSQL in CI); ADR 0018 records where the contracts differ from SPEC 5.1. Docs, wiki, changelog and status updated. Closes #7, #8, #9, #10, #11, #12, #13, #14 Co-Authored-By: Claude Opus 5.5 --- .editorconfig | 8 + CHANGELOG.md | 27 ++ Khaemhat.sln | 15 + docs/ARCHITECTURE.md | 17 +- docs/DATA-MODEL.md | 153 ++++++-- docs/DEVELOPMENT.md | 15 +- docs/SECURITY.md | 19 +- docs/STATUS.md | 19 +- docs/adr/0018-contratti-dello-strato-dati.md | 44 +++ docs/wiki/Changelog.md | 3 +- docs/wiki/Modello-dati.md | 16 +- scripts/gen-datamodel.sh | 13 +- scripts/new-migration.sh | 89 ++++- .../Common/ConcurrencyException.cs | 51 +++ src/Khaemhat.Core/Entities/Entity.cs | 27 ++ src/Khaemhat.Core/Entities/IAuditable.cs | 27 ++ src/Khaemhat.Core/Entities/IDemoMarkable.cs | 8 + src/Khaemhat.Core/Entities/IEntity.cs | 9 + src/Khaemhat.Core/Instance/Company.cs | 31 ++ src/Khaemhat.Core/Instance/CompanyStatus.cs | 17 + src/Khaemhat.Core/Instance/SystemSetting.cs | 16 + .../Security/ISecretProtector.cs | 22 ++ .../Security/SecretProtectionException.cs | 26 ++ .../PostgresAdvisoryLocks.cs | 26 ++ .../PostgresDatabaseCreator.cs | 92 +++++ .../PostgresDialect.cs | 126 +++++++ .../PostgresPartitioning.cs | 46 +++ .../PostgresProvider.cs | 116 ++++++ .../PostgresSchemaBuilder.cs | 18 + .../PostgresServerInfo.cs | 59 +++ .../SqliteDecimalConverter.cs | 39 ++ src/Khaemhat.Data.Sqlite/SqliteDialect.cs | 99 +++++ src/Khaemhat.Data.Sqlite/SqliteProvider.cs | 145 +++++++ .../SqliteSchemaBuilder.cs | 34 ++ src/Khaemhat.Data.Sqlite/SqliteWriteLock.cs | 41 ++ .../Companies/CompanyRegistry.cs | 200 ++++++++++ src/Khaemhat.Data/CompanyCodes.cs | 43 +++ src/Khaemhat.Data/DatabaseScope.cs | 26 ++ src/Khaemhat.Data/DatabaseTarget.cs | 36 ++ src/Khaemhat.Data/DatabaseUrl.cs | 259 +++++++++++++ src/Khaemhat.Data/DbEngine.cs | 11 + src/Khaemhat.Data/DbProviderCatalog.cs | 37 ++ src/Khaemhat.Data/DbServerInfo.cs | 28 ++ src/Khaemhat.Data/Execution/CommandCache.cs | 90 +++++ .../Execution/DataAccessContext.cs | 43 +++ src/Khaemhat.Data/Execution/DbExecutor.cs | 51 +++ src/Khaemhat.Data/Execution/ReaderMapper.cs | 154 ++++++++ src/Khaemhat.Data/Execution/Repository.cs | 140 +++++++ .../Execution/TransientRetryPolicy.cs | 45 +++ src/Khaemhat.Data/Execution/UnitOfWork.cs | 287 ++++++++++++++ .../Execution/UnitOfWorkFactory.cs | 18 + src/Khaemhat.Data/GlobalSuppressions.cs | 5 + src/Khaemhat.Data/IDbProvider.cs | 68 ++++ src/Khaemhat.Data/IRepository.cs | 62 +++ src/Khaemhat.Data/ISqlDialect.cs | 98 +++++ src/Khaemhat.Data/IUnitOfWork.cs | 93 +++++ src/Khaemhat.Data/Mapping/ClrConverter.cs | 63 ++++ src/Khaemhat.Data/Mapping/ColumnMap.cs | 207 ++++++++++ src/Khaemhat.Data/Mapping/EntityMap.cs | 353 ++++++++++++++++++ src/Khaemhat.Data/Mapping/MapRegistry.cs | 62 +++ .../Migrations/DataModelDocument.cs | 141 +++++++ src/Khaemhat.Data/Migrations/IMigration.cs | 56 +++ .../Migrations/MigrationCatalog.cs | 127 +++++++ .../Migrations/MigrationRunner.cs | 155 ++++++++ .../Migrations/SchemaVersionStore.cs | 95 +++++ src/Khaemhat.Data/PagedResult.cs | 13 + .../Querying/ExpressionTranslator.cs | 307 +++++++++++++++ src/Khaemhat.Data/Querying/Query.cs | 109 ++++++ src/Khaemhat.Data/Querying/QueryBuilder.cs | 75 ++++ src/Khaemhat.Data/Schema/ColumnType.cs | 107 ++++++ src/Khaemhat.Data/Schema/DdlGenerator.cs | 144 +++++++ src/Khaemhat.Data/Schema/Definitions.cs | 136 +++++++ src/Khaemhat.Data/Schema/ISchemaBuilder.cs | 53 +++ src/Khaemhat.Data/Schema/SchemaBuilder.cs | 53 +++ src/Khaemhat.Data/Schema/SchemaModel.cs | 86 +++++ src/Khaemhat.Data/Schema/SchemaOperations.cs | 30 ++ src/Khaemhat.Data/Schema/TableBuilder.cs | 347 +++++++++++++++++ src/Khaemhat.Data/SqlDialectBase.cs | 130 +++++++ src/Khaemhat.Data/SqlFragment.cs | 249 ++++++++++++ src/Khaemhat.Data/SqlNames.cs | 82 ++++ .../M20261111_001_CreateSysCompanies.cs | 28 ++ .../M20261111_002_CreateSysSettings.cs | 24 ++ .../SystemDatabase/SystemMaps.cs | 29 ++ src/Khaemhat.Data/UnitOfWorkExtensions.cs | 55 +++ .../Security/SecretProtector.cs | 189 ++++++++++ .../Architecture/DependencyRulesTests.cs | 1 + .../CompanyRegistryTests.cs | 125 +++++++ .../DataModelDocumentTests.cs | 48 +++ tests/Khaemhat.Data.Tests/DatabaseUrlTests.cs | 117 ++++++ tests/Khaemhat.Data.Tests/DdlTests.cs | 121 ++++++ tests/Khaemhat.Data.Tests/DialectTests.cs | 238 ++++++++++++ .../Khaemhat.Data.Tests.csproj | 1 + tests/Khaemhat.Data.Tests/MappingTests.cs | 82 ++++ tests/Khaemhat.Data.Tests/MigrationTests.cs | 154 ++++++++ .../PerformanceBaselineTests.cs | 64 ++++ .../Khaemhat.Data.Tests/ProjectSmokeTests.cs | 12 +- .../QueryTranslationTests.cs | 112 ++++++ tests/Khaemhat.Data.Tests/RepositoryTests.cs | 190 ++++++++++ tests/Khaemhat.Data.Tests/SqlFragmentTests.cs | 70 ++++ .../Support/TestDatabase.cs | 93 +++++ tests/Khaemhat.Data.Tests/Support/TestItem.cs | 79 ++++ .../ProjectSmokeTests.cs | 10 - .../SecretProtectorTests.cs | 133 +++++++ .../Khaemhat.DataModelTool.csproj | 14 + tools/Khaemhat.DataModelTool/Program.cs | 31 ++ 105 files changed, 8434 insertions(+), 73 deletions(-) create mode 100644 docs/adr/0018-contratti-dello-strato-dati.md create mode 100644 src/Khaemhat.Core/Common/ConcurrencyException.cs create mode 100644 src/Khaemhat.Core/Entities/Entity.cs create mode 100644 src/Khaemhat.Core/Entities/IAuditable.cs create mode 100644 src/Khaemhat.Core/Entities/IDemoMarkable.cs create mode 100644 src/Khaemhat.Core/Entities/IEntity.cs create mode 100644 src/Khaemhat.Core/Instance/Company.cs create mode 100644 src/Khaemhat.Core/Instance/CompanyStatus.cs create mode 100644 src/Khaemhat.Core/Instance/SystemSetting.cs create mode 100644 src/Khaemhat.Core/Security/ISecretProtector.cs create mode 100644 src/Khaemhat.Core/Security/SecretProtectionException.cs create mode 100644 src/Khaemhat.Data.PostgreSql/PostgresAdvisoryLocks.cs create mode 100644 src/Khaemhat.Data.PostgreSql/PostgresDatabaseCreator.cs create mode 100644 src/Khaemhat.Data.PostgreSql/PostgresDialect.cs create mode 100644 src/Khaemhat.Data.PostgreSql/PostgresPartitioning.cs create mode 100644 src/Khaemhat.Data.PostgreSql/PostgresProvider.cs create mode 100644 src/Khaemhat.Data.PostgreSql/PostgresSchemaBuilder.cs create mode 100644 src/Khaemhat.Data.PostgreSql/PostgresServerInfo.cs create mode 100644 src/Khaemhat.Data.Sqlite/SqliteDecimalConverter.cs create mode 100644 src/Khaemhat.Data.Sqlite/SqliteDialect.cs create mode 100644 src/Khaemhat.Data.Sqlite/SqliteProvider.cs create mode 100644 src/Khaemhat.Data.Sqlite/SqliteSchemaBuilder.cs create mode 100644 src/Khaemhat.Data.Sqlite/SqliteWriteLock.cs create mode 100644 src/Khaemhat.Data/Companies/CompanyRegistry.cs create mode 100644 src/Khaemhat.Data/CompanyCodes.cs create mode 100644 src/Khaemhat.Data/DatabaseScope.cs create mode 100644 src/Khaemhat.Data/DatabaseTarget.cs create mode 100644 src/Khaemhat.Data/DatabaseUrl.cs create mode 100644 src/Khaemhat.Data/DbEngine.cs create mode 100644 src/Khaemhat.Data/DbProviderCatalog.cs create mode 100644 src/Khaemhat.Data/DbServerInfo.cs create mode 100644 src/Khaemhat.Data/Execution/CommandCache.cs create mode 100644 src/Khaemhat.Data/Execution/DataAccessContext.cs create mode 100644 src/Khaemhat.Data/Execution/DbExecutor.cs create mode 100644 src/Khaemhat.Data/Execution/ReaderMapper.cs create mode 100644 src/Khaemhat.Data/Execution/Repository.cs create mode 100644 src/Khaemhat.Data/Execution/TransientRetryPolicy.cs create mode 100644 src/Khaemhat.Data/Execution/UnitOfWork.cs create mode 100644 src/Khaemhat.Data/Execution/UnitOfWorkFactory.cs create mode 100644 src/Khaemhat.Data/GlobalSuppressions.cs create mode 100644 src/Khaemhat.Data/IDbProvider.cs create mode 100644 src/Khaemhat.Data/IRepository.cs create mode 100644 src/Khaemhat.Data/ISqlDialect.cs create mode 100644 src/Khaemhat.Data/IUnitOfWork.cs create mode 100644 src/Khaemhat.Data/Mapping/ClrConverter.cs create mode 100644 src/Khaemhat.Data/Mapping/ColumnMap.cs create mode 100644 src/Khaemhat.Data/Mapping/EntityMap.cs create mode 100644 src/Khaemhat.Data/Mapping/MapRegistry.cs create mode 100644 src/Khaemhat.Data/Migrations/DataModelDocument.cs create mode 100644 src/Khaemhat.Data/Migrations/IMigration.cs create mode 100644 src/Khaemhat.Data/Migrations/MigrationCatalog.cs create mode 100644 src/Khaemhat.Data/Migrations/MigrationRunner.cs create mode 100644 src/Khaemhat.Data/Migrations/SchemaVersionStore.cs create mode 100644 src/Khaemhat.Data/PagedResult.cs create mode 100644 src/Khaemhat.Data/Querying/ExpressionTranslator.cs create mode 100644 src/Khaemhat.Data/Querying/Query.cs create mode 100644 src/Khaemhat.Data/Querying/QueryBuilder.cs create mode 100644 src/Khaemhat.Data/Schema/ColumnType.cs create mode 100644 src/Khaemhat.Data/Schema/DdlGenerator.cs create mode 100644 src/Khaemhat.Data/Schema/Definitions.cs create mode 100644 src/Khaemhat.Data/Schema/ISchemaBuilder.cs create mode 100644 src/Khaemhat.Data/Schema/SchemaBuilder.cs create mode 100644 src/Khaemhat.Data/Schema/SchemaModel.cs create mode 100644 src/Khaemhat.Data/Schema/SchemaOperations.cs create mode 100644 src/Khaemhat.Data/Schema/TableBuilder.cs create mode 100644 src/Khaemhat.Data/SqlDialectBase.cs create mode 100644 src/Khaemhat.Data/SqlFragment.cs create mode 100644 src/Khaemhat.Data/SqlNames.cs create mode 100644 src/Khaemhat.Data/SystemDatabase/Migrations/M20261111_001_CreateSysCompanies.cs create mode 100644 src/Khaemhat.Data/SystemDatabase/Migrations/M20261111_002_CreateSysSettings.cs create mode 100644 src/Khaemhat.Data/SystemDatabase/SystemMaps.cs create mode 100644 src/Khaemhat.Data/UnitOfWorkExtensions.cs create mode 100644 src/Khaemhat.Infrastructure/Security/SecretProtector.cs create mode 100644 tests/Khaemhat.Data.Tests/CompanyRegistryTests.cs create mode 100644 tests/Khaemhat.Data.Tests/DataModelDocumentTests.cs create mode 100644 tests/Khaemhat.Data.Tests/DatabaseUrlTests.cs create mode 100644 tests/Khaemhat.Data.Tests/DdlTests.cs create mode 100644 tests/Khaemhat.Data.Tests/DialectTests.cs create mode 100644 tests/Khaemhat.Data.Tests/MappingTests.cs create mode 100644 tests/Khaemhat.Data.Tests/MigrationTests.cs create mode 100644 tests/Khaemhat.Data.Tests/PerformanceBaselineTests.cs create mode 100644 tests/Khaemhat.Data.Tests/QueryTranslationTests.cs create mode 100644 tests/Khaemhat.Data.Tests/RepositoryTests.cs create mode 100644 tests/Khaemhat.Data.Tests/SqlFragmentTests.cs create mode 100644 tests/Khaemhat.Data.Tests/Support/TestDatabase.cs create mode 100644 tests/Khaemhat.Data.Tests/Support/TestItem.cs delete mode 100644 tests/Khaemhat.Infrastructure.Tests/ProjectSmokeTests.cs create mode 100644 tests/Khaemhat.Infrastructure.Tests/SecretProtectorTests.cs create mode 100644 tools/Khaemhat.DataModelTool/Khaemhat.DataModelTool.csproj create mode 100644 tools/Khaemhat.DataModelTool/Program.cs diff --git a/.editorconfig b/.editorconfig index 4ca6da5..eff14a3 100644 --- a/.editorconfig +++ b/.editorconfig @@ -70,4 +70,12 @@ dotnet_diagnostic.IDE0005.severity = warning [tests/**.cs] # CA1707 (underscores in names): test names read as Method_Condition_Result. dotnet_diagnostic.CA1707.severity = none +# CA1861/CA1847/CA1862: tests pass literal arrays and write the string predicates the query translator must handle. +dotnet_diagnostic.CA1861.severity = none +dotnet_diagnostic.CA1847.severity = none +dotnet_diagnostic.CA1862.severity = none # MSTEST0037 and friends stay at their defaults. + +# Migration classes are named after their version, M__ (SPEC 5.3). +[**/Migrations/*.cs] +dotnet_diagnostic.CA1707.severity = none diff --git a/CHANGELOG.md b/CHANGELOG.md index 21813ec..a596fa9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,33 @@ un'immagine `gitea.encke-hake.ts.net/balbosoft/khaemhat:X.Y.Z`. ## [Unreleased] +## [0.1.0] - 2026-11-11 + +M1 — Strato dati. Nessuna funzione visibile nell'interfaccia: è il motore su cui si appoggiano tutte le milestone +successive. + +### Added +- Micro-ORM interno in `Khaemhat.Data` (ADR 0004, ADR 0018): mappe esplicite `EntityMap`, query tipizzate + `Query.For().Where(...)` con un sottoinsieme sicuro di espressioni, `SqlFragment` con valori sempre + parametrizzati, `IUnitOfWork` con transazioni, `IRepository` con colonne di audit e concorrenza + ottimistica su `row_version` (`ConcurrencyException`), tentativi automatici sugli errori transitori. +- Provider SQLite (tabelle `STRICT`, WAL, decimali esatti come interi scalati x10^6, LIKE senza distinzione di + maiuscole anche per le lettere accentate, scritture serializzate) e PostgreSQL 16+ (`numeric(19,6)`, `timestamptz`, + `jsonb`, `ILIKE`, lock di transazione, creazione ed eliminazione dei database aziendali, informazioni sul server). +- Configurazione dei database da URL `postgres://` e `sqlite://` (`DatabaseUrl`), con la password mai mostrata nei + messaggi. +- Migrazioni in C#: cataloghi di sistema e aziendale, `schema_version` con checksum in ogni database, una + migrazione per transazione sotto lock, rifiuto delle versioni sconosciute; prime tabelle di sistema + `sys_companies` e `sys_settings`. +- Registro delle aziende (`CompanyRegistry`) con la posizione del database cifrata e risoluzione + `DatabaseScope.System` / `DatabaseScope.Company(codice)` in `IUnitOfWorkFactory`. +- Cifratura dei segreti a riposo con AES-256-GCM e chiave derivata da `KHAEMHAT_SECRET_KEY` (`SecretProtector`, + formato `khs1`), con il mattone della rotazione delle chiavi. +- `scripts/new-migration.sh` e `scripts/gen-datamodel.sh` (con `tools/Khaemhat.DataModelTool`): la sezione + *Schema attuale* di `docs/DATA-MODEL.md` è generata dalle migrazioni e un test ne verifica l'aggiornamento. +- Test su SQLite e PostgreSQL (tipi, decimali, paginazione, upsert, LIKE, ordinamento temporale, CRUD, concorrenza, + transazioni, input ostile, migrazioni, registro, cifratura) e misure di riferimento delle prestazioni. + ## [0.0.1] - 2026-11-11 Prima versione: M0 — Bootstrap. Nessuna funzione gestionale ancora; c'è lo scheletro su cui costruirle. diff --git a/Khaemhat.sln b/Khaemhat.sln index 3d70c39..d1ee8aa 100644 --- a/Khaemhat.sln +++ b/Khaemhat.sln @@ -41,6 +41,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tools", "tools", "{07C2787E EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Khaemhat.LicenseTool", "tools\Khaemhat.LicenseTool\Khaemhat.LicenseTool.csproj", "{4317FFD0-9473-43FD-8543-F80EF9BA812A}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Khaemhat.DataModelTool", "tools\Khaemhat.DataModelTool\Khaemhat.DataModelTool.csproj", "{7CA26565-5E1A-46AA-A2F7-C38FBADF0E3F}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -243,6 +245,18 @@ Global {4317FFD0-9473-43FD-8543-F80EF9BA812A}.Release|x64.Build.0 = Release|Any CPU {4317FFD0-9473-43FD-8543-F80EF9BA812A}.Release|x86.ActiveCfg = Release|Any CPU {4317FFD0-9473-43FD-8543-F80EF9BA812A}.Release|x86.Build.0 = Release|Any CPU + {7CA26565-5E1A-46AA-A2F7-C38FBADF0E3F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7CA26565-5E1A-46AA-A2F7-C38FBADF0E3F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7CA26565-5E1A-46AA-A2F7-C38FBADF0E3F}.Debug|x64.ActiveCfg = Debug|Any CPU + {7CA26565-5E1A-46AA-A2F7-C38FBADF0E3F}.Debug|x64.Build.0 = Debug|Any CPU + {7CA26565-5E1A-46AA-A2F7-C38FBADF0E3F}.Debug|x86.ActiveCfg = Debug|Any CPU + {7CA26565-5E1A-46AA-A2F7-C38FBADF0E3F}.Debug|x86.Build.0 = Debug|Any CPU + {7CA26565-5E1A-46AA-A2F7-C38FBADF0E3F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7CA26565-5E1A-46AA-A2F7-C38FBADF0E3F}.Release|Any CPU.Build.0 = Release|Any CPU + {7CA26565-5E1A-46AA-A2F7-C38FBADF0E3F}.Release|x64.ActiveCfg = Release|Any CPU + {7CA26565-5E1A-46AA-A2F7-C38FBADF0E3F}.Release|x64.Build.0 = Release|Any CPU + {7CA26565-5E1A-46AA-A2F7-C38FBADF0E3F}.Release|x86.ActiveCfg = Release|Any CPU + {7CA26565-5E1A-46AA-A2F7-C38FBADF0E3F}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -264,5 +278,6 @@ Global {C5C8124F-3B12-43A9-BD0F-5AA5F7261699} = {0AB3BF05-4346-4AA6-1389-037BE0695223} {FDCAC352-BB40-4B5E-948A-2CDAA48B9E6A} = {0AB3BF05-4346-4AA6-1389-037BE0695223} {4317FFD0-9473-43FD-8543-F80EF9BA812A} = {07C2787E-EAC7-C090-1BA3-A61EC2A24D84} + {7CA26565-5E1A-46AA-A2F7-C38FBADF0E3F} = {07C2787E-EAC7-C090-1BA3-A61EC2A24D84} EndGlobalSection EndGlobal diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 1618f05..07a371a 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,6 +1,6 @@ # Architettura di Khaemhat -> Stato: bozza · Revisione 2026-11-11 · Fonte: SPEC §1, §2, §3, §4, §5, §7, §9, §13, §15, §19, §21; ADR 0009-0017 +> Stato: bozza · Revisione 2026-11-11 · Fonte: SPEC §1, §2, §3, §4, §5, §7, §9, §13, §15, §19, §21; ADR 0009-0018 Questo documento descrive l'architettura **prevista** del gestionale: livelli, progetti, regole di dipendenza, composizione dei moduli, gestione multi-database, ciclo di una richiesta e servizi trasversali. Il codice non esiste @@ -74,6 +74,7 @@ client degli aggiornamenti è `GiteaPackagesClient`. | `Khaemhat.Application` | casi d'uso, moduli (`IModule`), motore documenti e contabile, seeding, report | M2 → M12 | | `Khaemhat.Web` | host ASP.NET Core 10, Blazor Interactive Server, Minimal API, MCP, autenticazione | M0 → M13 | | `tools/Khaemhat.LicenseTool` | CLI per chiavi ed emissione licenze (chiave privata mai nel repository; non distribuito) | M2 | +| `tools/Khaemhat.DataModelTool` | rigenera la sezione generata di `docs/DATA-MODEL.md` dalle migrazioni (non distribuito, ADR 0018) | M1 | | `tests/*` | `Core`, `Data`, `Infrastructure`, `Fiscal.Italy`, `Application`, `Web` (MSTest) | ogni milestone | ## 3. Regole di dipendenza @@ -171,11 +172,11 @@ tabelle è in [DATA-MODEL.md](DATA-MODEL.md). | Tipo | Progetto | Responsabilità | |---|---|---| -| `IDbProvider` | Data | motore (`DbEngine`), dialetto, `SchemaBuilder`, creazione connessioni, `EnsureDatabaseExistsAsync`, `DropDatabaseAsync`, `GetServerInfoAsync` | -| `ISqlDialect` | Data | quoting, parametri, paginazione, `InsertReturningId`, `Upsert`, `CaseInsensitiveLike`, `AdvisoryLock`, conversioni `ColumnType` | +| `IDbProvider` | Data | motore (`DbEngine`), dialetto, generatore DDL (`Ddl`), connection string da `DatabaseUrl`, creazione connessioni, `EnsureDatabaseExistsAsync`, `DatabaseExistsAsync`, `DropDatabaseAsync`, `GetServerInfoAsync`, `IsTransient`, `EnterWriteScopeAsync` (ADR 0018) | +| `ISqlDialect` | Data | quoting, parametri, paginazione, `InsertReturningId`, `Upsert`, `CaseInsensitiveLike` e `AdvisoryLock` come `SqlFragment`, conversioni `ColumnType` | | `IUnitOfWorkFactory` | Data | `CreateAsync(DatabaseScope scope)` dove lo scope è `System` oppure `Company(code)` | | `IUnitOfWork` / `IRepository` | Data | transazione, repository tipizzati, `ExecuteAsync`/`QueryAsync`/`StreamAsync` su `SqlFragment` | -| `ICompanyRegistry` (`CompanyRegistry`) | Data | risolve `companyCode → (IDbProvider, connection string decifrata)` leggendo `sys_companies` | +| `ICompanyRegistry` (`CompanyRegistry`) | Data | registra le aziende e risolve `companyCode → DatabaseTarget` (provider + connection string) decifrando l'URL salvato in `sys_companies` | | `ICompanyContext` | Core (interfaccia), Web (`CompanyContextAccessor`) | azienda corrente: **per circuito Blazor** e **per richiesta API** (header `X-Company`) | | `SecretProtector` | Infrastructure | AES-256-GCM con chiave derivata da `KHAEMHAT_SECRET_KEY` per connection string e credenziali | @@ -440,11 +441,11 @@ stando fuori dal pacchetto fiscale: è la collocazione indicata da SPEC §21. | Area | Classi | |---|---| -| Contratti | `IDbProvider`, `ISqlDialect`, `ISchemaBuilder`, `IUnitOfWork`, `IUnitOfWorkFactory`, `IRepository<,>`, `Repository<,>` | -| Registro database | `ICompanyRegistry`, `CompanyRegistry`, `DatabaseScope` | +| Contratti | `IDbProvider`, `ISqlDialect`, `ISchemaBuilder`, `IUnitOfWork`, `IUnitOfWorkFactory`, `IRepository<,>`, `Repository<,>`, `DatabaseUrl`, `DatabaseTarget`, `DbProviderCatalog` | +| Registro database | `ICompanyRegistry`, `CompanyRegistry`, `IDatabaseLocator`, `DatabaseLocator`, `DatabaseScope`, `CompanyCodes` | | Mappe e query | `EntityMap`, `ColumnMap`, `ColumnType`, `MapRegistry`, `QueryBuilder`, `Query`, `PagedResult`, `SqlFragment`, `ExpressionTranslator` | -| Esecuzione | `DbExecutor`, `ReaderMapper`, `CommandCache`, `TransientRetryPolicy` | -| Migrazioni | `IMigration`, `MigrationAttribute`, `MigrationRunner`, `MigrationCatalog` (System/Company), `SchemaVersionStore`, `TableDefinition`, `ColumnDefinition`, `IndexDefinition`, `ForeignKeyDefinition`, `DdlGenerator` | +| Esecuzione | `UnitOfWork`, `UnitOfWorkFactory`, `DataAccessContext` (orologio, utente, tentativi), `DbExecutor`, `ReaderMapper`, `CommandCache`, `TransientRetryPolicy` | +| Migrazioni | `IMigration`, `MigrationAttribute`, `MigrationRunner`, `MigrationReport`, `MigrationCatalog` (`ForSystem`/`ForCompany`), `SchemaVersionStore`, `SchemaModel`, `DataModelDocument`, `TableDefinition`, `ColumnDefinition`, `IndexDefinition`, `ForeignKeyDefinition`, `DdlGenerator` | | SQLite | `SqliteProvider`, `SqliteDialect`, `SqliteSchemaBuilder`, `SqliteWriteLock`, `SqliteDecimalConverter` | | PostgreSQL | `PostgresProvider`, `PostgresDialect`, `PostgresSchemaBuilder`, `PostgresDatabaseCreator`, `PostgresAdvisoryLocks`, `PostgresPartitioning`, `PostgresServerInfo` | diff --git a/docs/DATA-MODEL.md b/docs/DATA-MODEL.md index 57e58d0..46d5aa7 100644 --- a/docs/DATA-MODEL.md +++ b/docs/DATA-MODEL.md @@ -1,12 +1,13 @@ # Modello dati -> Stato: bozza · Revisione 2026-11-10 · Fonte: SPEC §5, §7, §8, §9, §11, §12, §13, §14, §21 +> Stato: in corso · Revisione 2026-11-11 (M1) · Fonte: SPEC §5, §7, §8, §9, §11, §12, §13, §14, §21 -> **Versione pianificata.** A partire dalla milestone 1 questo file sarà **generato** da `scripts/gen-datamodel.sh` -> leggendo le mappe `XxxMap : EntityMap` e le migrazioni: non andrà più modificato a mano. Finché il codice non -> esiste, questa è la versione pianificata ricavata dalla specifica: nomi e colonne sono indicativi e verranno -> confermati (o corretti) dalla prima generazione. I nomi di tabella contrassegnati con **†** non compaiono -> letteralmente in SPEC: sono derivati dal nome dell'entità di SPEC §21 secondo la convenzione snake_case plurale. +> **Due parti.** La sezione [8. Schema attuale](#8-schema-attuale-generato) è **generata** da +> `scripts/gen-datamodel.sh` leggendo le migrazioni: non si modifica a mano e un test fallisce se non è aggiornata. +> Le sezioni 3-4 sono la **pianificazione** ricavata dalla specifica: nomi e colonne sono indicativi e vengono +> confermati (o corretti) dalla migrazione che crea ciascuna tabella, milestone dopo milestone. I nomi di tabella +> contrassegnati con **†** non compaiono letteralmente in SPEC: sono derivati dal nome dell'entità di SPEC §21 +> secondo la convenzione snake_case plurale. Documenti collegati: [ARCHITECTURE.md](ARCHITECTURE.md) · [MODULES.md](MODULES.md) · [API.md](API.md). @@ -31,24 +32,29 @@ Documenti collegati: [ARCHITECTURE.md](ARCHITECTURE.md) · [MODULES.md](MODULES. | `ColumnType` | PostgreSQL | SQLite | Note | |---|---|---|---| -| `String(n)` | `varchar(n)` | `TEXT` | lunghezza verificata dall'applicazione su SQLite | +| `String(n)` | `varchar(n)` | `TEXT` + `CHECK (length(col) <= n)` | la lunghezza è imposta dal database su entrambi i motori | | `Text` | `text` | `TEXT` | | -| `Int32` | `int` | `INTEGER` | | +| `Int32` | `integer` | `INTEGER` | | | `Int64` | `bigint` | `INTEGER` | chiavi primarie ed esterne | | `Decimal(19,6)` | `numeric(19,6)` | `INTEGER` scalato x10⁶ | vedi §1.3 | -| `Bool` | `boolean` | `INTEGER` 0/1 | | +| `Bool` | `boolean` | `INTEGER` + `CHECK (col IN (0, 1))` | | | `Date` | `date` | `TEXT` `yyyy-MM-dd` | `DateOnly` in C# | -| `DateTimeUtc` | `timestamptz` | `TEXT` ISO 8601 UTC | `DateTime` UTC; conversione a `Europe/Rome` solo in presentazione | +| `DateTimeUtc` | `timestamptz` | `TEXT` `yyyy-MM-ddTHH:mm:ss.ffffffZ` | `DateTime` UTC al microsecondo su entrambi i motori; conversione a `Europe/Rome` solo in presentazione | | `Guid` | `uuid` | `TEXT` | | | `Bytes` | `bytea` | `BLOB` | | | `Json` | `jsonb` | `TEXT` | diff di audit, payload dei canali, dati di log | +Le tabelle SQLite sono `STRICT`: un valore del tipo sbagliato è rifiutato invece di essere convertito. Le enumerazioni +sono memorizzate **per nome** (`String(40)`), così un riordino dei valori in C# non altera i dati. + ### 1.3 Decimali scalati su SQLite SQLite non ha un tipo decimale esatto: i valori `Decimal(19,6)` sono memorizzati come **interi moltiplicati per 10⁶** (`SqliteDecimalConverter`). Conseguenze per chi scrive query: - `ISqlDialect.ToDbValue` / `FromDbValue` convertono in scrittura e in lettura; i parametri passano già scalati; +- l'arrotondamento al sesto decimale è *half away from zero* (`1,2345675` diventa `1,234568`), come `numeric(19,6)` + su PostgreSQL; un valore oltre l'intervallo solleva `OverflowException` invece di essere troncato; - le **somme in SQL** (`SUM`, `GROUP BY`) operano sugli interi scalati e vengono **riscalate in lettura**: niente aritmetica in virgola mobile; - prodotti e divisioni fra due colonne decimali non vanno fatti in SQL su SQLite (il fattore di scala si @@ -59,8 +65,14 @@ SQLite non ha un tipo decimale esatto: i valori `Decimal(19,6)` sono memorizzati - Ogni entità ha una mappa `XxxMap : EntityMap` (tabella, colonne, chiave, indici, FK, conversioni); nessuna riflessione a runtime oltre la costruzione iniziale (`MapRegistry`). -- `QueryBuilder`: uguaglianza, confronto, `Contains`/`StartsWith`, `In`, `And`/`Or`/`Not`, `IsNull`; il resto come - `SqlFragment` parametrizzato. Mai concatenare input utente in SQL. +- `Query.For().Where(x => ...)`: uguaglianza, confronto, `== null`, `Contains`/`StartsWith`/`EndsWith` (LIKE + senza distinzione di maiuscole e accenti compresi, con `%` e `_` dell'utente protetti), `lista.Contains(x.Col)` + (`IN`), `&&`/`||`/`!`, `string.IsNullOrEmpty`; il resto come `SqlFragment` parametrizzato. Un'espressione fuori da + questo insieme solleva `NotSupportedException`, non viene valutata in memoria. Mai concatenare input utente in SQL: + in `SqlFragment.Of($"... {valore}")` ogni valore interpolato diventa un parametro. +- `IRepository` compila le colonne di audit, imposta `row_version = 1` all'inserimento e la incrementa a ogni + modifica; `IUnitOfWork.AcquireLockAsync(chiave)` prende un lock di transazione (`pg_advisory_xact_lock` o il lock di + scrittura di SQLite) per numeratori e chiusure. - SQLite: `journal_mode=WAL; foreign_keys=ON; busy_timeout=5000`; scritture serializzate da `SqliteWriteLock`. - PostgreSQL: pool separato per database (`MaxPoolSize` da impostazioni), `statement_timeout` per sessione, `pg_advisory_xact_lock` per numeratori, chiusure e scheduler. @@ -68,22 +80,24 @@ SQLite non ha un tipo decimale esatto: i valori `Decimal(19,6)` sono memorizzati ## 2. Migrazioni (SPEC §5.3) ```csharp -[Migration(20261013_001, "create_parties")] -public sealed class M20261013_001 : IMigration +[Migration(20261013_001, "create_parties")] // Target = MigrationTarget.Company se omesso +public sealed class M20261013_001_CreateParties : IMigration { - public void Up(ISchemaBuilder sb) { /* tabelle, colonne, indici, FK in forma neutra */ } - public string? UpSql(ISqlDialect d) => null; // SQL specifico del dialetto, se serve + public void Up(ISchemaBuilder schema) { /* tabelle, colonne, indici, FK in forma neutra */ } + public string? UpSql(ISqlDialect dialect) => null; // SQL specifico del dialetto, se serve } ``` | Regola | Dettaglio | |---|---| -| Due cataloghi | `MigrationCatalog.System` (database di sistema) e `MigrationCatalog.Company` (ogni azienda) | -| Versioni | `schema_version(module, version, name, applied_at, checksum)` in **ogni** database | -| Esecuzione | `MigrationRunner` applica in transazione, è idempotente, registra l'esito nella sezione di log *Migrazioni* | -| Downgrade | non supportato: se trova versioni applicate non note rifiuta di partire | +| Due cataloghi | `MigrationCatalog.ForSystem()` (database di sistema) e `MigrationCatalog.ForCompany(assembly dei moduli)` | +| Versioni | `yyyymmdd_nnn` (11 cifre, data valida, `nnn` da 001), uniche per modulo; `schema_version(module, version, name, applied_at, checksum)` in **ogni** database, `checksum` = SHA-256 delle istruzioni generate | +| Esecuzione | `MigrationRunner` applica **una migrazione per transazione** sotto lock (più istanze che partono insieme non si pestano i piedi), è idempotente e restituisce un `MigrationReport`; la registrazione nella sezione di log *Migrazioni* arriva con il log (M2) | +| Checksum diverso | una migrazione già applicata e poi modificata produce un avviso, non un errore: le migrazioni applicate non si cambiano, se ne scrive una nuova | +| Downgrade | non supportato: se trova versioni applicate non note rifiuta di partire (`MigrationException`) | | Avvio | il sistema è migrato all'avvio; le aziende all'avvio (impostazione) o dal pannello (SPEC §12.1) in modalità manutenzione | -| Nuova migrazione | `scripts/new-migration.sh` | +| Nuova migrazione | `scripts/new-migration.sh [cartella] [modulo]` | +| Documentazione | `scripts/gen-datamodel.sh` rigenera la [sezione 8](#8-schema-attuale-generato); `--check` verifica soltanto | | Test obbligatorio | tutte le migrazioni da zero su SQLite e PostgreSQL, `schema_version` coerente, riapplicazione senza effetti | ## 3. Database di sistema (`khaemhat_system`) @@ -325,9 +339,106 @@ qualunque modifica ──► audit_log - Obiettivi di prestazione di riferimento (SPEC §18): bilancio di verifica su 2 M righe < 10 s, pagina log con 1 M righe < 1 s per filtro. +### 6.1 Misure di riferimento dello strato dati (M1) + +`PerformanceBaselineTests` inserisce righe con tutti i tipi di colonna tramite `IRepository` in una sola transazione, +poi le legge tutte e legge una pagina filtrata con il conteggio totale. Con `KHAEMHAT_PERF=1` usa 100.000 righe, +altrimenti 2.000 (prova di fumo, sempre eseguita anche in CI). + +```bash +KHAEMHAT_PERF=1 scripts/test.sh --filter TestCategory=Performance --logger "console;verbosity=detailed" +``` + +| Data | Motore | Macchina | Inserimenti | Lettura di tutte le righe | Pagina filtrata + conteggio | +|---|---|---|---|---|---| +| 2026-11-11 | SQLite (file in memoria) | Ryzen 5 2600X, Windows 11, .NET 10.0.403 | 100.000 in 5,6 s (circa 17.800/s) | 0,9 s | 17 ms | + +Le misure servono a riconoscere un peggioramento, non come soglia: il test non fallisce sui tempi. Le misure su +PostgreSQL si aggiungono dal job `postgres` della CI alla chiusura della milestone. + ## 7. Segreti nel database Cifrati con AES-256-GCM e chiave derivata da `KHAEMHAT_SECRET_KEY` (`SecretProtector`): connection string delle aziende (`sys_companies`), password SMTP/PEC/S3/WebDAV (`sys_settings`), token degli intermediari e credenziali dei canali SDI (`sdi_channels`), credenziali dei canali e-commerce (`sales_channels`), segreti dei webhook. La rotazione avviene con `KHAEMHAT_SECRET_KEY_NEW` (SPEC §12.9). Gli hash (password, chiavi API) non sono cifrati ma irreversibili. + +## 8. Schema attuale (generato) + +Questa sezione è scritta da `scripts/gen-datamodel.sh` a partire dalle migrazioni presenti nel codice; il test +`DataModelDocumentTests` fallisce se non è aggiornata. Comprende `schema_version`, presente in ogni database. + + + +### Database di sistema + +Migrazioni applicate in ordine: + +- `core` 20261111001 `create_sys_companies` (M20261111_001_CreateSysCompanies) +- `core` 20261111002 `create_sys_settings` (M20261111_002_CreateSysSettings) + +#### `schema_version` + +| Colonna | Tipo | Null | Default | Note | +|---|---|---|---|---| +| `module` | String(50) | no | | chiave primaria | +| `version` | Int64 | no | | chiave primaria | +| `name` | String(200) | no | | | +| `applied_at` | DateTimeUtc | no | | | +| `checksum` | String(64) | no | | | + +Chiave primaria: `module`, `version`. + +#### `sys_companies` + +| Colonna | Tipo | Null | Default | Note | +|---|---|---|---|---| +| `id` | Int64 | no | | chiave primaria, identity | +| `code` | String(20) | no | | | +| `name` | String(200) | no | | | +| `engine` | String(20) | no | | | +| `connection_string` | Text | no | | | +| `status` | String(40) | no | `'Active'` | | +| `is_default` | Bool | no | `false` | | +| `notes` | Text | sì | | | +| `created_at` | DateTimeUtc | no | | | +| `created_by` | String(100) | sì | | | +| `updated_at` | DateTimeUtc | sì | | | +| `updated_by` | String(100) | sì | | | +| `row_version` | Int64 | no | `1` | | + +- indice unico `ux_sys_companies_code` su `code` + +#### `sys_settings` + +| Colonna | Tipo | Null | Default | Note | +|---|---|---|---|---| +| `id` | Int64 | no | | chiave primaria, identity | +| `key` | String(200) | no | | | +| `value` | Text | sì | | | +| `is_secret` | Bool | no | `false` | | +| `created_at` | DateTimeUtc | no | | | +| `created_by` | String(100) | sì | | | +| `updated_at` | DateTimeUtc | sì | | | +| `updated_by` | String(100) | sì | | | +| `row_version` | Int64 | no | `1` | | + +- indice unico `ux_sys_settings_key` su `key` + +### Database aziendale + +Nessuna migrazione: la tabella `schema_version` è l'unica presente. + +#### `schema_version` + +| Colonna | Tipo | Null | Default | Note | +|---|---|---|---|---| +| `module` | String(50) | no | | chiave primaria | +| `version` | Int64 | no | | chiave primaria | +| `name` | String(200) | no | | | +| `applied_at` | DateTimeUtc | no | | | +| `checksum` | String(64) | no | | | + +Chiave primaria: `module`, `version`. + + diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index d25cfff..5fe8bae 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -30,9 +30,20 @@ Se manca l'SDK 10 ci si ferma e lo si chiede al titolare (SPEC §23). Su Windows | `docker compose -f deploy/docker-compose.sqlite.yml up` | demo autonoma (con `KHAEMHAT_VERSION` per scegliere l'immagine) | | `scripts/publish-image.sh vX.Y.Z [--arm64]` | pubblicazione dalla macchina di sviluppo se il runner non c'è (ADR 0015) | | `scripts/publish-wiki.sh` | pubblica `docs/wiki/` nel wiki Gitea senza passare dal runner | +| `scripts/new-migration.sh [cartella] [modulo]` | crea `M__.cs` con il primo numero libero del giorno (SPEC §5.3) | +| `scripts/gen-datamodel.sh [--check]` | rigenera la sezione generata di `docs/DATA-MODEL.md` dalle migrazioni; `--check` verifica soltanto | +| `KHAEMHAT_PERF=1 scripts/test.sh --filter TestCategory=Performance` | misure di riferimento dello strato dati con 100.000 righe (`docs/DATA-MODEL.md` §6.1) | -Gli stub `new-migration.sh` (M1), `gen-datamodel.sh` (M1), `seed.sh` (M4) e `restore-system.sh` (M12, incluso -nell'immagine, ADR 0017) spiegano l'uso previsto ed escono con errore finché la loro milestone non li completa. +Gli stub `seed.sh` (M4) e `restore-system.sh` (M12, incluso nell'immagine, ADR 0017) spiegano l'uso previsto ed +escono con errore finché la loro milestone non li completa. + +#### Test sui due motori + +I test dello strato dati girano con `[DataRow("Sqlite")]` e `[DataRow("PostgreSql")]`. SQLite usa un database in +memoria condiviso, sempre disponibile. PostgreSQL usa `KHAEMHAT_TEST_PG` (URL +`postgres://utente:password@host:5432/postgres` di un ruolo con `CREATEDB`): ogni test crea un database +`khaemhat_t_` e lo elimina alla fine; senza la variabile i test PostgreSQL risultano *saltati*. In CI il +job `postgres` li esegue su PostgreSQL 16 e 17 sui tag e all'avvio manuale del workflow. ### 1.2 Visual Studio Code diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 3597f6d..0d630ec 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -240,11 +240,17 @@ passaggio TOTP (prevenzione della session fixation). - `SecretProtector` cifra con **AES-256-GCM** (`System.Security.Cryptography.AesGcm`) i segreti salvati: connection string delle aziende (`sys_companies`), password SMTP/PEC/S3/WebDAV, token degli intermediari SDI, credenziali dei canali e-commerce. -- La chiave AES è **derivata** da `KHAEMHAT_SECRET_KEY`. Proposta: HKDF-SHA256 (in-box, `HKDF`) con etichette di - contesto distinte per uso (`secrets`, `backup`, `totp`), così una chiave non viene riusata fra domini. -- Proposta di formato del testo cifrato: `v1` (versione/identificativo chiave) ‖ nonce 12 byte casuale ‖ - ciphertext ‖ tag 16 byte, in base64. Il nonce non si riusa mai con la stessa chiave. Come dato associato (AAD) - usare tabella+colonna+id, così un segreto copiato in un'altra riga non si decifra. +- La chiave AES è **derivata** da `KHAEMHAT_SECRET_KEY` con HKDF-SHA256 (in-box, `HKDF`) ed etichetta di contesto + `khaemhat/secrets/aes-256-gcm/v1`; altri usi (backup, TOTP) avranno etichette proprie, così una chiave non viene + riusata fra domini. L'identificativo della chiave è una seconda derivazione HKDF di 4 byte (8 cifre esadecimali): + identifica la chiave senza rivelarla. +- **Formato del testo cifrato (M1):** `khs1..`. + Il nonce è casuale e non si riusa mai con la stessa chiave. Il dato associato (AAD) è lo **scopo** del segreto + (per esempio `company:ACME:database` per la connessione di un'azienda): un valore copiato in un'altra riga o + usato per un altro scopo non si decifra. Un valore cifrato con un'altra chiave è rifiutato con un messaggio che + indica i due identificativi (`SecretProtectionException`), senza mai ripetere il segreto. +- Implementazione: `Khaemhat.Infrastructure.Security.SecretProtector` (con `SecretKey` e `SecretRotation`), + interfaccia `ISecretProtector` in `Khaemhat.Core`. - La chiave e i segreti in chiaro **non vanno mai** in log, messaggi d'errore, diagnostica, risposte API o file su disco (SPEC §1.5: nessun `appsettings..json` con segreti, nessun file temporaneo). - Le varianti `_FILE` delle variabili (Docker secrets) sono previste per `KHAEMHAT_SYSTEM_DB` e le credenziali del @@ -258,6 +264,9 @@ passaggio TOTP (prevenzione della session fixation). 3. L'operatore sostituisce `KHAEMHAT_SECRET_KEY` con la nuova e rimuove `KHAEMHAT_SECRET_KEY_NEW`. 4. Evento in `sys_security_audit`. +In M1 esiste il mattone `SecretRotation` (`NeedsRotation` legge l'identificativo della chiave, `Rotate` decifra e +ri-cifra); la procedura che percorre tutte le colonne cifrate arriva in M12. + Da decidere con ADR: trattamento dei **backup già cifrati** con la vecchia chiave (ri-cifratura o conservazione della vecchia chiave come chiave di sola decifratura identificata dalla versione nel formato). diff --git a/docs/STATUS.md b/docs/STATUS.md index c91ddc8..26346ea 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -3,25 +3,32 @@ > Punto di ripresa per ogni sessione. Aggiornare a fine sessione. **Ultimo aggiornamento:** 2026-11-11 -**Milestone corrente:** M0 — Bootstrap (via allo sviluppo dato dal titolare il 2026-11-11) -**Prossimo passo:** chiudere M0 (PR `feat/m0-bootstrap` → `main`, CI verde, tag `v0.0.1`, immagine e release), poi -M1 — Fondamenta dati, partendo dall'issue #7 (colonna *Da fare* del progetto *Khaemhat 1.0 — Sviluppo*). +**Milestone corrente:** M1 — Strato dati (in chiusura sul ramo `feat/7-strato-dati`) +**Prossimo passo:** chiudere M1 (PR verso `main`, CI verde compreso il job `postgres` avviato a mano sul ramo, tag +`v0.1.0`, immagine e release; misure PostgreSQL in `docs/DATA-MODEL.md` §6.1), poi M2 — Sistema e sicurezza, +partendo dall'issue #15. ## Fatto - Specifica normativa in `docs/SPEC.md`; `CLAUDE.md` con emendamenti E1-E6. -- ADR 0001-0011. +- ADR 0001-0018. - Mockup in `docs/mockups/` (22 in alta risoluzione + galleria, 88 miniature, `mockup.css`, indice). - Icone in `assets/`; deploy in `deploy/` (compose Unraid, `.env.example`, template Unraid). - Documentazione tecnica: `ARCHITECTURE.md`, `DATA-MODEL.md` (provvisorio, sarà generato), `MODULES.md`, `API.md`, `MCP.md`, `DEPLOY.md`, `SECURITY.md`, `DEVELOPMENT.md`. - Wiki completa in bozza in `docs/wiki/`, pubblicata nel wiki Gitea. -- **M0 (in chiusura):** soluzione `Khaemhat.sln` con i 10 progetti di SPEC §3 e 6 progetti di test; regole di build +- **M0 (chiusa, `v0.0.1`):** soluzione `Khaemhat.sln` con i 10 progetti di SPEC §3 e 6 progetti di test; regole di build (`Directory.Build.props/targets`, Central Package Management con l'allowlist, `.editorconfig`, warning come errori); test architetturali delle dipendenze e dei pacchetti; `Khaemhat.Web` con pagina "in costruzione", icone, `/health/live`, `/health/ready`, `/health` (ADR 0012); `docker/Dockerfile`, `entrypoint.sh` con `PUID`/`PGID` (ADR 0013), `postgres/init.sql`; `deploy/docker-compose.yml` e `docker-compose.sqlite.yml`; Gitea Actions `ci.yml`, `release.yml`, `wiki.yml` con controllo YAML; script di build, test, avvio, pubblicazione e stub; ADR 0012-0017. +- **M1 (in chiusura, `v0.1.0`):** micro-ORM in `Khaemhat.Data` (mappe, query tipizzate, `SqlFragment`, unit of work, + repository con audit e `row_version`), provider SQLite e PostgreSQL, `DatabaseUrl`, migrazioni con + `schema_version`, tabelle `sys_companies` e `sys_settings`, `CompanyRegistry` con URL cifrato, `SecretProtector` + (AES-256-GCM, formato `khs1`), `tools/Khaemhat.DataModelTool` e gli script `new-migration.sh` e + `gen-datamodel.sh`; ADR 0018. Test: 85 su SQLite verdi in locale, 21 su PostgreSQL eseguiti solo in CI (in locale + Docker non è disponibile). - Su Gitea: etichette, milestone M0-M14, 114 issue con checklist, progetti *Khaemhat 1.0 — Sviluppo* (M0-M13) e *Khaemhat 1.1+ — Fase 2* (M14). Numerazione: M0 #1-#6 e #114, M1 #7-#14, M2 #15-#24, M3 #25-#36, M4 #37-#46, M5 #47-#54, M6 #55-#61, M7 #62-#68, M8 #69-#73, M9 #74-#81, M10 #82-#87, M11 #88-#91, @@ -62,7 +69,7 @@ Emersi durante la stesura della documentazione. Si decidono con un ADR quando si 6. Tag e doppia build: `publish-image.sh` (Appendice D) crea il tag git, ma con `release.yml` sui tag `v*` l'immagine verrebbe costruita due volte. Proposta: lo script non crea tag, oppure `release.yml` salta le versioni già nel registro. **Deciso in M0: ADR 0015.** 7. Formato tag: tag git `vX.Y.Z`, tag immagine `X.Y.Z` (senza `v`). **Deciso in M0: ADR 0015.** 8. Icona versionata: `build.sh` deve produrre PNG, serve un rasterizzatore; proposta: generarla sulla macchina di sviluppo e committarla prima del rilascio. **Deciso in M0: ADR 0016.** -9. Rotazione della password PostgreSQL: le connection string delle aziende sono cifrate per intero in `sys_companies`; serve una procedura di aggiornamento. +9. Rotazione della password PostgreSQL: l'URL del database di ogni azienda è cifrato per intero in `sys_companies` (ADR 0018); serve una procedura di aggiornamento (M12). 10. Il mockup 085 mostra ancora `ghcr.io`: l'app mostrerà il registro Gitea (ADR 0009). 11. Non definiti: schermata di attivazione TOTP, nome dell'impostazione "migra aziende all'avvio", recupero password del super utente senza SMTP, chi modifica le impostazioni di backup cloud (scritto: super utente), flusso di ripristino da backup logico. diff --git a/docs/adr/0018-contratti-dello-strato-dati.md b/docs/adr/0018-contratti-dello-strato-dati.md new file mode 100644 index 0000000..cc3de92 --- /dev/null +++ b/docs/adr/0018-contratti-dello-strato-dati.md @@ -0,0 +1,44 @@ +# ADR 0018 - Contratti dello strato dati: scostamenti da SPEC §5.1 + +- **Stato:** accettata +- **Data:** 2026-11-11 +- **Modifica:** SPEC §5.1-§5.4, §3 (con ADR 0004); milestone M1, issue #7-#14 + +## Contesto + +SPEC §5.1 elenca firme indicative di `IDbProvider`, `ISqlDialect`, `ICompanyRegistry` e del motore di migrazione. +Implementandole su due motori sono emersi punti in cui la firma letterale costringeva a soluzioni peggiori +(stringhe SQL da ricomporre, chiamate non annullabili, segreti in chiaro in memoria più del necessario). Questo ADR +fissa le firme effettive. + +## Decisione + +1. **`CancellationToken` ovunque.** Tutti i metodi asincroni di `IDbProvider`, `IUnitOfWork`, `IRepository`, + `ICompanyRegistry` e `MigrationRunner` accettano un `CancellationToken` (default `default`). +2. **DDL fuori dal builder.** `IDbProvider` espone `DdlGenerator Ddl` invece di `ISchemaBuilder SchemaBuilder`: + le migrazioni scrivono su un `ISchemaBuilder` neutro che **registra** operazioni (`SchemaOperation`), e il + generatore del provider le traduce in istruzioni. Così la stessa registrazione serve per eseguire, calcolare il + checksum e generare `docs/DATA-MODEL.md`. Le classi dei provider mantengono i nomi di SPEC + (`SqliteSchemaBuilder`, `PostgresSchemaBuilder`) ma sono generatori DDL. +3. **Frammenti, non stringhe.** `ISqlDialect.CaseInsensitiveLike` e `AdvisoryLock` restituiscono `SqlFragment` + (testo + parametri), non `string`, perché il pattern e la chiave del lock sono valori e devono restare parametri. +4. **Membri aggiunti a `IDbProvider`:** `BuildConnectionString(DatabaseUrl)` (configurazione da URL `postgres://` o + `sqlite://`, mai stringhe ADO.NET nelle variabili), `DatabaseExistsAsync`, `IsTransient(Exception)` per i + tentativi automatici, `EnterWriteScopeAsync` (il `SqliteWriteLock` su SQLite, nulla su PostgreSQL). +5. **Registro aziende cifrato per intero.** `sys_companies.connection_string` contiene l'URL del database cifrato + con `SecretProtector` e scopo `company::database`: una riga copiata su un'altra azienda non si decifra. + La rotazione della password PostgreSQL resta un punto aperto (`docs/STATUS.md`, punto 9). +6. **Esito delle migrazioni al chiamante.** `MigrationRunner.RunAsync` restituisce un `MigrationReport` (applicate con + durata, già presenti, avvisi); la scrittura nella sezione di log *Migrazioni* arriva con il log in M2. Ogni + migrazione gira in una propria transazione sotto lock, con un nuovo controllo di `schema_version` dopo il lock. +7. **Enumerazioni per nome.** Le proprietà enum sono salvate come testo (`String(40)`), non come intero. +8. **Strumento di sviluppo.** `tools/Khaemhat.DataModelTool` (dipende solo da `Khaemhat.Data`, non distribuito) + rigenera la sezione generata di `docs/DATA-MODEL.md`; la regola di dipendenza è nel test architetturale. + +## Conseguenze + +- Il codice sopra lo strato dati usa `DatabaseScope`, `IUnitOfWorkFactory` e `SqlFragment`; nessun riferimento a + Npgsql o Microsoft.Data.Sqlite fuori dai provider (verificato da un test). +- Chi scrive una migrazione usa `scripts/new-migration.sh` e poi `scripts/gen-datamodel.sh`; un test fallisce se il + documento non è aggiornato. +- Le firme di SPEC §5.1 restano il riferimento per i nomi; dove divergono vale questo ADR. diff --git a/docs/wiki/Changelog.md b/docs/wiki/Changelog.md index 409a1de..19b1938 100644 --- a/docs/wiki/Changelog.md +++ b/docs/wiki/Changelog.md @@ -1,6 +1,6 @@ # Changelog -> **Stato:** completa per 0.0.1 · **Ultima revisione:** 2026-11-11 +> **Stato:** completa per 0.1.0 · **Ultima revisione:** 2026-11-11 Le novità di ogni versione sono nel file [`CHANGELOG.md`](https://gitea.encke-hake.ts.net/BalboSoft/Khaemhat/src/branch/main/CHANGELOG.md) del repository @@ -11,6 +11,7 @@ Le novità di ogni versione sono nel file | Versione | Data | Contenuto | |---|---|---| +| 0.1.0 | 2026-11-11 | M1 — Strato dati: accesso ai database SQLite e PostgreSQL, migrazioni dello schema, registro delle aziende, cifratura dei dati riservati. Nessuna funzione visibile nell'interfaccia. | | 0.0.1 | 2026-11-11 | M0 — Bootstrap: struttura del codice, immagine Docker con pagina "in costruzione" ed endpoint di salute, compose, CI e rilascio su Gitea Actions. Nessuna funzione gestionale ancora. | | *preparazione* | 2026-11-10 | Documentazione, specifica, mockup, wiki e pianificazione su Gitea. Nessun software. | diff --git a/docs/wiki/Modello-dati.md b/docs/wiki/Modello-dati.md index d3a704c..8809548 100644 --- a/docs/wiki/Modello-dati.md +++ b/docs/wiki/Modello-dati.md @@ -1,12 +1,13 @@ # Modello dati -> **Stato:** bozza · **Ultima revisione:** 2026-11-10 · **Versione software:** non ancora rilasciata (milestone M1 e seguenti) +> **Stato:** bozza · **Ultima revisione:** 2026-11-11 · **Versione software:** 0.1.0 (strato dati, M1); le tabelle funzionali arrivano con le milestone successive > **Specifica:** SPEC §5, §11, §13, §14 Come Khaemhat organizza i dati, in sintesi. L'elenco completo di tabelle e colonne è in -[docs/DATA-MODEL.md](https://gitea.encke-hake.ts.net/BalboSoft/Khaemhat/src/branch/main/docs/DATA-MODEL.md): oggi è -la versione **pianificata**; dalla milestone M1 sarà generato automaticamente dal codice, quindi sempre allineato al -programma. +[docs/DATA-MODEL.md](https://gitea.encke-hake.ts.net/BalboSoft/Khaemhat/src/branch/main/docs/DATA-MODEL.md). La sua +sezione *Schema attuale* è **generata automaticamente** dalle migrazioni del programma, quindi è sempre allineata a +ciò che il programma crea davvero nei database; le altre sezioni descrivono le tabelle **pianificate**, che compaiono +nello schema attuale man mano che le milestone le realizzano. ## Due tipi di database @@ -62,8 +63,11 @@ settimana). Quando una nuova versione cambia la struttura dei dati, Khaemhat applica da solo le **migrazioni**: il database di sistema all'avvio, le aziende all'avvio o dal pannello **Amministrazione → Istanza e database**, mettendo l'azienda in -manutenzione per il tempo necessario. Non è possibile tornare a una versione precedente dello schema: per questo prima -di ogni aggiornamento conviene un backup ([Aggiornamento](Aggiornamento), [Backup-e-ripristino](Backup-e-ripristino)). +manutenzione per il tempo necessario. Ogni migrazione è applicata +per intero o per niente, e più copie del programma avviate insieme non la applicano due volte. Non è possibile +tornare a una versione precedente dello schema: se il database contiene una migrazione che il programma non conosce +(per esempio dopo aver rimesso un'immagine più vecchia) Khaemhat si rifiuta di partire invece di rischiare i dati. +Per questo prima di ogni aggiornamento conviene un backup ([Aggiornamento](Aggiornamento), [Backup-e-ripristino](Backup-e-ripristino)). ## Pagine correlate diff --git a/scripts/gen-datamodel.sh b/scripts/gen-datamodel.sh index 870b608..719c4a9 100755 --- a/scripts/gen-datamodel.sh +++ b/scripts/gen-datamodel.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash -# Regenerates docs/DATA-MODEL.md from the entity maps (SPEC §18). -# scripts/gen-datamodel.sh -# Stub: completed in milestone M1 (SPEC §20). Until then it explains what it will do and exits with an error. +# Regenerates the generated section of docs/DATA-MODEL.md from the migrations (SPEC §18). +# scripts/gen-datamodel.sh rewrite the section +# scripts/gen-datamodel.sh --check only verify it is up to date (exit 1 if not) set -euo pipefail -echo "gen-datamodel.sh: non ancora disponibile, arriva con la milestone M1." >&2 -echo "Uso previsto: scripts/gen-datamodel.sh" >&2 -exit 1 +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$root" + +dotnet run --project tools/Khaemhat.DataModelTool -c Release -- "$@" docs/DATA-MODEL.md diff --git a/scripts/new-migration.sh b/scripts/new-migration.sh index c841cf2..daa9eca 100755 --- a/scripts/new-migration.sh +++ b/scripts/new-migration.sh @@ -1,9 +1,88 @@ #!/usr/bin/env bash # Creates a new numbered migration for the system or company schema (SPEC §5.3). -# scripts/new-migration.sh -# Stub: completed in milestone M1 (SPEC §20). Until then it explains what it will do and exits with an error. +# scripts/new-migration.sh [cartella] [modulo] +# Example: scripts/new-migration.sh system create_sys_users +# The version is today's date plus the next free number of the day across the repository (yyyymmdd_nnn); the +# class is M__ in the given folder (default: the system or company migrations folder). set -euo pipefail -echo "new-migration.sh: non ancora disponibile, arriva con la milestone M1." >&2 -echo "Uso previsto: scripts/new-migration.sh " >&2 -exit 1 +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$root" + +usage() { + echo "Uso: scripts/new-migration.sh [cartella] [modulo]" >&2 + exit 2 +} + +[ $# -ge 2 ] || usage +target="$1" +name="$2" +module="${4:-core}" +case "$target" in + system) dir="${3:-src/Khaemhat.Data/SystemDatabase/Migrations}"; attribute_target="MigrationTarget.System" ;; + company) dir="${3:-src/Khaemhat.Application/Migrations}"; attribute_target="MigrationTarget.Company" ;; + *) usage ;; +esac + +if ! [[ "$name" =~ ^[a-z][a-z0-9_]{0,62}$ ]]; then + echo "Il nome deve essere snake_case (lettere minuscole, cifre, _): $name" >&2 + exit 2 +fi + +if ! [[ "$module" =~ ^[a-z][a-z0-9_]{0,39}$ ]]; then + echo "Il modulo deve essere snake_case: $module" >&2 + exit 2 +fi + +if [[ "$dir" != src/* ]]; then + echo "La cartella deve stare sotto src/: $dir" >&2 + exit 2 +fi + +day="$(date +%Y%m%d)" +last="$(grep -rhoE "\[Migration\(${day}_[0-9]{3}" src --include='*.cs' 2>/dev/null | sed -E 's/.*_([0-9]{3})$/\1/' | sort -n | tail -1 || true)" +next=$(printf '%03d' $((10#${last:-0} + 1))) +pascal="$(echo "$name" | awk -F_ '{ for (i = 1; i <= NF; i++) printf "%s%s", toupper(substr($i, 1, 1)), substr($i, 2) }')" +class="M${day}_${next}_${pascal}" +namespace="$(echo "${dir#src/}" | tr '/' '.')" +file="$dir/$class.cs" + +mkdir -p "$dir" +if [ -e "$file" ]; then + echo "Esiste già: $file" >&2 + exit 1 +fi + +if [ "$attribute_target" = "MigrationTarget.Company" ]; then + target_argument="" +else + target_argument=", Target = $attribute_target" +fi + +if [ "$module" = "core" ]; then + module_argument="" +else + module_argument=", Module = \"$module\"" +fi + +cat > "$file" <TODO: what this migration changes and why (SPEC section). +[Migration(${day}_${next}, "$name"$target_argument$module_argument)] +public sealed class $class : IMigration +{ + /// + public void Up(ISchemaBuilder schema) + { + ArgumentNullException.ThrowIfNull(schema); + // schema.CreateTable("table_name", t => { t.Id(); t.AuditColumns(); }); + } +} +CS + +echo "Creata $file (versione ${day}_${next}, modulo $module)." +echo "Dopo averla scritta: aggiornare la mappa dell'entità, eseguire scripts/gen-datamodel.sh e i test." diff --git a/src/Khaemhat.Core/Common/ConcurrencyException.cs b/src/Khaemhat.Core/Common/ConcurrencyException.cs new file mode 100644 index 0000000..608a9c5 --- /dev/null +++ b/src/Khaemhat.Core/Common/ConcurrencyException.cs @@ -0,0 +1,51 @@ +namespace Khaemhat.Core.Common; + +/// +/// Thrown when a row changed (or was deleted) after it was read: the update found no row with the expected +/// row_version (SPEC §5.2). The interface reloads and asks the user; the API answers 409. +/// +public sealed class ConcurrencyException : Exception +{ + /// Creates the exception for a row of . + /// The table name. + /// The primary key of the row, as text. + /// The row_version the caller read. + public ConcurrencyException(string table, string key, long expectedVersion) + : base($"The row {key} of {table} was changed or deleted by someone else (expected version {expectedVersion}).") + { + Table = table; + Key = key; + ExpectedVersion = expectedVersion; + } + + /// Creates the exception with a message only. + /// The message. + public ConcurrencyException(string message) : base(message) + { + Table = string.Empty; + Key = string.Empty; + } + + /// Creates the exception with a message and an inner exception. + /// The message. + /// The cause. + public ConcurrencyException(string message, Exception innerException) : base(message, innerException) + { + Table = string.Empty; + Key = string.Empty; + } + + /// Creates the exception without details. + public ConcurrencyException() : this("The row was changed or deleted by someone else.") + { + } + + /// The table of the row. + public string Table { get; } + + /// The primary key of the row, as text. + public string Key { get; } + + /// The row_version the caller read. + public long ExpectedVersion { get; } +} diff --git a/src/Khaemhat.Core/Entities/Entity.cs b/src/Khaemhat.Core/Entities/Entity.cs new file mode 100644 index 0000000..ea29134 --- /dev/null +++ b/src/Khaemhat.Core/Entities/Entity.cs @@ -0,0 +1,27 @@ +namespace Khaemhat.Core.Entities; + +/// +/// Convenience base class for entities with an identity key and the audit columns. Entities may also implement +/// and directly. +/// +/// The key type. +public abstract class Entity : IEntity, IAuditable +{ + /// + public TKey Id { get; set; } = default!; + + /// + public DateTime CreatedAt { get; set; } + + /// + public string? CreatedBy { get; set; } + + /// + public DateTime? UpdatedAt { get; set; } + + /// + public string? UpdatedBy { get; set; } + + /// + public long RowVersion { get; set; } +} diff --git a/src/Khaemhat.Core/Entities/IAuditable.cs b/src/Khaemhat.Core/Entities/IAuditable.cs new file mode 100644 index 0000000..ca90418 --- /dev/null +++ b/src/Khaemhat.Core/Entities/IAuditable.cs @@ -0,0 +1,27 @@ +namespace Khaemhat.Core.Entities; + +/// +/// An entity with the audit columns of SPEC §5.2 (created_at, created_by, updated_at, +/// updated_by) and the row_version used for optimistic concurrency. The data layer fills them; code above +/// it only reads them. +/// +public interface IAuditable +{ + /// When the row was inserted, in UTC. + DateTime CreatedAt { get; set; } + + /// Who inserted the row (user name, or a system actor such as system), when known. + string? CreatedBy { get; set; } + + /// When the row was last updated, in UTC; if it never was. + DateTime? UpdatedAt { get; set; } + + /// Who last updated the row, when known. + string? UpdatedBy { get; set; } + + /// + /// Incremented by every update. An update succeeds only if the stored value still equals this one; otherwise the + /// data layer throws . + /// + long RowVersion { get; set; } +} diff --git a/src/Khaemhat.Core/Entities/IDemoMarkable.cs b/src/Khaemhat.Core/Entities/IDemoMarkable.cs new file mode 100644 index 0000000..8d485ac --- /dev/null +++ b/src/Khaemhat.Core/Entities/IDemoMarkable.cs @@ -0,0 +1,8 @@ +namespace Khaemhat.Core.Entities; + +/// An entity whose rows can be created by the demo seeder and removed with it (SPEC §14, column is_demo). +public interface IDemoMarkable +{ + /// for rows created by the demo data seeder. + bool IsDemo { get; set; } +} diff --git a/src/Khaemhat.Core/Entities/IEntity.cs b/src/Khaemhat.Core/Entities/IEntity.cs new file mode 100644 index 0000000..f4c14f0 --- /dev/null +++ b/src/Khaemhat.Core/Entities/IEntity.cs @@ -0,0 +1,9 @@ +namespace Khaemhat.Core.Entities; + +/// An entity with an identity of type , stored in the id column (SPEC §5.2). +/// The key type: for identity keys, which is the default for every table. +public interface IEntity +{ + /// The primary key. For identity keys it is the default value until the row is inserted. + TKey Id { get; set; } +} diff --git a/src/Khaemhat.Core/Instance/Company.cs b/src/Khaemhat.Core/Instance/Company.cs new file mode 100644 index 0000000..85a8287 --- /dev/null +++ b/src/Khaemhat.Core/Instance/Company.cs @@ -0,0 +1,31 @@ +using Khaemhat.Core.Entities; + +namespace Khaemhat.Core.Instance; + +/// +/// A company in the instance registry (sys_companies, SPEC §5.4): its code, name and the location of its own +/// database. The location is stored encrypted and is never shown or logged. +/// +public sealed class Company : Entity +{ + /// Company code: upper-case letters, digits and underscore, 1 to 20 characters (database khaemhat_code). + public string Code { get; set; } = string.Empty; + + /// Company name (ragione sociale). + public string Name { get; set; } = string.Empty; + + /// The database engine name: Sqlite or PostgreSql. + public string Engine { get; set; } = string.Empty; + + /// The database URL, encrypted with the instance key. + public string ConnectionString { get; set; } = string.Empty; + + /// The company state. + public CompanyStatus Status { get; set; } + + /// Whether this is the company opened by default. + public bool IsDefault { get; set; } + + /// Free notes for administrators. + public string? Notes { get; set; } +} diff --git a/src/Khaemhat.Core/Instance/CompanyStatus.cs b/src/Khaemhat.Core/Instance/CompanyStatus.cs new file mode 100644 index 0000000..731d207 --- /dev/null +++ b/src/Khaemhat.Core/Instance/CompanyStatus.cs @@ -0,0 +1,17 @@ +namespace Khaemhat.Core.Instance; + +/// State of a company database in the registry (SPEC §5.4, §12.1). +public enum CompanyStatus +{ + /// In use. + Active = 0, + + /// Maintenance in progress (migration, restore): nobody but administrators can open it. + Maintenance = 1, + + /// Readable only, for example an archive. + ReadOnly = 2, + + /// Kept but not available. + Disabled = 3, +} diff --git a/src/Khaemhat.Core/Instance/SystemSetting.cs b/src/Khaemhat.Core/Instance/SystemSetting.cs new file mode 100644 index 0000000..eb36b72 --- /dev/null +++ b/src/Khaemhat.Core/Instance/SystemSetting.cs @@ -0,0 +1,16 @@ +using Khaemhat.Core.Entities; + +namespace Khaemhat.Core.Instance; + +/// An instance setting (sys_settings). Secret values are stored encrypted (). +public sealed class SystemSetting : Entity +{ + /// The setting key, for example smtp.host. + public string Key { get; set; } = string.Empty; + + /// The value; encrypted when is set. + public string? Value { get; set; } + + /// Whether is a secret encrypted with the instance key. + public bool IsSecret { get; set; } +} diff --git a/src/Khaemhat.Core/Security/ISecretProtector.cs b/src/Khaemhat.Core/Security/ISecretProtector.cs new file mode 100644 index 0000000..c8fb346 --- /dev/null +++ b/src/Khaemhat.Core/Security/ISecretProtector.cs @@ -0,0 +1,22 @@ +namespace Khaemhat.Core.Security; + +/// +/// Encrypts secrets stored in the databases (company connection strings, SMTP/PEC/S3/WebDAV passwords, tokens) with +/// the instance key KHAEMHAT_SECRET_KEY (SPEC §5.4). The purpose binds a ciphertext to its use: a value +/// protected for one purpose cannot be read back for another. +/// +public interface ISecretProtector +{ + /// Encrypts ; the result is printable text safe to store in a column. + /// The secret. + /// What the secret is for, for example company:DEMO:database. + /// The protected value. + string Protect(string plaintext, string purpose); + + /// Decrypts a value produced by with the same key and purpose. + /// The protected value. + /// The purpose given to . + /// The secret. + /// The value was protected with another key or purpose, or is damaged. + string Unprotect(string protectedValue, string purpose); +} diff --git a/src/Khaemhat.Core/Security/SecretProtectionException.cs b/src/Khaemhat.Core/Security/SecretProtectionException.cs new file mode 100644 index 0000000..7ea3c01 --- /dev/null +++ b/src/Khaemhat.Core/Security/SecretProtectionException.cs @@ -0,0 +1,26 @@ +namespace Khaemhat.Core.Security; + +/// +/// A secret could not be decrypted: wrong instance key, wrong purpose or damaged value. The message never contains +/// the secret or the ciphertext. +/// +public sealed class SecretProtectionException : Exception +{ + /// Creates the exception. + public SecretProtectionException() : base("The secret cannot be decrypted.") + { + } + + /// Creates the exception with a message. + /// The message (never the secret). + public SecretProtectionException(string message) : base(message) + { + } + + /// Creates the exception with a message and a cause. + /// The message (never the secret). + /// The cause. + public SecretProtectionException(string message, Exception innerException) : base(message, innerException) + { + } +} diff --git a/src/Khaemhat.Data.PostgreSql/PostgresAdvisoryLocks.cs b/src/Khaemhat.Data.PostgreSql/PostgresAdvisoryLocks.cs new file mode 100644 index 0000000..7a4cf8d --- /dev/null +++ b/src/Khaemhat.Data.PostgreSql/PostgresAdvisoryLocks.cs @@ -0,0 +1,26 @@ +namespace Khaemhat.Data.PostgreSql; + +/// +/// Application locks on PostgreSQL (SPEC §5.2) for numerators, closings and the scheduler. The text key is hashed to +/// 64 bits by the server (hashtextextended); transaction locks are released by commit or rollback. +/// +public static class PostgresAdvisoryLocks +{ + /// Waits for the lock and holds it until the end of the transaction. + /// The lock key. + /// The statement. + public static SqlFragment TransactionLock(string key) + { + ArgumentException.ThrowIfNullOrEmpty(key); + return SqlFragment.Of($"SELECT pg_advisory_xact_lock(hashtextextended({key}, 0))"); + } + + /// Takes the lock if free, without waiting; the statement returns a boolean. + /// The lock key. + /// The statement. + public static SqlFragment TryTransactionLock(string key) + { + ArgumentException.ThrowIfNullOrEmpty(key); + return SqlFragment.Of($"SELECT pg_try_advisory_xact_lock(hashtextextended({key}, 0))"); + } +} diff --git a/src/Khaemhat.Data.PostgreSql/PostgresDatabaseCreator.cs b/src/Khaemhat.Data.PostgreSql/PostgresDatabaseCreator.cs new file mode 100644 index 0000000..f1782f1 --- /dev/null +++ b/src/Khaemhat.Data.PostgreSql/PostgresDatabaseCreator.cs @@ -0,0 +1,92 @@ +using System.Diagnostics.CodeAnalysis; +using Npgsql; + +namespace Khaemhat.Data.PostgreSql; + +/// +/// Creates and drops company databases on the server of KHAEMHAT_DB_SERVER (SPEC §5.4). It connects to the +/// maintenance database postgres with the same credentials. Database names are validated identifiers: they +/// cannot be parameters in DDL, so nothing else may reach the statement. +/// +public static class PostgresDatabaseCreator +{ + private const string MaintenanceDatabase = "postgres"; + + /// Creates the database of if it does not exist (UTF-8, from template0). + /// The connection string of the database to create. + /// Cancellation. + /// A task. + public static async Task EnsureExistsAsync(string connectionString, CancellationToken cancellationToken = default) + { + var name = DatabaseName(connectionString); + if (await ExistsAsync(connectionString, cancellationToken).ConfigureAwait(false)) + { + return; + } + + try + { + await ExecuteOnMaintenanceAsync( + connectionString, $"CREATE DATABASE \"{name}\" TEMPLATE template0 ENCODING 'UTF8'", cancellationToken).ConfigureAwait(false); + } + catch (PostgresException ex) when (ex.SqlState == PostgresErrorCodes.DuplicateDatabase) + { + // Created concurrently by another instance: the goal is reached. + } + } + + /// Drops the database, closing its connections (WITH (FORCE)). Used only by the registry and restores. + /// The connection string of the database to drop. + /// Cancellation. + /// A task. + public static async Task DropAsync(string connectionString, CancellationToken cancellationToken = default) + { + var name = DatabaseName(connectionString); + if (name == MaintenanceDatabase) + { + throw new InvalidOperationException("The maintenance database cannot be dropped."); + } + + using (var connection = new NpgsqlConnection(connectionString)) + { + NpgsqlConnection.ClearPool(connection); + } + + await ExecuteOnMaintenanceAsync(connectionString, $"DROP DATABASE IF EXISTS \"{name}\" WITH (FORCE)", cancellationToken).ConfigureAwait(false); + } + + /// Whether the database exists. + /// The connection string of the database. + /// Cancellation. + /// if it exists. + public static async Task ExistsAsync(string connectionString, CancellationToken cancellationToken = default) + { + var name = DatabaseName(connectionString); + await using var connection = new NpgsqlConnection(MaintenanceConnectionString(connectionString)); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + await using var command = new NpgsqlCommand("SELECT 1 FROM pg_database WHERE datname = @name", connection); + command.Parameters.AddWithValue("name", name); + return await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false) is not null; + } + + private static string DatabaseName(string connectionString) + { + var name = new NpgsqlConnectionStringBuilder(connectionString).Database ?? string.Empty; + return SqlNames.IsValidIdentifier(name) + ? name + : throw new ArgumentException("The database name must be lower-case letters, digits and underscore.", nameof(connectionString)); + } + + private static string MaintenanceConnectionString(string connectionString) => + new NpgsqlConnectionStringBuilder(connectionString) { Database = MaintenanceDatabase, Pooling = false }.ToString(); + + [SuppressMessage("Security", "CA2100:Review SQL queries for security vulnerabilities", + Justification = "The only variable part is a database name validated as a snake_case identifier.")] + private static async Task ExecuteOnMaintenanceAsync(string connectionString, string sql, CancellationToken cancellationToken) + { + await using var connection = new NpgsqlConnection(MaintenanceConnectionString(connectionString)); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + await using var command = new NpgsqlCommand(sql, connection); + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/Khaemhat.Data.PostgreSql/PostgresDialect.cs b/src/Khaemhat.Data.PostgreSql/PostgresDialect.cs new file mode 100644 index 0000000..e403c17 --- /dev/null +++ b/src/Khaemhat.Data.PostgreSql/PostgresDialect.cs @@ -0,0 +1,126 @@ +using System.Data.Common; +using System.Globalization; +using Khaemhat.Data.Schema; +using Npgsql; +using NpgsqlTypes; + +namespace Khaemhat.Data.PostgreSql; + +/// +/// The PostgreSQL dialect (SPEC §5.2): varchar(n)/text/integer/bigint/numeric(19,6)/boolean/date/timestamptz/uuid/ +/// bytea/jsonb, ILIKE for case-insensitive search and pg_advisory_xact_lock for application locks. +/// Parameters get an explicit driver type, so jsonb and timestamptz columns accept them without casts. +/// +public sealed class PostgresDialect : SqlDialectBase +{ + /// The shared instance. + public static PostgresDialect Instance { get; } = new(); + + /// + public override DbEngine Engine => DbEngine.PostgreSql; + + /// + public override string CurrentTimestampUtc => "now()"; + + /// + public override SqlFragment CaseInsensitiveLike(SqlFragment column, SqlFragment pattern) => + column + SqlFragment.Literal(" ILIKE ") + pattern + SqlFragment.Literal(" ESCAPE '\\'"); + + /// + public override SqlFragment AdvisoryLock(string key) => PostgresAdvisoryLocks.TransactionLock(key); + + /// + public override object ToDbValue(object? value, ColumnType type) + { + if (value is null or DBNull) + { + return DBNull.Value; + } + + return type switch + { + ColumnType.String or ColumnType.Text or ColumnType.Json => value as string ?? Convert.ToString(value, CultureInfo.InvariantCulture)!, + ColumnType.Int32 => value is Enum ? Convert.ToInt32(value, CultureInfo.InvariantCulture) : ConvertTo(value), + ColumnType.Int64 => value is Enum ? Convert.ToInt64(value, CultureInfo.InvariantCulture) : ConvertTo(value), + ColumnType.Decimal => ConvertTo(value), + ColumnType.Bool => ConvertTo(value), + ColumnType.Date => ToDate(value), + ColumnType.DateTimeUtc => ToUtc(value), + ColumnType.Guid => value is Guid g ? g : Guid.Parse((string)value, CultureInfo.InvariantCulture), + ColumnType.Bytes => (byte[])value, + _ => throw new ArgumentOutOfRangeException(nameof(type), type, null), + }; + } + + /// + public override object? FromDbValue(object? value, ColumnType type) + { + if (value is null or DBNull) + { + return null; + } + + return type switch + { + ColumnType.String or ColumnType.Text or ColumnType.Json => value as string ?? Convert.ToString(value, CultureInfo.InvariantCulture), + ColumnType.Int32 => ConvertTo(value), + ColumnType.Int64 => ConvertTo(value), + ColumnType.Decimal => ConvertTo(value), + ColumnType.Bool => ConvertTo(value), + ColumnType.Date => ToDate(value), + ColumnType.DateTimeUtc => ToUtc(value), + ColumnType.Guid => value is Guid g ? g : Guid.Parse(Convert.ToString(value, CultureInfo.InvariantCulture)!, CultureInfo.InvariantCulture), + ColumnType.Bytes => (byte[])value, + _ => throw new ArgumentOutOfRangeException(nameof(type), type, null), + }; + } + + /// + public override string SqlType(ColumnType type, int? length, int? scale) => type switch + { + ColumnType.String => length is null ? "text" : string.Create(CultureInfo.InvariantCulture, $"varchar({length})"), + ColumnType.Text => "text", + ColumnType.Int32 => "integer", + ColumnType.Int64 => "bigint", + ColumnType.Decimal => string.Create(CultureInfo.InvariantCulture, $"numeric({ColumnTypes.DecimalPrecision},{scale ?? ColumnTypes.DecimalScale})"), + ColumnType.Bool => "boolean", + ColumnType.Date => "date", + ColumnType.DateTimeUtc => "timestamptz", + ColumnType.Guid => "uuid", + ColumnType.Bytes => "bytea", + ColumnType.Json => "jsonb", + _ => throw new ArgumentOutOfRangeException(nameof(type), type, null), + }; + + /// + public override string Literal(object? value, ColumnType type) => value switch + { + bool b => b ? "TRUE" : "FALSE", + decimal d => d.ToString(CultureInfo.InvariantCulture), + _ => base.Literal(value, type), + }; + + /// + public override void ConfigureParameter(DbParameter parameter, object? value, ColumnType? type) + { + base.ConfigureParameter(parameter, value, type); + var logical = type ?? (value is null ? null : ColumnTypes.FromClrType(value.GetType())); + if (parameter is NpgsqlParameter npgsql && logical is not null) + { + npgsql.NpgsqlDbType = logical.Value switch + { + ColumnType.String or ColumnType.Text => NpgsqlDbType.Text, + ColumnType.Int32 => NpgsqlDbType.Integer, + ColumnType.Int64 => NpgsqlDbType.Bigint, + ColumnType.Decimal => NpgsqlDbType.Numeric, + ColumnType.Bool => NpgsqlDbType.Boolean, + ColumnType.Date => NpgsqlDbType.Date, + ColumnType.DateTimeUtc => NpgsqlDbType.TimestampTz, + ColumnType.Guid => NpgsqlDbType.Uuid, + ColumnType.Bytes => NpgsqlDbType.Bytea, + ColumnType.Json => NpgsqlDbType.Jsonb, + _ => npgsql.NpgsqlDbType, + }; + } + } +} diff --git a/src/Khaemhat.Data.PostgreSql/PostgresPartitioning.cs b/src/Khaemhat.Data.PostgreSql/PostgresPartitioning.cs new file mode 100644 index 0000000..7020756 --- /dev/null +++ b/src/Khaemhat.Data.PostgreSql/PostgresPartitioning.cs @@ -0,0 +1,46 @@ +using System.Globalization; + +namespace Khaemhat.Data.PostgreSql; + +/// +/// Yearly range partitioning for the large tables (SPEC §5.2: gl_entry_lines, inventory_entry_lines, +/// audit_log, sys_log). Prepared, not active: tables record their partition column with +/// TableBuilder.PartitionByYear; when one exceeds 50 million rows a migration (through +/// ISchemaBuilder.Sql) recreates it partitioned using these statements and moves the rows. +/// +public static class PostgresPartitioning +{ + /// The clause that makes a CREATE TABLE partitioned by range of . + /// The date or timestamp column. + /// The clause, starting with a space. + public static string PartitionClause(string column) => " PARTITION BY RANGE (" + PostgresDialect.Instance.Quote(column) + ")"; + + /// The name of the partition of a year, for example gl_entry_lines_2026. + /// The partitioned table. + /// The year. + /// The partition name. + public static string PartitionName(string table, int year) => table + "_" + year.ToString(CultureInfo.InvariantCulture); + + /// The statement creating the partition of one year if it does not exist. + /// The partitioned table. + /// The year. + /// The statement. + public static string CreateYearPartition(string table, int year) + { + ArgumentOutOfRangeException.ThrowIfLessThan(year, 1900); + ArgumentOutOfRangeException.ThrowIfGreaterThan(year, 9998); + var dialect = PostgresDialect.Instance; + return string.Create( + CultureInfo.InvariantCulture, + $"CREATE TABLE IF NOT EXISTS {dialect.Quote(PartitionName(table, year))} PARTITION OF {dialect.Quote(table)} FOR VALUES FROM ('{year:D4}-01-01') TO ('{year + 1:D4}-01-01')"); + } + + /// The statement creating the default partition, for rows outside the yearly ones. + /// The partitioned table. + /// The statement. + public static string CreateDefaultPartition(string table) + { + var dialect = PostgresDialect.Instance; + return $"CREATE TABLE IF NOT EXISTS {dialect.Quote(table + "_default")} PARTITION OF {dialect.Quote(table)} DEFAULT"; + } +} diff --git a/src/Khaemhat.Data.PostgreSql/PostgresProvider.cs b/src/Khaemhat.Data.PostgreSql/PostgresProvider.cs new file mode 100644 index 0000000..9f62c2d --- /dev/null +++ b/src/Khaemhat.Data.PostgreSql/PostgresProvider.cs @@ -0,0 +1,116 @@ +using System.Data.Common; +using System.Globalization; +using Khaemhat.Data.Schema; +using Npgsql; + +namespace Khaemhat.Data.PostgreSql; + +/// Settings of the PostgreSQL provider (SPEC §5.2), from the instance settings. +public sealed record PostgresProviderOptions +{ + /// Maximum pooled connections per database (each database has its own pool). + public int MaxPoolSize { get; init; } = 20; + + /// Server-side limit of a statement (statement_timeout of every session). + public TimeSpan StatementTimeout { get; init; } = TimeSpan.FromSeconds(60); + + /// The application name shown in pg_stat_activity. + public string ApplicationName { get; init; } = "khaemhat"; +} + +/// +/// The PostgreSQL engine (SPEC §1.3, §5.2), version 16 or later: one connection pool per database, a statement timeout +/// per session, databases created with CREATE DATABASE on the server (the role needs CREATEDB). +/// +/// The provider settings. +public sealed class PostgresProvider(PostgresProviderOptions options) : IDbProvider +{ + /// The oldest supported major version. + public const int MinimumMajorVersion = 16; + + /// Creates the provider with default settings. + public PostgresProvider() + : this(new PostgresProviderOptions()) + { + } + + /// + public DbEngine Engine => DbEngine.PostgreSql; + + /// + public ISqlDialect Dialect => PostgresDialect.Instance; + + /// + public DdlGenerator Ddl { get; } = new PostgresSchemaBuilder(); + + /// The settings. + public PostgresProviderOptions Options { get; } = options ?? throw new ArgumentNullException(nameof(options)); + + /// + public string BuildConnectionString(DatabaseUrl url) + { + ArgumentNullException.ThrowIfNull(url); + if (url.Engine != DbEngine.PostgreSql) + { + throw new ArgumentException("Not a PostgreSQL URL.", nameof(url)); + } + + var builder = new NpgsqlConnectionStringBuilder + { + Host = url.Host, + Port = url.Port ?? 5432, + Username = url.UserName, + Password = url.Password, + Database = url.Database.Length == 0 ? "postgres" : url.Database, + MaxPoolSize = Options.MaxPoolSize, + ApplicationName = Options.ApplicationName, + Options = string.Create(CultureInfo.InvariantCulture, $"-c statement_timeout={(long)Options.StatementTimeout.TotalMilliseconds}"), + }; + foreach (var (key, value) in url.Options) + { + try + { + builder[key] = value; + } + catch (ArgumentException) + { + throw new FormatException($"The PostgreSQL URL option '{key}' is not supported."); + } + } + + return builder.ToString(); + } + + /// + public DbConnection CreateConnection(string connectionString) => new NpgsqlConnection(connectionString); + + /// + public Task EnsureDatabaseExistsAsync(string connectionString, CancellationToken cancellationToken = default) => + PostgresDatabaseCreator.EnsureExistsAsync(connectionString, cancellationToken); + + /// + public Task DropDatabaseAsync(string connectionString, CancellationToken cancellationToken = default) => + PostgresDatabaseCreator.DropAsync(connectionString, cancellationToken); + + /// + public Task DatabaseExistsAsync(string connectionString, CancellationToken cancellationToken = default) => + PostgresDatabaseCreator.ExistsAsync(connectionString, cancellationToken); + + /// + public Task GetServerInfoAsync(string connectionString, CancellationToken cancellationToken = default) => + PostgresServerInfo.ReadAsync(connectionString, cancellationToken); + + /// + public bool IsTransient(Exception exception) => exception is NpgsqlException { IsTransient: true }; + + /// + public ValueTask EnterWriteScopeAsync(string connectionString, CancellationToken cancellationToken = default) => + ValueTask.FromResult(NoScope.Instance); + + private sealed class NoScope : IAsyncDisposable + { + public static NoScope Instance { get; } = new(); + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } +} diff --git a/src/Khaemhat.Data.PostgreSql/PostgresSchemaBuilder.cs b/src/Khaemhat.Data.PostgreSql/PostgresSchemaBuilder.cs new file mode 100644 index 0000000..380864e --- /dev/null +++ b/src/Khaemhat.Data.PostgreSql/PostgresSchemaBuilder.cs @@ -0,0 +1,18 @@ +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.PostgreSql; + +/// +/// DDL for PostgreSQL (SPEC §21 PostgresSchemaBuilder): identity keys are +/// bigint GENERATED BY DEFAULT AS IDENTITY, so restores and imports may still write explicit ids. +/// +public sealed class PostgresSchemaBuilder() : DdlGenerator(PostgresDialect.Instance) +{ + /// + protected override string IdentityPrimaryKey(ColumnDefinition column) + { + ArgumentNullException.ThrowIfNull(column); + var type = column.Type == ColumnType.Int32 ? "integer" : "bigint"; + return Dialect.Quote(column.Name) + " " + type + " GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY"; + } +} diff --git a/src/Khaemhat.Data.PostgreSql/PostgresServerInfo.cs b/src/Khaemhat.Data.PostgreSql/PostgresServerInfo.cs new file mode 100644 index 0000000..cd1ab27 --- /dev/null +++ b/src/Khaemhat.Data.PostgreSql/PostgresServerInfo.cs @@ -0,0 +1,59 @@ +using System.Globalization; +using Npgsql; + +namespace Khaemhat.Data.PostgreSql; + +/// Reads version, role, connections, size and extensions of a PostgreSQL database (SPEC §12.1). +public static class PostgresServerInfo +{ + private const string Query = """ + SELECT current_setting('server_version'), + current_setting('server_version_num')::int, + current_database(), + pg_database_size(current_database()), + current_user, + (SELECT rolcreatedb FROM pg_roles WHERE rolname = current_user), + (SELECT numbackends FROM pg_stat_database WHERE datname = current_database()), + current_setting('max_connections')::int, + (SELECT coalesce(string_agg(extname || ' ' || extversion, ',' ORDER BY extname), '') FROM pg_extension) + """; + + /// Reads the information. + /// The connection string of the database. + /// Cancellation. + /// The information, with a warning below version 16 and when the role cannot create databases. + public static async Task ReadAsync(string connectionString, CancellationToken cancellationToken = default) + { + await using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + await using var command = new NpgsqlCommand(Query, connection); + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + await reader.ReadAsync(cancellationToken).ConfigureAwait(false); + + var versionNumber = reader.GetInt32(1); + var canCreate = !reader.IsDBNull(5) && reader.GetBoolean(5); + var warnings = new List(); + if (versionNumber / 10000 < PostgresProvider.MinimumMajorVersion) + { + warnings.Add(string.Create( + CultureInfo.InvariantCulture, + $"PostgreSQL {reader.GetString(0)} is older than {PostgresProvider.MinimumMajorVersion}, the oldest supported version; upgrade it (17 is recommended).")); + } + + if (!canCreate) + { + warnings.Add($"The role {reader.GetString(4)} has no CREATEDB: new company databases must be created by an administrator."); + } + + var extensions = reader.GetString(8); + return new DbServerInfo(DbEngine.PostgreSql, reader.GetString(0), versionNumber, reader.GetString(2), reader.GetInt64(3)) + { + UserName = reader.GetString(4), + CanCreateDatabases = canCreate, + ActiveConnections = reader.IsDBNull(6) ? null : reader.GetInt32(6), + MaxConnections = reader.GetInt32(7), + Details = extensions.Length == 0 ? [] : extensions.Split(','), + Warnings = warnings, + }; + } +} diff --git a/src/Khaemhat.Data.Sqlite/SqliteDecimalConverter.cs b/src/Khaemhat.Data.Sqlite/SqliteDecimalConverter.cs new file mode 100644 index 0000000..f13bd5c --- /dev/null +++ b/src/Khaemhat.Data.Sqlite/SqliteDecimalConverter.cs @@ -0,0 +1,39 @@ +namespace Khaemhat.Data.Sqlite; + +/// +/// SQLite has no exact decimal type: Decimal(19,6) values are stored as integers multiplied by 10⁶ (SPEC §5.2). +/// Sums in SQL therefore stay exact integers and are scaled back when read. Values are rounded to 6 decimals half +/// away from zero, like PostgreSQL numeric(19,6). The usable range is ±9.2 x 10¹². +/// +public static class SqliteDecimalConverter +{ + /// The scale factor, 10⁶. + public const long Factor = 1_000_000; + + /// The largest absolute value that can be stored. + public static readonly decimal MaxValue = long.MaxValue / (decimal)Factor; + + /// Converts a decimal to its scaled integer. + /// The value. + /// The scaled integer. + /// The value is beyond ±9.2 x 10¹². + public static long ToScaled(decimal value) + { + var scaled = decimal.Round(value * Factor, 0, MidpointRounding.AwayFromZero); + return scaled is > long.MaxValue or < long.MinValue + ? throw new OverflowException($"The amount exceeds the SQLite decimal range of ±{MaxValue:N0}.") + : (long)scaled; + } + + /// Converts a scaled integer (or a scaled sum read as integer or real) back to a decimal. + /// The stored value. + /// The decimal. + public static decimal FromScaled(object value) => value switch + { + long l => (decimal)l / Factor, + int i => (decimal)i / Factor, + double d => (decimal)d / Factor, + decimal m => m / Factor, + _ => throw new InvalidCastException($"A {value.GetType().Name} is not a scaled SQLite decimal."), + }; +} diff --git a/src/Khaemhat.Data.Sqlite/SqliteDialect.cs b/src/Khaemhat.Data.Sqlite/SqliteDialect.cs new file mode 100644 index 0000000..69c8299 --- /dev/null +++ b/src/Khaemhat.Data.Sqlite/SqliteDialect.cs @@ -0,0 +1,99 @@ +using System.Globalization; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.Sqlite; + +/// +/// The SQLite dialect (SPEC §5.2): STRICT tables with TEXT, INTEGER and BLOB; decimals as +/// integers x10⁶, dates yyyy-MM-dd, timestamps ISO 8601 UTC with microseconds, booleans 0/1, GUIDs as text. +/// Case-insensitive search uses the kh_lower function registered on every connection, which lowers Unicode text +/// (SQLite's own LIKE and lower only fold ASCII). +/// +public sealed class SqliteDialect : SqlDialectBase +{ + /// The format of stored timestamps: fixed width, so text order is time order. + public const string TimestampFormat = "yyyy-MM-dd'T'HH:mm:ss.ffffff'Z'"; + + /// The name of the Unicode lower-case function registered on connections. + public const string LowerFunction = "kh_lower"; + + /// The shared instance. + public static SqliteDialect Instance { get; } = new(); + + /// + public override DbEngine Engine => DbEngine.Sqlite; + + /// + public override string CurrentTimestampUtc => "strftime('%Y-%m-%dT%H:%M:%f000Z', 'now')"; + + /// + public override SqlFragment CaseInsensitiveLike(SqlFragment column, SqlFragment pattern) => + SqlFragment.Literal(LowerFunction + "(") + column + SqlFragment.Literal(") LIKE " + LowerFunction + "(") + pattern + + SqlFragment.Literal(") ESCAPE '\\'"); + + /// + public override SqlFragment AdvisoryLock(string key) => SqlFragment.Empty; + + /// + public override object ToDbValue(object? value, ColumnType type) + { + if (value is null or DBNull) + { + return DBNull.Value; + } + + return type switch + { + ColumnType.String or ColumnType.Text or ColumnType.Json => value as string ?? Convert.ToString(value, CultureInfo.InvariantCulture)!, + ColumnType.Int32 or ColumnType.Int64 => value is Enum ? Convert.ToInt64(value, CultureInfo.InvariantCulture) : ConvertTo(value), + ColumnType.Decimal => SqliteDecimalConverter.ToScaled(ConvertTo(value)), + ColumnType.Bool => ConvertTo(value) ? 1L : 0L, + ColumnType.Date => ToDate(value).ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), + ColumnType.DateTimeUtc => ToUtc(value).ToString(TimestampFormat, CultureInfo.InvariantCulture), + ColumnType.Guid => (value is Guid g ? g : Guid.Parse((string)value, CultureInfo.InvariantCulture)).ToString("D"), + ColumnType.Bytes => (byte[])value, + _ => throw new ArgumentOutOfRangeException(nameof(type), type, null), + }; + } + + /// + public override object? FromDbValue(object? value, ColumnType type) + { + if (value is null or DBNull) + { + return null; + } + + return type switch + { + ColumnType.String or ColumnType.Text or ColumnType.Json => value as string ?? Convert.ToString(value, CultureInfo.InvariantCulture), + ColumnType.Int32 => ConvertTo(value), + ColumnType.Int64 => ConvertTo(value), + ColumnType.Decimal => SqliteDecimalConverter.FromScaled(value), + ColumnType.Bool => ConvertTo(value) != 0, + ColumnType.Date => ToDate(value), + ColumnType.DateTimeUtc => value is string s + ? DateTime.Parse(s, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal) + : ToUtc(value), + ColumnType.Guid => value is Guid g ? g : Guid.Parse((string)value, CultureInfo.InvariantCulture), + ColumnType.Bytes => (byte[])value, + _ => throw new ArgumentOutOfRangeException(nameof(type), type, null), + }; + } + + /// + public override string SqlType(ColumnType type, int? length, int? scale) => type switch + { + ColumnType.Int32 or ColumnType.Int64 or ColumnType.Decimal or ColumnType.Bool => "INTEGER", + ColumnType.Bytes => "BLOB", + _ => "TEXT", + }; + + /// + public override string Literal(object? value, ColumnType type) => value switch + { + bool b => b ? "1" : "0", + decimal d when type == ColumnType.Decimal => SqliteDecimalConverter.ToScaled(d).ToString(CultureInfo.InvariantCulture), + _ => base.Literal(value, type), + }; +} diff --git a/src/Khaemhat.Data.Sqlite/SqliteProvider.cs b/src/Khaemhat.Data.Sqlite/SqliteProvider.cs new file mode 100644 index 0000000..0632fd7 --- /dev/null +++ b/src/Khaemhat.Data.Sqlite/SqliteProvider.cs @@ -0,0 +1,145 @@ +using System.Data; +using System.Data.Common; +using System.Globalization; +using Khaemhat.Data.Schema; +using Microsoft.Data.Sqlite; + +namespace Khaemhat.Data.Sqlite; + +/// +/// The SQLite engine (SPEC §1.3, §5.2): tests, demo, trial and read-only archives. Every connection gets +/// foreign_keys=ON and busy_timeout=5000; file databases use journal_mode=WAL. Databases are files +/// under KHAEMHAT_DATA_DIR, or in-memory databases shared by the connections of the process (tests). +/// +public sealed class SqliteProvider : IDbProvider +{ + private const int BusyTimeoutMilliseconds = 5000; + + /// + public DbEngine Engine => DbEngine.Sqlite; + + /// + public ISqlDialect Dialect => SqliteDialect.Instance; + + /// + public DdlGenerator Ddl { get; } = new SqliteSchemaBuilder(); + + /// + public string BuildConnectionString(DatabaseUrl url) + { + ArgumentNullException.ThrowIfNull(url); + if (url.Engine != DbEngine.Sqlite) + { + throw new ArgumentException("Not a SQLite URL.", nameof(url)); + } + + var builder = url.IsInMemory + ? new SqliteConnectionStringBuilder { DataSource = url.Database, Mode = SqliteOpenMode.Memory, Cache = SqliteCacheMode.Shared } + : new SqliteConnectionStringBuilder { DataSource = Path.GetFullPath(url.Database), Mode = SqliteOpenMode.ReadWriteCreate }; + builder.Pooling = true; + builder.DefaultTimeout = 30; + return builder.ToString(); + } + + /// + public DbConnection CreateConnection(string connectionString) + { + var connection = new SqliteConnection(connectionString); + connection.CreateFunction(SqliteDialect.LowerFunction, static s => s?.ToLowerInvariant(), isDeterministic: true); + var isFile = IsFile(connectionString); + connection.StateChange += (sender, args) => + { + if (args.CurrentState == ConnectionState.Open && sender is SqliteConnection open) + { + ApplyPragmas(open, isFile); + } + }; + return connection; + } + + /// + public async Task EnsureDatabaseExistsAsync(string connectionString, CancellationToken cancellationToken = default) + { + if (!IsFile(connectionString)) + { + return; + } + + var path = new SqliteConnectionStringBuilder(connectionString).DataSource; + Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(path))!); + await using var connection = CreateConnection(connectionString); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + } + + /// + public Task DropDatabaseAsync(string connectionString, CancellationToken cancellationToken = default) + { + if (!IsFile(connectionString)) + { + return Task.CompletedTask; + } + + using (var connection = new SqliteConnection(connectionString)) + { + SqliteConnection.ClearPool(connection); + } + + var path = new SqliteConnectionStringBuilder(connectionString).DataSource; + foreach (var file in new[] { path, path + "-wal", path + "-shm", path + "-journal" }) + { + File.Delete(file); + } + + return Task.CompletedTask; + } + + /// + public Task DatabaseExistsAsync(string connectionString, CancellationToken cancellationToken = default) => + Task.FromResult(!IsFile(connectionString) || File.Exists(new SqliteConnectionStringBuilder(connectionString).DataSource)); + + /// + public async Task GetServerInfoAsync(string connectionString, CancellationToken cancellationToken = default) + { + await using var connection = CreateConnection(connectionString); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + var version = (string)(await ScalarAsync(connection, "SELECT sqlite_version()", cancellationToken).ConfigureAwait(false))!; + var pages = Convert.ToInt64(await ScalarAsync(connection, "PRAGMA page_count", cancellationToken).ConfigureAwait(false), CultureInfo.InvariantCulture); + var pageSize = Convert.ToInt64(await ScalarAsync(connection, "PRAGMA page_size", cancellationToken).ConfigureAwait(false), CultureInfo.InvariantCulture); + var journal = Convert.ToString(await ScalarAsync(connection, "PRAGMA journal_mode", cancellationToken).ConfigureAwait(false), CultureInfo.InvariantCulture); + var parts = version.Split('.').Select(p => int.Parse(p, CultureInfo.InvariantCulture)).Concat([0, 0, 0]).ToArray(); + var source = new SqliteConnectionStringBuilder(connectionString).DataSource; + return new DbServerInfo( + DbEngine.Sqlite, version, (parts[0] * 1_000_000) + (parts[1] * 1_000) + parts[2], + IsFile(connectionString) ? Path.GetFileNameWithoutExtension(source) : source, pages * pageSize) + { + Details = ["journal_mode=" + journal], + }; + } + + /// + public bool IsTransient(Exception exception) => + exception is SqliteException { SqliteErrorCode: 5 or 6 }; // SQLITE_BUSY, SQLITE_LOCKED + + /// + public ValueTask EnterWriteScopeAsync(string connectionString, CancellationToken cancellationToken = default) => + SqliteWriteLock.AcquireAsync(new SqliteConnectionStringBuilder(connectionString).DataSource, cancellationToken); + + private static bool IsFile(string connectionString) => + new SqliteConnectionStringBuilder(connectionString).Mode != SqliteOpenMode.Memory; + + private static void ApplyPragmas(SqliteConnection connection, bool isFile) + { + using var command = connection.CreateCommand(); + command.CommandText = isFile + ? $"PRAGMA journal_mode=WAL; PRAGMA foreign_keys=ON; PRAGMA busy_timeout={BusyTimeoutMilliseconds};" + : $"PRAGMA foreign_keys=ON; PRAGMA busy_timeout={BusyTimeoutMilliseconds};"; + command.ExecuteNonQuery(); + } + + private static async Task ScalarAsync(DbConnection connection, string sql, CancellationToken cancellationToken) + { + await using var command = connection.CreateCommand(); + command.CommandText = sql; + return await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/Khaemhat.Data.Sqlite/SqliteSchemaBuilder.cs b/src/Khaemhat.Data.Sqlite/SqliteSchemaBuilder.cs new file mode 100644 index 0000000..a490c0e --- /dev/null +++ b/src/Khaemhat.Data.Sqlite/SqliteSchemaBuilder.cs @@ -0,0 +1,34 @@ +using System.Globalization; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.Sqlite; + +/// +/// DDL for SQLite (SPEC §21 SqliteSchemaBuilder): STRICT tables, INTEGER PRIMARY KEY for identity +/// keys, and CHECK constraints that give text lengths and booleans the same limits as on PostgreSQL. +/// +public sealed class SqliteSchemaBuilder() : DdlGenerator(SqliteDialect.Instance) +{ + /// + protected override string TableOptions(TableDefinition table) => " STRICT"; + + /// + protected override string IdentityPrimaryKey(ColumnDefinition column) + { + ArgumentNullException.ThrowIfNull(column); + return Dialect.Quote(column.Name) + " INTEGER PRIMARY KEY"; + } + + /// + protected override string ColumnConstraints(ColumnDefinition column) + { + ArgumentNullException.ThrowIfNull(column); + var name = Dialect.Quote(column.Name); + return column.Type switch + { + ColumnType.String => string.Create(CultureInfo.InvariantCulture, $" CHECK (length({name}) <= {column.Length})"), + ColumnType.Bool => $" CHECK ({name} IN (0, 1))", + _ => string.Empty, + }; + } +} diff --git a/src/Khaemhat.Data.Sqlite/SqliteWriteLock.cs b/src/Khaemhat.Data.Sqlite/SqliteWriteLock.cs new file mode 100644 index 0000000..b768d5e --- /dev/null +++ b/src/Khaemhat.Data.Sqlite/SqliteWriteLock.cs @@ -0,0 +1,41 @@ +using System.Collections.Concurrent; + +namespace Khaemhat.Data.Sqlite; + +/// +/// Serializes the writers of one SQLite database inside the process (SPEC §5.2). SQLite allows a single writer; waiting +/// here is fairer and cheaper than retrying on SQLITE_BUSY. Other processes are kept out by +/// BEGIN IMMEDIATE and busy_timeout. Do not open a second writing unit of work on the same database while +/// holding one in the same flow: it would wait for itself. +/// +public static class SqliteWriteLock +{ + private static readonly ConcurrentDictionary Locks = new(StringComparer.OrdinalIgnoreCase); + + /// Waits for the write lock of a database. + /// The database key: full file path, or the in-memory name. + /// Cancellation. + /// The held lock; dispose it to release. + public static async ValueTask AcquireAsync(string database, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrEmpty(database); + var semaphore = Locks.GetOrAdd(database, static _ => new SemaphoreSlim(1, 1)); + await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + return new Releaser(semaphore); + } + + private sealed class Releaser(SemaphoreSlim semaphore) : IAsyncDisposable + { + private int _released; + + public ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _released, 1) == 0) + { + semaphore.Release(); + } + + return ValueTask.CompletedTask; + } + } +} diff --git a/src/Khaemhat.Data/Companies/CompanyRegistry.cs b/src/Khaemhat.Data/Companies/CompanyRegistry.cs new file mode 100644 index 0000000..19bcde9 --- /dev/null +++ b/src/Khaemhat.Data/Companies/CompanyRegistry.cs @@ -0,0 +1,200 @@ +using System.Collections.Concurrent; +using Khaemhat.Core.Instance; +using Khaemhat.Core.Security; +using Khaemhat.Data.Execution; +using Khaemhat.Data.Mapping; +using Khaemhat.Data.Querying; + +namespace Khaemhat.Data.Companies; + +/// A company to add to the registry. +/// The company code. +/// The company name. +/// Where its database is (printed redacted). +/// Whether it becomes the default company. +/// Notes for administrators. +public sealed record CompanyRegistration(string Code, string Name, DatabaseUrl Database, bool IsDefault = false, string? Notes = null); + +/// +/// The company registry in the system database (SPEC §5.4): it resolves company code → (provider, connection +/// string), decrypting the location stored in sys_companies. +/// +public interface ICompanyRegistry +{ + /// All companies, by code. + /// Cancellation. + /// The companies (their location stays encrypted). + Task> ListAsync(CancellationToken cancellationToken = default); + + /// A company by code. + /// The code. + /// Cancellation. + /// The company, or . + Task FindAsync(string code, CancellationToken cancellationToken = default); + + /// Adds a company whose database already exists or will be created by the caller. + /// The company. + /// Cancellation. + /// The stored company. + Task RegisterAsync(CompanyRegistration registration, CancellationToken cancellationToken = default); + + /// The database of a company. + /// The code. + /// Cancellation. + /// The target. + /// No company has that code. + Task ResolveAsync(string code, CancellationToken cancellationToken = default); + + /// Changes the state of a company. + /// The code. + /// The new state. + /// Cancellation. + /// A task. + Task SetStatusAsync(string code, CompanyStatus status, CancellationToken cancellationToken = default); +} + +/// +/// on sys_companies. The database URL is encrypted with +/// for the purpose company:<code>:database, so a row copied to another company +/// cannot be decrypted. Resolved targets are cached until the row changes through this registry. +/// +public sealed class CompanyRegistry : ICompanyRegistry +{ + private readonly DatabaseTarget _system; + private readonly DbProviderCatalog _providers; + private readonly ISecretProtector _protector; + private readonly MapRegistry _maps; + private readonly DataAccessContext _context; + private readonly ConcurrentDictionary _resolved = new(StringComparer.Ordinal); + + /// Creates the registry. + /// The system database. + /// The providers by engine. + /// Encrypts the database locations. + /// The entity maps, including . + /// Clock, user and retry policy. + public CompanyRegistry(DatabaseTarget system, DbProviderCatalog providers, ISecretProtector protector, MapRegistry maps, DataAccessContext context) + { + ArgumentNullException.ThrowIfNull(system); + ArgumentNullException.ThrowIfNull(providers); + ArgumentNullException.ThrowIfNull(protector); + ArgumentNullException.ThrowIfNull(maps); + ArgumentNullException.ThrowIfNull(context); + _system = system; + _providers = providers; + _protector = protector; + _maps = maps; + _context = context; + } + + /// The purpose used to encrypt the database location of a company. + /// The normalized code. + /// The purpose. + public static string Purpose(string code) => "company:" + code + ":database"; + + /// + public async Task> ListAsync(CancellationToken cancellationToken = default) + { + await using var unitOfWork = await OpenAsync(cancellationToken).ConfigureAwait(false); + var page = await unitOfWork.Repository().ListAsync(Query.For().OrderBy(c => c.Code), cancellationToken).ConfigureAwait(false); + return page.Items; + } + + /// + public async Task FindAsync(string code, CancellationToken cancellationToken = default) + { + var normalized = CompanyCodes.Normalize(code); + await using var unitOfWork = await OpenAsync(cancellationToken).ConfigureAwait(false); + return await unitOfWork.Repository().FindAsync(Query.For().Where(c => c.Code == normalized), cancellationToken) + .ConfigureAwait(false); + } + + /// + public async Task RegisterAsync(CompanyRegistration registration, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(registration); + ArgumentException.ThrowIfNullOrWhiteSpace(registration.Name); + var code = CompanyCodes.Normalize(registration.Code); + _ = _providers.Get(registration.Database.Engine); + + var company = new Company + { + Code = code, + Name = registration.Name.Trim(), + Engine = registration.Database.Engine.ToString(), + ConnectionString = _protector.Protect(registration.Database.ToUnredactedString(), Purpose(code)), + Status = CompanyStatus.Active, + IsDefault = registration.IsDefault, + Notes = registration.Notes, + }; + + await using var unitOfWork = await OpenAsync(cancellationToken).ConfigureAwait(false); + var companies = unitOfWork.Repository(); + await unitOfWork.InTransactionAsync( + async ct => + { + if (await companies.CountAsync(Query.For().Where(c => c.Code == code), ct).ConfigureAwait(false) > 0) + { + throw new InvalidOperationException($"A company with code {code} already exists."); + } + + if (company.IsDefault) + { + await unitOfWork.ExecuteAsync( + SqlFragment.Of($"UPDATE {SqlFragment.Identifier("sys_companies")} SET is_default = {false} WHERE is_default = {true}"), ct) + .ConfigureAwait(false); + } + + await companies.InsertAsync(company, ct).ConfigureAwait(false); + }, + cancellationToken).ConfigureAwait(false); + + _resolved.TryRemove(code, out _); + return company; + } + + /// + public async Task ResolveAsync(string code, CancellationToken cancellationToken = default) + { + var normalized = CompanyCodes.Normalize(code); + if (_resolved.TryGetValue(normalized, out var cached)) + { + return cached; + } + + var company = await FindAsync(normalized, cancellationToken).ConfigureAwait(false) + ?? throw new KeyNotFoundException($"No company has code {normalized}."); + var url = DatabaseUrl.Parse(_protector.Unprotect(company.ConnectionString, Purpose(normalized))); + var target = _providers.Resolve(url); + _resolved[normalized] = target; + return target; + } + + /// + public async Task SetStatusAsync(string code, CompanyStatus status, CancellationToken cancellationToken = default) + { + var normalized = CompanyCodes.Normalize(code); + await using var unitOfWork = await OpenAsync(cancellationToken).ConfigureAwait(false); + var companies = unitOfWork.Repository(); + var company = await companies.FindAsync(Query.For().Where(c => c.Code == normalized), cancellationToken).ConfigureAwait(false) + ?? throw new KeyNotFoundException($"No company has code {normalized}."); + company.Status = status; + await companies.UpdateAsync(company, cancellationToken).ConfigureAwait(false); + _resolved.TryRemove(normalized, out _); + } + + private Task OpenAsync(CancellationToken cancellationToken) => UnitOfWork.OpenAsync(_system, _maps, _context, cancellationToken); +} + +/// Finds the system database, or a company database through the registry. +/// The system database. +/// The company registry. +public sealed class DatabaseLocator(DatabaseTarget system, ICompanyRegistry companies) : IDatabaseLocator +{ + /// + public async ValueTask LocateAsync(DatabaseScope scope, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(scope); + return scope.IsSystem ? system : await companies.ResolveAsync(scope.CompanyCode!, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/Khaemhat.Data/CompanyCodes.cs b/src/Khaemhat.Data/CompanyCodes.cs new file mode 100644 index 0000000..8146bc0 --- /dev/null +++ b/src/Khaemhat.Data/CompanyCodes.cs @@ -0,0 +1,43 @@ +using System.Text.RegularExpressions; + +namespace Khaemhat.Data; + +/// +/// Rules for company codes and the names of their databases (SPEC §5.4): the code is 1 to 20 upper-case letters, +/// digits or underscores starting with a letter; the database is khaemhat_<code in lower case>. +/// +public static partial class CompanyCodes +{ + /// The prefix of every company database name. + public const string DatabasePrefix = "khaemhat_"; + + /// Whether (already normalized) is a valid company code. + /// The code. + /// if valid. + public static bool IsValid(string? code) => code is not null && CodePattern().IsMatch(code); + + /// Trims and upper-cases a code, then validates it. + /// The code as typed. + /// The normalized code. + /// The code is not valid. + public static string Normalize(string code) + { + ArgumentNullException.ThrowIfNull(code); + var normalized = code.Trim().ToUpperInvariant(); + if (!IsValid(normalized)) + { + throw new ArgumentException( + "A company code has 1 to 20 characters: letters, digits or underscore, starting with a letter.", nameof(code)); + } + + return normalized; + } + + /// The database name of a company, for example khaemhat_demo for DEMO. + /// The company code. + /// The database name. + public static string DatabaseName(string code) => DatabasePrefix + Normalize(code).ToLowerInvariant(); + + [GeneratedRegex("^[A-Z][A-Z0-9_]{0,19}$", RegexOptions.CultureInvariant)] + private static partial Regex CodePattern(); +} diff --git a/src/Khaemhat.Data/DatabaseScope.cs b/src/Khaemhat.Data/DatabaseScope.cs new file mode 100644 index 0000000..bfb2f78 --- /dev/null +++ b/src/Khaemhat.Data/DatabaseScope.cs @@ -0,0 +1,26 @@ +namespace Khaemhat.Data; + +/// +/// Which database a unit of work opens (SPEC §5.4): the instance system database or the database of one company. +/// +public sealed record DatabaseScope +{ + private DatabaseScope(string? companyCode) => CompanyCode = companyCode; + + /// The system database (KHAEMHAT_SYSTEM_DB). + public static DatabaseScope System { get; } = new((string?)null); + + /// The company code, or for the system database. + public string? CompanyCode { get; } + + /// Whether this is the system database. + public bool IsSystem => CompanyCode is null; + + /// The database of the company with the given code. + /// The company code (case-insensitive; stored upper-case). + /// The scope. + public static DatabaseScope Company(string code) => new(CompanyCodes.Normalize(code)); + + /// + public override string ToString() => CompanyCode is null ? "system" : "company " + CompanyCode; +} diff --git a/src/Khaemhat.Data/DatabaseTarget.cs b/src/Khaemhat.Data/DatabaseTarget.cs new file mode 100644 index 0000000..a4d5258 --- /dev/null +++ b/src/Khaemhat.Data/DatabaseTarget.cs @@ -0,0 +1,36 @@ +namespace Khaemhat.Data; + +/// +/// A database ready to be opened: its provider and driver connection string. shows only the +/// engine and the database name, never the connection string. +/// +public sealed class DatabaseTarget +{ + /// Creates a target. + /// The provider. + /// The driver connection string. + /// The database name, for messages. + public DatabaseTarget(IDbProvider provider, string connectionString, string databaseName) + { + ArgumentNullException.ThrowIfNull(provider); + ArgumentException.ThrowIfNullOrEmpty(connectionString); + Provider = provider; + ConnectionString = connectionString; + DatabaseName = databaseName ?? string.Empty; + } + + /// The provider. + public IDbProvider Provider { get; } + + /// The driver connection string. It contains credentials: never log it. + public string ConnectionString { get; } + + /// The database name. + public string DatabaseName { get; } + + /// The engine. + public DbEngine Engine => Provider.Engine; + + /// + public override string ToString() => Engine + " " + DatabaseName; +} diff --git a/src/Khaemhat.Data/DatabaseUrl.cs b/src/Khaemhat.Data/DatabaseUrl.cs new file mode 100644 index 0000000..d27d254 --- /dev/null +++ b/src/Khaemhat.Data/DatabaseUrl.cs @@ -0,0 +1,259 @@ +using System.Globalization; +using System.Text; + +namespace Khaemhat.Data; + +/// +/// A database location in URL form, as given by KHAEMHAT_SYSTEM_DB, KHAEMHAT_DB_SERVER and the company +/// registry (SPEC §6): +/// +/// postgres://user:password@host:5432/database?sslmode=require (also postgresql://; the database +/// may be empty for a server URL); +/// sqlite:///absolute/path/file.db (on Windows sqlite:///C:/path/file.db); +/// sqlite://memory/name, an in-memory database shared by the connections of the process (tests). +/// +/// never shows the password; parsing errors never echo the input. +/// +public sealed class DatabaseUrl +{ + private const string SqliteMemoryPrefix = "sqlite://memory/"; + + private DatabaseUrl( + DbEngine engine, string database, string? host, int? port, string? userName, string? password, + IReadOnlyDictionary options, bool isInMemory) + { + Engine = engine; + Database = database; + Host = host; + Port = port; + UserName = userName; + Password = password; + Options = options; + IsInMemory = isInMemory; + } + + /// The engine. + public DbEngine Engine { get; } + + /// + /// PostgreSQL: the database name (empty for a server URL). SQLite: the file path, or the name of the in-memory database. + /// + public string Database { get; } + + /// PostgreSQL host. + public string? Host { get; } + + /// PostgreSQL port, when given. + public int? Port { get; } + + /// PostgreSQL user. + public string? UserName { get; } + + /// PostgreSQL password. Never printed by . + public string? Password { get; } + + /// Query-string options, for example sslmode. + public IReadOnlyDictionary Options { get; } + + /// Whether this is an in-memory SQLite database. + public bool IsInMemory { get; } + + /// + /// The short database name: the PostgreSQL database, or the SQLite file name without extension (or the in-memory name). + /// + public string DatabaseName => Engine == DbEngine.PostgreSql || IsInMemory + ? Database + : Path.GetFileNameWithoutExtension(Database); + + /// Parses a database URL. + /// The URL. + /// The parsed URL. + /// The URL is not valid (the message does not repeat it). + public static DatabaseUrl Parse(string url) + { + ArgumentNullException.ThrowIfNull(url); + return TryParse(url, out var result, out var error) ? result : throw new FormatException(error); + } + + /// Tries to parse a database URL. + /// The URL. + /// The parsed URL. + /// if the URL is valid. + public static bool TryParse(string? url, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out DatabaseUrl? result) => + TryParse(url, out result, out _); + + /// + /// The same server with another database: on PostgreSQL the database name changes; on SQLite the file + /// <name>.db in the same directory (or another in-memory database). + /// + /// The database name, for example khaemhat_demo. + /// The new URL. + public DatabaseUrl WithDatabase(string databaseName) + { + ArgumentException.ThrowIfNullOrWhiteSpace(databaseName); + if (Engine == DbEngine.PostgreSql || IsInMemory) + { + return new DatabaseUrl(Engine, databaseName, Host, Port, UserName, Password, Options, IsInMemory); + } + + var directory = Path.GetDirectoryName(Database) ?? string.Empty; + var path = directory.Length == 0 ? databaseName + ".db" : directory.Replace('\\', '/') + "/" + databaseName + ".db"; + return new DatabaseUrl(Engine, path, Host, Port, UserName, Password, Options, IsInMemory); + } + + /// + /// The complete URL including the password, for storing it encrypted. Never log or display the result. + /// + /// The URL. + public string ToUnredactedString() => Format(includePassword: true); + + /// The URL with the password replaced by ***. + /// The redacted URL. + public override string ToString() => Format(includePassword: false); + + private string Format(bool includePassword) + { + if (Engine == DbEngine.Sqlite) + { + return IsInMemory ? SqliteMemoryPrefix + Database : "sqlite://" + (Database.StartsWith('/') ? Database : "/" + Database); + } + + var builder = new StringBuilder("postgres://"); + if (UserName is not null) + { + builder.Append(Uri.EscapeDataString(UserName)); + if (Password is not null) + { + builder.Append(':').Append(includePassword ? Uri.EscapeDataString(Password) : "***"); + } + + builder.Append('@'); + } + + builder.Append(Host); + if (Port is not null) + { + builder.Append(':').Append(Port.Value.ToString(CultureInfo.InvariantCulture)); + } + + builder.Append('/').Append(Uri.EscapeDataString(Database)); + var separator = '?'; + foreach (var (key, value) in Options) + { + builder.Append(separator).Append(Uri.EscapeDataString(key)).Append('=').Append(Uri.EscapeDataString(value)); + separator = '&'; + } + + return builder.ToString(); + } + + private static bool TryParse(string? url, out DatabaseUrl result, out string error) + { + result = null!; + url = url?.Trim(); + if (string.IsNullOrEmpty(url)) + { + error = "The database URL is empty."; + return false; + } + + if (url.StartsWith("sqlite:", StringComparison.OrdinalIgnoreCase)) + { + return TryParseSqlite(url, out result, out error); + } + + if (url.StartsWith("postgres://", StringComparison.OrdinalIgnoreCase) + || url.StartsWith("postgresql://", StringComparison.OrdinalIgnoreCase)) + { + return TryParsePostgres(url, out result, out error); + } + + error = "The database URL must start with postgres:// or sqlite://."; + return false; + } + + private static bool TryParseSqlite(string url, out DatabaseUrl result, out string error) + { + result = null!; + var empty = new Dictionary(StringComparer.OrdinalIgnoreCase); + if (url.StartsWith(SqliteMemoryPrefix, StringComparison.OrdinalIgnoreCase)) + { + var name = url[SqliteMemoryPrefix.Length..]; + if (name.Length == 0 || name.Any(c => !(char.IsAsciiLetterOrDigit(c) || c is '_' or '-'))) + { + error = "An in-memory SQLite URL is sqlite://memory/ with letters, digits, '-' or '_'."; + return false; + } + + result = new DatabaseUrl(DbEngine.Sqlite, name, null, null, null, null, empty, isInMemory: true); + error = string.Empty; + return true; + } + + if (!url.StartsWith("sqlite:///", StringComparison.OrdinalIgnoreCase)) + { + error = "A SQLite URL is sqlite:/// (three slashes)."; + return false; + } + + var path = url["sqlite://".Length..].Replace('\\', '/'); + // sqlite:///C:/data/x.db -> C:/data/x.db + if (path.Length >= 3 && path[0] == '/' && char.IsAsciiLetter(path[1]) && path[2] == ':') + { + path = path[1..]; + } + + if (path.Length < 2 || path.EndsWith('/')) + { + error = "The SQLite URL has no file name."; + return false; + } + + result = new DatabaseUrl(DbEngine.Sqlite, path, null, null, null, null, empty, isInMemory: false); + error = string.Empty; + return true; + } + + private static bool TryParsePostgres(string url, out DatabaseUrl result, out string error) + { + result = null!; + if (!Uri.TryCreate(url, UriKind.Absolute, out var uri) || string.IsNullOrEmpty(uri.Host)) + { + error = "The PostgreSQL URL is not valid: expected postgres://user:password@host:port/database."; + return false; + } + + string? user = null; + string? password = null; + if (uri.UserInfo.Length > 0) + { + var parts = uri.UserInfo.Split(':', 2); + user = Uri.UnescapeDataString(parts[0]); + password = parts.Length == 2 ? Uri.UnescapeDataString(parts[1]) : null; + } + + var database = Uri.UnescapeDataString(uri.AbsolutePath.Trim('/')); + if (database.Contains('/', StringComparison.Ordinal)) + { + error = "The PostgreSQL URL path must be a single database name."; + return false; + } + + var options = new Dictionary(StringComparer.OrdinalIgnoreCase); + var query = uri.Query.TrimStart('?'); + if (query.Length > 0) + { + foreach (var pair in query.Split('&', StringSplitOptions.RemoveEmptyEntries)) + { + var kv = pair.Split('=', 2); + options[Uri.UnescapeDataString(kv[0])] = kv.Length == 2 ? Uri.UnescapeDataString(kv[1]) : string.Empty; + } + } + + result = new DatabaseUrl( + DbEngine.PostgreSql, database, uri.Host, uri.IsDefaultPort || uri.Port < 0 ? null : uri.Port, user, password, + options, isInMemory: false); + error = string.Empty; + return true; + } +} diff --git a/src/Khaemhat.Data/DbEngine.cs b/src/Khaemhat.Data/DbEngine.cs new file mode 100644 index 0000000..893e158 --- /dev/null +++ b/src/Khaemhat.Data/DbEngine.cs @@ -0,0 +1,11 @@ +namespace Khaemhat.Data; + +/// The database engines Khaemhat supports (SPEC §1.3). +public enum DbEngine +{ + /// SQLite: tests, demo, trial and read-only archives. + Sqlite = 0, + + /// PostgreSQL 16 or later (17 recommended): real installations. + PostgreSql = 1, +} diff --git a/src/Khaemhat.Data/DbProviderCatalog.cs b/src/Khaemhat.Data/DbProviderCatalog.cs new file mode 100644 index 0000000..1d9a85d --- /dev/null +++ b/src/Khaemhat.Data/DbProviderCatalog.cs @@ -0,0 +1,37 @@ +namespace Khaemhat.Data; + +/// The providers available to the instance, by engine. The web host registers SQLite and PostgreSQL. +public sealed class DbProviderCatalog +{ + private readonly Dictionary _providers = []; + + /// Creates a catalog with the given providers. + /// The providers. + public DbProviderCatalog(params IEnumerable providers) + { + ArgumentNullException.ThrowIfNull(providers); + foreach (var provider in providers) + { + _providers[provider.Engine] = provider; + } + } + + /// The provider of an engine. + /// The engine. + /// The provider. + /// No provider is registered for the engine. + public IDbProvider Get(DbEngine engine) => + _providers.TryGetValue(engine, out var provider) + ? provider + : throw new InvalidOperationException($"No database provider is registered for {engine}."); + + /// The target for a database URL: its provider and connection string. + /// The URL. + /// The target. + public DatabaseTarget Resolve(DatabaseUrl url) + { + ArgumentNullException.ThrowIfNull(url); + var provider = Get(url.Engine); + return new DatabaseTarget(provider, provider.BuildConnectionString(url), url.DatabaseName); + } +} diff --git a/src/Khaemhat.Data/DbServerInfo.cs b/src/Khaemhat.Data/DbServerInfo.cs new file mode 100644 index 0000000..92cd119 --- /dev/null +++ b/src/Khaemhat.Data/DbServerInfo.cs @@ -0,0 +1,28 @@ +namespace Khaemhat.Data; + +/// Information about a database and its server, for the system status page (SPEC §12.1). +/// The engine. +/// The server version as text, for example 17.2 or 3.46.1. +/// The version as a number: PostgreSQL server_version_num (170002), SQLite (3046001). +/// The database name. +/// The database size in bytes, when known. +public sealed record DbServerInfo(DbEngine Engine, string Version, int VersionNumber, string Database, long? SizeBytes) +{ + /// The user the application connects as (PostgreSQL). + public string? UserName { get; init; } + + /// Whether that user may create databases (PostgreSQL CREATEDB). + public bool? CanCreateDatabases { get; init; } + + /// Open connections to this database (PostgreSQL). + public int? ActiveConnections { get; init; } + + /// The server connection limit (PostgreSQL max_connections). + public int? MaxConnections { get; init; } + + /// Installed extensions (PostgreSQL) or the journal mode (SQLite). + public IReadOnlyList Details { get; init; } = []; + + /// Problems to show to administrators, for example a PostgreSQL version below 16. + public IReadOnlyList Warnings { get; init; } = []; +} diff --git a/src/Khaemhat.Data/Execution/CommandCache.cs b/src/Khaemhat.Data/Execution/CommandCache.cs new file mode 100644 index 0000000..0c2b31d --- /dev/null +++ b/src/Khaemhat.Data/Execution/CommandCache.cs @@ -0,0 +1,90 @@ +using System.Collections.Concurrent; +using System.Globalization; +using Khaemhat.Data.Mapping; + +namespace Khaemhat.Data.Execution; + +/// The kinds of statements the repositories cache. +public enum CachedStatementKind +{ + /// INSERT, with RETURNING for identity keys. + Insert = 0, + + /// UPDATE … WHERE key = … [AND row_version = …]. + Update = 1, + + /// DELETE … WHERE key = …. + Delete = 2, +} + +/// A cached statement: SQL text whose parameter p{i} takes the value of [i]. +/// The SQL text. +/// The columns whose values fill the parameters, in order. +public sealed record CachedStatement(string Text, IReadOnlyList Parameters); + +/// +/// The SQL text of the repository statements, built once per entity, statement and dialect and reused (SPEC §21, +/// CommandCache). +/// +public static class CommandCache +{ + private static readonly ConcurrentDictionary<(Type Entity, CachedStatementKind Kind, DbEngine Engine), CachedStatement> Statements = new(); + + /// The statement of a kind for an entity map and dialect. + /// The entity map. + /// The statement kind. + /// The dialect. + /// The statement. + public static CachedStatement Get(IEntityMap map, CachedStatementKind kind, ISqlDialect dialect) + { + ArgumentNullException.ThrowIfNull(map); + ArgumentNullException.ThrowIfNull(dialect); + return Statements.GetOrAdd((map.EntityType, kind, dialect.Engine), _ => Build(map, kind, dialect)); + } + + private static CachedStatement Build(IEntityMap map, CachedStatementKind kind, ISqlDialect dialect) + { + string P(int index) => dialect.Parameter("p" + index.ToString(CultureInfo.InvariantCulture)); + switch (kind) + { + case CachedStatementKind.Insert: + { + var columns = map.Columns.Where(c => !(c.IsKey && c.IsIdentity)).ToList(); + var names = columns.Select(c => c.ColumnName).ToList(); + var placeholders = columns.Select((_, i) => P(i)).ToList(); + var text = map.Key.IsIdentity + ? dialect.InsertReturningId(map.Table, names, placeholders, map.Key.ColumnName) + : InsertOnly(dialect, map.Table, names, placeholders); + return new CachedStatement(text, columns); + } + + case CachedStatementKind.Update: + { + var set = map.Columns + .Where(c => !c.IsKey && c.Audit is not (Mapping.AuditRole.CreatedAt or Mapping.AuditRole.CreatedBy or Mapping.AuditRole.RowVersion)) + .ToList(); + var assignments = set.Select((c, i) => dialect.Quote(c.ColumnName) + " = " + P(i)).ToList(); + var parameters = new List(set) { map.Key }; + var where = dialect.Quote(map.Key.ColumnName) + " = " + P(set.Count); + if (map.RowVersion is { } version) + { + assignments.Add(dialect.Quote(version.ColumnName) + " = " + dialect.Quote(version.ColumnName) + " + 1"); + where += " AND " + dialect.Quote(version.ColumnName) + " = " + P(parameters.Count); + parameters.Add(version); + } + + var text = "UPDATE " + dialect.Quote(map.Table) + " SET " + string.Join(", ", assignments) + " WHERE " + where; + return new CachedStatement(text, parameters); + } + + case CachedStatementKind.Delete: + return new CachedStatement( + "DELETE FROM " + dialect.Quote(map.Table) + " WHERE " + dialect.Quote(map.Key.ColumnName) + " = " + P(0), [map.Key]); + default: + throw new ArgumentOutOfRangeException(nameof(kind), kind, null); + } + } + + private static string InsertOnly(ISqlDialect dialect, string table, IReadOnlyList columns, IReadOnlyList placeholders) => + "INSERT INTO " + dialect.Quote(table) + " (" + string.Join(", ", columns.Select(dialect.Quote)) + ") VALUES (" + string.Join(", ", placeholders) + ")"; +} diff --git a/src/Khaemhat.Data/Execution/DataAccessContext.cs b/src/Khaemhat.Data/Execution/DataAccessContext.cs new file mode 100644 index 0000000..255a0c2 --- /dev/null +++ b/src/Khaemhat.Data/Execution/DataAccessContext.cs @@ -0,0 +1,43 @@ +namespace Khaemhat.Data.Execution; + +/// +/// What the data layer needs from the running application: the clock for audit timestamps, the current user for +/// created_by/updated_by, and the retry policy. The web host builds it from ICurrentUser (M2). +/// +public sealed class DataAccessContext +{ + private readonly Func _currentUser; + + /// Creates a context. + /// The clock. + /// Returns the current user name, or . + /// The retry policy for transient errors outside transactions. + public DataAccessContext(TimeProvider clock, Func currentUser, TransientRetryPolicy? retryPolicy = null) + { + ArgumentNullException.ThrowIfNull(clock); + ArgumentNullException.ThrowIfNull(currentUser); + Clock = clock; + _currentUser = currentUser; + RetryPolicy = retryPolicy ?? TransientRetryPolicy.Default; + } + + /// System clock, no user, default retries. + public static DataAccessContext Default { get; } = new(TimeProvider.System, static () => null); + + /// The clock. + public TimeProvider Clock { get; } + + /// The retry policy. + public TransientRetryPolicy RetryPolicy { get; } + + /// The current user name, or . + public string? CurrentUser => _currentUser(); + + /// The current UTC time, truncated to microseconds (the precision of PostgreSQL and of SQLite storage). + public DateTime UtcNow => Truncate(Clock.GetUtcNow().UtcDateTime); + + /// Truncates a timestamp to microseconds. + /// The timestamp. + /// The truncated timestamp. + public static DateTime Truncate(DateTime value) => new(value.Ticks - (value.Ticks % 10), value.Kind); +} diff --git a/src/Khaemhat.Data/Execution/DbExecutor.cs b/src/Khaemhat.Data/Execution/DbExecutor.cs new file mode 100644 index 0000000..4805d5c --- /dev/null +++ b/src/Khaemhat.Data/Execution/DbExecutor.cs @@ -0,0 +1,51 @@ +using System.Data.Common; +using System.Diagnostics.CodeAnalysis; + +namespace Khaemhat.Data.Execution; + +/// +/// Creates driver commands from rendered SQL. This is the only place where command text is assigned: the text comes +/// from or from the , both built from code and mapped +/// names, and every value travels as a parameter. +/// +/// The dialect that configures parameters. +public sealed class DbExecutor(ISqlDialect dialect) +{ + /// Creates a command for a fragment. + /// The open connection. + /// The transaction, if any. + /// The fragment. + /// The command; the caller disposes it. + public DbCommand CreateCommand(DbConnection connection, DbTransaction? transaction, SqlFragment sql) + { + ArgumentNullException.ThrowIfNull(sql); + var rendered = sql.Render(dialect); + return CreateCommand(connection, transaction, rendered.Text, rendered.Parameters); + } + + /// Creates a command for rendered text and its parameters. + /// The open connection. + /// The transaction, if any. + /// SQL text produced by the data layer. + /// The parameters. + /// The command; the caller disposes it. + [SuppressMessage("Security", "CA2100:Review SQL queries for security vulnerabilities", + Justification = "Text comes only from SqlFragment rendering or the command cache; values are always parameters (SPEC §5.2).")] + public DbCommand CreateCommand(DbConnection connection, DbTransaction? transaction, string text, IEnumerable parameters) + { + ArgumentNullException.ThrowIfNull(connection); + ArgumentNullException.ThrowIfNull(parameters); + var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = text; + foreach (var parameter in parameters) + { + var dbParameter = command.CreateParameter(); + dbParameter.ParameterName = parameter.Name; + dialect.ConfigureParameter(dbParameter, parameter.Value, parameter.Type); + command.Parameters.Add(dbParameter); + } + + return command; + } +} diff --git a/src/Khaemhat.Data/Execution/ReaderMapper.cs b/src/Khaemhat.Data/Execution/ReaderMapper.cs new file mode 100644 index 0000000..e3c09f8 --- /dev/null +++ b/src/Khaemhat.Data/Execution/ReaderMapper.cs @@ -0,0 +1,154 @@ +using System.Collections.Concurrent; +using System.Data.Common; +using System.Linq.Expressions; +using System.Reflection; +using Khaemhat.Data.Mapping; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.Execution; + +/// +/// Maps reader rows to objects (SPEC §21, ReaderMapper<T>). The plan for a type and a column layout is +/// built once and cached; rows are then mapped with compiled delegates. Supported targets: +/// +/// scalars (numbers, text, dates, GUIDs, enums, nullable or not): the first column; +/// entities with a registered map: columns matched by column name; +/// other classes and records: settable properties, or the parameters of the public constructor, matched by +/// snake_case name. +/// +/// Values pass through , so for example scaled SQLite decimals (sums included) +/// come back as . +/// +public static class ReaderMapper +{ + private static readonly ConcurrentDictionary<(Type Type, DbEngine Engine, string Layout), Delegate> Plans = new(); + + /// The row mapper for and the columns of . + /// The target type. + /// The reader, positioned before or on a row. + /// The dialect. + /// The entity maps. + /// A function mapping the current row. + public static Func For(DbDataReader reader, ISqlDialect dialect, MapRegistry maps) + { + ArgumentNullException.ThrowIfNull(reader); + ArgumentNullException.ThrowIfNull(dialect); + ArgumentNullException.ThrowIfNull(maps); + var columns = Enumerable.Range(0, reader.FieldCount).Select(reader.GetName).ToArray(); + var key = (typeof(T), dialect.Engine, string.Join('|', columns)); + return (Func)Plans.GetOrAdd(key, _ => Build(columns, dialect, maps)); + } + + private static Func Build(string[] columns, ISqlDialect dialect, MapRegistry maps) + { + var target = typeof(T); + if (IsScalar(target)) + { + var type = LogicalType(target); + return reader => (T)ClrConverter.ConvertTo(dialect.FromDbValue(reader.GetValue(0), type), target)!; + } + + if (maps.Find(target) is { } map) + { + return BuildEntity(columns, dialect, map); + } + + var constructor = target.GetConstructor(Type.EmptyTypes); + return constructor is not null ? BuildProperties(columns, dialect) : BuildConstructor(columns, dialect); + } + + private static Func BuildEntity(string[] columns, ISqlDialect dialect, IEntityMap map) + { + var slots = columns + .Select((name, ordinal) => (Ordinal: ordinal, Column: map.FindByColumn(name))) + .Where(s => s.Column is not null) + .Select(s => (s.Ordinal, Column: s.Column!)) + .ToArray(); + return reader => + { + var entity = map.CreateEntity(); + foreach (var (ordinal, column) in slots) + { + column.SetValue(entity, column.FromProvider(dialect.FromDbValue(reader.GetValue(ordinal), column.Type))); + } + + return (T)entity; + }; + } + + private static Func BuildProperties(string[] columns, ISqlDialect dialect) + { + var properties = typeof(T).GetProperties(BindingFlags.Public | BindingFlags.Instance) + .Where(p => p.CanWrite && p.GetSetMethod() is not null) + .ToDictionary(p => SqlNames.ToSnakeCase(p.Name), StringComparer.OrdinalIgnoreCase); + var create = Expression.Lambda>(Expression.New(typeof(T))).Compile(); + var slots = new List<(int Ordinal, ColumnType Type, Type Target, Action Set)>(); + for (var ordinal = 0; ordinal < columns.Length; ordinal++) + { + if (!properties.TryGetValue(columns[ordinal], out var property)) + { + continue; + } + + var instance = Expression.Parameter(typeof(T)); + var value = Expression.Parameter(typeof(object)); + var set = Expression.Lambda>( + Expression.Assign(Expression.Property(instance, property), Expression.Convert(value, property.PropertyType)), instance, value).Compile(); + slots.Add((ordinal, LogicalType(property.PropertyType), property.PropertyType, set)); + } + + return reader => + { + var item = create(); + foreach (var (ordinal, type, target, set) in slots) + { + set(item, ClrConverter.ConvertTo(dialect.FromDbValue(reader.GetValue(ordinal), type), target)); + } + + return item; + }; + } + + private static Func BuildConstructor(string[] columns, ISqlDialect dialect) + { + var constructor = typeof(T).GetConstructors().OrderByDescending(c => c.GetParameters().Length).FirstOrDefault() + ?? throw new InvalidOperationException($"{typeof(T).Name} has no public constructor to map rows to."); + var parameters = constructor.GetParameters(); + var ordinals = parameters.Select(p => + { + var index = Array.FindIndex(columns, c => string.Equals(c, SqlNames.ToSnakeCase(p.Name!), StringComparison.OrdinalIgnoreCase)); + return index >= 0 + ? index + : throw new InvalidOperationException($"{typeof(T).Name}: no column {SqlNames.ToSnakeCase(p.Name!)} for constructor parameter {p.Name}."); + }).ToArray(); + var types = parameters.Select(p => LogicalType(p.ParameterType)).ToArray(); + var arguments = Expression.Parameter(typeof(object?[])); + var create = Expression.Lambda>( + Expression.New(constructor, parameters.Select((p, i) => + (Expression)Expression.Convert(Expression.ArrayIndex(arguments, Expression.Constant(i)), p.ParameterType))), + arguments).Compile(); + + return reader => + { + var values = new object?[parameters.Length]; + for (var i = 0; i < parameters.Length; i++) + { + values[i] = ClrConverter.ConvertTo(dialect.FromDbValue(reader.GetValue(ordinals[i]), types[i]), parameters[i].ParameterType); + } + + return create(values); + }; + } + + private static bool IsScalar(Type type) + { + var underlying = Nullable.GetUnderlyingType(type) ?? type; + return underlying.IsEnum || ColumnTypes.FromClrType(underlying) is not null; + } + + private static ColumnType LogicalType(Type type) + { + var underlying = Nullable.GetUnderlyingType(type) ?? type; + return underlying.IsEnum ? ColumnType.Text : ColumnTypes.FromClrType(underlying) ?? ColumnType.Text; + } +} diff --git a/src/Khaemhat.Data/Execution/Repository.cs b/src/Khaemhat.Data/Execution/Repository.cs new file mode 100644 index 0000000..bece971 --- /dev/null +++ b/src/Khaemhat.Data/Execution/Repository.cs @@ -0,0 +1,140 @@ +using System.Globalization; +using Khaemhat.Core.Common; +using Khaemhat.Core.Entities; +using Khaemhat.Data.Mapping; +using Khaemhat.Data.Querying; + +namespace Khaemhat.Data.Execution; + +/// +/// The repository of SPEC §5.1 for one entity type in one unit of work. Insert, update and delete statements are +/// cached per dialect (); queries go through . +/// +/// The entity type. +/// The key type. +public sealed class Repository : IRepository + where T : class, IEntity, new() +{ + private readonly UnitOfWork _unitOfWork; + private readonly EntityMap _map; + private readonly QueryBuilder _builder; + + internal Repository(UnitOfWork unitOfWork, EntityMap map) + { + _unitOfWork = unitOfWork; + _map = map; + _builder = new QueryBuilder(map, unitOfWork.Dialect); + } + + /// + public Task GetAsync(TKey id, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(id); + return _unitOfWork.QuerySingleAsync(_builder.SelectByKey(id), cancellationToken); + } + + /// + public Task FindAsync(Query query, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(query); + return _unitOfWork.QuerySingleAsync(_builder.Select(query.Take(1)), cancellationToken); + } + + /// + public async Task> ListAsync(Query query, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(query); + var items = await _unitOfWork.QueryAsync(_builder.Select(query), cancellationToken).ConfigureAwait(false); + long total = query.Limit is null && query.Offset == 0 + ? items.Count + : await CountAsync(query, cancellationToken).ConfigureAwait(false); + return new PagedResult(items, total, query.Offset, query.Limit); + } + + /// + public async Task CountAsync(Query query, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(query); + return (int)await _unitOfWork.QuerySingleAsync(_builder.Count(query), cancellationToken).ConfigureAwait(false); + } + + /// + public async Task InsertAsync(T entity, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(entity); + if (entity is IAuditable audit) + { + audit.CreatedAt = _unitOfWork.Context.UtcNow; + audit.CreatedBy = _unitOfWork.Context.CurrentUser; + audit.UpdatedAt = null; + audit.UpdatedBy = null; + audit.RowVersion = 1; + } + + var statement = CommandCache.Get(_map, CachedStatementKind.Insert, _unitOfWork.Dialect); + var parameters = Parameters(statement, entity, overrides: null); + if (!_map.Key.IsIdentity) + { + await _unitOfWork.WriteAsync(statement.Text, parameters, (c, ct) => c.ExecuteNonQueryAsync(ct), cancellationToken).ConfigureAwait(false); + return entity.Id; + } + + var generated = await _unitOfWork.WriteAsync(statement.Text, parameters, (c, ct) => c.ExecuteScalarAsync(ct), cancellationToken) + .ConfigureAwait(false); + var key = (TKey)ClrConverter.ConvertTo(_unitOfWork.Dialect.FromDbValue(generated, _map.Key.Type), typeof(TKey))!; + entity.Id = key; + return key; + } + + /// + public async Task UpdateAsync(T entity, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(entity); + var statement = CommandCache.Get(_map, CachedStatementKind.Update, _unitOfWork.Dialect); + var now = _unitOfWork.Context.UtcNow; + var user = _unitOfWork.Context.CurrentUser; + var overrides = new Dictionary { [AuditRole.UpdatedAt] = now, [AuditRole.UpdatedBy] = user }; + var parameters = Parameters(statement, entity, overrides); + + var affected = await _unitOfWork.WriteAsync(statement.Text, parameters, (c, ct) => c.ExecuteNonQueryAsync(ct), cancellationToken) + .ConfigureAwait(false); + if (affected == 0) + { + var version = entity is IAuditable a ? a.RowVersion : 0; + throw new ConcurrencyException(_map.Table, Convert.ToString(entity.Id, CultureInfo.InvariantCulture) ?? string.Empty, version); + } + + if (entity is IAuditable audit) + { + audit.UpdatedAt = now; + audit.UpdatedBy = user; + audit.RowVersion++; + } + } + + /// + public async Task DeleteAsync(TKey id, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(id); + var statement = CommandCache.Get(_map, CachedStatementKind.Delete, _unitOfWork.Dialect); + var parameters = new[] { new SqlParameterValue("p0", _map.Key.ToProvider(id), _map.Key.Type) }; + var affected = await _unitOfWork.WriteAsync(statement.Text, parameters, (c, ct) => c.ExecuteNonQueryAsync(ct), cancellationToken) + .ConfigureAwait(false); + return affected > 0; + } + + private static List Parameters(CachedStatement statement, T entity, Dictionary? overrides) + { + var parameters = new List(statement.Parameters.Count); + for (var i = 0; i < statement.Parameters.Count; i++) + { + var column = statement.Parameters[i]; + var value = overrides is not null && column.Audit != AuditRole.None && overrides.TryGetValue(column.Audit, out var replaced) + ? replaced + : column.GetValue(entity); + parameters.Add(new SqlParameterValue("p" + i.ToString(CultureInfo.InvariantCulture), column.ToProvider(value), column.Type)); + } + + return parameters; + } +} diff --git a/src/Khaemhat.Data/Execution/TransientRetryPolicy.cs b/src/Khaemhat.Data/Execution/TransientRetryPolicy.cs new file mode 100644 index 0000000..8ffecd8 --- /dev/null +++ b/src/Khaemhat.Data/Execution/TransientRetryPolicy.cs @@ -0,0 +1,45 @@ +namespace Khaemhat.Data.Execution; + +/// +/// Retries an operation that failed with a transient error (database busy, connection lost). It is applied only +/// outside transactions: inside one, a retry could repeat half of the work, so the error goes to the caller. +/// +/// Total attempts, at least 1. +/// Delay before the second attempt; it doubles each time. +public sealed class TransientRetryPolicy(int maxAttempts, TimeSpan baseDelay) +{ + /// Three attempts, 100 ms then 200 ms apart. + public static TransientRetryPolicy Default { get; } = new(3, TimeSpan.FromMilliseconds(100)); + + /// No retries. + public static TransientRetryPolicy None { get; } = new(1, TimeSpan.Zero); + + /// Total attempts. + public int MaxAttempts { get; } = maxAttempts >= 1 ? maxAttempts : throw new ArgumentOutOfRangeException(nameof(maxAttempts)); + + /// Runs , retrying when says the error is transient. + /// The result type. + /// The operation. + /// Classifies exceptions. + /// Cancellation. + /// The result. + public async Task ExecuteAsync( + Func> operation, Func isTransient, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(operation); + ArgumentNullException.ThrowIfNull(isTransient); + var delay = baseDelay; + for (var attempt = 1; ; attempt++) + { + try + { + return await operation(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (attempt < MaxAttempts && !cancellationToken.IsCancellationRequested && isTransient(ex)) + { + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + delay += delay; + } + } + } +} diff --git a/src/Khaemhat.Data/Execution/UnitOfWork.cs b/src/Khaemhat.Data/Execution/UnitOfWork.cs new file mode 100644 index 0000000..2e5bedc --- /dev/null +++ b/src/Khaemhat.Data/Execution/UnitOfWork.cs @@ -0,0 +1,287 @@ +using System.Data.Common; +using Khaemhat.Core.Entities; +using Khaemhat.Data.Mapping; + +namespace Khaemhat.Data.Execution; + +/// +/// The unit of work of SPEC §5.1 over one open connection. Statements outside a transaction are retried on transient +/// errors; on SQLite every write enters the database write scope (the whole transaction, or the single statement). +/// +public sealed class UnitOfWork : IUnitOfWork +{ + private readonly Dictionary _repositories = []; + private IAsyncDisposable? _writeScope; + private bool _disposed; + + private UnitOfWork(DatabaseTarget target, DbConnection connection, MapRegistry maps, DataAccessContext context) + { + Target = target; + Connection = connection; + Maps = maps; + Context = context; + Executor = new DbExecutor(target.Provider.Dialect); + } + + /// The database. + public DatabaseTarget Target { get; } + + /// The provider. + public IDbProvider Provider => Target.Provider; + + /// + public DbConnection Connection { get; } + + /// + public DbTransaction? Transaction { get; private set; } + + /// + public ISqlDialect Dialect => Provider.Dialect; + + /// The entity maps. + public MapRegistry Maps { get; } + + /// Clock, user and retry policy. + public DataAccessContext Context { get; } + + /// Creates commands. + public DbExecutor Executor { get; } + + /// Opens a unit of work on a database. + /// The database. + /// The entity maps. + /// Clock, user and retry policy. + /// Cancellation. + /// The open unit of work. + public static async Task OpenAsync( + DatabaseTarget target, MapRegistry maps, DataAccessContext context, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(target); + ArgumentNullException.ThrowIfNull(maps); + ArgumentNullException.ThrowIfNull(context); + var connection = target.Provider.CreateConnection(target.ConnectionString); + try + { + await context.RetryPolicy.ExecuteAsync( + async ct => + { + await connection.OpenAsync(ct).ConfigureAwait(false); + return true; + }, + target.Provider.IsTransient, + cancellationToken).ConfigureAwait(false); + } + catch + { + await connection.DisposeAsync().ConfigureAwait(false); + throw; + } + + return new UnitOfWork(target, connection, maps, context); + } + + /// + public async Task BeginAsync(CancellationToken cancellationToken = default) + { + ObjectDisposedException.ThrowIf(_disposed, this); + if (Transaction is not null) + { + throw new InvalidOperationException("A transaction is already open on this unit of work."); + } + + _writeScope = await Provider.EnterWriteScopeAsync(Target.ConnectionString, cancellationToken).ConfigureAwait(false); + try + { + Transaction = await Connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + } + catch + { + await ReleaseWriteScopeAsync().ConfigureAwait(false); + throw; + } + } + + /// + public async Task CommitAsync(CancellationToken cancellationToken = default) + { + var transaction = Transaction ?? throw new InvalidOperationException("No transaction is open."); + try + { + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + } + finally + { + await EndTransactionAsync().ConfigureAwait(false); + } + } + + /// + public async Task RollbackAsync(CancellationToken cancellationToken = default) + { + var transaction = Transaction ?? throw new InvalidOperationException("No transaction is open."); + try + { + await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false); + } + finally + { + await EndTransactionAsync().ConfigureAwait(false); + } + } + + /// + public IRepository Repository() + where T : class, IEntity, new() + { + ObjectDisposedException.ThrowIf(_disposed, this); + if (!_repositories.TryGetValue(typeof(T), out var repository)) + { + repository = new Repository(this, Maps.Get()); + _repositories[typeof(T)] = repository; + } + + return (IRepository)repository; + } + + /// + public Task ExecuteAsync(SqlFragment sql, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(sql); + var rendered = sql.Render(Dialect); + return WriteAsync(rendered.Text, rendered.Parameters, (command, ct) => command.ExecuteNonQueryAsync(ct), cancellationToken); + } + + /// + public async Task QuerySingleAsync(SqlFragment sql, CancellationToken cancellationToken = default) + { + var rows = await ReadAsync(sql, single: true, cancellationToken).ConfigureAwait(false); + return rows.Count == 0 ? default : rows[0]; + } + + /// + public Task> QueryAsync(SqlFragment sql, CancellationToken cancellationToken = default) => + ReadAsync(sql, single: false, cancellationToken); + + /// + public async Task StreamAsync(SqlFragment sql, CancellationToken cancellationToken = default) + { + ObjectDisposedException.ThrowIf(_disposed, this); + var command = Executor.CreateCommand(Connection, Transaction, sql); + try + { + return await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + } + catch + { + await command.DisposeAsync().ConfigureAwait(false); + throw; + } + } + + /// + public async ValueTask DisposeAsync() + { + if (_disposed) + { + return; + } + + _disposed = true; + if (Transaction is not null) + { + try + { + await Transaction.RollbackAsync().ConfigureAwait(false); + } + catch (DbException) + { + // The connection may already be broken; disposing it releases the transaction anyway. + } + + await EndTransactionAsync().ConfigureAwait(false); + } + + await Connection.DisposeAsync().ConfigureAwait(false); + } + + /// + /// Runs a writing command: inside the transaction if there is one; otherwise inside the write scope, with retries + /// on transient errors. + /// + /// The result type. + /// The SQL text. + /// The parameters. + /// Runs the command. + /// Cancellation. + /// The result. + internal async Task WriteAsync( + string text, IEnumerable parameters, Func> run, CancellationToken cancellationToken) + { + ObjectDisposedException.ThrowIf(_disposed, this); + if (Transaction is not null) + { + await using var command = Executor.CreateCommand(Connection, Transaction, text, parameters); + return await run(command, cancellationToken).ConfigureAwait(false); + } + + var materialized = parameters.ToList(); + return await Context.RetryPolicy.ExecuteAsync( + async ct => + { + await using var scope = await Provider.EnterWriteScopeAsync(Target.ConnectionString, ct).ConfigureAwait(false); + await using var command = Executor.CreateCommand(Connection, null, text, materialized); + return await run(command, ct).ConfigureAwait(false); + }, + Provider.IsTransient, + cancellationToken).ConfigureAwait(false); + } + + private async Task> ReadAsync(SqlFragment sql, bool single, CancellationToken cancellationToken) + { + ObjectDisposedException.ThrowIf(_disposed, this); + ArgumentNullException.ThrowIfNull(sql); + + async Task> Run(CancellationToken ct) + { + await using var command = Executor.CreateCommand(Connection, Transaction, sql); + await using var reader = await command.ExecuteReaderAsync(ct).ConfigureAwait(false); + var rows = new List(); + Func? map = null; + while (await reader.ReadAsync(ct).ConfigureAwait(false)) + { + map ??= ReaderMapper.For(reader, Dialect, Maps); + rows.Add(map(reader)); + if (single) + { + break; + } + } + + return rows; + } + + return Transaction is not null + ? await Run(cancellationToken).ConfigureAwait(false) + : await Context.RetryPolicy.ExecuteAsync(Run, Provider.IsTransient, cancellationToken).ConfigureAwait(false); + } + + private async Task EndTransactionAsync() + { + if (Transaction is not null) + { + await Transaction.DisposeAsync().ConfigureAwait(false); + Transaction = null; + } + + await ReleaseWriteScopeAsync().ConfigureAwait(false); + } + + private async Task ReleaseWriteScopeAsync() + { + if (_writeScope is not null) + { + await _writeScope.DisposeAsync().ConfigureAwait(false); + _writeScope = null; + } + } +} diff --git a/src/Khaemhat.Data/Execution/UnitOfWorkFactory.cs b/src/Khaemhat.Data/Execution/UnitOfWorkFactory.cs new file mode 100644 index 0000000..87f1dd4 --- /dev/null +++ b/src/Khaemhat.Data/Execution/UnitOfWorkFactory.cs @@ -0,0 +1,18 @@ +using Khaemhat.Data.Mapping; + +namespace Khaemhat.Data.Execution; + +/// Opens s on the database of a scope (SPEC §5.1). +/// Finds the database of a scope. +/// The entity maps. +/// Clock, user and retry policy. +public sealed class UnitOfWorkFactory(IDatabaseLocator locator, MapRegistry maps, DataAccessContext context) : IUnitOfWorkFactory +{ + /// + public async Task CreateAsync(DatabaseScope scope, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(scope); + var target = await locator.LocateAsync(scope, cancellationToken).ConfigureAwait(false); + return await UnitOfWork.OpenAsync(target, maps, context, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/Khaemhat.Data/GlobalSuppressions.cs b/src/Khaemhat.Data/GlobalSuppressions.cs new file mode 100644 index 0000000..ce3c5b5 --- /dev/null +++ b/src/Khaemhat.Data/GlobalSuppressions.cs @@ -0,0 +1,5 @@ +using System.Diagnostics.CodeAnalysis; + +[assembly: SuppressMessage( + "Naming", "CA1720:Identifier contains type name", Scope = "namespaceanddescendants", Target = "~N:Khaemhat.Data.Schema", + Justification = "The logical column types are named as in SPEC §5.2 (String, Int32, Int64, Decimal, Guid): they are the vocabulary of migrations and docs.")] diff --git a/src/Khaemhat.Data/IDbProvider.cs b/src/Khaemhat.Data/IDbProvider.cs new file mode 100644 index 0000000..90fac46 --- /dev/null +++ b/src/Khaemhat.Data/IDbProvider.cs @@ -0,0 +1,68 @@ +using System.Data.Common; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data; + +/// +/// A database engine (SPEC §5.1): connections, DDL, creation and removal of databases, server information. Implemented +/// by SqliteProvider and PostgresProvider. +/// +public interface IDbProvider +{ + /// The engine. + DbEngine Engine { get; } + + /// The SQL dialect. + ISqlDialect Dialect { get; } + + /// The DDL generator used by migrations. + DdlGenerator Ddl { get; } + + /// The driver connection string for a database URL. + /// The database URL (its engine must be ). + /// The connection string. It contains credentials: never log it. + string BuildConnectionString(DatabaseUrl url); + + /// Creates a closed connection. Session settings (pragmas, timeouts) are applied when it opens. + /// The connection string. + /// The connection. + DbConnection CreateConnection(string connectionString); + + /// Creates the database if it does not exist (CREATE DATABASE, or the SQLite file). + /// The connection string of the database. + /// Cancellation. + /// A task. + Task EnsureDatabaseExistsAsync(string connectionString, CancellationToken cancellationToken = default); + + /// Drops the database. Only the company registry and the restore service call it. + /// The connection string of the database. + /// Cancellation. + /// A task. + Task DropDatabaseAsync(string connectionString, CancellationToken cancellationToken = default); + + /// Whether the database exists. + /// The connection string of the database. + /// Cancellation. + /// if it exists. + Task DatabaseExistsAsync(string connectionString, CancellationToken cancellationToken = default); + + /// Version, size, connections and warnings of the server and database (SPEC §12.1). + /// The connection string of the database. + /// Cancellation. + /// The information. + Task GetServerInfoAsync(string connectionString, CancellationToken cancellationToken = default); + + /// Whether an exception is transient (busy, connection lost) and the statement may be retried outside a transaction. + /// The exception. + /// if transient. + bool IsTransient(Exception exception); + + /// + /// Enters the write scope of a database: on SQLite it serializes writers of the process (SqliteWriteLock); + /// on PostgreSQL it does nothing. Dispose the result to leave. + /// + /// The connection string of the database. + /// Cancellation. + /// The scope. + ValueTask EnterWriteScopeAsync(string connectionString, CancellationToken cancellationToken = default); +} diff --git a/src/Khaemhat.Data/IRepository.cs b/src/Khaemhat.Data/IRepository.cs new file mode 100644 index 0000000..7a332f8 --- /dev/null +++ b/src/Khaemhat.Data/IRepository.cs @@ -0,0 +1,62 @@ +using Khaemhat.Core.Entities; +using Khaemhat.Data.Querying; + +namespace Khaemhat.Data; + +/// +/// Reads and writes the rows of one entity type in the database of a unit of work (SPEC §5.1). Updates check +/// row_version and throw on conflict. +/// +/// The entity type. +/// The key type. +public interface IRepository + where T : class, IEntity +{ + /// The row with the given key. + /// The key. + /// Cancellation. + /// The entity, or . + Task GetAsync(TKey id, CancellationToken cancellationToken = default); + + /// The first row of a query. + /// The query. + /// Cancellation. + /// The entity, or . + Task FindAsync(Query query, CancellationToken cancellationToken = default); + + /// The rows of a query, with the total count when it is paged. + /// The query. + /// Cancellation. + /// The page. + Task> ListAsync(Query query, CancellationToken cancellationToken = default); + + /// The number of rows matching a query. + /// The query. + /// Cancellation. + /// The count. + Task CountAsync(Query query, CancellationToken cancellationToken = default); + + /// + /// Inserts a row. Audit columns are filled, row_version starts at 1, an identity key is generated and set + /// on the entity. + /// + /// The entity. + /// Cancellation. + /// The key. + Task InsertAsync(T entity, CancellationToken cancellationToken = default); + + /// + /// Updates a row if its row_version is still the one on the entity, then increments it on the entity. + /// + /// The entity. + /// Cancellation. + /// A task. + /// The row changed or no longer exists. + Task UpdateAsync(T entity, CancellationToken cancellationToken = default); + + /// Deletes the row with the given key; nothing happens if it does not exist. + /// The key. + /// Cancellation. + /// if a row was deleted. + Task DeleteAsync(TKey id, CancellationToken cancellationToken = default); +} diff --git a/src/Khaemhat.Data/ISqlDialect.cs b/src/Khaemhat.Data/ISqlDialect.cs new file mode 100644 index 0000000..b789be2 --- /dev/null +++ b/src/Khaemhat.Data/ISqlDialect.cs @@ -0,0 +1,98 @@ +using System.Data.Common; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data; + +/// +/// What differs between SQL engines (SPEC §5.1): quoting, parameters, paging, insert/upsert forms, case-insensitive +/// search, advisory locks, value conversion and physical types. Everything above the providers uses only this +/// interface, so no code outside Khaemhat.Data.Sqlite and Khaemhat.Data.PostgreSql knows the engine. +/// +public interface ISqlDialect +{ + /// The engine. + DbEngine Engine { get; } + + /// The current UTC timestamp, in the same form the dialect stores . + string CurrentTimestampUtc { get; } + + /// Quotes a validated identifier. + /// The identifier (snake_case). + /// The quoted identifier. + string Quote(string identifier); + + /// The placeholder of a named parameter, for example @p0. + /// The parameter name without prefix. + /// The placeholder. + string Parameter(string name); + + /// The paging clause, appended after ORDER BY. + /// Rows to skip. + /// Rows to return. + /// The clause, starting with a space. + string Paging(int offset, int limit); + + /// An insert that returns the generated key. + /// The table. + /// The inserted columns. + /// The parameter placeholders, one per column. + /// The generated key column. + /// The statement. + string InsertReturningId(string table, IReadOnlyList columns, IReadOnlyList parameters, string keyColumn); + + /// An insert that updates the existing row on a conflict of . + /// The table. + /// The inserted columns. + /// The parameter placeholders, one per column. + /// The columns of the unique key. + /// The columns to update on conflict. + /// The statement. + string Upsert( + string table, IReadOnlyList columns, IReadOnlyList parameters, + IReadOnlyList conflictColumns, IReadOnlyList updateColumns); + + /// A case-insensitive LIKE with \ as escape character. + /// The column (or expression). + /// The pattern, normally a parameter. + /// The condition. + SqlFragment CaseInsensitiveLike(SqlFragment column, SqlFragment pattern); + + /// + /// A statement that takes a lock held until the end of the current transaction (numerators, closings, scheduler); + /// empty where writes are already serialized. + /// + /// The lock key, for example numerator:FV:2026. + /// The statement, possibly empty. + SqlFragment AdvisoryLock(string key); + + /// Converts a CLR value to what the driver stores for ; null becomes . + /// The value. + /// The logical type. + /// The value for the driver. + object ToDbValue(object? value, ColumnType type); + + /// Converts a value read from the driver to the canonical CLR type of (see ). + /// The value read. + /// The logical type. + /// The CLR value, or . + object? FromDbValue(object? value, ColumnType type); + + /// The physical type of a column. + /// The logical type. + /// The length for . + /// The scale for . + /// The SQL type. + string SqlType(ColumnType type, int? length, int? scale); + + /// A constant as SQL literal, for DDL defaults only (values in statements are always parameters). + /// The constant. + /// The logical type. + /// The literal. + string Literal(object? value, ColumnType type); + + /// Sets the value (converted with ) and the driver type of a parameter. + /// The parameter. + /// The CLR value. + /// The logical type, or to infer it from the value. + void ConfigureParameter(DbParameter parameter, object? value, ColumnType? type); +} diff --git a/src/Khaemhat.Data/IUnitOfWork.cs b/src/Khaemhat.Data/IUnitOfWork.cs new file mode 100644 index 0000000..a547fd0 --- /dev/null +++ b/src/Khaemhat.Data/IUnitOfWork.cs @@ -0,0 +1,93 @@ +using System.Data.Common; +using Khaemhat.Core.Entities; + +namespace Khaemhat.Data; + +/// +/// An open connection to one database, with an optional explicit transaction (SPEC §5.1). Application services open +/// one per operation, call for writes that must be atomic, and dispose it; a transaction not +/// committed is rolled back on dispose. +/// +public interface IUnitOfWork : IAsyncDisposable +{ + /// The open connection. + DbConnection Connection { get; } + + /// The current transaction, if any. + DbTransaction? Transaction { get; } + + /// The dialect of the database. + ISqlDialect Dialect { get; } + + /// Starts a transaction. On SQLite it also enters the write scope of the database. + /// Cancellation. + /// A task. + Task BeginAsync(CancellationToken cancellationToken = default); + + /// Commits the transaction. + /// Cancellation. + /// A task. + Task CommitAsync(CancellationToken cancellationToken = default); + + /// Rolls back the transaction. + /// Cancellation. + /// A task. + Task RollbackAsync(CancellationToken cancellationToken = default); + + /// The repository of an entity type. + /// The entity type, which needs a registered map. + /// The key type. + /// The repository. + IRepository Repository() + where T : class, IEntity, new(); + + /// Executes a statement that changes data or schema. + /// The statement. + /// Cancellation. + /// The number of affected rows. + Task ExecuteAsync(SqlFragment sql, CancellationToken cancellationToken = default); + + /// Reads the first row (or ), mapped to . + /// A scalar, a mapped entity, or a class/record whose members match the columns by snake_case name. + /// The query. + /// Cancellation. + /// The row, or . + Task QuerySingleAsync(SqlFragment sql, CancellationToken cancellationToken = default); + + /// Reads all rows, mapped to . + /// A scalar, a mapped entity, or a class/record whose members match the columns by snake_case name. + /// The query. + /// Cancellation. + /// The rows. + Task> QueryAsync(SqlFragment sql, CancellationToken cancellationToken = default); + + /// + /// Opens a reader for large results (exports, reports) without loading them in memory. Dispose the reader before + /// running anything else on this unit of work. Values are raw driver values: convert them with + /// . + /// + /// The query. + /// Cancellation. + /// The reader. + Task StreamAsync(SqlFragment sql, CancellationToken cancellationToken = default); +} + +/// Opens units of work on the system database or on a company database (SPEC §5.1). +public interface IUnitOfWorkFactory +{ + /// Opens a unit of work. + /// The system database, or a company. + /// Cancellation. + /// The open unit of work. + Task CreateAsync(DatabaseScope scope, CancellationToken cancellationToken = default); +} + +/// Finds the database of a scope: the system database, or a company through the registry. +public interface IDatabaseLocator +{ + /// The database of a scope. + /// The scope. + /// Cancellation. + /// The target. + ValueTask LocateAsync(DatabaseScope scope, CancellationToken cancellationToken = default); +} diff --git a/src/Khaemhat.Data/Mapping/ClrConverter.cs b/src/Khaemhat.Data/Mapping/ClrConverter.cs new file mode 100644 index 0000000..9b516af --- /dev/null +++ b/src/Khaemhat.Data/Mapping/ClrConverter.cs @@ -0,0 +1,63 @@ +using System.Globalization; + +namespace Khaemhat.Data.Mapping; + +/// +/// Converts the canonical values returned by (or raw driver values) to the CLR +/// type of a property, constructor parameter or scalar result. +/// +public static class ClrConverter +{ + /// Converts to . + /// The value, possibly or . + /// The target type, nullable or not. + /// The converted value; for a null input and a nullable target. + /// A null value for a non-nullable value type, or an impossible conversion. + public static object? ConvertTo(object? value, Type target) + { + ArgumentNullException.ThrowIfNull(target); + var underlying = Nullable.GetUnderlyingType(target); + if (value is null or DBNull) + { + return target.IsValueType && underlying is null + ? throw new InvalidCastException($"NULL cannot be assigned to {target.Name}.") + : null; + } + + var type = underlying ?? target; + if (type.IsInstanceOfType(value)) + { + return type == typeof(DateTime) && value is DateTime { Kind: not DateTimeKind.Utc } d ? DateTime.SpecifyKind(d, DateTimeKind.Utc) : value; + } + + if (type.IsEnum) + { + return value is string name ? Enum.Parse(type, name, ignoreCase: false) : Enum.ToObject(type, value); + } + + return type switch + { + _ when type == typeof(DateOnly) => value switch + { + DateTime dt => DateOnly.FromDateTime(dt), + string s => DateOnly.ParseExact(s, "yyyy-MM-dd", CultureInfo.InvariantCulture), + _ => throw Impossible(value, type), + }, + _ when type == typeof(DateTime) => value switch + { + DateTimeOffset o => o.UtcDateTime, + DateOnly d => d.ToDateTime(TimeOnly.MinValue, DateTimeKind.Utc), + string s => DateTime.Parse(s, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal), + _ => throw Impossible(value, type), + }, + _ when type == typeof(Guid) => value is string g ? Guid.Parse(g, CultureInfo.InvariantCulture) : throw Impossible(value, type), + _ when type == typeof(bool) && value is IConvertible => Convert.ToInt64(value, CultureInfo.InvariantCulture) != 0, + _ when type == typeof(string) => Convert.ToString(value, CultureInfo.InvariantCulture), + _ when value is IConvertible => Convert.ChangeType(value, type, CultureInfo.InvariantCulture), + _ => throw Impossible(value, type), + }; + } + + private static InvalidCastException Impossible(object value, Type type) => + new($"A {value.GetType().Name} value cannot be converted to {type.Name}."); +} diff --git a/src/Khaemhat.Data/Mapping/ColumnMap.cs b/src/Khaemhat.Data/Mapping/ColumnMap.cs new file mode 100644 index 0000000..bf4de94 --- /dev/null +++ b/src/Khaemhat.Data/Mapping/ColumnMap.cs @@ -0,0 +1,207 @@ +using System.Reflection; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.Mapping; + +/// The role of a column among the audit columns of SPEC §5.2. +public enum AuditRole +{ + /// Not an audit column. + None = 0, + + /// created_at. + CreatedAt = 1, + + /// created_by. + CreatedBy = 2, + + /// updated_at. + UpdatedAt = 3, + + /// updated_by. + UpdatedBy = 4, + + /// row_version. + RowVersion = 5, +} + +/// +/// How a property is stored: column name, logical type, nullability, key and audit role, plus compiled accessors. +/// Configured in the constructor of an , then frozen. +/// +public sealed class ColumnMap +{ + private readonly Func _getter; + private readonly Action _setter; + private bool _frozen; + private bool? _nullable; + + internal ColumnMap(PropertyInfo property, Func getter, Action setter) + { + PropertyName = property.Name; + PropertyType = property.PropertyType; + ValueType = Nullable.GetUnderlyingType(property.PropertyType) ?? property.PropertyType; + ColumnName = SqlNames.ToSnakeCase(property.Name); + _getter = getter; + _setter = setter; + } + + /// The C# property name. + public string PropertyName { get; } + + /// The declared property type. + public Type PropertyType { get; } + + /// The property type without . + public Type ValueType { get; } + + /// The column name (snake_case of the property name unless set with ). + public string ColumnName { get; private set; } + + /// The logical type. + public ColumnType Type { get; private set; } + + /// The maximum length, for . + public int? Length { get; private set; } + + /// The scale, for . + public int? Scale { get; private set; } + + /// Whether the column accepts NULL. + public bool IsNullable => _nullable ?? false; + + /// Whether this is the primary key. + public bool IsKey { get; internal set; } + + /// Whether the key is generated by the database. + public bool IsIdentity { get; internal set; } + + /// The audit role. + public AuditRole Audit { get; internal set; } + + /// The logical type as documented, for example String(20). + public string TypeDescription => ColumnTypes.Describe(Type, Length, Scale); + + internal bool HasExplicitType { get; private set; } + + internal bool HasExplicitNullability => _nullable is not null; + + /// Sets the column name. + /// The column name (snake_case). + /// This map. + public ColumnMap Named(string column) + { + EnsureNotFrozen(); + SqlNames.EnsureIdentifier(column); + ColumnName = column; + return this; + } + + /// Stores a string as with a maximum length (an enum by its name). + /// The maximum length. + /// This map. + public ColumnMap MaxLength(int length) + { + ArgumentOutOfRangeException.ThrowIfLessThan(length, 1); + return SetType(ColumnType.String, length, null); + } + + /// Stores a string as (the default for strings). + /// This map. + public ColumnMap Text() => SetType(ColumnType.Text, null, null); + + /// Stores a string as . + /// This map. + public ColumnMap Json() => SetType(ColumnType.Json, null, null); + + /// Sets the scale of a decimal (default 6). + /// Digits after the point, 0 to 6. + /// This map. + public ColumnMap Scaled(int scale) + { + ArgumentOutOfRangeException.ThrowIfNegative(scale); + ArgumentOutOfRangeException.ThrowIfGreaterThan(scale, ColumnTypes.DecimalScale); + return SetType(ColumnType.Decimal, null, scale); + } + + /// The column refuses NULL, whatever the property nullability says. + /// This map. + public ColumnMap Required() + { + EnsureNotFrozen(); + _nullable = false; + return this; + } + + /// The column accepts NULL. + /// This map. + public ColumnMap Optional() + { + EnsureNotFrozen(); + _nullable = true; + return this; + } + + /// Reads the property of an entity. + /// The entity. + /// The property value. + public object? GetValue(object entity) => _getter(entity); + + /// Writes the property of an entity. + /// The entity. + /// The value, already of the property type. + public void SetValue(object entity, object? value) + { + if (value is null && PropertyType.IsValueType && Nullable.GetUnderlyingType(PropertyType) is null) + { + throw new InvalidCastException($"Column {ColumnName} is NULL but property {PropertyName} cannot hold null."); + } + + _setter(entity, value); + } + + /// Converts a property value to the value given to the dialect (enums become their name). + /// The property value. + /// The value for . + public object? ToProvider(object? value) => value is Enum e && ValueType.IsEnum ? e.ToString() : value; + + /// Converts a canonical value from to the property type. + /// The canonical value. + /// The property value. + public object? FromProvider(object? value) => ClrConverter.ConvertTo(value, PropertyType); + + /// + public override string ToString() => ColumnName + " " + TypeDescription; + + internal void SetDefaults(ColumnType type, int? length, bool nullable) + { + if (!HasExplicitType) + { + Type = type; + Length = length; + Scale = type == ColumnType.Decimal ? ColumnTypes.DecimalScale : null; + } + + _nullable ??= nullable; + } + + internal void Freeze() => _frozen = true; + + private ColumnMap SetType(ColumnType type, int? length, int? scale) + { + EnsureNotFrozen(); + Type = type; + Length = length; + Scale = scale; + HasExplicitType = true; + return this; + } + + private void EnsureNotFrozen() + { + if (_frozen) + { + throw new InvalidOperationException("A column map cannot change after its entity map is registered."); + } + } +} diff --git a/src/Khaemhat.Data/Mapping/EntityMap.cs b/src/Khaemhat.Data/Mapping/EntityMap.cs new file mode 100644 index 0000000..adcfe5b --- /dev/null +++ b/src/Khaemhat.Data/Mapping/EntityMap.cs @@ -0,0 +1,353 @@ +using System.Linq.Expressions; +using System.Reflection; +using Khaemhat.Core.Entities; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.Mapping; + +/// A map from an entity type to its table (non-generic view used by the data layer). +public interface IEntityMap +{ + /// The entity type. + Type EntityType { get; } + + /// The table name. + string Table { get; } + + /// The primary key column. + ColumnMap Key { get; } + + /// All columns, key first, audit columns last. + IReadOnlyList Columns { get; } + + /// The row_version column, if the entity is . + ColumnMap? RowVersion { get; } + + /// Finds the column of a property. + /// The property name. + /// The column, or . + ColumnMap? FindByProperty(string propertyName); + + /// Finds a column by name. + /// The column name. + /// The column, or . + ColumnMap? FindByColumn(string columnName); + + /// Creates an empty entity. + /// The entity. + object CreateEntity(); + + /// Completes the map by convention and makes it read-only. Called by . + void Freeze(); +} + +/// +/// The map of an entity to its table (SPEC §5.2: XxxMap : EntityMap<Xxx>). The constructor names the table +/// and states only what differs from the conventions; every other public read/write property is mapped by convention: +/// +/// column name = snake_case of the property; +/// Id is the key, generated by the database when it is or ; +/// types from , strings as unless +/// is set, enums by name as String(40); +/// nullability from the C# declaration; +/// the audit columns of and is_demo of . +/// +/// Reflection is used only here, once; reading and writing use compiled accessors. +/// +/// The entity type. +public abstract class EntityMap : IEntityMap + where T : class, new() +{ + private const int EnumLength = 40; + private static readonly NullabilityInfoContext NullabilityContext = new(); + + private readonly Dictionary _configured = new(StringComparer.Ordinal); + private readonly HashSet _ignored = new(StringComparer.Ordinal); + private readonly object _gate = new(); + private string? _table; + private string _keyProperty = "Id"; + private bool? _keyIdentity; + private List? _columns; + private ColumnMap? _key; + private ColumnMap? _rowVersion; + + /// + public Type EntityType => typeof(T); + + /// + public string Table => _table ?? throw new InvalidOperationException($"{GetType().Name} does not call ToTable."); + + /// + public ColumnMap Key + { + get + { + Freeze(); + return _key!; + } + } + + /// + public IReadOnlyList Columns + { + get + { + Freeze(); + return _columns!; + } + } + + /// + public ColumnMap? RowVersion + { + get + { + Freeze(); + return _rowVersion; + } + } + + /// + public ColumnMap? FindByProperty(string propertyName) => Columns.FirstOrDefault(c => c.PropertyName == propertyName); + + /// + public ColumnMap? FindByColumn(string columnName) => Columns.FirstOrDefault(c => c.ColumnName == columnName); + + /// Creates an empty entity. + /// The entity. + public T Create() => new(); + + /// + object IEntityMap.CreateEntity() => new T(); + + /// The column of a property. + /// The property type. + /// The property, for example x => x.Code. + /// The column. + public ColumnMap Column(Expression> property) => + FindByProperty(PropertyOf(property).Name) ?? throw new InvalidOperationException($"{typeof(T).Name}.{PropertyOf(property).Name} is not mapped."); + + /// + public void Freeze() + { + if (_columns is not null) + { + return; + } + + lock (_gate) + { + if (_columns is null) + { + Build(); + } + } + } + + /// Sets the table name. + /// The table (snake_case; sys_ prefix in the system database). + protected void ToTable(string table) + { + SqlNames.EnsureIdentifier(table); + _table = table; + } + + /// Configures the column of a property. + /// The property type. + /// The property, for example x => x.Code. + /// The column map to configure. + protected ColumnMap Map(Expression> property) + { + if (_columns is not null) + { + throw new InvalidOperationException("The map is frozen."); + } + + var info = PropertyOf(property); + if (!_configured.TryGetValue(info.Name, out var column)) + { + column = CreateColumn(info); + _configured[info.Name] = column; + } + + return column; + } + + /// Sets a key other than Id. + /// The property type. + /// The key property. + /// Whether the database generates it. + protected void HasKey(Expression> property, bool identity) + { + _keyProperty = PropertyOf(property).Name; + _keyIdentity = identity; + } + + /// Excludes a property from the table. + /// The property type. + /// The property. + protected void Ignore(Expression> property) => _ignored.Add(PropertyOf(property).Name); + + private static PropertyInfo PropertyOf(Expression> property) + { + ArgumentNullException.ThrowIfNull(property); + var body = property.Body is UnaryExpression { NodeType: ExpressionType.Convert } convert ? convert.Operand : property.Body; + return body is MemberExpression { Member: PropertyInfo info } member && member.Expression == property.Parameters[0] + ? info + : throw new ArgumentException("Use a property of the entity, for example x => x.Code.", nameof(property)); + } + + private static ColumnMap CreateColumn(PropertyInfo property) + { + var entity = Expression.Parameter(typeof(object), "entity"); + var value = Expression.Parameter(typeof(object), "value"); + var typed = Expression.Convert(entity, typeof(T)); + var getter = Expression.Lambda>( + Expression.Convert(Expression.Property(typed, property), typeof(object)), entity).Compile(); + var setter = Expression.Lambda>( + Expression.Assign(Expression.Property(typed, property), Expression.Convert(value, property.PropertyType)), entity, value).Compile(); + return new ColumnMap(property, getter, setter); + } + + private void Build() + { + _ = Table; + var properties = typeof(T).GetProperties(BindingFlags.Public | BindingFlags.Instance) + .Where(p => p.CanRead && p.CanWrite && p.GetSetMethod() is not null && p.GetIndexParameters().Length == 0) + .ToList(); + var key = new List(1); + var regular = new List(); + var audit = new List(); + + foreach (var property in properties) + { + if (_ignored.Contains(property.Name)) + { + continue; + } + + var column = _configured.TryGetValue(property.Name, out var configured) ? configured : CreateColumn(property); + var role = AuditRoleOf(property.Name); + if (property.Name == _keyProperty) + { + column.IsKey = true; + column.IsIdentity = _keyIdentity ?? (column.ValueType == typeof(long) || column.ValueType == typeof(int)); + Infer(column, property, forceNotNull: true); + key.Add(column); + } + else if (role != AuditRole.None) + { + column.Audit = role; + ApplyAuditDefaults(column, role); + audit.Add(column); + } + else + { + if (property.Name == nameof(IDemoMarkable.IsDemo) && typeof(IDemoMarkable).IsAssignableFrom(typeof(T))) + { + column.SetDefaults(ColumnType.Bool, null, nullable: false); + } + + Infer(column, property, forceNotNull: false); + regular.Add(column); + } + } + + if (key.Count == 0) + { + throw new InvalidOperationException($"{typeof(T).Name} has no key property {_keyProperty}."); + } + + var unknown = _configured.Keys.Except(properties.Select(p => p.Name)).ToList(); + if (unknown.Count > 0) + { + throw new InvalidOperationException($"{typeof(T).Name}: cannot map {string.Join(", ", unknown)} (needs a public getter and setter)."); + } + + audit.Sort((a, b) => a.Audit.CompareTo(b.Audit)); + var columns = key.Concat(regular).Concat(audit).ToList(); + var duplicate = columns.GroupBy(c => c.ColumnName).FirstOrDefault(g => g.Count() > 1); + if (duplicate is not null) + { + throw new InvalidOperationException($"{typeof(T).Name}: column {duplicate.Key} is mapped twice."); + } + + foreach (var column in columns) + { + column.Freeze(); + } + + _key = key[0]; + _rowVersion = audit.Find(c => c.Audit == AuditRole.RowVersion); + _columns = columns; + } + + private static void Infer(ColumnMap column, PropertyInfo property, bool forceNotNull) + { + var nullable = !forceNotNull && IsNullable(property); + if (column.ValueType.IsEnum) + { + column.SetDefaults(ColumnType.String, EnumLength, nullable); + return; + } + + var type = ColumnTypes.FromClrType(column.ValueType) + ?? throw new InvalidOperationException( + $"{typeof(T).Name}.{property.Name}: type {column.ValueType.Name} has no column type; map or ignore it."); + column.SetDefaults(type, null, nullable); + if (column.Type == ColumnType.String && column.Length is null) + { + throw new InvalidOperationException($"{typeof(T).Name}.{property.Name}: a String column needs MaxLength."); + } + } + + private static bool IsNullable(PropertyInfo property) + { + if (property.PropertyType.IsValueType) + { + return Nullable.GetUnderlyingType(property.PropertyType) is not null; + } + + lock (NullabilityContext) + { + return NullabilityContext.Create(property).WriteState != NullabilityState.NotNull; + } + } + + private static AuditRole AuditRoleOf(string propertyName) + { + if (!typeof(IAuditable).IsAssignableFrom(typeof(T))) + { + return AuditRole.None; + } + + return propertyName switch + { + nameof(IAuditable.CreatedAt) => AuditRole.CreatedAt, + nameof(IAuditable.CreatedBy) => AuditRole.CreatedBy, + nameof(IAuditable.UpdatedAt) => AuditRole.UpdatedAt, + nameof(IAuditable.UpdatedBy) => AuditRole.UpdatedBy, + nameof(IAuditable.RowVersion) => AuditRole.RowVersion, + _ => AuditRole.None, + }; + } + + private static void ApplyAuditDefaults(ColumnMap column, AuditRole role) + { + switch (role) + { + case AuditRole.CreatedAt: + column.SetDefaults(ColumnType.DateTimeUtc, null, nullable: false); + break; + case AuditRole.UpdatedAt: + column.SetDefaults(ColumnType.DateTimeUtc, null, nullable: true); + break; + case AuditRole.CreatedBy or AuditRole.UpdatedBy: + column.SetDefaults(ColumnType.String, AuditColumnLengths.User, nullable: true); + break; + case AuditRole.RowVersion: + column.SetDefaults(ColumnType.Int64, null, nullable: false); + break; + } + } +} diff --git a/src/Khaemhat.Data/Mapping/MapRegistry.cs b/src/Khaemhat.Data/Mapping/MapRegistry.cs new file mode 100644 index 0000000..7b3e6cc --- /dev/null +++ b/src/Khaemhat.Data/Mapping/MapRegistry.cs @@ -0,0 +1,62 @@ +using System.Collections.Concurrent; +using System.Reflection; + +namespace Khaemhat.Data.Mapping; + +/// +/// The entity maps known to the instance (SPEC §5.2). Maps are completed and frozen when added; afterwards the data +/// layer only reads them. +/// +public sealed class MapRegistry +{ + private readonly ConcurrentDictionary _maps = new(); + + /// All registered maps. + public IReadOnlyCollection Maps => [.. _maps.Values.OrderBy(m => m.Table, StringComparer.Ordinal)]; + + /// Adds a map. + /// The map. + /// This registry. + /// The entity already has a map, or the map is not valid. + public MapRegistry Add(IEntityMap map) + { + ArgumentNullException.ThrowIfNull(map); + map.Freeze(); + if (!_maps.TryAdd(map.EntityType, map)) + { + throw new InvalidOperationException($"{map.EntityType.Name} has two maps."); + } + + return this; + } + + /// Adds every non-abstract with a public parameterless constructor found in an assembly. + /// The assembly. + /// This registry. + public MapRegistry AddFromAssembly(Assembly assembly) + { + ArgumentNullException.ThrowIfNull(assembly); + var types = assembly.GetTypes() + .Where(t => t is { IsClass: true, IsAbstract: false } && typeof(IEntityMap).IsAssignableFrom(t) && t.GetConstructor(Type.EmptyTypes) is not null) + .OrderBy(t => t.FullName, StringComparer.Ordinal); + foreach (var type in types) + { + Add((IEntityMap)Activator.CreateInstance(type)!); + } + + return this; + } + + /// The map of . + /// The entity type. + /// The map. + /// No map is registered. + public EntityMap Get() + where T : class, new() => + (EntityMap)(Find(typeof(T)) ?? throw new InvalidOperationException($"{typeof(T).Name} has no entity map.")); + + /// The map of a type, if any. + /// The entity type. + /// The map, or . + public IEntityMap? Find(Type entityType) => _maps.GetValueOrDefault(entityType); +} diff --git a/src/Khaemhat.Data/Migrations/DataModelDocument.cs b/src/Khaemhat.Data/Migrations/DataModelDocument.cs new file mode 100644 index 0000000..93e5ab5 --- /dev/null +++ b/src/Khaemhat.Data/Migrations/DataModelDocument.cs @@ -0,0 +1,141 @@ +using System.Globalization; +using System.Text; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.Migrations; + +/// +/// Renders the schema produced by the migrations as Markdown for docs/DATA-MODEL.md (SPEC §18), between two +/// markers, so the document can never drift from the code. Used by scripts/gen-datamodel.sh and checked by a +/// test. The text is Italian because it is documentation. +/// +public static class DataModelDocument +{ + /// The line that opens the generated section. + public const string BeginMarker = ""; + + /// The line that closes the generated section. + public const string EndMarker = ""; + + /// Renders both catalogues. + /// The system database migrations. + /// The company database migrations. + /// The Markdown, without the markers, ending with a newline. + public static string Render(MigrationCatalog system, MigrationCatalog company) + { + ArgumentNullException.ThrowIfNull(system); + ArgumentNullException.ThrowIfNull(company); + var text = new StringBuilder(); + RenderCatalog(text, "Database di sistema", system); + text.Append('\n'); + RenderCatalog(text, "Database aziendale", company); + return text.ToString(); + } + + /// Replaces the generated section of a document. + /// The current document. + /// The output of . + /// The updated document. + /// The markers are missing or out of order. + public static string Update(string document, string generated) + { + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(generated); + var begin = document.IndexOf(BeginMarker, StringComparison.Ordinal); + var end = document.IndexOf(EndMarker, StringComparison.Ordinal); + if (begin < 0 || end < begin) + { + throw new InvalidOperationException($"The document must contain '{BeginMarker}' followed by '{EndMarker}'."); + } + + var start = begin + BeginMarker.Length; + return document[..start] + "\n\n" + generated + "\n" + document[end..]; + } + + private static void RenderCatalog(StringBuilder text, string title, MigrationCatalog catalog) + { + var model = catalog.BuildModel(); + Line(text, $"### {title}"); + text.Append('\n'); + if (catalog.Migrations.Count == 0) + { + Line(text, "Nessuna migrazione: la tabella `schema_version` è l'unica presente."); + } + else + { + Line(text, "Migrazioni applicate in ordine:"); + text.Append('\n'); + foreach (var migration in catalog.Migrations) + { + Line(text, $"- `{migration.Module}` {migration.Version} `{migration.Name}` ({migration.Type.Name})"); + } + } + + foreach (var table in model.Tables.OrderBy(t => t.Name, StringComparer.Ordinal)) + { + text.Append('\n'); + RenderTable(text, table); + } + } + + private static void RenderTable(StringBuilder text, TableDefinition table) + { + Line(text, $"#### `{table.Name}`"); + text.Append('\n'); + Line(text, "| Colonna | Tipo | Null | Default | Note |"); + Line(text, "|---|---|---|---|---|"); + foreach (var column in table.Columns) + { + var notes = new List(); + if (table.PrimaryKey.Contains(column.Name)) + { + notes.Add("chiave primaria"); + } + + if (column.IsIdentity) + { + notes.Add("identity"); + } + + notes.AddRange(table.ForeignKeys + .Where(fk => fk.Columns.Contains(column.Name)) + .Select(fk => $"-> `{fk.ReferencedTable}` ({fk.OnDelete})")); + Line(text, $"| `{column.Name}` | {column.TypeDescription} | {(column.IsNullable ? "sì" : "no")} | {Default(column)} | {string.Join(", ", notes)} |"); + } + + if (table.PrimaryKey.Count > 1) + { + text.Append('\n'); + Line(text, $"Chiave primaria: {Columns(table.PrimaryKey)}."); + } + + if (table.Indexes.Count > 0) + { + text.Append('\n'); + foreach (var index in table.Indexes.OrderBy(i => i.Name, StringComparer.Ordinal)) + { + Line(text, $"- indice {(index.IsUnique ? "unico " : string.Empty)}`{index.Name}` su {Columns(index.Columns)}"); + } + } + + if (table.PartitionByYearColumn is { } partition) + { + text.Append('\n'); + Line(text, $"Partizionata per anno su `{partition}` (solo PostgreSQL)."); + } + } + + private static string Default(ColumnDefinition column) => column switch + { + { DefaultsToCurrentTimestamp: true } => "ora corrente (UTC)", + { DefaultValue: null } => string.Empty, + { DefaultValue: string s } => $"`'{s}'`", + { DefaultValue: bool b } => b ? "`true`" : "`false`", + { DefaultValue: IFormattable f } => $"`{f.ToString(null, CultureInfo.InvariantCulture)}`", + _ => $"`{column.DefaultValue}`", + }; + + private static string Columns(IEnumerable names) => string.Join(", ", names.Select(n => $"`{n}`")); + + private static void Line(StringBuilder text, string line) => text.Append(line).Append('\n'); +} diff --git a/src/Khaemhat.Data/Migrations/IMigration.cs b/src/Khaemhat.Data/Migrations/IMigration.cs new file mode 100644 index 0000000..8887b55 --- /dev/null +++ b/src/Khaemhat.Data/Migrations/IMigration.cs @@ -0,0 +1,56 @@ +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.Migrations; + +/// +/// A schema migration (SPEC §5.3). Describe the change in neutral form in ; add dialect-specific SQL in +/// only for what the neutral form cannot express. Once released, a migration never changes: write a +/// new one. Create the file with scripts/new-migration.sh. +/// +/// [Migration(20261111_002, "create_sys_settings", Target = MigrationTarget.System)] +/// public sealed class M20261111_002_CreateSysSettings : IMigration +/// { +/// public void Up(ISchemaBuilder schema) => schema.CreateTable("sys_settings", t => { … }); +/// } +/// +/// +public interface IMigration +{ + /// Describes the schema change. + /// The schema builder. + void Up(ISchemaBuilder schema); + + /// Dialect-specific SQL run after , or . + /// The dialect of the database being migrated. + /// Trusted SQL (several statements separated by semicolons are allowed), or . + string? UpSql(ISqlDialect dialect) => null; +} + +/// Which catalog a migration belongs to. +public enum MigrationTarget +{ + /// Every company database (khaemhat_<code>). + Company = 0, + + /// The system database (khaemhat_system). + System = 1, +} + +/// Marks an with its version and name. +/// The version yyyymmdd_nnn written as a number, for example 20261111_001. +/// The name in snake_case, for example create_sys_companies. +[AttributeUsage(AttributeTargets.Class, AllowMultiple = false, Inherited = false)] +public sealed class MigrationAttribute(long version, string name) : Attribute +{ + /// The version yyyymmddnnn. + public long Version { get; } = version; + + /// The name. + public string Name { get; } = name; + + /// The catalog (default ). + public MigrationTarget Target { get; init; } = MigrationTarget.Company; + + /// The module that owns the migration, as stored in schema_version.module (default core). + public string Module { get; init; } = "core"; +} diff --git a/src/Khaemhat.Data/Migrations/MigrationCatalog.cs b/src/Khaemhat.Data/Migrations/MigrationCatalog.cs new file mode 100644 index 0000000..227ba19 --- /dev/null +++ b/src/Khaemhat.Data/Migrations/MigrationCatalog.cs @@ -0,0 +1,127 @@ +using System.Globalization; +using System.Reflection; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.Migrations; + +/// A migration found in a catalog. +/// The version yyyymmddnnn. +/// The name. +/// The owning module. +/// The catalog. +/// The migration class. +public sealed record MigrationInfo(long Version, string Name, string Module, MigrationTarget Target, Type Type) +{ + /// Creates the migration. + /// The migration. + public IMigration Create() => (IMigration)Activator.CreateInstance(Type)!; + + /// Records the operations of the migration. + /// The operations of . + public IReadOnlyList Operations() + { + var builder = new SchemaBuilder(); + Create().Up(builder); + return builder.Operations; + } + + /// + public override string ToString() => string.Create(CultureInfo.InvariantCulture, $"{Module}/{Version} {Name}"); +} + +/// +/// The migrations of one database kind (SPEC §5.3): for the system database, +/// for every company. Built by scanning assemblies for . +/// +public sealed class MigrationCatalog +{ + private MigrationCatalog(MigrationTarget target, IReadOnlyList migrations) + { + Target = target; + Migrations = migrations; + } + + /// The catalog kind. + public MigrationTarget Target { get; } + + /// The migrations, ordered by version then module. + public IReadOnlyList Migrations { get; } + + /// The system catalog with the migrations of this assembly and of . + /// Further assemblies with system migrations (security, licences, logs...). + /// The catalog. + public static MigrationCatalog ForSystem(params Assembly[] assemblies) => + Discover(MigrationTarget.System, [typeof(MigrationCatalog).Assembly, .. assemblies]); + + /// The company catalog with the migrations of . + /// Assemblies with company migrations (the application modules). + /// The catalog. + public static MigrationCatalog ForCompany(params Assembly[] assemblies) => + Discover(MigrationTarget.Company, [typeof(MigrationCatalog).Assembly, .. assemblies]); + + /// Finds the migrations of in assemblies and validates them. + /// The catalog kind. + /// The assemblies. + /// The catalog. + /// A migration is not valid or a version is used twice in a module. + public static MigrationCatalog Discover(MigrationTarget target, IEnumerable assemblies) + { + ArgumentNullException.ThrowIfNull(assemblies); + var migrations = new List(); + foreach (var type in assemblies.Distinct().SelectMany(a => a.GetTypes())) + { + if (type.GetCustomAttribute() is not { } attribute || attribute.Target != target) + { + continue; + } + + if (!typeof(IMigration).IsAssignableFrom(type) || type.IsAbstract || type.GetConstructor(Type.EmptyTypes) is null) + { + throw new InvalidOperationException($"{type.Name} has [Migration] but is not a concrete IMigration with a parameterless constructor."); + } + + Validate(type, attribute); + migrations.Add(new MigrationInfo(attribute.Version, attribute.Name, attribute.Module, target, type)); + } + + var duplicate = migrations.GroupBy(m => (m.Module, m.Version)).FirstOrDefault(g => g.Count() > 1); + if (duplicate is not null) + { + throw new InvalidOperationException( + $"Version {duplicate.Key.Version} of module {duplicate.Key.Module} is used by {string.Join(", ", duplicate.Select(m => m.Type.Name))}."); + } + + return new MigrationCatalog( + target, [.. migrations.OrderBy(m => m.Version).ThenBy(m => m.Module, StringComparer.Ordinal)]); + } + + /// The schema produced by all migrations from an empty database, plus schema_version. + /// The schema model. + public SchemaModel BuildModel() + { + var model = new SchemaModel(); + model.Apply([SchemaVersionStore.CreateOperation]); + foreach (var migration in Migrations) + { + model.Apply(migration.Operations()); + } + + return model; + } + + private static void Validate(Type type, MigrationAttribute attribute) + { + var text = attribute.Version.ToString(CultureInfo.InvariantCulture); + if (text.Length != 11 + || !DateOnly.TryParseExact(text[..8], "yyyyMMdd", CultureInfo.InvariantCulture, DateTimeStyles.None, out _) + || text[8..] == "000") + { + throw new InvalidOperationException($"{type.Name}: the version must be yyyymmdd_nnn (for example 20261111_001), not {text}."); + } + + if (!SqlNames.IsValidIdentifier(attribute.Name) || !SqlNames.IsValidIdentifier(attribute.Module) || attribute.Module.Length > 50) + { + throw new InvalidOperationException($"{type.Name}: name and module must be snake_case."); + } + } +} diff --git a/src/Khaemhat.Data/Migrations/MigrationRunner.cs b/src/Khaemhat.Data/Migrations/MigrationRunner.cs new file mode 100644 index 0000000..941f80a --- /dev/null +++ b/src/Khaemhat.Data/Migrations/MigrationRunner.cs @@ -0,0 +1,155 @@ +using System.Diagnostics; +using System.Security.Cryptography; +using System.Text; +using Khaemhat.Data.Execution; +using Khaemhat.Data.Mapping; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.Migrations; + +/// The outcome of a migration run, for the log section Migrazioni (written by the caller). +/// The database name. +/// The migrations applied now, with their duration. +/// How many catalog migrations were already applied. +/// +/// Applied migrations whose statements differ from what the code generates now (a changed migration or DDL generator). +/// +public sealed record MigrationReport( + string Database, IReadOnlyList<(MigrationInfo Migration, TimeSpan Duration)> Applied, int AlreadyApplied, IReadOnlyList Warnings); + +/// A database cannot be migrated, for example because it is newer than the application. +public sealed class MigrationException : Exception +{ + /// Creates the exception. + public MigrationException() + { + } + + /// Creates the exception with a message. + /// The message. + public MigrationException(string message) : base(message) + { + } + + /// Creates the exception with a message and a cause. + /// The message. + /// The cause. + public MigrationException(string message, Exception innerException) : base(message, innerException) + { + } +} + +/// +/// Applies the pending migrations of a catalog to a database (SPEC §5.3): each in its own transaction together with its +/// schema_version row, under an application lock so that two instances never migrate at the same time. +/// Re-running does nothing. A database with applied versions unknown to the catalog (a downgrade) is refused. +/// +/// Clock and retry policy. +public sealed class MigrationRunner(DataAccessContext context) +{ + private const string LockKey = "khaemhat:migrations"; + private static readonly MapRegistry NoMaps = new(); + + /// Migrates a database. + /// The database. + /// The catalog for this kind of database. + /// Cancellation. + /// What was done. + /// The database has versions unknown to this application. + public async Task RunAsync(DatabaseTarget target, MigrationCatalog catalog, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(target); + ArgumentNullException.ThrowIfNull(catalog); + await using var unitOfWork = await UnitOfWork.OpenAsync(target, NoMaps, context, cancellationToken).ConfigureAwait(false); + + await unitOfWork.InTransactionAsync(ct => SchemaVersionStore.EnsureCreatedAsync(unitOfWork, ct), cancellationToken).ConfigureAwait(false); + var applied = await SchemaVersionStore.ReadAsync(unitOfWork, cancellationToken).ConfigureAwait(false); + + var known = catalog.Migrations.Select(m => (m.Module, m.Version)).ToHashSet(); + var unknown = applied.Where(a => !known.Contains((a.Module, a.Version))).ToList(); + if (unknown.Count > 0) + { + throw new MigrationException( + $"Database {target.DatabaseName} has migrations this version of Khaemhat does not know (" + + string.Join(", ", unknown.Select(u => $"{u.Module}/{u.Version} {u.Name}")) + + "): it was used by a newer version. Downgrades are not supported; restore a backup or install the newer version."); + } + + var appliedByKey = applied.ToDictionary(a => (a.Module, a.Version)); + var done = new List<(MigrationInfo, TimeSpan)>(); + var warnings = new List(); + var alreadyApplied = 0; + foreach (var migration in catalog.Migrations) + { + var statements = Statements(migration, unitOfWork.Provider); + var checksum = Checksum(statements); + if (appliedByKey.TryGetValue((migration.Module, migration.Version), out var row)) + { + alreadyApplied++; + if (!string.Equals(row.Checksum, checksum, StringComparison.Ordinal)) + { + warnings.Add($"{migration}: the statements differ from those applied on {row.AppliedAt:yyyy-MM-dd}."); + } + + continue; + } + + var watch = Stopwatch.StartNew(); + var appliedNow = false; + await unitOfWork.InTransactionAsync( + async ct => + { + await unitOfWork.AcquireLockAsync(LockKey, ct).ConfigureAwait(false); + if (await SchemaVersionStore.IsAppliedAsync(unitOfWork, migration.Module, migration.Version, ct).ConfigureAwait(false)) + { + return; // another instance applied it meanwhile + } + + foreach (var statement in statements) + { + await unitOfWork.ExecuteAsync(SqlFragment.Literal(statement), ct).ConfigureAwait(false); + } + + await SchemaVersionStore.RecordAsync( + unitOfWork, new AppliedMigration(migration.Module, migration.Version, migration.Name, context.UtcNow, checksum), ct) + .ConfigureAwait(false); + appliedNow = true; + }, + cancellationToken).ConfigureAwait(false); + + if (appliedNow) + { + done.Add((migration, watch.Elapsed)); + } + else + { + alreadyApplied++; + } + } + + return new MigrationReport(target.DatabaseName, done, alreadyApplied, warnings); + } + + /// The statements a migration runs on a provider. + /// The migration. + /// The provider. + /// The statements. + public static IReadOnlyList Statements(MigrationInfo migration, IDbProvider provider) + { + ArgumentNullException.ThrowIfNull(migration); + ArgumentNullException.ThrowIfNull(provider); + var instance = migration.Create(); + var builder = new SchemaBuilder(); + instance.Up(builder); + var statements = provider.Ddl.Generate(builder.Operations).ToList(); + if (instance.UpSql(provider.Dialect) is { Length: > 0 } sql) + { + statements.Add(sql); + } + + return statements; + } + + private static string Checksum(IEnumerable statements) => + Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(string.Join(";\n", statements)))); +} diff --git a/src/Khaemhat.Data/Migrations/SchemaVersionStore.cs b/src/Khaemhat.Data/Migrations/SchemaVersionStore.cs new file mode 100644 index 0000000..45f3090 --- /dev/null +++ b/src/Khaemhat.Data/Migrations/SchemaVersionStore.cs @@ -0,0 +1,95 @@ +using Khaemhat.Data.Execution; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.Migrations; + +/// A row of schema_version. +/// The owning module. +/// The version. +/// The name. +/// When it was applied (UTC). +/// SHA-256 of the statements that were run, in hexadecimal. +public sealed record AppliedMigration(string Module, long Version, string Name, DateTime AppliedAt, string Checksum); + +/// +/// The schema_version(module, version, name, applied_at, checksum) table present in every database (SPEC §5.3). +/// +public static class SchemaVersionStore +{ + /// The table name. + public const string Table = "schema_version"; + + /// The operation creating the table if it is missing. + public static CreateTableOperation CreateOperation { get; } = BuildCreateOperation(); + + /// Creates the table if it is missing. + /// The unit of work. + /// Cancellation. + /// A task. + public static async Task EnsureCreatedAsync(UnitOfWork unitOfWork, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(unitOfWork); + foreach (var statement in unitOfWork.Provider.Ddl.Generate([CreateOperation])) + { + await unitOfWork.ExecuteAsync(SqlFragment.Literal(statement), cancellationToken).ConfigureAwait(false); + } + } + + /// All applied migrations. + /// The unit of work. + /// Cancellation. + /// The rows, ordered by version. + public static Task> ReadAsync(UnitOfWork unitOfWork, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(unitOfWork); + return unitOfWork.QueryAsync( + SqlFragment.Literal("SELECT module, version, name, applied_at, checksum FROM ") + SqlFragment.Identifier(Table) + + SqlFragment.Literal(" ORDER BY version, module"), + cancellationToken); + } + + /// Whether a version is applied (read inside the migration transaction). + /// The unit of work. + /// The module. + /// The version. + /// Cancellation. + /// if applied. + public static async Task IsAppliedAsync(UnitOfWork unitOfWork, string module, long version, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(unitOfWork); + var count = await unitOfWork.QuerySingleAsync( + SqlFragment.Of($"SELECT COUNT(*) FROM {SqlFragment.Identifier(Table)} WHERE module = {module} AND version = {version}"), + cancellationToken).ConfigureAwait(false); + return count > 0; + } + + /// Records an applied migration. + /// The unit of work, inside the migration transaction. + /// The row. + /// Cancellation. + /// A task. + public static Task RecordAsync(UnitOfWork unitOfWork, AppliedMigration applied, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(unitOfWork); + ArgumentNullException.ThrowIfNull(applied); + return unitOfWork.ExecuteAsync( + SqlFragment.Of( + $"INSERT INTO {SqlFragment.Identifier(Table)} (module, version, name, applied_at, checksum) VALUES ({applied.Module}, {applied.Version}, {applied.Name}, {applied.AppliedAt}, {applied.Checksum})"), + cancellationToken); + } + + private static CreateTableOperation BuildCreateOperation() + { + var builder = new SchemaBuilder(); + builder.CreateTable(Table, t => + { + t.String("module", 50).NotNull(); + t.Int64("version").NotNull(); + t.String("name", 200).NotNull(); + t.DateTimeUtc("applied_at").NotNull(); + t.String("checksum", 64).NotNull(); + t.PrimaryKey("module", "version"); + }); + return ((CreateTableOperation)builder.Operations[0]) with { IfNotExists = true }; + } +} diff --git a/src/Khaemhat.Data/PagedResult.cs b/src/Khaemhat.Data/PagedResult.cs new file mode 100644 index 0000000..7172c25 --- /dev/null +++ b/src/Khaemhat.Data/PagedResult.cs @@ -0,0 +1,13 @@ +namespace Khaemhat.Data; + +/// A page of results with the total number of matching rows. +/// The item type. +/// The rows of the page. +/// All rows matching the filters. +/// Rows skipped. +/// Page size, or when not paged. +public sealed record PagedResult(IReadOnlyList Items, long TotalCount, int Offset, int? Limit) +{ + /// Whether more rows follow this page. + public bool HasMore => Offset + Items.Count < TotalCount; +} diff --git a/src/Khaemhat.Data/Querying/ExpressionTranslator.cs b/src/Khaemhat.Data/Querying/ExpressionTranslator.cs new file mode 100644 index 0000000..0f2368b --- /dev/null +++ b/src/Khaemhat.Data/Querying/ExpressionTranslator.cs @@ -0,0 +1,307 @@ +using System.Collections; +using System.Linq.Expressions; +using System.Reflection; +using Khaemhat.Data.Mapping; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.Querying; + +/// +/// Translates predicates of the safe subset of SPEC §5.2 into parameterized SQL. Every value — constants, captured +/// variables, list items — becomes a parameter; only mapped column names reach the SQL text. Anything outside the +/// subset throws : use a condition instead. +/// +/// The map of the queried entity. +/// The dialect, for the case-insensitive LIKE. +public sealed class ExpressionTranslator(IEntityMap map, ISqlDialect dialect) +{ + private static readonly SqlFragment True = SqlFragment.Literal("1 = 1"); + private static readonly SqlFragment False = SqlFragment.Literal("1 = 0"); + + /// Translates a predicate. + /// A lambda with one parameter of the entity type returning . + /// The condition. + public SqlFragment Translate(LambdaExpression predicate) + { + ArgumentNullException.ThrowIfNull(predicate); + return Condition(predicate.Body, predicate.Parameters[0]); + } + + /// The column of a property selector such as x => x.Name. + /// The selector. + /// The column. + public ColumnMap Column(LambdaExpression selector) + { + ArgumentNullException.ThrowIfNull(selector); + return TryColumn(selector.Body, selector.Parameters[0]) + ?? throw new NotSupportedException($"Only mapped properties of {map.EntityType.Name} can be used here: {selector}."); + } + + private SqlFragment Condition(Expression node, ParameterExpression entity) + { + switch (node) + { + case BinaryExpression { NodeType: ExpressionType.AndAlso } and: + return (Condition(and.Left, entity) + SqlFragment.Literal(" AND ") + Condition(and.Right, entity)).Parenthesize(); + case BinaryExpression { NodeType: ExpressionType.OrElse } or: + return (Condition(or.Left, entity) + SqlFragment.Literal(" OR ") + Condition(or.Right, entity)).Parenthesize(); + case UnaryExpression { NodeType: ExpressionType.Not } not: + return SqlFragment.Literal("NOT ") + Condition(not.Operand, entity).Parenthesize(); + case BinaryExpression binary when Operator(binary.NodeType) is not null: + return Comparison(binary, entity); + case MethodCallExpression call: + return Method(call, entity); + case ConstantExpression { Value: bool constant }: + return constant ? True : False; + default: + if (node.Type == typeof(bool) && TryColumn(node, entity) is { } flag) + { + return Name(flag) + SqlFragment.Literal(" = ") + Value(flag, true); + } + + if (node.Type == typeof(bool) && !References(node, entity)) + { + return (bool)Evaluate(node)! ? True : False; + } + + throw Unsupported(node); + } + } + + private SqlFragment Comparison(BinaryExpression binary, ParameterExpression entity) + { + var op = Operator(binary.NodeType)!; + var leftColumn = TryColumn(binary.Left, entity); + var rightColumn = TryColumn(binary.Right, entity); + + if (leftColumn is not null && rightColumn is not null) + { + return Name(leftColumn) + SqlFragment.Literal(" " + op + " ") + Name(rightColumn); + } + + if (leftColumn is null && rightColumn is not null) + { + // value op column -> column (mirrored op) value + (leftColumn, op) = (rightColumn, Mirror(op)); + return CompareWithValue(leftColumn, op, binary.Left, entity); + } + + return leftColumn is not null ? CompareWithValue(leftColumn, op, binary.Right, entity) : throw Unsupported(binary); + } + + private SqlFragment CompareWithValue(ColumnMap column, string op, Expression valueNode, ParameterExpression entity) + { + if (References(valueNode, entity)) + { + throw Unsupported(valueNode); + } + + var value = Evaluate(valueNode); + if (value is null) + { + return op switch + { + "=" => Name(column) + SqlFragment.Literal(" IS NULL"), + "<>" => Name(column) + SqlFragment.Literal(" IS NOT NULL"), + _ => throw new NotSupportedException($"Cannot compare {column.PropertyName} with null using {op}."), + }; + } + + return Name(column) + SqlFragment.Literal(" " + op + " ") + Value(column, value); + } + + private SqlFragment Method(MethodCallExpression call, ParameterExpression entity) + { + // column.Contains / StartsWith / EndsWith (value): case-insensitive LIKE + if (call.Method.DeclaringType == typeof(string) && call.Object is not null && call.Arguments.Count >= 1 + && call.Method.Name is nameof(string.Contains) or nameof(string.StartsWith) or nameof(string.EndsWith) + && TryColumn(call.Object, entity) is { } textColumn) + { + if (References(call.Arguments[0], entity)) + { + throw Unsupported(call); + } + + var text = Evaluate(call.Arguments[0]) as string ?? string.Empty; + var escaped = EscapeLike(text); + var pattern = call.Method.Name switch + { + nameof(string.StartsWith) => escaped + "%", + nameof(string.EndsWith) => "%" + escaped, + _ => "%" + escaped + "%", + }; + return Like(textColumn, pattern); + } + + // string.IsNullOrEmpty(column) + if (call.Method.DeclaringType == typeof(string) && call.Method.Name == nameof(string.IsNullOrEmpty) + && TryColumn(call.Arguments[0], entity) is { } emptyColumn) + { + return (Name(emptyColumn) + SqlFragment.Literal(" IS NULL OR ") + Name(emptyColumn) + SqlFragment.Literal(" = ") + + SqlFragment.Parameter(string.Empty, emptyColumn.Type)).Parenthesize(); + } + + // values.Contains(column) -> column IN (...) + Expression? collection = null; + Expression? item = null; + if (call.Method.Name == nameof(Enumerable.Contains)) + { + if (call.Object is null && call.Arguments.Count == 2) + { + (collection, item) = (call.Arguments[0], call.Arguments[1]); + } + else if (call.Object is not null && call.Arguments.Count == 1) + { + (collection, item) = (call.Object, call.Arguments[0]); + } + } + + if (collection is not null && item is not null && TryColumn(item, entity) is { } inColumn && !References(collection, entity)) + { + collection = UnwrapSpan(collection); + var values = (Evaluate(collection) as IEnumerable)?.Cast().ToList() ?? []; + if (values.Count == 0) + { + return False; + } + + var nonNull = values.Where(v => v is not null).Select(v => Value(inColumn, v!)).ToList(); + var condition = nonNull.Count == 0 + ? SqlFragment.Empty + : Name(inColumn) + SqlFragment.Literal(" IN (") + SqlFragment.Join(", ", nonNull) + SqlFragment.Literal(")"); + if (values.Count != nonNull.Count) + { + var isNull = Name(inColumn) + SqlFragment.Literal(" IS NULL"); + condition = condition.IsEmpty ? isNull : (condition + SqlFragment.Literal(" OR ") + isNull).Parenthesize(); + } + + return condition; + } + + if (call.Type == typeof(bool) && !References(call, entity)) + { + return (bool)Evaluate(call)! ? True : False; + } + + throw Unsupported(call); + } + + /// + /// C# 14 binds array.Contains(x) to MemoryExtensions.Contains(ReadOnlySpan<T>, T) through an + /// implicit conversion; spans cannot be evaluated by the expression interpreter, so read the array behind it. + /// + private static Expression UnwrapSpan(Expression node) + { + while (IsSpan(node.Type)) + { + switch (node) + { + case MethodCallExpression { Method.Name: "op_Implicit", Arguments.Count: 1 } conversion: + node = conversion.Arguments[0]; + break; + case UnaryExpression { NodeType: ExpressionType.Convert } convert: + node = convert.Operand; + break; + default: + return node; + } + } + + return node; + + static bool IsSpan(Type type) => + type.IsGenericType && (type.GetGenericTypeDefinition() == typeof(ReadOnlySpan<>) || type.GetGenericTypeDefinition() == typeof(Span<>)); + } + + private SqlFragment Like(ColumnMap column, string pattern) => + dialect.CaseInsensitiveLike(Name(column), SqlFragment.Parameter(pattern, ColumnType.Text)); + + private ColumnMap? TryColumn(Expression node, ParameterExpression entity) + { + while (node is UnaryExpression { NodeType: ExpressionType.Convert or ExpressionType.ConvertChecked } convert) + { + node = convert.Operand; + } + + return node is MemberExpression { Member: PropertyInfo property } member && member.Expression == entity + ? map.FindByProperty(property.Name) ?? throw new NotSupportedException($"{map.EntityType.Name}.{property.Name} is not mapped.") + : null; + } + + private static SqlFragment Name(ColumnMap column) => SqlFragment.Identifier(column.ColumnName); + + private static SqlFragment Value(ColumnMap column, object value) + { + if (column.ValueType.IsEnum && value is not Enum) + { + value = Enum.ToObject(column.ValueType, value); + } + + return SqlFragment.Parameter(column.ToProvider(value), column.Type); + } + + private static string? Operator(ExpressionType type) => type switch + { + ExpressionType.Equal => "=", + ExpressionType.NotEqual => "<>", + ExpressionType.LessThan => "<", + ExpressionType.LessThanOrEqual => "<=", + ExpressionType.GreaterThan => ">", + ExpressionType.GreaterThanOrEqual => ">=", + _ => null, + }; + + private static string Mirror(string op) => op switch + { + "<" => ">", + "<=" => ">=", + ">" => "<", + ">=" => "<=", + _ => op, + }; + + private static string EscapeLike(string text) => + text.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("%", "\\%", StringComparison.Ordinal).Replace("_", "\\_", StringComparison.Ordinal); + + private static bool References(Expression node, ParameterExpression entity) + { + var finder = new ParameterFinder(entity); + finder.Visit(node); + return finder.Found; + } + + private static object? Evaluate(Expression node) + { + switch (node) + { + case ConstantExpression constant: + return constant.Value; + case MemberExpression { Member: FieldInfo field } member when member.Expression is null or ConstantExpression: + return field.GetValue((member.Expression as ConstantExpression)?.Value); + case MemberExpression { Member: PropertyInfo property } member when member.Expression is null or ConstantExpression: + return property.GetValue((member.Expression as ConstantExpression)?.Value); + case MemberExpression { Member: FieldInfo field } member: + return field.GetValue(Evaluate(member.Expression!)); + case MemberExpression { Member: PropertyInfo property } member: + return property.GetValue(Evaluate(member.Expression!)); + case UnaryExpression { NodeType: ExpressionType.Convert } convert when convert.Type == typeof(object): + return Evaluate(convert.Operand); + default: + return Expression.Lambda>(Expression.Convert(node, typeof(object))).Compile(preferInterpretation: true)(); + } + } + + private NotSupportedException Unsupported(Expression node) => + new($"The expression '{node}' is outside the supported query subset for {map.EntityType.Name}; use a SqlFragment condition."); + + private sealed class ParameterFinder(ParameterExpression parameter) : ExpressionVisitor + { + public bool Found { get; private set; } + + protected override Expression VisitParameter(ParameterExpression node) + { + Found |= node == parameter; + return node; + } + } +} diff --git a/src/Khaemhat.Data/Querying/Query.cs b/src/Khaemhat.Data/Querying/Query.cs new file mode 100644 index 0000000..cfe1917 --- /dev/null +++ b/src/Khaemhat.Data/Querying/Query.cs @@ -0,0 +1,109 @@ +using System.Collections.Immutable; +using System.Linq.Expressions; + +namespace Khaemhat.Data.Querying; + +/// Starts queries: Query.For<Party>().Where(p => p.Name.Contains(text)).OrderBy(p => p.Name).Page(0, 50). +public static class Query +{ + /// A query over all rows of . + /// The entity type. + /// The query. + public static Query For() + where T : class => Query.All; +} + +/// +/// An immutable query over the table of (SPEC §5.2): filters in the safe expression subset +/// (equality, comparisons, Contains/StartsWith/EndsWith, In as list.Contains(x.Prop), +/// &&, ||, !, == null), parameterized conditions, ordering and +/// paging. Each method returns a new query. +/// +/// The entity type. +public sealed class Query + where T : class +{ + private Query( + ImmutableList filters, ImmutableList conditions, ImmutableList order, int offset, int? limit) + { + Filters = filters; + Conditions = conditions; + Order = order; + Offset = offset; + Limit = limit; + } + + internal static Query All { get; } = new([], [], [], 0, null); + + /// The expression filters, combined with AND. + public ImmutableList Filters { get; } + + /// The SQL conditions, combined with AND. + public ImmutableList Conditions { get; } + + /// The ordering, in priority order. + public ImmutableList Order { get; } + + /// Rows to skip. + public int Offset { get; } + + /// Rows to return at most, or for all. + public int? Limit { get; } + + /// Adds a filter. + /// The predicate, in the supported subset. + /// The new query. + public Query Where(Expression> predicate) + { + ArgumentNullException.ThrowIfNull(predicate); + return new(Filters.Add(predicate), Conditions, Order, Offset, Limit); + } + + /// Adds a parameterized SQL condition, for what expressions cannot say. + /// The condition, for example SqlFragment.Of($"{SqlFragment.Identifier("total")} > {min}"). + /// The new query. + public Query Where(SqlFragment condition) + { + ArgumentNullException.ThrowIfNull(condition); + return condition.IsEmpty ? this : new(Filters, Conditions.Add(condition), Order, Offset, Limit); + } + + /// Orders by a property, ascending; further calls add secondary orderings. + /// The property type. + /// The property. + /// The new query. + public Query OrderBy(Expression> property) => AddOrder(property, descending: false); + + /// Orders by a property, descending; further calls add secondary orderings. + /// The property type. + /// The property. + /// The new query. + public Query OrderByDescending(Expression> property) => AddOrder(property, descending: true); + + /// Returns at most rows after skipping . + /// Rows to skip. + /// Rows to return. + /// The new query. + public Query Page(int offset, int limit) + { + ArgumentOutOfRangeException.ThrowIfNegative(offset); + ArgumentOutOfRangeException.ThrowIfNegative(limit); + return new(Filters, Conditions, Order, offset, limit); + } + + /// Returns at most rows. + /// Rows to return. + /// The new query. + public Query Take(int limit) => Page(Offset, limit); + + private Query AddOrder(LambdaExpression property, bool descending) + { + ArgumentNullException.ThrowIfNull(property); + return new(Filters, Conditions, Order.Add(new QueryOrder(property, descending)), Offset, Limit); + } +} + +/// An ordering of a query. +/// The property selector. +/// Whether the order is descending. +public sealed record QueryOrder(LambdaExpression Property, bool Descending); diff --git a/src/Khaemhat.Data/Querying/QueryBuilder.cs b/src/Khaemhat.Data/Querying/QueryBuilder.cs new file mode 100644 index 0000000..c5c3e82 --- /dev/null +++ b/src/Khaemhat.Data/Querying/QueryBuilder.cs @@ -0,0 +1,75 @@ +using Khaemhat.Data.Mapping; + +namespace Khaemhat.Data.Querying; + +/// Turns a into parameterized SELECT and COUNT statements for one dialect. +/// The entity type. +/// The map of the entity. +/// The dialect. +public sealed class QueryBuilder(EntityMap map, ISqlDialect dialect) + where T : class, new() +{ + private readonly ExpressionTranslator _translator = new(map, dialect); + + /// The map. + public EntityMap Map { get; } = map; + + /// The list of mapped columns, in map order. + public SqlFragment ColumnList => SqlFragment.Join(", ", Map.Columns.Select(c => SqlFragment.Identifier(c.ColumnName))); + + /// The WHERE clause (with leading space), or empty when the query has no filters. + /// The query. + /// The clause. + public SqlFragment Where(Query query) + { + ArgumentNullException.ThrowIfNull(query); + var conditions = query.Filters.Select(_translator.Translate).Concat(query.Conditions.Select(c => c.Parenthesize())).ToList(); + return conditions.Count == 0 ? SqlFragment.Empty : SqlFragment.Literal(" WHERE ") + SqlFragment.Join(" AND ", conditions); + } + + /// + /// The SELECT statement with ordering and paging. A paged query without ordering is ordered by the key, + /// so pages are stable. + /// + /// The query. + /// The statement. + public SqlFragment Select(Query query) + { + ArgumentNullException.ThrowIfNull(query); + var sql = SqlFragment.Literal("SELECT ") + ColumnList + SqlFragment.Literal(" FROM ") + SqlFragment.Identifier(Map.Table) + Where(query); + var order = query.Order + .Select(o => SqlFragment.Identifier(_translator.Column(o.Property).ColumnName) + SqlFragment.Literal(o.Descending ? " DESC" : " ASC")) + .ToList(); + var paged = query.Limit is not null || query.Offset > 0; + if (order.Count == 0 && paged) + { + order.Add(SqlFragment.Identifier(Map.Key.ColumnName) + SqlFragment.Literal(" ASC")); + } + + if (order.Count > 0) + { + sql += SqlFragment.Literal(" ORDER BY ") + SqlFragment.Join(", ", order); + } + + if (paged) + { + sql += SqlFragment.Literal(dialect.Paging(query.Offset, query.Limit ?? int.MaxValue)); + } + + return sql; + } + + /// The COUNT(*) statement for the filters of the query (ordering and paging ignored). + /// The query. + /// The statement. + public SqlFragment Count(Query query) => + SqlFragment.Literal("SELECT COUNT(*) FROM ") + SqlFragment.Identifier(Map.Table) + Where(query); + + /// The SELECT of one row by key. + /// The key value. + /// The statement. + public SqlFragment SelectByKey(object key) => + SqlFragment.Literal("SELECT ") + ColumnList + SqlFragment.Literal(" FROM ") + SqlFragment.Identifier(Map.Table) + + SqlFragment.Literal(" WHERE ") + SqlFragment.Identifier(Map.Key.ColumnName) + SqlFragment.Literal(" = ") + + SqlFragment.Parameter(Map.Key.ToProvider(key), Map.Key.Type); +} diff --git a/src/Khaemhat.Data/Schema/ColumnType.cs b/src/Khaemhat.Data/Schema/ColumnType.cs new file mode 100644 index 0000000..b499418 --- /dev/null +++ b/src/Khaemhat.Data/Schema/ColumnType.cs @@ -0,0 +1,107 @@ +namespace Khaemhat.Data.Schema; + +/// +/// Logical column types (SPEC §5.2), mapped by each dialect to a physical type: +/// PostgreSQL varchar(n)/text/int/bigint/numeric(19,6)/boolean/date/timestamptz/uuid/bytea/jsonb; +/// SQLite TEXT/TEXT/INTEGER/INTEGER/INTEGER x10⁶/INTEGER 0-1/TEXT yyyy-MM-dd/TEXT ISO 8601 UTC/TEXT/BLOB/TEXT. +/// +public enum ColumnType +{ + /// Text with a maximum length n (). + String = 0, + + /// Text without a length limit (). + Text = 1, + + /// 32-bit integer (). + Int32 = 2, + + /// 64-bit integer (): primary and foreign keys. + Int64 = 3, + + /// Exact decimal with 19 digits, 6 after the point (): amounts and quantities. + Decimal = 4, + + /// Boolean (). + Bool = 5, + + /// Calendar date without time (). + Date = 6, + + /// Instant in UTC ( with ). + DateTimeUtc = 7, + + /// GUID (). + Guid = 8, + + /// Binary data (byte[]). + Bytes = 9, + + /// JSON document stored as text (). + Json = 10, +} + +/// Helpers on . +public static class ColumnTypes +{ + /// The fixed precision of . + public const int DecimalPrecision = 19; + + /// The default scale of . + public const int DecimalScale = 6; + + /// + /// The logical type for a CLR type, or when there is none (the value is then passed to the + /// driver as it is). maps to ; enums are not mapped here + /// (entity maps store them by name). + /// + /// The CLR type, nullable or not. + /// The logical type, if any. + public static ColumnType? FromClrType(Type type) + { + ArgumentNullException.ThrowIfNull(type); + type = Nullable.GetUnderlyingType(type) ?? type; + return type switch + { + _ when type == typeof(string) => ColumnType.Text, + _ when type == typeof(int) || type == typeof(short) || type == typeof(byte) => ColumnType.Int32, + _ when type == typeof(long) => ColumnType.Int64, + _ when type == typeof(decimal) => ColumnType.Decimal, + _ when type == typeof(bool) => ColumnType.Bool, + _ when type == typeof(DateOnly) => ColumnType.Date, + _ when type == typeof(DateTime) || type == typeof(DateTimeOffset) => ColumnType.DateTimeUtc, + _ when type == typeof(Guid) => ColumnType.Guid, + _ when type == typeof(byte[]) => ColumnType.Bytes, + _ => null, + }; + } + + /// The canonical CLR type the dialects return for a logical type. + /// The logical type. + /// The CLR type. + public static Type ClrType(ColumnType type) => type switch + { + ColumnType.String or ColumnType.Text or ColumnType.Json => typeof(string), + ColumnType.Int32 => typeof(int), + ColumnType.Int64 => typeof(long), + ColumnType.Decimal => typeof(decimal), + ColumnType.Bool => typeof(bool), + ColumnType.Date => typeof(DateOnly), + ColumnType.DateTimeUtc => typeof(DateTime), + ColumnType.Guid => typeof(Guid), + ColumnType.Bytes => typeof(byte[]), + _ => throw new ArgumentOutOfRangeException(nameof(type), type, null), + }; + + /// The logical type as written in the documentation, for example String(20) or Decimal(19,6). + /// The logical type. + /// The length for . + /// The scale for . + /// The description. + public static string Describe(ColumnType type, int? length, int? scale) => type switch + { + ColumnType.String => FormattableString.Invariant($"String({length})"), + ColumnType.Decimal => FormattableString.Invariant($"Decimal({DecimalPrecision},{scale ?? DecimalScale})"), + _ => type.ToString(), + }; +} diff --git a/src/Khaemhat.Data/Schema/DdlGenerator.cs b/src/Khaemhat.Data/Schema/DdlGenerator.cs new file mode 100644 index 0000000..9ce5a0b --- /dev/null +++ b/src/Khaemhat.Data/Schema/DdlGenerator.cs @@ -0,0 +1,144 @@ +using System.Text; + +namespace Khaemhat.Data.Schema; + +/// +/// Turns recorded s into DDL statements for one engine (SPEC §5.3, DdlGenerator). +/// The common SQL lives here; providers override what differs (identity columns, table options, constraints). +/// +/// The dialect of the engine. +public abstract class DdlGenerator(ISqlDialect dialect) +{ + /// The dialect. + public ISqlDialect Dialect { get; } = dialect ?? throw new ArgumentNullException(nameof(dialect)); + + /// Generates the statements for , in order. + /// The operations. + /// The statements, without trailing semicolons. + public IReadOnlyList Generate(IEnumerable operations) + { + ArgumentNullException.ThrowIfNull(operations); + var statements = new List(); + foreach (var operation in operations) + { + switch (operation) + { + case CreateTableOperation create: + statements.Add(CreateTable(create.Table, create.IfNotExists)); + statements.AddRange(create.Table.Indexes.Select(i => CreateIndex(i, create.IfNotExists))); + break; + case DropTableOperation drop: + statements.Add("DROP TABLE " + Dialect.Quote(drop.Table)); + break; + case AddColumnOperation add: + statements.Add("ALTER TABLE " + Dialect.Quote(add.Table) + " ADD COLUMN " + Column(add.Column, inlinePrimaryKey: false)); + break; + case CreateIndexOperation index: + statements.Add(CreateIndex(index.Index, ifNotExists: false)); + break; + case DropIndexOperation dropIndex: + statements.Add("DROP INDEX " + Dialect.Quote(dropIndex.Name)); + break; + case RawSqlOperation raw when raw.Sql(Dialect) is { Length: > 0 } sql: + statements.Add(sql); + break; + } + } + + return statements; + } + + /// The CREATE TABLE statement. + /// The table. + /// Whether to add IF NOT EXISTS. + /// The statement. + protected virtual string CreateTable(TableDefinition table, bool ifNotExists) + { + ArgumentNullException.ThrowIfNull(table); + var identityKey = table.PrimaryKey.Count == 1 && table.Column(table.PrimaryKey[0])?.IsIdentity == true; + var lines = table.Columns.Select(c => Column(c, inlinePrimaryKey: identityKey && c.Name == table.PrimaryKey[0])).ToList(); + if (!identityKey && table.PrimaryKey.Count > 0) + { + lines.Add("CONSTRAINT " + Dialect.Quote("pk_" + table.Name) + " PRIMARY KEY (" + string.Join(", ", table.PrimaryKey.Select(Dialect.Quote)) + ")"); + } + + foreach (var foreignKey in table.ForeignKeys) + { + lines.Add( + "CONSTRAINT " + Dialect.Quote(foreignKey.Name) + " FOREIGN KEY (" + string.Join(", ", foreignKey.Columns.Select(Dialect.Quote)) + + ") REFERENCES " + Dialect.Quote(foreignKey.ReferencedTable) + " (" + string.Join(", ", foreignKey.ReferencedColumns.Select(Dialect.Quote)) + + ") ON DELETE " + foreignKey.OnDelete switch + { + ReferentialAction.Cascade => "CASCADE", + ReferentialAction.SetNull => "SET NULL", + _ => "RESTRICT", + }); + } + + var builder = new StringBuilder("CREATE TABLE "); + if (ifNotExists) + { + builder.Append("IF NOT EXISTS "); + } + + builder.Append(Dialect.Quote(table.Name)).Append(" (\n ").AppendJoin(",\n ", lines).Append("\n)").Append(TableOptions(table)); + return builder.ToString(); + } + + /// Text appended after the closing parenthesis of CREATE TABLE (SQLite: STRICT). + /// The table. + /// The options, possibly empty. + protected virtual string TableOptions(TableDefinition table) => string.Empty; + + /// A column definition. + /// The column. + /// Whether the column is the identity primary key. + /// The SQL. + protected virtual string Column(ColumnDefinition column, bool inlinePrimaryKey) + { + ArgumentNullException.ThrowIfNull(column); + if (inlinePrimaryKey) + { + return IdentityPrimaryKey(column); + } + + var builder = new StringBuilder(Dialect.Quote(column.Name)).Append(' ').Append(Dialect.SqlType(column.Type, column.Length, column.Scale)); + if (!column.IsNullable) + { + builder.Append(" NOT NULL"); + } + + if (column.DefaultsToCurrentTimestamp) + { + builder.Append(" DEFAULT (").Append(Dialect.CurrentTimestampUtc).Append(')'); + } + else if (column.DefaultValue is not null) + { + builder.Append(" DEFAULT ").Append(Dialect.Literal(column.DefaultValue, column.Type)); + } + + builder.Append(ColumnConstraints(column)); + return builder.ToString(); + } + + /// Extra constraints after a column, for example SQLite checks for lengths and booleans. + /// The column. + /// The constraints, possibly empty. + protected virtual string ColumnConstraints(ColumnDefinition column) => string.Empty; + + /// The definition of an identity primary key column. + /// The column. + /// The SQL. + protected abstract string IdentityPrimaryKey(ColumnDefinition column); + + /// The CREATE INDEX statement. + /// The index. + /// Whether to add IF NOT EXISTS. + /// The statement. + protected virtual string CreateIndex(IndexDefinition index, bool ifNotExists) + { + ArgumentNullException.ThrowIfNull(index); + return "CREATE " + (index.IsUnique ? "UNIQUE " : string.Empty) + "INDEX " + (ifNotExists ? "IF NOT EXISTS " : string.Empty) + + Dialect.Quote(index.Name) + " ON " + Dialect.Quote(index.Table) + " (" + string.Join(", ", index.Columns.Select(Dialect.Quote)) + ")"; + } +} diff --git a/src/Khaemhat.Data/Schema/Definitions.cs b/src/Khaemhat.Data/Schema/Definitions.cs new file mode 100644 index 0000000..68efae2 --- /dev/null +++ b/src/Khaemhat.Data/Schema/Definitions.cs @@ -0,0 +1,136 @@ +namespace Khaemhat.Data.Schema; + +/// A column of a table, in engine-neutral form. +public sealed class ColumnDefinition +{ + /// Creates a column definition. + /// The column name (snake_case). + /// The logical type. + public ColumnDefinition(string name, ColumnType type) + { + SqlNames.EnsureIdentifier(name); + Name = name; + Type = type; + } + + /// The column name. + public string Name { get; } + + /// The logical type. + public ColumnType Type { get; } + + /// The maximum length, for . + public int? Length { get; init; } + + /// The scale, for (default 6). + public int? Scale { get; init; } + + /// Whether the column accepts NULL. + public bool IsNullable { get; init; } = true; + + /// Whether the column is an identity primary key (id). + public bool IsIdentity { get; init; } + + /// A constant default value written as an SQL literal, or . + public object? DefaultValue { get; init; } + + /// Whether the default is the current UTC timestamp. + public bool DefaultsToCurrentTimestamp { get; init; } + + /// The logical type as documented, for example String(20). + public string TypeDescription => ColumnTypes.Describe(Type, Length, Scale); +} + +/// An index. +/// The index name (for example ix_parties_name or ux_parties_code). +/// The table. +/// The indexed columns, in order. +/// Whether the index is unique. +public sealed record IndexDefinition(string Name, string Table, IReadOnlyList Columns, bool IsUnique); + +/// What happens to referencing rows when the referenced row is deleted. +public enum ReferentialAction +{ + /// The delete is refused while referencing rows exist (default). + Restrict = 0, + + /// Referencing rows are deleted too. + Cascade = 1, + + /// The referencing column is set to NULL. + SetNull = 2, +} + +/// A foreign key. +/// The constraint name (for example fk_party_addresses_party). +/// The referencing columns. +/// The referenced table. +/// The referenced columns (usually id). +/// The action on delete. +public sealed record ForeignKeyDefinition( + string Name, IReadOnlyList Columns, string ReferencedTable, IReadOnlyList ReferencedColumns, + ReferentialAction OnDelete = ReferentialAction.Restrict); + +/// A table, in engine-neutral form. +public sealed class TableDefinition +{ + private readonly List _columns = []; + private readonly List _indexes = []; + private readonly List _foreignKeys = []; + + /// Creates a table definition. + /// The table name (snake_case). + public TableDefinition(string name) + { + SqlNames.EnsureIdentifier(name); + Name = name; + } + + /// The table name. + public string Name { get; } + + /// The columns in order. + public IReadOnlyList Columns => _columns; + + /// The primary key columns (usually id). + public IReadOnlyList PrimaryKey { get; internal set; } = []; + + /// The indexes. + public IReadOnlyList Indexes => _indexes; + + /// The foreign keys. + public IReadOnlyList ForeignKeys => _foreignKeys; + + /// + /// The date column by whose year PostgreSQL may partition the table (SPEC §5.2, large tables); recorded now, + /// applied by a later migration with PostgresPartitioning. + /// + public string? PartitionByYearColumn { get; internal set; } + + /// Finds a column by name. + /// The column name. + /// The column, or . + public ColumnDefinition? Column(string name) => _columns.Find(c => c.Name == name); + + internal void AddColumn(ColumnDefinition column) + { + if (Column(column.Name) is not null) + { + throw new InvalidOperationException($"Column {column.Name} is defined twice in table {Name}."); + } + + _columns.Add(column); + } + + internal void ReplaceColumn(ColumnDefinition column) + { + var index = _columns.FindIndex(c => c.Name == column.Name); + _columns[index] = column; + } + + internal void AddIndex(IndexDefinition index) => _indexes.Add(index); + + internal void RemoveIndex(string name) => _indexes.RemoveAll(i => i.Name == name); + + internal void AddForeignKey(ForeignKeyDefinition foreignKey) => _foreignKeys.Add(foreignKey); +} diff --git a/src/Khaemhat.Data/Schema/ISchemaBuilder.cs b/src/Khaemhat.Data/Schema/ISchemaBuilder.cs new file mode 100644 index 0000000..bec308b --- /dev/null +++ b/src/Khaemhat.Data/Schema/ISchemaBuilder.cs @@ -0,0 +1,53 @@ +namespace Khaemhat.Data.Schema; + +/// +/// What a migration uses to describe schema changes in engine-neutral form (SPEC §5.3). The calls are recorded as +/// s; each provider's turns them into DDL for its engine. +/// +/// schema.CreateTable("sys_settings", t => +/// { +/// t.Id(); +/// t.String("key", 200).NotNull(); +/// t.Text("value"); +/// t.AuditColumns(); +/// t.Unique("ux_sys_settings_key", "key"); +/// }); +/// +/// +public interface ISchemaBuilder +{ + /// Creates a table. + /// The table name. + /// Defines columns, keys, indexes and foreign keys. + void CreateTable(string name, Action define); + + /// + /// Changes an existing table: adds columns and indexes. Foreign keys can only be declared when the table is + /// created (SQLite cannot add them later). + /// + /// The table name. + /// Adds columns and indexes. + void AlterTable(string name, Action define); + + /// Drops a table. + /// The table name. + void DropTable(string name); + + /// Creates an index. + /// The table. + /// The index name. + /// Whether it is unique. + /// The columns. + void CreateIndex(string table, string name, bool unique, params string[] columns); + + /// Drops an index. + /// The index name. + void DropIndex(string name); + + /// + /// Adds a dialect-specific statement at this point, for what the neutral operations cannot express. The function + /// returns for dialects where nothing is needed. + /// + /// Produces the trusted SQL for a dialect. + void Sql(Func sql); +} diff --git a/src/Khaemhat.Data/Schema/SchemaBuilder.cs b/src/Khaemhat.Data/Schema/SchemaBuilder.cs new file mode 100644 index 0000000..3d546ea --- /dev/null +++ b/src/Khaemhat.Data/Schema/SchemaBuilder.cs @@ -0,0 +1,53 @@ +namespace Khaemhat.Data.Schema; + +/// Records the calls of a migration as s. +public sealed class SchemaBuilder : ISchemaBuilder +{ + private readonly List _operations = []; + + /// The recorded operations, in order. + public IReadOnlyList Operations => _operations; + + /// + public void CreateTable(string name, Action define) + { + ArgumentNullException.ThrowIfNull(define); + var builder = new TableBuilder(name, alter: false); + define(builder); + _operations.AddRange(builder.Build()); + } + + /// + public void AlterTable(string name, Action define) + { + ArgumentNullException.ThrowIfNull(define); + var builder = new TableBuilder(name, alter: true); + define(builder); + _operations.AddRange(builder.Build()); + } + + /// + public void DropTable(string name) + { + SqlNames.EnsureIdentifier(name); + _operations.Add(new DropTableOperation(name)); + } + + /// + public void CreateIndex(string table, string name, bool unique, params string[] columns) => + AlterTable(table, t => _ = unique ? t.Unique(name, columns) : t.Index(name, columns)); + + /// + public void DropIndex(string name) + { + SqlNames.EnsureIdentifier(name); + _operations.Add(new DropIndexOperation(name)); + } + + /// + public void Sql(Func sql) + { + ArgumentNullException.ThrowIfNull(sql); + _operations.Add(new RawSqlOperation(sql)); + } +} diff --git a/src/Khaemhat.Data/Schema/SchemaModel.cs b/src/Khaemhat.Data/Schema/SchemaModel.cs new file mode 100644 index 0000000..808defd --- /dev/null +++ b/src/Khaemhat.Data/Schema/SchemaModel.cs @@ -0,0 +1,86 @@ +namespace Khaemhat.Data.Schema; + +/// +/// The schema that results from applying operations, kept in memory: used to document the data model +/// (scripts/gen-datamodel.sh) and to check that entity maps agree with the migrations. Dialect-specific +/// statements () are not modelled. +/// +public sealed class SchemaModel +{ + private readonly SortedDictionary _tables = new(StringComparer.Ordinal); + + /// The tables, by name. + public IReadOnlyCollection Tables => _tables.Values; + + /// Finds a table. + /// The table name. + /// The table, or . + public TableDefinition? Table(string name) => _tables.GetValueOrDefault(name); + + /// Applies operations in order. + /// The operations. + public void Apply(IEnumerable operations) + { + ArgumentNullException.ThrowIfNull(operations); + foreach (var operation in operations) + { + Apply(operation); + } + } + + private void Apply(SchemaOperation operation) + { + switch (operation) + { + case CreateTableOperation create when create.IfNotExists && _tables.ContainsKey(create.Table.Name): + break; + case CreateTableOperation create: + if (!_tables.TryAdd(create.Table.Name, Copy(create.Table))) + { + throw new InvalidOperationException($"Table {create.Table.Name} is created twice."); + } + + break; + case DropTableOperation drop: + _tables.Remove(drop.Table); + break; + case AddColumnOperation add: + Require(add.Table).AddColumn(add.Column); + break; + case CreateIndexOperation index: + Require(index.Index.Table).AddIndex(index.Index); + break; + case DropIndexOperation dropIndex: + foreach (var table in _tables.Values) + { + table.RemoveIndex(dropIndex.Name); + } + + break; + } + } + + private TableDefinition Require(string name) => + _tables.GetValueOrDefault(name) ?? throw new InvalidOperationException($"Table {name} does not exist."); + + private static TableDefinition Copy(TableDefinition source) + { + var copy = new TableDefinition(source.Name) { PrimaryKey = source.PrimaryKey, PartitionByYearColumn = source.PartitionByYearColumn }; + foreach (var column in source.Columns) + { + copy.AddColumn(column); + } + + foreach (var index in source.Indexes) + { + copy.AddIndex(index); + } + + foreach (var foreignKey in source.ForeignKeys) + { + copy.AddForeignKey(foreignKey); + } + + return copy; + } +} diff --git a/src/Khaemhat.Data/Schema/SchemaOperations.cs b/src/Khaemhat.Data/Schema/SchemaOperations.cs new file mode 100644 index 0000000..71eb310 --- /dev/null +++ b/src/Khaemhat.Data/Schema/SchemaOperations.cs @@ -0,0 +1,30 @@ +namespace Khaemhat.Data.Schema; + +/// A schema change recorded by . +public abstract record SchemaOperation; + +/// Creates a table. +/// The table. +/// Whether to skip the creation when the table exists (only for schema_version). +public sealed record CreateTableOperation(TableDefinition Table, bool IfNotExists = false) : SchemaOperation; + +/// Drops a table. +/// The table name. +public sealed record DropTableOperation(string Table) : SchemaOperation; + +/// Adds a column to an existing table. +/// The table name. +/// The column. +public sealed record AddColumnOperation(string Table, ColumnDefinition Column) : SchemaOperation; + +/// Creates an index. +/// The index. +public sealed record CreateIndexOperation(IndexDefinition Index) : SchemaOperation; + +/// Drops an index. +/// The index name. +public sealed record DropIndexOperation(string Name) : SchemaOperation; + +/// A dialect-specific statement. +/// Produces the SQL for a dialect, or . +public sealed record RawSqlOperation(Func Sql) : SchemaOperation; diff --git a/src/Khaemhat.Data/Schema/TableBuilder.cs b/src/Khaemhat.Data/Schema/TableBuilder.cs new file mode 100644 index 0000000..370ff84 --- /dev/null +++ b/src/Khaemhat.Data/Schema/TableBuilder.cs @@ -0,0 +1,347 @@ +namespace Khaemhat.Data.Schema; + +/// Defines the columns, keys, indexes and foreign keys of a table inside . +public sealed class TableBuilder +{ + private readonly List _columns = []; + private readonly List _indexes = []; + private readonly List _foreignKeys = []; + private readonly bool _alter; + private string[] _primaryKey = []; + private string? _partitionColumn; + + internal TableBuilder(string name, bool alter) + { + SqlNames.EnsureIdentifier(name); + Name = name; + _alter = alter; + } + + /// The table name. + public string Name { get; } + + /// The standard primary key: id, 64-bit identity. + /// This builder. + public TableBuilder Id() + { + var column = Add("id", ColumnType.Int64).NotNull(); + column.Identity = true; + _primaryKey = ["id"]; + return this; + } + + /// Adds a column of at most characters. + /// The column name. + /// The maximum length (1 to 10485760). + /// The column builder. + public ColumnBuilder String(string name, int length) + { + ArgumentOutOfRangeException.ThrowIfLessThan(length, 1); + ArgumentOutOfRangeException.ThrowIfGreaterThan(length, 10_485_760); + var column = Add(name, ColumnType.String); + column.Length = length; + return column; + } + + /// Adds a column. + /// The column name. + /// The column builder. + public ColumnBuilder Text(string name) => Add(name, ColumnType.Text); + + /// Adds an column. + /// The column name. + /// The column builder. + public ColumnBuilder Int32(string name) => Add(name, ColumnType.Int32); + + /// Adds an column (for example a foreign key party_id). + /// The column name. + /// The column builder. + public ColumnBuilder Int64(string name) => Add(name, ColumnType.Int64); + + /// Adds a column, precision 19. + /// The column name. + /// Digits after the point, 0 to 6. + /// The column builder. + public ColumnBuilder Decimal(string name, int scale = ColumnTypes.DecimalScale) + { + ArgumentOutOfRangeException.ThrowIfNegative(scale); + ArgumentOutOfRangeException.ThrowIfGreaterThan(scale, ColumnTypes.DecimalScale); + var column = Add(name, ColumnType.Decimal); + column.Scale = scale; + return column; + } + + /// Adds a column. + /// The column name. + /// The column builder. + public ColumnBuilder Bool(string name) => Add(name, ColumnType.Bool); + + /// Adds a column. + /// The column name. + /// The column builder. + public ColumnBuilder Date(string name) => Add(name, ColumnType.Date); + + /// Adds a column. + /// The column name. + /// The column builder. + public ColumnBuilder DateTimeUtc(string name) => Add(name, ColumnType.DateTimeUtc); + + /// Adds a column. + /// The column name. + /// The column builder. + public ColumnBuilder Guid(string name) => Add(name, ColumnType.Guid); + + /// Adds a column. + /// The column name. + /// The column builder. + public ColumnBuilder Bytes(string name) => Add(name, ColumnType.Bytes); + + /// Adds a column. + /// The column name. + /// The column builder. + public ColumnBuilder Json(string name) => Add(name, ColumnType.Json); + + /// + /// Adds the audit columns of SPEC §5.2: created_at, created_by, updated_at, updated_by, + /// row_version (default 1). + /// + /// This builder. + public TableBuilder AuditColumns() + { + DateTimeUtc("created_at").NotNull(); + String("created_by", AuditColumnLengths.User); + DateTimeUtc("updated_at"); + String("updated_by", AuditColumnLengths.User); + Int64("row_version").NotNull().Default(1L); + return this; + } + + /// Adds is_demo (boolean, default false) for rows created by the demo seeder (SPEC §14). + /// This builder. + public TableBuilder DemoColumn() + { + Bool("is_demo").NotNull().Default(false); + return this; + } + + /// Sets a primary key other than , for example a composite key. + /// The key columns. + /// This builder. + public TableBuilder PrimaryKey(params string[] columns) + { + EnsureCreate("A primary key"); + ArgumentNullException.ThrowIfNull(columns); + ArgumentOutOfRangeException.ThrowIfZero(columns.Length); + _primaryKey = columns; + return this; + } + + /// Adds a non-unique index. + /// The index name, for example ix_parties_name. + /// The columns. + /// This builder. + public TableBuilder Index(string name, params string[] columns) => AddIndex(name, false, columns); + + /// Adds a unique index. + /// The index name, for example ux_parties_code. + /// The columns. + /// This builder. + public TableBuilder Unique(string name, params string[] columns) => AddIndex(name, true, columns); + + /// Adds a foreign key from to the id of . + /// The constraint name, for example fk_party_addresses_party. + /// The referencing column. + /// The referenced table. + /// The action on delete. + /// This builder. + public TableBuilder ForeignKey(string name, string column, string referencedTable, ReferentialAction onDelete = ReferentialAction.Restrict) + { + EnsureCreate("A foreign key"); + SqlNames.EnsureIdentifier(name); + SqlNames.EnsureIdentifier(column); + SqlNames.EnsureIdentifier(referencedTable); + _foreignKeys.Add(new ForeignKeyDefinition(name, [column], referencedTable, ["id"], onDelete)); + return this; + } + + /// + /// Records that PostgreSQL may partition this table by the year of (SPEC §5.2). Nothing + /// changes now; a later migration activates it beyond 50 million rows. + /// + /// A or column. + /// This builder. + public TableBuilder PartitionByYear(string column) + { + SqlNames.EnsureIdentifier(column); + _partitionColumn = column; + return this; + } + + internal IEnumerable Build() + { + if (!_alter) + { + var table = new TableDefinition(Name) { PrimaryKey = _primaryKey, PartitionByYearColumn = _partitionColumn }; + foreach (var column in _columns) + { + table.AddColumn(column.ToDefinition()); + } + + foreach (var index in _indexes) + { + EnsureColumnsExist(table, index.Columns); + table.AddIndex(index); + } + + foreach (var foreignKey in _foreignKeys) + { + EnsureColumnsExist(table, foreignKey.Columns); + table.AddForeignKey(foreignKey); + } + + EnsureColumnsExist(table, table.PrimaryKey); + yield return new CreateTableOperation(table); + yield break; + } + + foreach (var column in _columns) + { + yield return new AddColumnOperation(Name, column.ToDefinition()); + } + + foreach (var index in _indexes) + { + yield return new CreateIndexOperation(index); + } + } + + private static void EnsureColumnsExist(TableDefinition table, IEnumerable columns) + { + foreach (var column in columns) + { + if (table.Column(column) is null) + { + throw new InvalidOperationException($"Table {table.Name} has no column {column}."); + } + } + } + + private ColumnBuilder Add(string name, ColumnType type) + { + if (_columns.Exists(c => c.Name == name)) + { + throw new InvalidOperationException($"Column {name} is defined twice in table {Name}."); + } + + var column = new ColumnBuilder(name, type); + _columns.Add(column); + return column; + } + + private TableBuilder AddIndex(string name, bool unique, string[] columns) + { + SqlNames.EnsureIdentifier(name); + ArgumentNullException.ThrowIfNull(columns); + ArgumentOutOfRangeException.ThrowIfZero(columns.Length); + foreach (var column in columns) + { + SqlNames.EnsureIdentifier(column); + } + + _indexes.Add(new IndexDefinition(name, Name, columns, unique)); + return this; + } + + private void EnsureCreate(string what) + { + if (_alter) + { + throw new NotSupportedException(what + " can only be declared when the table is created."); + } + } +} + +/// Configures a column inside . +public sealed class ColumnBuilder +{ + internal ColumnBuilder(string name, ColumnType type) + { + SqlNames.EnsureIdentifier(name); + Name = name; + Type = type; + } + + /// The column name. + public string Name { get; } + + /// The logical type. + public ColumnType Type { get; } + + internal int? Length { get; set; } + + internal int? Scale { get; set; } + + internal bool Identity { get; set; } + + private bool Nullable { get; set; } = true; + + private object? DefaultValue { get; set; } + + private bool DefaultTimestamp { get; set; } + + /// The column refuses NULL. + /// This builder. + public ColumnBuilder NotNull() + { + Nullable = false; + return this; + } + + /// The column accepts NULL (the default). + /// This builder. + public ColumnBuilder Null() + { + Nullable = true; + return this; + } + + /// A constant default value (string, number, boolean or decimal). + /// The value. + /// This builder. + public ColumnBuilder Default(object value) + { + ArgumentNullException.ThrowIfNull(value); + DefaultValue = value; + return this; + } + + /// The current UTC timestamp as default, for . + /// This builder. + public ColumnBuilder DefaultCurrentTimestamp() + { + if (Type != ColumnType.DateTimeUtc) + { + throw new InvalidOperationException("Only DateTimeUtc columns can default to the current timestamp."); + } + + DefaultTimestamp = true; + return this; + } + + internal ColumnDefinition ToDefinition() => new(Name, Type) + { + Length = Length, + Scale = Scale, + IsNullable = Nullable, + IsIdentity = Identity, + DefaultValue = DefaultValue, + DefaultsToCurrentTimestamp = DefaultTimestamp, + }; +} + +/// Lengths of the audit columns. +public static class AuditColumnLengths +{ + /// Length of created_by and updated_by. + public const int User = 100; +} diff --git a/src/Khaemhat.Data/SqlDialectBase.cs b/src/Khaemhat.Data/SqlDialectBase.cs new file mode 100644 index 0000000..2da8862 --- /dev/null +++ b/src/Khaemhat.Data/SqlDialectBase.cs @@ -0,0 +1,130 @@ +using System.Data.Common; +using System.Globalization; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data; + +/// +/// The parts of that SQLite (3.35 or later) and PostgreSQL write the same way: double-quoted +/// identifiers, @name parameters, LIMIT/OFFSET, RETURNING and ON CONFLICT … DO UPDATE. +/// +public abstract class SqlDialectBase : ISqlDialect +{ + /// + public abstract DbEngine Engine { get; } + + /// + public abstract string CurrentTimestampUtc { get; } + + /// + public string Quote(string identifier) + { + SqlNames.EnsureIdentifier(identifier); + return "\"" + identifier + "\""; + } + + /// + public string Parameter(string name) => "@" + name; + + /// + public string Paging(int offset, int limit) + { + ArgumentOutOfRangeException.ThrowIfNegative(offset); + ArgumentOutOfRangeException.ThrowIfNegative(limit); + return string.Create(CultureInfo.InvariantCulture, $" LIMIT {limit} OFFSET {offset}"); + } + + /// + public string InsertReturningId(string table, IReadOnlyList columns, IReadOnlyList parameters, string keyColumn) => + Insert(table, columns, parameters) + " RETURNING " + Quote(keyColumn); + + /// + public string Upsert( + string table, IReadOnlyList columns, IReadOnlyList parameters, + IReadOnlyList conflictColumns, IReadOnlyList updateColumns) + { + ArgumentNullException.ThrowIfNull(conflictColumns); + ArgumentNullException.ThrowIfNull(updateColumns); + var conflict = string.Join(", ", conflictColumns.Select(Quote)); + var action = updateColumns.Count == 0 + ? "DO NOTHING" + : "DO UPDATE SET " + string.Join(", ", updateColumns.Select(c => Quote(c) + " = excluded." + Quote(c))); + return Insert(table, columns, parameters) + " ON CONFLICT (" + conflict + ") " + action; + } + + /// + public abstract SqlFragment CaseInsensitiveLike(SqlFragment column, SqlFragment pattern); + + /// + public abstract SqlFragment AdvisoryLock(string key); + + /// + public abstract object ToDbValue(object? value, ColumnType type); + + /// + public abstract object? FromDbValue(object? value, ColumnType type); + + /// + public abstract string SqlType(ColumnType type, int? length, int? scale); + + /// + public virtual string Literal(object? value, ColumnType type) => value switch + { + null => "NULL", + string s => "'" + s.Replace("'", "''", StringComparison.Ordinal) + "'", + Enum e => "'" + e.ToString() + "'", + IFormattable f when type is ColumnType.Int32 or ColumnType.Int64 => f.ToString(null, CultureInfo.InvariantCulture), + _ => throw new NotSupportedException($"No SQL literal for a {value.GetType().Name} default of a {type} column."), + }; + + /// + public virtual void ConfigureParameter(DbParameter parameter, object? value, ColumnType? type) + { + ArgumentNullException.ThrowIfNull(parameter); + var logical = type ?? (value is null ? null : ColumnTypes.FromClrType(value.GetType())); + parameter.Value = logical is null ? value ?? DBNull.Value : ToDbValue(value, logical.Value); + } + + /// Converts a value to with invariant culture, as drivers may widen numbers. + /// The target type. + /// The value. + /// The converted value. + protected static T ConvertTo(object value) => (T)Convert.ChangeType(value, typeof(T), CultureInfo.InvariantCulture); + + /// Converts a or to UTC. An unspecified kind is taken as UTC. + /// The value. + /// The UTC instant. + protected static DateTime ToUtc(object value) => value switch + { + DateTime { Kind: DateTimeKind.Utc } d => d, + DateTime { Kind: DateTimeKind.Local } d => d.ToUniversalTime(), + DateTime d => DateTime.SpecifyKind(d, DateTimeKind.Utc), + DateTimeOffset o => o.UtcDateTime, + _ => throw new InvalidCastException($"A {value.GetType().Name} is not a timestamp."), + }; + + /// Converts a value to . + /// A , or yyyy-MM-dd text. + /// The date. + protected static DateOnly ToDate(object value) => value switch + { + DateOnly d => d, + DateTime d => DateOnly.FromDateTime(d), + string s => DateOnly.ParseExact(s, "yyyy-MM-dd", CultureInfo.InvariantCulture), + _ => throw new InvalidCastException($"A {value.GetType().Name} is not a date."), + }; + + private string Insert(string table, IReadOnlyList columns, IReadOnlyList parameters) + { + ArgumentNullException.ThrowIfNull(columns); + ArgumentNullException.ThrowIfNull(parameters); + if (columns.Count != parameters.Count) + { + throw new ArgumentException("One parameter per column is required.", nameof(parameters)); + } + + return columns.Count == 0 + ? "INSERT INTO " + Quote(table) + " DEFAULT VALUES" + : "INSERT INTO " + Quote(table) + " (" + string.Join(", ", columns.Select(Quote)) + ") VALUES (" + string.Join(", ", parameters) + ")"; + } +} diff --git a/src/Khaemhat.Data/SqlFragment.cs b/src/Khaemhat.Data/SqlFragment.cs new file mode 100644 index 0000000..b9a9a09 --- /dev/null +++ b/src/Khaemhat.Data/SqlFragment.cs @@ -0,0 +1,249 @@ +using System.Collections.Immutable; +using System.Globalization; +using System.Runtime.CompilerServices; +using System.Text; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data; + +/// +/// A piece of SQL whose values are always parameters (SPEC §5.2: never concatenate user input into SQL). +/// Build it with an interpolated string: the literal text is code written by us, every hole becomes a parameter. +/// +/// var sql = SqlFragment.Of($"SELECT name FROM parties WHERE code = {code} AND is_active = {true}"); +/// +/// A hole can also hold another (inserted as SQL, its parameters renumbered) or an +/// identifier made with (quoted by the dialect). The text is produced only by +/// , for one dialect. +/// +public sealed class SqlFragment +{ + private readonly ImmutableArray _parts; + + private SqlFragment(ImmutableArray parts) => _parts = parts; + + /// The empty fragment. + public static SqlFragment Empty { get; } = new([]); + + /// Whether the fragment contains nothing. + public bool IsEmpty => _parts.IsEmpty; + + /// The number of parameters. + public int ParameterCount => _parts.Count(p => p is ParameterPart); + + internal ImmutableArray Parts => _parts; + + internal static SqlFragment FromParts(ImmutableArray parts) => parts.IsEmpty ? Empty : new(parts); + + /// Builds a fragment from an interpolated string: holes become parameters. + /// The interpolated string. + /// The fragment. + public static SqlFragment Of(ref SqlInterpolatedStringHandler handler) => handler.ToFragment(); + + /// + /// A fragment of trusted SQL text written in the code (keywords, operators, DDL generated from definitions). + /// Never pass text that comes from users, files or the network: use parameters for values. + /// + /// The SQL text. + /// The fragment. + public static SqlFragment Literal(string trustedSql) + { + ArgumentNullException.ThrowIfNull(trustedSql); + return trustedSql.Length == 0 ? Empty : new([new TextPart(trustedSql)]); + } + + /// A table or column name, quoted by the dialect when rendered. + /// The identifier: lower-case letters, digits and underscore, optionally table.column. + /// The fragment. + public static SqlFragment Identifier(string name) + { + SqlNames.EnsureQualifiedIdentifier(name); + return new([new IdentifierPart(name)]); + } + + /// A single parameter, optionally with its logical type (needed for example for ). + /// The value. + /// The logical type, or to infer it from the CLR type. + /// The fragment. + public static SqlFragment Parameter(object? value, ColumnType? type = null) => new([new ParameterPart(value, type)]); + + /// Joins fragments with a trusted separator such as ", " or " AND ". + /// The separator text. + /// The fragments; empty ones are skipped. + /// The joined fragment. + public static SqlFragment Join(string separator, IEnumerable fragments) + { + ArgumentNullException.ThrowIfNull(fragments); + var builder = ImmutableArray.CreateBuilder(); + var first = true; + foreach (var fragment in fragments) + { + if (fragment.IsEmpty) + { + continue; + } + + if (!first && separator.Length > 0) + { + builder.Add(new TextPart(separator)); + } + + builder.AddRange(fragment._parts); + first = false; + } + + return new(builder.ToImmutable()); + } + + /// Concatenates two fragments. + /// The first fragment. + /// The second fragment. + /// The concatenation. + public static SqlFragment operator +(SqlFragment left, SqlFragment right) + { + ArgumentNullException.ThrowIfNull(left); + ArgumentNullException.ThrowIfNull(right); + return left.Append(right); + } + + /// Appends another fragment. + /// The fragment to append. + /// The concatenation. + public SqlFragment Append(SqlFragment other) + { + ArgumentNullException.ThrowIfNull(other); + if (other.IsEmpty) + { + return this; + } + + return IsEmpty ? other : new(_parts.AddRange(other._parts)); + } + + /// Wraps the fragment in parentheses. + /// The wrapped fragment. + public SqlFragment Parenthesize() => IsEmpty ? this : Literal("(") + this + Literal(")"); + + /// Produces the SQL text and the parameters for a dialect. Parameters are named p0, p1, … + /// The dialect. + /// The text and the parameters. + public RenderedSql Render(ISqlDialect dialect) + { + ArgumentNullException.ThrowIfNull(dialect); + var text = new StringBuilder(); + var parameters = new List(); + foreach (var part in _parts) + { + switch (part) + { + case TextPart t: + text.Append(t.Text); + break; + case IdentifierPart i: + text.Append(QuoteQualified(dialect, i.Name)); + break; + case ParameterPart p: + var name = "p" + parameters.Count.ToString(CultureInfo.InvariantCulture); + text.Append(dialect.Parameter(name)); + parameters.Add(new SqlParameterValue(name, p.Value, p.Type)); + break; + } + } + + return new RenderedSql(text.ToString(), parameters); + } + + /// A readable form for diagnostics: parameters appear as @p0, never with their values. + /// The text. + public override string ToString() + { + var text = new StringBuilder(); + var index = 0; + foreach (var part in _parts) + { + text.Append(part switch + { + TextPart t => t.Text, + IdentifierPart i => "\"" + i.Name + "\"", + _ => "@p" + (index++).ToString(CultureInfo.InvariantCulture), + }); + } + + return text.ToString(); + } + + private static string QuoteQualified(ISqlDialect dialect, string name) + { + var dot = name.IndexOf('.', StringComparison.Ordinal); + return dot < 0 ? dialect.Quote(name) : dialect.Quote(name[..dot]) + "." + dialect.Quote(name[(dot + 1)..]); + } + + internal abstract record SqlPart; + + internal sealed record TextPart(string Text) : SqlPart; + + internal sealed record IdentifierPart(string Name) : SqlPart; + + internal sealed record ParameterPart(object? Value, ColumnType? Type) : SqlPart; +} + +/// SQL text and parameters produced by . +/// The SQL text. +/// The parameters in order. +public sealed record RenderedSql(string Text, IReadOnlyList Parameters); + +/// A parameter of a rendered statement. +/// The name without prefix, for example p0. +/// The CLR value. +/// The logical type, when known. +public sealed record SqlParameterValue(string Name, object? Value, ColumnType? Type) +{ + /// + public override string ToString() => "@" + Name; +} + +/// +/// Interpolated string handler for : literal parts are kept as SQL text, holes become +/// parameters (or nested fragments). +/// +[InterpolatedStringHandler] +public ref struct SqlInterpolatedStringHandler +{ + private readonly ImmutableArray.Builder _parts; + + /// Called by the compiler. + /// Total length of the literal parts. + /// Number of holes. + public SqlInterpolatedStringHandler(int literalLength, int formattedCount) + { + _ = literalLength; + _parts = ImmutableArray.CreateBuilder((formattedCount * 2) + 1); + } + + /// Called by the compiler for literal text. + /// The literal. + public readonly void AppendLiteral(string value) + { + if (value.Length > 0) + { + _parts.Add(new SqlFragment.TextPart(value)); + } + } + + /// Called by the compiler for a hole: a parameter, or a nested fragment. + /// The value type. + /// The value. + public readonly void AppendFormatted(T value) + { + if (value is SqlFragment fragment) + { + _parts.AddRange(fragment.Parts); + } + else + { + _parts.Add(new SqlFragment.ParameterPart(value, null)); + } + } + + internal readonly SqlFragment ToFragment() => SqlFragment.FromParts(_parts.ToImmutable()); +} diff --git a/src/Khaemhat.Data/SqlNames.cs b/src/Khaemhat.Data/SqlNames.cs new file mode 100644 index 0000000..3cdbdb4 --- /dev/null +++ b/src/Khaemhat.Data/SqlNames.cs @@ -0,0 +1,82 @@ +using System.Text; +using System.Text.RegularExpressions; + +namespace Khaemhat.Data; + +/// +/// Naming rules for tables, columns, indexes and constraints (SPEC §5.2): snake_case, lower-case ASCII letters, +/// digits and underscore, at most 63 characters (the PostgreSQL limit). Identifiers are validated before being quoted, +/// so they can never carry SQL. +/// +public static partial class SqlNames +{ + /// Whether is a valid identifier. + /// The name. + /// if valid. + public static bool IsValidIdentifier(string? name) => name is not null && IdentifierPattern().IsMatch(name); + + /// Throws if is not a valid identifier. + /// The name. + /// Not a valid identifier. + public static void EnsureIdentifier(string name) + { + if (!IsValidIdentifier(name)) + { + throw new ArgumentException( + $"'{name}' is not a valid SQL identifier: use lower-case letters, digits and underscore (snake_case).", nameof(name)); + } + } + + /// Throws if is neither an identifier nor table.column. + /// The name. + public static void EnsureQualifiedIdentifier(string name) + { + ArgumentNullException.ThrowIfNull(name); + var dot = name.IndexOf('.', StringComparison.Ordinal); + if (dot < 0) + { + EnsureIdentifier(name); + return; + } + + EnsureIdentifier(name[..dot]); + EnsureIdentifier(name[(dot + 1)..]); + } + + /// + /// Converts a C# name to snake_case: RowVersion → row_version, VatID → vat_id, + /// HTTPCode → http_code, Address2 → address2. + /// + /// The C# name. + /// The snake_case name. + public static string ToSnakeCase(string name) + { + ArgumentException.ThrowIfNullOrEmpty(name); + var builder = new StringBuilder(name.Length + 8); + for (var i = 0; i < name.Length; i++) + { + var c = name[i]; + if (char.IsUpper(c)) + { + var previous = i > 0 ? name[i - 1] : '\0'; + var next = i + 1 < name.Length ? name[i + 1] : '\0'; + var startsWord = i > 0 && (char.IsLower(previous) || char.IsDigit(previous) || (char.IsUpper(previous) && char.IsLower(next))); + if (startsWord && previous != '_') + { + builder.Append('_'); + } + + builder.Append(char.ToLowerInvariant(c)); + } + else + { + builder.Append(c); + } + } + + return builder.ToString(); + } + + [GeneratedRegex("^[a-z_][a-z0-9_]{0,62}$", RegexOptions.CultureInvariant)] + private static partial Regex IdentifierPattern(); +} diff --git a/src/Khaemhat.Data/SystemDatabase/Migrations/M20261111_001_CreateSysCompanies.cs b/src/Khaemhat.Data/SystemDatabase/Migrations/M20261111_001_CreateSysCompanies.cs new file mode 100644 index 0000000..cc5ed88 --- /dev/null +++ b/src/Khaemhat.Data/SystemDatabase/Migrations/M20261111_001_CreateSysCompanies.cs @@ -0,0 +1,28 @@ +using Khaemhat.Data.Migrations; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.SystemDatabase.Migrations; + +/// The company registry (SPEC §5.4): one row per company database, its location stored encrypted. +[Migration(20261111_001, "create_sys_companies", Target = MigrationTarget.System)] +public sealed class M20261111_001_CreateSysCompanies : IMigration +{ + /// + public void Up(ISchemaBuilder schema) + { + ArgumentNullException.ThrowIfNull(schema); + schema.CreateTable("sys_companies", t => + { + t.Id(); + t.String("code", 20).NotNull(); + t.String("name", 200).NotNull(); + t.String("engine", 20).NotNull(); + t.Text("connection_string").NotNull(); + t.String("status", 40).NotNull().Default("Active"); + t.Bool("is_default").NotNull().Default(false); + t.Text("notes"); + t.AuditColumns(); + t.Unique("ux_sys_companies_code", "code"); + }); + } +} diff --git a/src/Khaemhat.Data/SystemDatabase/Migrations/M20261111_002_CreateSysSettings.cs b/src/Khaemhat.Data/SystemDatabase/Migrations/M20261111_002_CreateSysSettings.cs new file mode 100644 index 0000000..e0eea22 --- /dev/null +++ b/src/Khaemhat.Data/SystemDatabase/Migrations/M20261111_002_CreateSysSettings.cs @@ -0,0 +1,24 @@ +using Khaemhat.Data.Migrations; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.SystemDatabase.Migrations; + +/// Instance settings (SPEC §5.4): key/value pairs, secrets encrypted with the instance key. +[Migration(20261111_002, "create_sys_settings", Target = MigrationTarget.System)] +public sealed class M20261111_002_CreateSysSettings : IMigration +{ + /// + public void Up(ISchemaBuilder schema) + { + ArgumentNullException.ThrowIfNull(schema); + schema.CreateTable("sys_settings", t => + { + t.Id(); + t.String("key", 200).NotNull(); + t.Text("value"); + t.Bool("is_secret").NotNull().Default(false); + t.AuditColumns(); + t.Unique("ux_sys_settings_key", "key"); + }); + } +} diff --git a/src/Khaemhat.Data/SystemDatabase/SystemMaps.cs b/src/Khaemhat.Data/SystemDatabase/SystemMaps.cs new file mode 100644 index 0000000..6b7a734 --- /dev/null +++ b/src/Khaemhat.Data/SystemDatabase/SystemMaps.cs @@ -0,0 +1,29 @@ +using Khaemhat.Core.Instance; +using Khaemhat.Data.Mapping; + +namespace Khaemhat.Data.SystemDatabase; + +/// Map of to sys_companies. +public sealed class CompanyMap : EntityMap +{ + /// Creates the map. + public CompanyMap() + { + ToTable("sys_companies"); + Map(x => x.Code).MaxLength(20); + Map(x => x.Name).MaxLength(200); + Map(x => x.Engine).MaxLength(20); + Map(x => x.Status).MaxLength(40); + } +} + +/// Map of to sys_settings. +public sealed class SystemSettingMap : EntityMap +{ + /// Creates the map. + public SystemSettingMap() + { + ToTable("sys_settings"); + Map(x => x.Key).MaxLength(200); + } +} diff --git a/src/Khaemhat.Data/UnitOfWorkExtensions.cs b/src/Khaemhat.Data/UnitOfWorkExtensions.cs new file mode 100644 index 0000000..5550fa3 --- /dev/null +++ b/src/Khaemhat.Data/UnitOfWorkExtensions.cs @@ -0,0 +1,55 @@ +namespace Khaemhat.Data; + +/// Helpers on . +public static class UnitOfWorkExtensions +{ + /// + /// Takes the application lock until the end of the current transaction (numerators, closings, + /// scheduler): pg_advisory_xact_lock on PostgreSQL; on SQLite the transaction already holds the write lock. + /// + /// The unit of work, with an open transaction. + /// The lock key, for example numerator:FV:2026. + /// Cancellation. + /// A task completed when the lock is held. + public static async Task AcquireLockAsync(this IUnitOfWork unitOfWork, string key, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(unitOfWork); + ArgumentException.ThrowIfNullOrEmpty(key); + if (unitOfWork.Transaction is null) + { + throw new InvalidOperationException("An application lock is held until the end of a transaction: call BeginAsync first."); + } + + var statement = unitOfWork.Dialect.AdvisoryLock(key); + if (!statement.IsEmpty) + { + await unitOfWork.ExecuteAsync(statement, cancellationToken).ConfigureAwait(false); + } + } + + /// Runs in a transaction: commits if it completes, rolls back if it throws. + /// The unit of work. + /// The work. + /// Cancellation. + /// A task. + public static async Task InTransactionAsync(this IUnitOfWork unitOfWork, Func work, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(unitOfWork); + ArgumentNullException.ThrowIfNull(work); + await unitOfWork.BeginAsync(cancellationToken).ConfigureAwait(false); + try + { + await work(cancellationToken).ConfigureAwait(false); + await unitOfWork.CommitAsync(cancellationToken).ConfigureAwait(false); + } + catch + { + if (unitOfWork.Transaction is not null) + { + await unitOfWork.RollbackAsync(CancellationToken.None).ConfigureAwait(false); + } + + throw; + } + } +} diff --git a/src/Khaemhat.Infrastructure/Security/SecretProtector.cs b/src/Khaemhat.Infrastructure/Security/SecretProtector.cs new file mode 100644 index 0000000..663eac0 --- /dev/null +++ b/src/Khaemhat.Infrastructure/Security/SecretProtector.cs @@ -0,0 +1,189 @@ +using System.Security.Cryptography; +using System.Text; +using Khaemhat.Core.Security; + +namespace Khaemhat.Infrastructure.Security; + +/// +/// The instance key from KHAEMHAT_SECRET_KEY: base64 of at least 32 random bytes (SPEC §6). The encryption key +/// is derived with HKDF-SHA256, so the configured bytes are never used directly. The key is never printed. +/// +public sealed class SecretKey +{ + /// The minimum length of the configured key, in bytes. + public const int MinimumLength = 32; + + private static readonly byte[] EncryptionInfo = "khaemhat/secrets/aes-256-gcm/v1"u8.ToArray(); + private static readonly byte[] IdentifierInfo = "khaemhat/secrets/key-id/v1"u8.ToArray(); + + private SecretKey(byte[] encryptionKey, string id) + { + EncryptionKey = encryptionKey; + Id = id; + } + + /// A short public identifier of the key (8 hexadecimal digits), stored with each ciphertext. + public string Id { get; } + + internal byte[] EncryptionKey { get; } + + /// Reads a key in base64. + /// The configured value. + /// The key. + /// Not base64, or shorter than 32 bytes (the message never repeats the value). + public static SecretKey FromBase64(string base64) + { + if (string.IsNullOrWhiteSpace(base64)) + { + throw new FormatException("KHAEMHAT_SECRET_KEY is empty: set it to the base64 of at least 32 random bytes."); + } + + byte[] material; + try + { + material = Convert.FromBase64String(base64.Trim()); + } + catch (FormatException) + { + throw new FormatException("KHAEMHAT_SECRET_KEY is not valid base64."); + } + + if (material.Length < MinimumLength) + { + throw new FormatException($"KHAEMHAT_SECRET_KEY must be the base64 of at least {MinimumLength} bytes (for example: openssl rand -base64 32)."); + } + + var encryption = HKDF.DeriveKey(HashAlgorithmName.SHA256, material, 32, salt: null, info: EncryptionInfo); + var id = Convert.ToHexStringLower(HKDF.DeriveKey(HashAlgorithmName.SHA256, material, 4, salt: null, info: IdentifierInfo)); + CryptographicOperations.ZeroMemory(material); + return new SecretKey(encryption, id); + } + + /// Creates a new random key in base64, for setup instructions and tests. + /// The base64 text of 32 random bytes. + public static string GenerateBase64() => Convert.ToBase64String(RandomNumberGenerator.GetBytes(MinimumLength)); + + /// + public override string ToString() => "SecretKey " + Id; +} + +/// +/// with AES-256-GCM (SPEC §5.4). A protected value is +/// khs1.<key id>.<base64url(nonce | tag | ciphertext)>; the purpose is authenticated as associated +/// data, so a value cannot be moved to another purpose, and a value of another key is recognized by its key id. +/// +/// The instance key. +public sealed class SecretProtector(SecretKey key) : ISecretProtector +{ + private const string Prefix = "khs1"; + private const int NonceSize = 12; + private const int TagSize = 16; + + /// The key. + public SecretKey Key { get; } = key ?? throw new ArgumentNullException(nameof(key)); + + /// The key id stored in a protected value, or if it is not one. + /// The value. + /// The key id. + public static string? KeyIdOf(string? protectedValue) + { + var parts = protectedValue?.Split('.'); + return parts is [Prefix, var id, _] ? id : null; + } + + /// + public string Protect(string plaintext, string purpose) + { + ArgumentNullException.ThrowIfNull(plaintext); + ArgumentException.ThrowIfNullOrEmpty(purpose); + var plain = Encoding.UTF8.GetBytes(plaintext); + var buffer = new byte[NonceSize + TagSize + plain.Length]; + var nonce = buffer.AsSpan(0, NonceSize); + RandomNumberGenerator.Fill(nonce); + using (var aes = new AesGcm(Key.EncryptionKey, TagSize)) + { + aes.Encrypt(nonce, plain, buffer.AsSpan(NonceSize + TagSize), buffer.AsSpan(NonceSize, TagSize), Encoding.UTF8.GetBytes(purpose)); + } + + CryptographicOperations.ZeroMemory(plain); + return Prefix + "." + Key.Id + "." + Base64Url(buffer); + } + + /// + public string Unprotect(string protectedValue, string purpose) + { + ArgumentException.ThrowIfNullOrEmpty(purpose); + var parts = protectedValue?.Split('.'); + if (parts is not [Prefix, var id, var payload]) + { + throw new SecretProtectionException("The value is not a protected secret."); + } + + if (!string.Equals(id, Key.Id, StringComparison.Ordinal)) + { + throw new SecretProtectionException( + $"The secret was protected with another instance key ({id}, current {Key.Id}): check KHAEMHAT_SECRET_KEY or run the key rotation."); + } + + byte[] buffer; + try + { + buffer = FromBase64Url(payload); + } + catch (FormatException ex) + { + throw new SecretProtectionException("The protected secret is damaged.", ex); + } + + if (buffer.Length < NonceSize + TagSize) + { + throw new SecretProtectionException("The protected secret is damaged."); + } + + var plain = new byte[buffer.Length - NonceSize - TagSize]; + try + { + using var aes = new AesGcm(Key.EncryptionKey, TagSize); + aes.Decrypt( + buffer.AsSpan(0, NonceSize), buffer.AsSpan(NonceSize + TagSize), buffer.AsSpan(NonceSize, TagSize), plain, Encoding.UTF8.GetBytes(purpose)); + return Encoding.UTF8.GetString(plain); + } + catch (AuthenticationTagMismatchException ex) + { + throw new SecretProtectionException("The secret cannot be decrypted: damaged value or different purpose.", ex); + } + finally + { + CryptographicOperations.ZeroMemory(plain); + } + } + + private static string Base64Url(byte[] bytes) => Convert.ToBase64String(bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_'); + + private static byte[] FromBase64Url(string text) + { + var base64 = text.Replace('-', '+').Replace('_', '/'); + return Convert.FromBase64String(base64.PadRight(base64.Length + ((4 - (base64.Length % 4)) % 4), '=')); + } +} + +/// +/// Re-encryption of secrets from the current key to KHAEMHAT_SECRET_KEY_NEW (SPEC §12.9). This is the building +/// block; the procedure that walks every encrypted column of every database arrives in milestone M12. +/// +/// The protector with the current key. +/// The protector with the new key. +public sealed class SecretRotation(SecretProtector current, SecretProtector next) +{ + /// Whether a value still needs to be re-encrypted with the new key. + /// The protected value. + /// if it is not protected with the new key yet. + public bool NeedsRotation(string protectedValue) => SecretProtector.KeyIdOf(protectedValue) != next.Key.Id; + + /// Re-encrypts a value with the new key; values already rotated are returned unchanged. + /// The protected value. + /// Its purpose. + /// The value protected with the new key. + public string Rotate(string protectedValue, string purpose) => + NeedsRotation(protectedValue) ? next.Protect(current.Unprotect(protectedValue, purpose), purpose) : protectedValue; +} diff --git a/tests/Khaemhat.Core.Tests/Architecture/DependencyRulesTests.cs b/tests/Khaemhat.Core.Tests/Architecture/DependencyRulesTests.cs index 79e870d..747f364 100644 --- a/tests/Khaemhat.Core.Tests/Architecture/DependencyRulesTests.cs +++ b/tests/Khaemhat.Core.Tests/Architecture/DependencyRulesTests.cs @@ -22,6 +22,7 @@ public sealed class DependencyRulesTests ["Khaemhat.Application"] = ["Khaemhat.Core", "Khaemhat.Data", "Khaemhat.Infrastructure", "Khaemhat.Fiscal.Abstractions"], ["Khaemhat.Web"] = ["Khaemhat.Application", "Khaemhat.Data.Sqlite", "Khaemhat.Data.PostgreSql", "Khaemhat.Fiscal.Italy"], ["Khaemhat.LicenseTool"] = ["Khaemhat.Core", "Khaemhat.Infrastructure"], + ["Khaemhat.DataModelTool"] = ["Khaemhat.Data"], }; // The only packages, and where each may be referenced. diff --git a/tests/Khaemhat.Data.Tests/CompanyRegistryTests.cs b/tests/Khaemhat.Data.Tests/CompanyRegistryTests.cs new file mode 100644 index 0000000..a9921b4 --- /dev/null +++ b/tests/Khaemhat.Data.Tests/CompanyRegistryTests.cs @@ -0,0 +1,125 @@ +using Khaemhat.Core.Instance; +using Khaemhat.Core.Security; +using Khaemhat.Data.Companies; +using Khaemhat.Data.Execution; +using Khaemhat.Data.Migrations; +using Khaemhat.Data.Tests.Support; +using Khaemhat.Infrastructure.Security; + +namespace Khaemhat.Data.Tests; + +[TestClass] +public sealed class CompanyRegistryTests +{ + private const string CompanyPassword = "Company-Db-Secret-42"; + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task Register_StoresTheLocationEncrypted_AndResolvesIt(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + var registry = await CreateRegistryAsync(db, SecretKey.GenerateBase64()); + var location = DatabaseUrl.Parse($"postgres://acme:{CompanyPassword}@db:5432/khaemhat_acme"); + + var company = await registry.RegisterAsync(new CompanyRegistration("acme", "ACME S.r.l.", location, IsDefault: true)); + var target = await registry.ResolveAsync("ACME"); + + Assert.AreEqual("ACME", company.Code); + Assert.AreEqual("PostgreSql", company.Engine); + StringAssert.StartsWith(company.ConnectionString, "khs1."); + Assert.DoesNotContain(CompanyPassword, company.ConnectionString); + Assert.AreEqual(DbEngine.PostgreSql, target.Engine); + Assert.AreEqual("khaemhat_acme", target.DatabaseName); + StringAssert.Contains(target.ConnectionString, "Database=khaemhat_acme"); + Assert.AreEqual("PostgreSql khaemhat_acme", target.ToString()); + + await using var unitOfWork = await db.OpenAsync(); + var stored = await unitOfWork.QuerySingleAsync(SqlFragment.Literal("SELECT connection_string FROM sys_companies")); + Assert.DoesNotContain(CompanyPassword, stored!); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task Status_Default_AndDuplicates(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + var registry = await CreateRegistryAsync(db, SecretKey.GenerateBase64()); + var sqlite = DatabaseUrl.Parse("sqlite:///data/system.db"); + + await registry.RegisterAsync(new CompanyRegistration("ONE", "One", sqlite.WithDatabase("khaemhat_one"), IsDefault: true)); + await registry.RegisterAsync(new CompanyRegistration("TWO", "Two", sqlite.WithDatabase("khaemhat_two"), IsDefault: true)); + await registry.SetStatusAsync("one", CompanyStatus.ReadOnly); + + var companies = await registry.ListAsync(); + Assert.AreEqual("ONE,TWO", string.Join(",", companies.Select(c => c.Code))); + Assert.IsFalse(companies[0].IsDefault, "registering a new default clears the old one"); + Assert.IsTrue(companies[1].IsDefault); + Assert.AreEqual(CompanyStatus.ReadOnly, companies[0].Status); + Assert.AreEqual(2L, companies[0].RowVersion); + await Assert.ThrowsExactlyAsync( + () => registry.RegisterAsync(new CompanyRegistration("one", "Again", sqlite.WithDatabase("khaemhat_one")))); + await Assert.ThrowsExactlyAsync(() => registry.ResolveAsync("NOPE")); + } + + [TestMethod] + public async Task WrongInstanceKey_IsRejected_WithoutLeakingTheSecret() + { + await using var db = await TestDatabase.CreateAsync(TestDatabase.Sqlite); + var registry = await CreateRegistryAsync(db, SecretKey.GenerateBase64()); + await registry.RegisterAsync(new CompanyRegistration("ACME", "ACME", DatabaseUrl.Parse($"postgres://u:{CompanyPassword}@db/khaemhat_acme"))); + + var otherKey = await CreateRegistryAsync(db, SecretKey.GenerateBase64(), migrate: false); + var error = await Assert.ThrowsExactlyAsync(() => otherKey.ResolveAsync("ACME")); + + StringAssert.Contains(error.Message, "another instance key"); + Assert.DoesNotContain(CompanyPassword, error.ToString()); + } + + [TestMethod] + public async Task ACopiedRow_CannotBeReadForAnotherCompany() + { + await using var db = await TestDatabase.CreateAsync(TestDatabase.Sqlite); + var key = SecretKey.GenerateBase64(); + var registry = await CreateRegistryAsync(db, key); + await registry.RegisterAsync(new CompanyRegistration("ACME", "ACME", DatabaseUrl.Parse("sqlite:///data/khaemhat_acme.db"))); + await registry.RegisterAsync(new CompanyRegistration("EVIL", "Evil", DatabaseUrl.Parse("sqlite:///data/khaemhat_evil.db"))); + await using (var unitOfWork = await db.OpenAsync()) + { + await unitOfWork.ExecuteAsync(SqlFragment.Literal( + "UPDATE sys_companies SET connection_string = (SELECT connection_string FROM sys_companies WHERE code = 'ACME') WHERE code = 'EVIL'")); + } + + var fresh = await CreateRegistryAsync(db, key, migrate: false); + await Assert.ThrowsExactlyAsync(() => fresh.ResolveAsync("EVIL")); + } + + [TestMethod] + public async Task Locator_OpensTheSystemOrACompany() + { + await using var system = await TestDatabase.CreateAsync(TestDatabase.Sqlite); + await using var company = await TestDatabase.CreateAsync(TestDatabase.Sqlite); + var registry = await CreateRegistryAsync(system, SecretKey.GenerateBase64()); + await registry.RegisterAsync(new CompanyRegistration("DEMO", "Demo", company.Url)); + var factory = new UnitOfWorkFactory(new DatabaseLocator(system.Target, registry), system.Maps, system.Context); + + await using var systemWork = await factory.CreateAsync(DatabaseScope.System); + await using var companyWork = await factory.CreateAsync(DatabaseScope.Company("demo")); + + Assert.AreEqual(1L, await systemWork.QuerySingleAsync(SqlFragment.Literal("SELECT COUNT(*) FROM sys_companies"))); + Assert.AreEqual(0L, await companyWork.QuerySingleAsync( + SqlFragment.Literal("SELECT COUNT(*) FROM sqlite_master WHERE name = 'sys_companies'"))); + Assert.AreEqual("company DEMO", DatabaseScope.Company("demo").ToString()); + } + + private static async Task CreateRegistryAsync(TestDatabase db, string key, bool migrate = true) + { + if (migrate) + { + await new MigrationRunner(db.Context).RunAsync(db.Target, MigrationCatalog.ForSystem()); + } + + return new CompanyRegistry(db.Target, db.Providers, new SecretProtector(SecretKey.FromBase64(key)), db.Maps, db.Context); + } +} diff --git a/tests/Khaemhat.Data.Tests/DataModelDocumentTests.cs b/tests/Khaemhat.Data.Tests/DataModelDocumentTests.cs new file mode 100644 index 0000000..4da0e6b --- /dev/null +++ b/tests/Khaemhat.Data.Tests/DataModelDocumentTests.cs @@ -0,0 +1,48 @@ +using Khaemhat.Data.Migrations; + +namespace Khaemhat.Data.Tests; + +/// docs/DATA-MODEL.md never drifts from the migrations (SPEC §18). +[TestClass] +public sealed class DataModelDocumentTests +{ + [TestMethod] + public void GeneratedSection_IsUpToDate() + { + var path = Path.Combine(RepositoryRoot(), "docs", "DATA-MODEL.md"); + if (!File.Exists(path)) + { + Assert.Inconclusive("docs/ is not part of this checkout (the Docker build context excludes it)."); + } + + var current = File.ReadAllText(path).ReplaceLineEndings("\n"); + + var expected = DataModelDocument.Update(current, DataModelDocument.Render(MigrationCatalog.ForSystem(), MigrationCatalog.ForCompany())); + + Assert.AreEqual(expected, current, "docs/DATA-MODEL.md is out of date: run scripts/gen-datamodel.sh"); + } + + [TestMethod] + public void Update_ReplacesOnlyTheGeneratedSection() + { + var document = $"intro\n{DataModelDocument.BeginMarker}\nold\n{DataModelDocument.EndMarker}\nfooter\n"; + + var updated = DataModelDocument.Update(document, "new\n"); + + Assert.AreEqual($"intro\n{DataModelDocument.BeginMarker}\n\nnew\n\n{DataModelDocument.EndMarker}\nfooter\n", updated); + Assert.ThrowsExactly(() => DataModelDocument.Update("no markers", "x")); + } + + private static string RepositoryRoot() + { + for (var dir = new DirectoryInfo(AppContext.BaseDirectory); dir is not null; dir = dir.Parent) + { + if (File.Exists(Path.Combine(dir.FullName, "Khaemhat.sln"))) + { + return dir.FullName; + } + } + + throw new InvalidOperationException("Khaemhat.sln not found above " + AppContext.BaseDirectory); + } +} diff --git a/tests/Khaemhat.Data.Tests/DatabaseUrlTests.cs b/tests/Khaemhat.Data.Tests/DatabaseUrlTests.cs new file mode 100644 index 0000000..b0f461e --- /dev/null +++ b/tests/Khaemhat.Data.Tests/DatabaseUrlTests.cs @@ -0,0 +1,117 @@ +using Khaemhat.Data.PostgreSql; +using Khaemhat.Data.Sqlite; + +namespace Khaemhat.Data.Tests; + +[TestClass] +public sealed class DatabaseUrlTests +{ + [TestMethod] + public void Postgres_IsParsedWithEncodedPasswordAndOptions() + { + var url = DatabaseUrl.Parse("postgres://khaemhat:p%40ss%3Aword@khaemhat-db:5433/khaemhat_system?sslmode=require"); + + Assert.AreEqual(DbEngine.PostgreSql, url.Engine); + Assert.AreEqual("khaemhat-db", url.Host); + Assert.AreEqual(5433, url.Port); + Assert.AreEqual("khaemhat", url.UserName); + Assert.AreEqual("p@ss:word", url.Password); + Assert.AreEqual("khaemhat_system", url.Database); + Assert.AreEqual("require", url.Options["sslmode"]); + } + + [TestMethod] + public void Postgres_ServerUrl_HasEmptyDatabase() + { + var url = DatabaseUrl.Parse("postgresql://u:p@db/"); + + Assert.AreEqual(string.Empty, url.Database); + Assert.IsNull(url.Port); + Assert.AreEqual("khaemhat_demo", url.WithDatabase(CompanyCodes.DatabaseName("demo")).Database); + } + + [TestMethod] + public void ToString_NeverShowsThePassword() + { + var url = DatabaseUrl.Parse("postgres://khaemhat:VerySecret1@db:5432/khaemhat_system"); + + Assert.AreEqual("postgres://khaemhat:***@db:5432/khaemhat_system", url.ToString()); + Assert.DoesNotContain("VerySecret1", url.ToString()); + Assert.Contains("VerySecret1", url.ToUnredactedString()); + Assert.AreEqual(url.ToUnredactedString(), DatabaseUrl.Parse(url.ToUnredactedString()).ToUnredactedString()); + } + + [TestMethod] + public void InvalidUrl_ErrorDoesNotRepeatTheInput() + { + var error = Assert.ThrowsExactly(() => DatabaseUrl.Parse("mysql://root:TopSecret@db/x")); + + Assert.DoesNotContain("TopSecret", error.Message); + } + + [TestMethod] + [DataRow("sqlite:///data/system.db", "/data/system.db", "system")] + [DataRow("sqlite:///C:/Khaemhat/data/system.db", "C:/Khaemhat/data/system.db", "system")] + [DataRow("sqlite:///B:\\dev\\.data\\system.db", "B:/dev/.data/system.db", "system")] + public void Sqlite_FilePaths(string text, string path, string name) + { + var url = DatabaseUrl.Parse(text); + + Assert.AreEqual(DbEngine.Sqlite, url.Engine); + Assert.AreEqual(path, url.Database); + Assert.AreEqual(name, url.DatabaseName); + Assert.IsFalse(url.IsInMemory); + } + + [TestMethod] + public void Sqlite_CompanyDatabase_IsASiblingFile() + { + var url = DatabaseUrl.Parse("sqlite:///data/system.db").WithDatabase(CompanyCodes.DatabaseName("Demo")); + + Assert.AreEqual("/data/khaemhat_demo.db", url.Database); + Assert.AreEqual("sqlite:///data/khaemhat_demo.db", url.ToString()); + } + + [TestMethod] + public void Sqlite_InMemory() + { + var url = DatabaseUrl.Parse("sqlite://memory/tests_1"); + + Assert.IsTrue(url.IsInMemory); + Assert.AreEqual("tests_1", url.DatabaseName); + StringAssert.Contains(new SqliteProvider().BuildConnectionString(url), "Mode=Memory"); + } + + [TestMethod] + [DataRow("")] + [DataRow("sqlite://relative.db")] + [DataRow("sqlite:///")] + [DataRow("sqlite://memory/bad name")] + [DataRow("postgres://")] + [DataRow("postgres://u:p@host/a/b")] + public void Invalid_AreRejected(string text) => Assert.IsFalse(DatabaseUrl.TryParse(text, out _)); + + [TestMethod] + public void PostgresConnectionString_HasPoolAndStatementTimeout() + { + var provider = new PostgresProvider(new PostgresProviderOptions { MaxPoolSize = 7, StatementTimeout = TimeSpan.FromSeconds(30) }); + + var cs = provider.BuildConnectionString(DatabaseUrl.Parse("postgres://u:p@db/khaemhat_demo?sslmode=disable")); + + StringAssert.Contains(cs, "Maximum Pool Size=7"); + StringAssert.Contains(cs, "statement_timeout=30000"); + StringAssert.Contains(cs, "Database=khaemhat_demo"); + } + + [TestMethod] + [DataRow("DEMO", "khaemhat_demo")] + [DataRow(" acme_2 ", "khaemhat_acme_2")] + public void CompanyCodes_DatabaseNames(string code, string database) => Assert.AreEqual(database, CompanyCodes.DatabaseName(code)); + + [TestMethod] + [DataRow("")] + [DataRow("2ACME")] + [DataRow("AC-ME")] + [DataRow("ABCDEFGHIJKLMNOPQRSTU")] + public void CompanyCodes_InvalidAreRejected(string code) => Assert.ThrowsExactly(() => CompanyCodes.Normalize(code)); +} diff --git a/tests/Khaemhat.Data.Tests/DdlTests.cs b/tests/Khaemhat.Data.Tests/DdlTests.cs new file mode 100644 index 0000000..54ce6ef --- /dev/null +++ b/tests/Khaemhat.Data.Tests/DdlTests.cs @@ -0,0 +1,121 @@ +using Khaemhat.Data.Migrations; +using Khaemhat.Data.PostgreSql; +using Khaemhat.Data.Schema; +using Khaemhat.Data.Sqlite; +using Khaemhat.Data.SystemDatabase.Migrations; + +namespace Khaemhat.Data.Tests; + +[TestClass] +public sealed class DdlTests +{ + private static readonly MigrationInfo Companies = MigrationCatalog.ForSystem().Migrations.Single(m => m.Type == typeof(M20261111_001_CreateSysCompanies)); + + [TestMethod] + public void Sqlite_UsesStrictTables_IntegerKeys_AndChecks() + { + var sql = MigrationRunner.Statements(Companies, new SqliteProvider()); + + Assert.AreEqual( + """ + CREATE TABLE "sys_companies" ( + "id" INTEGER PRIMARY KEY, + "code" TEXT NOT NULL CHECK (length("code") <= 20), + "name" TEXT NOT NULL CHECK (length("name") <= 200), + "engine" TEXT NOT NULL CHECK (length("engine") <= 20), + "connection_string" TEXT NOT NULL, + "status" TEXT NOT NULL DEFAULT 'Active' CHECK (length("status") <= 40), + "is_default" INTEGER NOT NULL DEFAULT 0 CHECK ("is_default" IN (0, 1)), + "notes" TEXT, + "created_at" TEXT NOT NULL, + "created_by" TEXT CHECK (length("created_by") <= 100), + "updated_at" TEXT, + "updated_by" TEXT CHECK (length("updated_by") <= 100), + "row_version" INTEGER NOT NULL DEFAULT 1 + ) STRICT + """.ReplaceLineEndings("\n"), + sql[0]); + Assert.AreEqual("CREATE UNIQUE INDEX \"ux_sys_companies_code\" ON \"sys_companies\" (\"code\")", sql[1]); + } + + [TestMethod] + public void Postgres_UsesIdentity_AndNativeTypes() + { + var sql = MigrationRunner.Statements(Companies, new PostgresProvider()); + + StringAssert.Contains(sql[0], "\"id\" bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY"); + StringAssert.Contains(sql[0], "\"code\" varchar(20) NOT NULL"); + StringAssert.Contains(sql[0], "\"is_default\" boolean NOT NULL DEFAULT FALSE"); + StringAssert.Contains(sql[0], "\"created_at\" timestamptz NOT NULL"); + Assert.DoesNotContain("STRICT", sql[0]); + } + + [TestMethod] + public void AllTypes_MapAsSpecSection5_2Says() + { + var expected = new Dictionary + { + [ColumnType.String] = ("varchar(30)", "TEXT"), + [ColumnType.Text] = ("text", "TEXT"), + [ColumnType.Int32] = ("integer", "INTEGER"), + [ColumnType.Int64] = ("bigint", "INTEGER"), + [ColumnType.Decimal] = ("numeric(19,6)", "INTEGER"), + [ColumnType.Bool] = ("boolean", "INTEGER"), + [ColumnType.Date] = ("date", "TEXT"), + [ColumnType.DateTimeUtc] = ("timestamptz", "TEXT"), + [ColumnType.Guid] = ("uuid", "TEXT"), + [ColumnType.Bytes] = ("bytea", "BLOB"), + [ColumnType.Json] = ("jsonb", "TEXT"), + }; + + foreach (var (type, (postgres, sqlite)) in expected) + { + var length = type == ColumnType.String ? 30 : (int?)null; + Assert.AreEqual(postgres, PostgresDialect.Instance.SqlType(type, length, null), type.ToString()); + Assert.AreEqual(sqlite, SqliteDialect.Instance.SqlType(type, length, null), type.ToString()); + } + } + + [TestMethod] + public void ForeignKeys_CompositeKeys_AndAlterTable() + { + var schema = new SchemaBuilder(); + schema.CreateTable("parent_rows", t => t.Id()); + schema.CreateTable("child_rows", t => + { + t.Id(); + t.Int64("parent_row_id").NotNull(); + t.ForeignKey("fk_child_rows_parent", "parent_row_id", "parent_rows", ReferentialAction.Cascade); + }); + schema.AlterTable("child_rows", t => + { + t.Decimal("price", 2).NotNull().Default(0m); + t.Index("ix_child_rows_price", "price"); + }); + + var sql = new PostgresSchemaBuilder().Generate(schema.Operations); + + StringAssert.Contains(sql[1], "CONSTRAINT \"fk_child_rows_parent\" FOREIGN KEY (\"parent_row_id\") REFERENCES \"parent_rows\" (\"id\") ON DELETE CASCADE"); + Assert.AreEqual("ALTER TABLE \"child_rows\" ADD COLUMN \"price\" numeric(19,2) NOT NULL DEFAULT 0", sql[2]); + Assert.AreEqual("CREATE INDEX \"ix_child_rows_price\" ON \"child_rows\" (\"price\")", sql[3]); + Assert.ThrowsExactly(() => schema.AlterTable("child_rows", t => t.ForeignKey("fk_x", "id", "parent_rows"))); + } + + [TestMethod] + public void SchemaVersion_HasCompositeKey_AndIsCreatedIfMissing() + { + var sql = new SqliteSchemaBuilder().Generate([SchemaVersionStore.CreateOperation]); + + StringAssert.StartsWith(sql[0], "CREATE TABLE IF NOT EXISTS \"schema_version\""); + StringAssert.Contains(sql[0], "CONSTRAINT \"pk_schema_version\" PRIMARY KEY (\"module\", \"version\")"); + } + + [TestMethod] + public void Partitioning_IsPrepared() + { + Assert.AreEqual( + "CREATE TABLE IF NOT EXISTS \"gl_entry_lines_2026\" PARTITION OF \"gl_entry_lines\" FOR VALUES FROM ('2026-01-01') TO ('2027-01-01')", + PostgresPartitioning.CreateYearPartition("gl_entry_lines", 2026)); + Assert.AreEqual(" PARTITION BY RANGE (\"entry_date\")", PostgresPartitioning.PartitionClause("entry_date")); + } +} diff --git a/tests/Khaemhat.Data.Tests/DialectTests.cs b/tests/Khaemhat.Data.Tests/DialectTests.cs new file mode 100644 index 0000000..4b8e684 --- /dev/null +++ b/tests/Khaemhat.Data.Tests/DialectTests.cs @@ -0,0 +1,238 @@ +using Khaemhat.Data.Querying; +using Khaemhat.Data.Schema; +using Khaemhat.Data.Sqlite; +using Khaemhat.Data.Tests.Support; + +namespace Khaemhat.Data.Tests; + +/// The same behaviour on both engines: types, decimals, paging, upsert, LIKE, timestamps. +[TestClass] +public sealed class DialectTests +{ + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task EveryType_RoundTrips(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + await db.CreateTestTableAsync(); + var at = new DateTime(2026, 11, 11, 9, 30, 15, DateTimeKind.Utc).AddTicks(1234560); + var item = new TestItem + { + Code = "ALL", + Name = "Caffè «speciale» ✓", + Quantity = int.MaxValue, + Big = long.MinValue, + Amount = -123_456_789_012.345678m, + Active = true, + Day = new DateOnly(2026, 2, 28), + At = at, + Uid = Guid.Parse("7a1b0c3e-9d2f-4e5a-8b6c-1d2e3f4a5b6c"), + Data = [0, 1, 2, 254, 255], + Payload = "{\"a\": 1}", + Kind = TestKind.Kit, + IsDemo = true, + }; + + await using var unitOfWork = await db.OpenAsync(); + var items = unitOfWork.Repository(); + var id = await items.InsertAsync(item); + var read = await items.GetAsync(id); + + Assert.IsNotNull(read); + Assert.AreEqual(item.Name, read.Name); + Assert.AreEqual(int.MaxValue, read.Quantity); + Assert.AreEqual(long.MinValue, read.Big); + Assert.AreEqual(item.Amount, read.Amount); + Assert.IsTrue(read.Active); + Assert.AreEqual(item.Day, read.Day); + Assert.AreEqual(at, read.At); + Assert.AreEqual(DateTimeKind.Utc, read.At!.Value.Kind); + Assert.AreEqual(item.Uid, read.Uid); + CollectionAssert.AreEqual(item.Data, read.Data); + StringAssert.Contains(read.Payload, "\"a\""); + Assert.AreEqual(TestKind.Kit, read.Kind); + Assert.IsTrue(read.IsDemo); + Assert.AreEqual(TestDatabase.TestUser, read.CreatedBy); + Assert.AreEqual(1L, read.RowVersion); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task Decimals_AreRoundedToSixPlaces_AndSumsAreExact(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + await db.CreateTestTableAsync(); + await using var unitOfWork = await db.OpenAsync(); + var items = unitOfWork.Repository(); + + decimal[] amounts = [999_999_999_999.999999m, 0.000001m, 1.2345675m, -1.0000005m, 0.1m, 0.2m]; + await unitOfWork.InTransactionAsync(async ct => + { + for (var i = 0; i < amounts.Length; i++) + { + await items.InsertAsync(new TestItem { Code = "D" + i, Amount = amounts[i] }, ct); + } + }); + + var stored = (await items.ListAsync(Query.For().OrderBy(x => x.Id))).Items.Select(x => x.Amount).ToArray(); + CollectionAssert.AreEqual(new[] { 999_999_999_999.999999m, 0.000001m, 1.234568m, -1.000001m, 0.1m, 0.2m }, stored); + + var sum = await unitOfWork.QuerySingleAsync(SqlFragment.Literal("SELECT SUM(amount) FROM test_items")); + Assert.AreEqual(1_000_000_000_000.534567m, sum); + + var small = await unitOfWork.QuerySingleAsync(SqlFragment.Of($"SELECT SUM(amount) FROM test_items WHERE amount < {1m}")); + Assert.AreEqual(-0.7m, small); + } + + [TestMethod] + public void SqliteDecimalConverter_RoundsHalfAwayFromZero_AndRefusesOverflow() + { + Assert.AreEqual(1_234_568L, SqliteDecimalConverter.ToScaled(1.2345675m)); + Assert.AreEqual(-1_000_001L, SqliteDecimalConverter.ToScaled(-1.0000005m)); + Assert.AreEqual(1_000_000_000_000_000_000L, SqliteDecimalConverter.ToScaled(1_000_000_000_000m)); + Assert.AreEqual(123.456789m, SqliteDecimalConverter.FromScaled(123_456_789L)); + Assert.ThrowsExactly(() => SqliteDecimalConverter.ToScaled(10_000_000_000_000m)); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task Like_IsCaseInsensitive_UnicodeAware_AndEscaped(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + await db.CreateTestTableAsync(); + await using var unitOfWork = await db.OpenAsync(); + var items = unitOfWork.Repository(); + string[] names = ["Àlbero", "albero", "ALBERGO", "100% Bio", "a_b", "axb"]; + for (var i = 0; i < names.Length; i++) + { + await items.InsertAsync(new TestItem { Code = "L" + i, Name = names[i] }); + } + + async Task Names(Query query) => + [.. (await items.ListAsync(query.OrderBy(x => x.Code))).Items.Select(x => x.Name!)]; + + CollectionAssert.AreEqual(new[] { "Àlbero" }, await Names(Query.For().Where(x => x.Name!.Contains("àL")))); + CollectionAssert.AreEqual(new[] { "albero", "ALBERGO" }, await Names(Query.For().Where(x => x.Name!.StartsWith("alb")))); + CollectionAssert.AreEqual(new[] { "100% Bio" }, await Names(Query.For().Where(x => x.Name!.Contains("%")))); + CollectionAssert.AreEqual(new[] { "a_b" }, await Names(Query.For().Where(x => x.Name!.Contains("_")))); + CollectionAssert.AreEqual(new[] { "Àlbero", "albero" }, await Names(Query.For().Where(x => x.Name!.EndsWith("BERO")))); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task Timestamps_SortChronologically_AndCurrentTimestampIsUtc(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + await db.CreateTestTableAsync(); + await using var unitOfWork = await db.OpenAsync(); + var items = unitOfWork.Repository(); + var baseTime = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + TimeSpan[] offsets = [TimeSpan.FromHours(10), TimeSpan.FromMilliseconds(1), TimeSpan.FromDays(400), TimeSpan.FromTicks(10), TimeSpan.Zero]; + for (var i = 0; i < offsets.Length; i++) + { + await items.InsertAsync(new TestItem { Code = "T" + i, At = baseTime + offsets[i] }); + } + + var sorted = (await items.ListAsync(Query.For().OrderBy(x => x.At))).Items.Select(x => x.Code).ToArray(); + var after = (await items.ListAsync(Query.For().Where(x => x.At > baseTime).OrderBy(x => x.At))).Items.Select(x => x.Code).ToArray(); + var now = await unitOfWork.QuerySingleAsync(SqlFragment.Literal("SELECT " + db.Dialect.CurrentTimestampUtc)); + + CollectionAssert.AreEqual(new[] { "T4", "T3", "T1", "T0", "T2" }, sorted); + CollectionAssert.AreEqual(new[] { "T3", "T1", "T0", "T2" }, after); + Assert.AreEqual(DateTimeKind.Utc, now.Kind); + Assert.IsLessThan(TimeSpan.FromMinutes(1), (now - DateTime.UtcNow).Duration()); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task Paging_ReturnsStablePagesAndTotal(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + await db.CreateTestTableAsync(); + await using var unitOfWork = await db.OpenAsync(); + var items = unitOfWork.Repository(); + await unitOfWork.InTransactionAsync(async ct => + { + for (var i = 0; i < 25; i++) + { + await items.InsertAsync(new TestItem { Code = "P" + i.ToString("D2", System.Globalization.CultureInfo.InvariantCulture), Quantity = i }, ct); + } + }); + + var page = await items.ListAsync(Query.For().Where(x => x.Quantity >= 5).OrderBy(x => x.Code).Page(10, 4)); + + Assert.AreEqual(20, page.TotalCount); + CollectionAssert.AreEqual(new[] { "P15", "P16", "P17", "P18" }, page.Items.Select(x => x.Code).ToArray()); + Assert.IsTrue(page.HasMore); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task Upsert_InsertsThenUpdates(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + await using var unitOfWork = await db.OpenAsync(); + await unitOfWork.ExecuteAsync(SqlFragment.Literal(string.Join(";", db.Provider.Ddl.Generate(UpsertTable())))); + var text = db.Dialect.Upsert("kv", ["k", "v"], ["@p0", "@p1"], ["k"], ["v"]); + + foreach (var value in new[] { "first", "second" }) + { + await using var command = unitOfWork.Executor.CreateCommand( + unitOfWork.Connection, null, text, [new SqlParameterValue("p0", "key", ColumnType.Text), new SqlParameterValue("p1", value, ColumnType.Text)]); + await command.ExecuteNonQueryAsync(); + } + + Assert.AreEqual(1L, await unitOfWork.QuerySingleAsync(SqlFragment.Literal("SELECT COUNT(*) FROM kv"))); + Assert.AreEqual("second", await unitOfWork.QuerySingleAsync(SqlFragment.Literal("SELECT v FROM kv"))); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task StringLength_IsEnforcedOnBothEngines(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + await db.CreateTestTableAsync(); + await using var unitOfWork = await db.OpenAsync(); + + await Assert.ThrowsAsync( + () => unitOfWork.Repository().InsertAsync(new TestItem { Code = new string('x', 21) })); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task ServerInfo_IsReported(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + + var info = await db.Provider.GetServerInfoAsync(db.Target.ConnectionString); + + Assert.AreEqual(db.Provider.Engine, info.Engine); + Assert.IsGreaterThan(0, info.VersionNumber); + if (engine == TestDatabase.Postgres) + { + Assert.IsGreaterThanOrEqualTo(160000, info.VersionNumber, "CI runs PostgreSQL 16 and 17"); + Assert.AreEqual(db.Url.Database, info.Database); + Assert.IsEmpty(info.Warnings); + } + } + + private static IReadOnlyList UpsertTable() + { + var schema = new SchemaBuilder(); + schema.CreateTable("kv", t => + { + t.String("k", 50).NotNull(); + t.Text("v"); + t.PrimaryKey("k"); + }); + return schema.Operations; + } +} diff --git a/tests/Khaemhat.Data.Tests/Khaemhat.Data.Tests.csproj b/tests/Khaemhat.Data.Tests/Khaemhat.Data.Tests.csproj index 201710e..d595a2e 100644 --- a/tests/Khaemhat.Data.Tests/Khaemhat.Data.Tests.csproj +++ b/tests/Khaemhat.Data.Tests/Khaemhat.Data.Tests.csproj @@ -8,6 +8,7 @@ + diff --git a/tests/Khaemhat.Data.Tests/MappingTests.cs b/tests/Khaemhat.Data.Tests/MappingTests.cs new file mode 100644 index 0000000..8c6fc90 --- /dev/null +++ b/tests/Khaemhat.Data.Tests/MappingTests.cs @@ -0,0 +1,82 @@ +using Khaemhat.Core.Instance; +using Khaemhat.Data.Mapping; +using Khaemhat.Data.Schema; +using Khaemhat.Data.SystemDatabase; +using Khaemhat.Data.Tests.Support; + +namespace Khaemhat.Data.Tests; + +[TestClass] +public sealed class MappingTests +{ + [TestMethod] + [DataRow("RowVersion", "row_version")] + [DataRow("IsDemo", "is_demo")] + [DataRow("VatID", "vat_id")] + [DataRow("HTTPCode", "http_code")] + [DataRow("Address2", "address2")] + [DataRow("Id", "id")] + public void SnakeCase(string name, string expected) => Assert.AreEqual(expected, SqlNames.ToSnakeCase(name)); + + [TestMethod] + public void Conventions_KeyTypesNullabilityAndAudit() + { + var map = new TestItemMap(); + map.Freeze(); + + Assert.AreEqual("id", map.Key.ColumnName); + Assert.IsTrue(map.Key.IsIdentity); + Assert.AreEqual("id", map.Columns[0].ColumnName); + Assert.AreEqual("row_version", map.Columns[^1].ColumnName); + Assert.AreSame(map.Columns[^1], map.RowVersion); + + Assert.AreEqual("String(20)", map.Column(x => x.Code).TypeDescription); + Assert.IsFalse(map.Column(x => x.Code).IsNullable); + Assert.IsTrue(map.Column(x => x.Name).IsNullable); + Assert.AreEqual(ColumnType.Decimal, map.Column(x => x.Amount).Type); + Assert.AreEqual(ColumnType.Date, map.Column(x => x.Day).Type); + Assert.IsTrue(map.Column(x => x.Day).IsNullable); + Assert.AreEqual(ColumnType.Json, map.Column(x => x.Payload).Type); + Assert.AreEqual("String(40)", map.Column(x => x.Kind).TypeDescription); + Assert.AreEqual(ColumnType.Bool, map.Column(x => x.IsDemo).Type); + Assert.AreEqual(AuditRole.CreatedBy, map.Column(x => x.CreatedBy).Audit); + Assert.AreEqual("String(100)", map.Column(x => x.CreatedBy).TypeDescription); + } + + [TestMethod] + public void Enums_AreStoredByName() + { + var map = new CompanyMap(); + map.Freeze(); + var status = map.Column(x => x.Status); + + Assert.AreEqual("Maintenance", status.ToProvider(CompanyStatus.Maintenance)); + Assert.AreEqual(CompanyStatus.ReadOnly, status.FromProvider("ReadOnly")); + } + + [TestMethod] + public void Registry_RejectsTwoMapsForOneEntity() + { + var registry = new MapRegistry().Add(new CompanyMap()); + + Assert.ThrowsExactly(() => registry.Add(new CompanyMap())); + } + + [TestMethod] + public void Registry_FindsTheSystemMaps() + { + var registry = new MapRegistry().AddFromAssembly(typeof(CompanyMap).Assembly); + + CollectionAssert.AreEquivalent(new[] { "sys_companies", "sys_settings" }, registry.Maps.Select(m => m.Table).ToArray()); + } + + [TestMethod] + public void StringWithoutLength_IsText_AndFrozenMapsCannotChange() + { + var map = new SystemSettingMap(); + map.Freeze(); + + Assert.AreEqual(ColumnType.Text, map.Column(x => x.Value).Type); + Assert.ThrowsExactly(() => map.Column(x => x.Key).MaxLength(10)); + } +} diff --git a/tests/Khaemhat.Data.Tests/MigrationTests.cs b/tests/Khaemhat.Data.Tests/MigrationTests.cs new file mode 100644 index 0000000..364c2d6 --- /dev/null +++ b/tests/Khaemhat.Data.Tests/MigrationTests.cs @@ -0,0 +1,154 @@ +using Khaemhat.Data.Mapping; +using Khaemhat.Data.Migrations; +using Khaemhat.Data.Schema; +using Khaemhat.Data.SystemDatabase; +using Khaemhat.Data.Tests.Support; + +namespace Khaemhat.Data.Tests; + +[TestClass] +public sealed class MigrationTests +{ + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task SystemMigrations_FromZero_ThenReapplied_DoNothing(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + var catalog = MigrationCatalog.ForSystem(); + var runner = new MigrationRunner(db.Context); + + var first = await runner.RunAsync(db.Target, catalog); + var second = await runner.RunAsync(db.Target, catalog); + + Assert.HasCount(catalog.Migrations.Count, first.Applied); + Assert.AreEqual(0, first.AlreadyApplied); + Assert.IsEmpty(second.Applied); + Assert.AreEqual(catalog.Migrations.Count, second.AlreadyApplied); + Assert.IsEmpty(second.Warnings); + + await using var unitOfWork = await db.OpenAsync(); + var applied = await SchemaVersionStore.ReadAsync(unitOfWork); + CollectionAssert.AreEqual( + catalog.Migrations.Select(m => (m.Module, m.Version, m.Name)).ToArray(), + applied.Select(a => (a.Module, a.Version, a.Name)).ToArray()); + Assert.IsTrue(applied.All(a => a.Checksum.Length == 64)); + + // The tables work through their maps. + var settings = unitOfWork.Repository(); + await settings.InsertAsync(new Core.Instance.SystemSetting { Key = "smtp.host", Value = "mail.example.com" }); + Assert.AreEqual(1, await settings.CountAsync(Querying.Query.For())); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task UnknownAppliedVersion_IsRefused(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + var runner = new MigrationRunner(db.Context); + await runner.RunAsync(db.Target, MigrationCatalog.ForSystem()); + await using (var unitOfWork = await db.OpenAsync()) + { + await SchemaVersionStore.RecordAsync(unitOfWork, new AppliedMigration("core", 20991231_001, "from_the_future", DateTime.UtcNow, new string('0', 64))); + } + + var error = await Assert.ThrowsExactlyAsync(() => runner.RunAsync(db.Target, MigrationCatalog.ForSystem())); + + StringAssert.Contains(error.Message, "20991231001 from_the_future"); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task FailingMigration_LeavesNoTrace(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + var catalog = MigrationCatalog.Discover(MigrationTarget.Company, [typeof(MigrationTests).Assembly]); + var runner = new MigrationRunner(db.Context); + + await Assert.ThrowsAsync(() => runner.RunAsync(db.Target, catalog)); + + await using var unitOfWork = await db.OpenAsync(); + Assert.IsEmpty(await SchemaVersionStore.ReadAsync(unitOfWork)); + var exists = await unitOfWork.QueryAsync( + SqlFragment.Of($"SELECT COUNT(*) FROM test_failing")).ContinueWith(t => t.IsCompletedSuccessfully, TaskScheduler.Default); + Assert.IsFalse(exists, "the table of the failed migration was rolled back"); + } + + [TestMethod] + public void Catalog_IsOrdered_AndVersionsAreValid() + { + var catalog = MigrationCatalog.ForSystem(); + + Assert.IsGreaterThanOrEqualTo(2, catalog.Migrations.Count); + CollectionAssert.AreEqual(catalog.Migrations.OrderBy(m => m.Version).ToArray(), catalog.Migrations.ToArray()); + Assert.IsTrue(catalog.Migrations.All(m => m.Target == MigrationTarget.System)); + } + + [TestMethod] + public void SystemMaps_AgreeWithTheMigratedSchema() + { + var model = MigrationCatalog.ForSystem().BuildModel(); + var maps = new MapRegistry().AddFromAssembly(typeof(CompanyMap).Assembly); + var problems = new List(); + + foreach (var map in maps.Maps) + { + var table = model.Table(map.Table); + if (table is null) + { + problems.Add($"{map.Table}: no migration creates it"); + continue; + } + + foreach (var column in map.Columns) + { + var definition = table.Column(column.ColumnName); + if (definition is null) + { + problems.Add($"{map.Table}.{column.ColumnName}: missing in the schema"); + } + else if (definition.TypeDescription != column.TypeDescription || definition.IsNullable != column.IsNullable) + { + problems.Add( + $"{map.Table}.{column.ColumnName}: map {column.TypeDescription} null={column.IsNullable}, schema {definition.TypeDescription} null={definition.IsNullable}"); + } + } + + problems.AddRange(table.Columns.Where(c => map.FindByColumn(c.Name) is null).Select(c => $"{map.Table}.{c.Name}: not mapped")); + } + + Assert.IsEmpty(problems, string.Join("\n", problems)); + } + + [TestMethod] + public void InvalidMigrations_AreRejectedByTheCatalog() + { + Assert.ThrowsExactly( + () => MigrationCatalog.Discover(MigrationTarget.System, [typeof(MigrationTests).Assembly])); + } +} + +/// Creates a table, then fails: used by . +[Migration(20261111_001, "failing_on_purpose", Module = "tests")] +public sealed class M20261111_001_FailingOnPurpose : IMigration +{ + /// + public void Up(ISchemaBuilder schema) + { + ArgumentNullException.ThrowIfNull(schema); + schema.CreateTable("test_failing", t => t.Id()); + schema.Sql(_ => "SELECT * FROM table_that_does_not_exist"); + } +} + +/// A badly numbered system migration: used by . +[Migration(2026_001, "bad_version", Target = MigrationTarget.System, Module = "tests")] +public sealed class BadlyNumberedMigration : IMigration +{ + /// + public void Up(ISchemaBuilder schema) + { + } +} diff --git a/tests/Khaemhat.Data.Tests/PerformanceBaselineTests.cs b/tests/Khaemhat.Data.Tests/PerformanceBaselineTests.cs new file mode 100644 index 0000000..cb31f9a --- /dev/null +++ b/tests/Khaemhat.Data.Tests/PerformanceBaselineTests.cs @@ -0,0 +1,64 @@ +using System.Diagnostics; +using System.Globalization; +using Khaemhat.Data.Querying; +using Khaemhat.Data.Tests.Support; + +namespace Khaemhat.Data.Tests; + +/// +/// Reference numbers for the data layer (issue M1 #13): insert and read rows through the repository. With +/// KHAEMHAT_PERF=1 it uses 100,000 rows and prints the timings; otherwise 2,000 rows as a smoke test. No +/// threshold is asserted: the numbers are recorded in docs/DATA-MODEL.md. +/// +[TestClass] +public sealed class PerformanceBaselineTests +{ + public TestContext TestContext { get; set; } = null!; + + [TestMethod] + [TestCategory("Performance")] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task InsertAndQuery_Baseline(string engine) + { + var rows = Environment.GetEnvironmentVariable("KHAEMHAT_PERF") == "1" ? 100_000 : 2_000; + await using var db = await TestDatabase.CreateAsync(engine); + await db.CreateTestTableAsync(); + await using var unitOfWork = await db.OpenAsync(); + var items = unitOfWork.Repository(); + + var insert = Stopwatch.StartNew(); + await unitOfWork.InTransactionAsync(async ct => + { + for (var i = 0; i < rows; i++) + { + await items.InsertAsync( + new TestItem + { + Code = "K" + i.ToString(CultureInfo.InvariantCulture), + Name = "Item " + i.ToString(CultureInfo.InvariantCulture), + Quantity = i % 100, + Amount = i * 1.5m, + At = DateTime.UtcNow, + }, + ct); + } + }); + insert.Stop(); + + var query = Stopwatch.StartNew(); + var all = await items.ListAsync(Query.For()); + query.Stop(); + + var page = Stopwatch.StartNew(); + var filtered = await items.ListAsync(Query.For().Where(x => x.Quantity == 42).OrderBy(x => x.Code).Page(0, 50)); + page.Stop(); + + Assert.HasCount(rows, all.Items); + Assert.AreEqual(rows / 100, filtered.TotalCount); + TestContext.WriteLine(string.Create( + CultureInfo.InvariantCulture, + $"[perf] {engine}: {rows} inserts in one transaction {insert.ElapsedMilliseconds} ms ({rows * 1000.0 / Math.Max(1, insert.ElapsedMilliseconds):F0}/s); " + + $"read all {query.ElapsedMilliseconds} ms; filtered page+count {page.ElapsedMilliseconds} ms")); + } +} diff --git a/tests/Khaemhat.Data.Tests/ProjectSmokeTests.cs b/tests/Khaemhat.Data.Tests/ProjectSmokeTests.cs index b713c1a..b0ee2ce 100644 --- a/tests/Khaemhat.Data.Tests/ProjectSmokeTests.cs +++ b/tests/Khaemhat.Data.Tests/ProjectSmokeTests.cs @@ -1,10 +1,16 @@ namespace Khaemhat.Data.Tests; -/// Keeps the test project runnable until the milestone that fills it (see docs/STATUS.md). +/// Architecture rule of issue M1 #7: the contracts know no engine. [TestClass] public sealed class ProjectSmokeTests { [TestMethod] - public void TestedAssembly_IsKhaemhat() => - StringAssert.StartsWith(typeof(ProjectSmokeTests).Assembly.GetName().Name, "Khaemhat."); + public void DataContracts_DoNotReferenceAnyDriver() + { + var references = typeof(IDbProvider).Assembly.GetReferencedAssemblies().Select(a => a.Name).ToArray(); + + Assert.DoesNotContain("Npgsql", references); + Assert.DoesNotContain("Microsoft.Data.Sqlite", references); + Assert.Contains("Khaemhat.Core", references); + } } diff --git a/tests/Khaemhat.Data.Tests/QueryTranslationTests.cs b/tests/Khaemhat.Data.Tests/QueryTranslationTests.cs new file mode 100644 index 0000000..7561e99 --- /dev/null +++ b/tests/Khaemhat.Data.Tests/QueryTranslationTests.cs @@ -0,0 +1,112 @@ +using Khaemhat.Data.PostgreSql; +using Khaemhat.Data.Querying; +using Khaemhat.Data.Sqlite; +using Khaemhat.Data.Tests.Support; + +namespace Khaemhat.Data.Tests; + +[TestClass] +public sealed class QueryTranslationTests +{ + private static readonly TestItemMap Map = CreateMap(); + + [TestMethod] + public void Comparisons_AndLogic_UseParameters() + { + var min = 5; + var code = "A'1"; + var sql = Where(Query.For().Where(x => (x.Quantity >= min && x.Code != code) || !x.Active)); + + Assert.AreEqual(" WHERE ((\"quantity\" >= @p0 AND \"code\" <> @p1) OR NOT (\"active\" = @p2))", sql.Text); + CollectionAssert.AreEqual(new object?[] { 5, "A'1", true }, sql.Parameters.Select(p => p.Value).ToArray()); + } + + [TestMethod] + public void ValueOnTheLeft_IsMirrored() + { + var sql = Where(Query.For().Where(x => 10 < x.Quantity)); + + Assert.AreEqual(" WHERE \"quantity\" > @p0", sql.Text); + } + + [TestMethod] + public void Null_BecomesIsNull() + { + string? none = null; + var sql = Where(Query.For().Where(x => x.Name == none && x.Day != null)); + + Assert.AreEqual(" WHERE (\"name\" IS NULL AND \"day\" IS NOT NULL)", sql.Text); + Assert.IsEmpty(sql.Parameters); + } + + [TestMethod] + public void Enums_AreComparedByName() + { + var sql = Where(Query.For().Where(x => x.Kind == TestKind.Service)); + + Assert.AreEqual("Service", sql.Parameters[0].Value); + } + + [TestMethod] + public void In_FromListContains_AndEmptyListIsFalse() + { + var codes = new List { "A", "B" }; + var sql = Where(Query.For().Where(x => codes.Contains(x.Code))); + var none = Where(Query.For().Where(x => Array.Empty().Contains(x.Id))); + + Assert.AreEqual(" WHERE \"code\" IN (@p0, @p1)", sql.Text); + Assert.AreEqual(" WHERE 1 = 0", none.Text); + } + + [TestMethod] + public void Like_IsCaseInsensitiveAndEscaped_PerDialect() + { + var query = Query.For().Where(x => x.Name!.Contains("10%_off")); + + var sqlite = new QueryBuilder(Map, SqliteDialect.Instance).Where(query).Render(SqliteDialect.Instance); + var postgres = new QueryBuilder(Map, PostgresDialect.Instance).Where(query).Render(PostgresDialect.Instance); + + Assert.AreEqual(" WHERE kh_lower(\"name\") LIKE kh_lower(@p0) ESCAPE '\\'", sqlite.Text); + Assert.AreEqual(" WHERE \"name\" ILIKE @p0 ESCAPE '\\'", postgres.Text); + Assert.AreEqual("%10\\%\\_off%", sqlite.Parameters[0].Value); + } + + [TestMethod] + public void UnsupportedExpressions_AreRefused() + { + var query = Query.For().Where(x => x.Code.ToUpperInvariant() == "A"); + + Assert.ThrowsExactly(() => Where(query)); + } + + [TestMethod] + public void Select_OrdersAndPages_AndDefaultsToKeyOrder() + { + var builder = new QueryBuilder(Map, SqliteDialect.Instance); + + var ordered = builder.Select(Query.For().OrderByDescending(x => x.Amount).OrderBy(x => x.Code).Page(20, 10)).Render(SqliteDialect.Instance); + var paged = builder.Select(Query.For().Take(5)).Render(SqliteDialect.Instance); + + StringAssert.EndsWith(ordered.Text, " ORDER BY \"amount\" DESC, \"code\" ASC LIMIT 10 OFFSET 20"); + StringAssert.EndsWith(paged.Text, " ORDER BY \"id\" ASC LIMIT 5 OFFSET 0"); + } + + [TestMethod] + public void SqlConditions_AreParenthesizedAndCombined() + { + var max = 100m; + var sql = Where(Query.For().Where(x => x.Active).Where(SqlFragment.Of($"{SqlFragment.Identifier("amount")} < {max}"))); + + Assert.AreEqual(" WHERE \"active\" = @p0 AND (\"amount\" < @p1)", sql.Text); + } + + private static TestItemMap CreateMap() + { + var map = new TestItemMap(); + map.Freeze(); + return map; + } + + private static RenderedSql Where(Query query) => + new QueryBuilder(Map, SqliteDialect.Instance).Where(query).Render(SqliteDialect.Instance); +} diff --git a/tests/Khaemhat.Data.Tests/RepositoryTests.cs b/tests/Khaemhat.Data.Tests/RepositoryTests.cs new file mode 100644 index 0000000..89e7690 --- /dev/null +++ b/tests/Khaemhat.Data.Tests/RepositoryTests.cs @@ -0,0 +1,190 @@ +using Khaemhat.Core.Common; +using Khaemhat.Data.Querying; +using Khaemhat.Data.Tests.Support; + +namespace Khaemhat.Data.Tests; + +[TestClass] +public sealed class RepositoryTests +{ + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task Crud_WithAuditColumns(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + await db.CreateTestTableAsync(); + await using var unitOfWork = await db.OpenAsync(); + var items = unitOfWork.Repository(); + + var item = new TestItem { Code = "C1", Name = "First", Amount = 10.5m }; + var id = await items.InsertAsync(item); + Assert.IsGreaterThan(0L, id); + Assert.AreEqual(id, item.Id); + Assert.AreEqual(1L, item.RowVersion); + Assert.IsNull(item.UpdatedAt); + + item.Name = "Changed"; + await items.UpdateAsync(item); + Assert.AreEqual(2L, item.RowVersion); + Assert.IsNotNull(item.UpdatedAt); + + var read = await items.GetAsync(id); + Assert.AreEqual("Changed", read!.Name); + Assert.AreEqual(2L, read.RowVersion); + Assert.AreEqual(item.CreatedAt, read.CreatedAt); + Assert.AreEqual(item.UpdatedAt, read.UpdatedAt); + Assert.AreEqual(TestDatabase.TestUser, read.UpdatedBy); + + Assert.IsTrue(await items.DeleteAsync(id)); + Assert.IsFalse(await items.DeleteAsync(id)); + Assert.IsNull(await items.GetAsync(id)); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task Update_OfAStaleCopy_ThrowsConcurrencyException(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + await db.CreateTestTableAsync(); + long id; + await using (var setup = await db.OpenAsync()) + { + id = await setup.Repository().InsertAsync(new TestItem { Code = "CC", Name = "v1" }); + } + + await using var first = await db.OpenAsync(); + await using var second = await db.OpenAsync(); + var copyA = await first.Repository().GetAsync(id); + var copyB = await second.Repository().GetAsync(id); + + copyA!.Name = "by A"; + await first.Repository().UpdateAsync(copyA); + + copyB!.Name = "by B"; + var conflict = await Assert.ThrowsExactlyAsync(() => second.Repository().UpdateAsync(copyB)); + Assert.AreEqual("test_items", conflict.Table); + Assert.AreEqual(1L, conflict.ExpectedVersion); + Assert.AreEqual("by A", (await second.Repository().GetAsync(id))!.Name); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task Update_OfADeletedRow_ThrowsConcurrencyException(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + await db.CreateTestTableAsync(); + await using var unitOfWork = await db.OpenAsync(); + var items = unitOfWork.Repository(); + var item = new TestItem { Code = "GONE" }; + await items.InsertAsync(item); + await items.DeleteAsync(item.Id); + + await Assert.ThrowsExactlyAsync(() => items.UpdateAsync(item)); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task Transactions_CommitAndRollback(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + await db.CreateTestTableAsync(); + await using var unitOfWork = await db.OpenAsync(); + var items = unitOfWork.Repository(); + + await unitOfWork.BeginAsync(); + await items.InsertAsync(new TestItem { Code = "ROLLED" }); + await unitOfWork.RollbackAsync(); + + await Assert.ThrowsExactlyAsync( + () => unitOfWork.InTransactionAsync(async ct => + { + await items.InsertAsync(new TestItem { Code = "FAILED" }, ct); + throw new InvalidOperationException("business rule"); + })); + + await unitOfWork.InTransactionAsync(async ct => + { + await unitOfWork.AcquireLockAsync("numerator:FV:2026", ct); + await items.InsertAsync(new TestItem { Code = "KEPT" }, ct); + }); + + var codes = (await items.ListAsync(Query.For())).Items.Select(x => x.Code).ToArray(); + CollectionAssert.AreEqual(new[] { "KEPT" }, codes); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task HostileInput_IsStoredAndFound_AsData(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + await db.CreateTestTableAsync(); + await using var unitOfWork = await db.OpenAsync(); + var items = unitOfWork.Repository(); + const string hostile = "x'); DROP TABLE test_items; --"; + + await items.InsertAsync(new TestItem { Code = "H1", Name = hostile }); + var found = await items.FindAsync(Query.For().Where(x => x.Name == hostile)); + var like = await items.CountAsync(Query.For().Where(x => x.Name!.Contains("'); DROP"))); + var raw = await unitOfWork.QueryAsync(SqlFragment.Of($"SELECT code FROM test_items WHERE name = {hostile}")); + + Assert.AreEqual("H1", found?.Code); + Assert.AreEqual(1, like); + CollectionAssert.AreEqual(new[] { "H1" }, raw.ToArray()); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task Queries_MapToRecordsClassesAndScalars(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + await db.CreateTestTableAsync(); + await using var unitOfWork = await db.OpenAsync(); + var items = unitOfWork.Repository(); + await items.InsertAsync(new TestItem { Code = "R1", Amount = 1.25m, Kind = TestKind.Service }); + await items.InsertAsync(new TestItem { Code = "R2", Amount = 2.5m, Kind = TestKind.Service }); + await items.InsertAsync(new TestItem { Code = "R3", Amount = 4m, Kind = TestKind.Goods }); + + var totals = await unitOfWork.QueryAsync( + SqlFragment.Literal("SELECT kind, SUM(amount) AS total_amount, COUNT(*) AS item_count FROM test_items GROUP BY kind ORDER BY kind")); + var rows = await unitOfWork.QueryAsync(SqlFragment.Literal("SELECT code, amount FROM test_items ORDER BY code")); + var kinds = await unitOfWork.QueryAsync(SqlFragment.Literal("SELECT DISTINCT kind FROM test_items ORDER BY kind")); + var ids = await unitOfWork.QueryAsync(SqlFragment.Literal("SELECT id FROM test_items ORDER BY id")); + var inList = await items.CountAsync(Query.For().Where(x => ids.Contains(x.Id) && x.Kind == TestKind.Service)); + + CollectionAssert.AreEqual( + new[] { new KindTotal(TestKind.Goods, 4m, 1), new KindTotal(TestKind.Service, 3.75m, 2) }, totals.ToArray()); + Assert.AreEqual(2.5m, rows[1].Amount); + CollectionAssert.AreEqual(new[] { TestKind.Goods, TestKind.Service }, kinds.ToArray()); + Assert.AreEqual(2, inList); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task Stream_ReadsWithoutMapping(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + await db.CreateTestTableAsync(); + await using var unitOfWork = await db.OpenAsync(); + await unitOfWork.Repository().InsertAsync(new TestItem { Code = "S1", Amount = 7.000001m }); + + await using var reader = await unitOfWork.StreamAsync(SqlFragment.Literal("SELECT amount FROM test_items")); + Assert.IsTrue(await reader.ReadAsync()); + Assert.AreEqual(7.000001m, db.Dialect.FromDbValue(reader.GetValue(0), Schema.ColumnType.Decimal)); + } + + private sealed record KindTotal(TestKind Kind, decimal TotalAmount, long ItemCount); + + private sealed class CodeRow + { + public string Code { get; set; } = string.Empty; + + public decimal Amount { get; set; } + } +} diff --git a/tests/Khaemhat.Data.Tests/SqlFragmentTests.cs b/tests/Khaemhat.Data.Tests/SqlFragmentTests.cs new file mode 100644 index 0000000..fd5611a --- /dev/null +++ b/tests/Khaemhat.Data.Tests/SqlFragmentTests.cs @@ -0,0 +1,70 @@ +using Khaemhat.Data.PostgreSql; +using Khaemhat.Data.Schema; +using Khaemhat.Data.Sqlite; + +namespace Khaemhat.Data.Tests; + +[TestClass] +public sealed class SqlFragmentTests +{ + [TestMethod] + public void Holes_BecomeParameters_AndNeverReachTheText() + { + const string hostile = "x'); DROP TABLE parties; --"; + var fragment = SqlFragment.Of($"SELECT * FROM parties WHERE name = {hostile} AND id > {42L}"); + + var rendered = fragment.Render(SqliteDialect.Instance); + + Assert.AreEqual("SELECT * FROM parties WHERE name = @p0 AND id > @p1", rendered.Text); + Assert.HasCount(2, rendered.Parameters); + Assert.AreEqual(hostile, rendered.Parameters[0].Value); + Assert.AreEqual(42L, rendered.Parameters[1].Value); + Assert.DoesNotContain("DROP", rendered.Text); + } + + [TestMethod] + public void NestedFragments_AreRenumbered() + { + var inner = SqlFragment.Of($"code = {"A"}"); + var outer = SqlFragment.Of($"SELECT 1 WHERE {inner} OR name = {"B"}"); + + var rendered = outer.Render(PostgresDialect.Instance); + + Assert.AreEqual("SELECT 1 WHERE code = @p0 OR name = @p1", rendered.Text); + CollectionAssert.AreEqual(new object?[] { "A", "B" }, rendered.Parameters.Select(p => p.Value).ToArray()); + } + + [TestMethod] + public void Identifiers_AreValidatedAndQuoted() + { + var fragment = SqlFragment.Of($"SELECT {SqlFragment.Identifier("parties.vat_number")} FROM {SqlFragment.Identifier("parties")}"); + + Assert.AreEqual("SELECT \"parties\".\"vat_number\" FROM \"parties\"", fragment.Render(SqliteDialect.Instance).Text); + Assert.ThrowsExactly(() => SqlFragment.Identifier("parties; DROP TABLE x")); + Assert.ThrowsExactly(() => SqlFragment.Identifier("Name")); + } + + [TestMethod] + public void ToString_ShowsPlaceholdersButNotValues() + { + var fragment = SqlFragment.Of($"UPDATE sys_settings SET value = {"s3cr3t-password"}"); + + Assert.AreEqual("UPDATE sys_settings SET value = @p0", fragment.ToString()); + } + + [TestMethod] + public void TypedParameter_KeepsItsType() + { + var fragment = SqlFragment.Of($"INSERT INTO t (data) VALUES ({SqlFragment.Parameter("{}", ColumnType.Json)})"); + + Assert.AreEqual(ColumnType.Json, fragment.Render(PostgresDialect.Instance).Parameters[0].Type); + } + + [TestMethod] + public void Join_SkipsEmptyFragments() + { + var joined = SqlFragment.Join(" AND ", [SqlFragment.Literal("a = 1"), SqlFragment.Empty, SqlFragment.Of($"b = {2}")]); + + Assert.AreEqual("a = 1 AND b = @p0", joined.Render(SqliteDialect.Instance).Text); + } +} diff --git a/tests/Khaemhat.Data.Tests/Support/TestDatabase.cs b/tests/Khaemhat.Data.Tests/Support/TestDatabase.cs new file mode 100644 index 0000000..b4f89cf --- /dev/null +++ b/tests/Khaemhat.Data.Tests/Support/TestDatabase.cs @@ -0,0 +1,93 @@ +using System.Data.Common; +using Khaemhat.Data.Execution; +using Khaemhat.Data.Mapping; +using Khaemhat.Data.PostgreSql; +using Khaemhat.Data.Sqlite; +using Khaemhat.Data.SystemDatabase; + +namespace Khaemhat.Data.Tests.Support; + +/// +/// A fresh, empty database for one test: SQLite in memory, or a temporary PostgreSQL database on the server of +/// KHAEMHAT_TEST_PG (dropped at the end). PostgreSQL tests are inconclusive (skipped) when the variable is unset. +/// +internal sealed class TestDatabase : IAsyncDisposable +{ + public const string Sqlite = "Sqlite"; + public const string Postgres = "PostgreSql"; + public const string TestUser = "tester"; + + private readonly DbConnection? _keeper; + + private TestDatabase(DatabaseUrl url, DatabaseTarget target, DbProviderCatalog providers, DbConnection? keeper) + { + Url = url; + Target = target; + Providers = providers; + _keeper = keeper; + } + + public static string? PostgresServer => Environment.GetEnvironmentVariable("KHAEMHAT_TEST_PG") is { Length: > 0 } value ? value : null; + + public DatabaseUrl Url { get; } + + public DatabaseTarget Target { get; } + + public DbProviderCatalog Providers { get; } + + public IDbProvider Provider => Target.Provider; + + public ISqlDialect Dialect => Provider.Dialect; + + public MapRegistry Maps { get; } = new MapRegistry().AddFromAssembly(typeof(CompanyMap).Assembly).Add(new TestItemMap()); + + public DataAccessContext Context { get; } = new(TimeProvider.System, () => TestUser); + + public static DbProviderCatalog AllProviders() => new(new SqliteProvider(), new PostgresProvider()); + + public static async Task CreateAsync(string engine) + { + var providers = AllProviders(); + if (engine == Sqlite) + { + var url = DatabaseUrl.Parse("sqlite://memory/t" + Guid.NewGuid().ToString("N")); + var target = providers.Resolve(url); + var keeper = target.Provider.CreateConnection(target.ConnectionString); + await keeper.OpenAsync(); + return new TestDatabase(url, target, providers, keeper); + } + + if (PostgresServer is null) + { + Assert.Inconclusive("KHAEMHAT_TEST_PG is not set: PostgreSQL test skipped."); + } + + var pgUrl = DatabaseUrl.Parse(PostgresServer!).WithDatabase("khaemhat_t_" + Guid.NewGuid().ToString("N")[..12]); + var pgTarget = providers.Resolve(pgUrl); + await pgTarget.Provider.EnsureDatabaseExistsAsync(pgTarget.ConnectionString); + return new TestDatabase(pgUrl, pgTarget, providers, keeper: null); + } + + public Task OpenAsync() => UnitOfWork.OpenAsync(Target, Maps, Context); + + /// Creates the test_items table used by the repository and dialect tests. + public async Task CreateTestTableAsync() + { + await using var unitOfWork = await OpenAsync(); + foreach (var statement in Provider.Ddl.Generate(TestItemMap.Schema())) + { + await unitOfWork.ExecuteAsync(SqlFragment.Literal(statement)); + } + } + + public async ValueTask DisposeAsync() + { + if (_keeper is not null) + { + await _keeper.DisposeAsync(); + return; + } + + await Provider.DropDatabaseAsync(Target.ConnectionString); + } +} diff --git a/tests/Khaemhat.Data.Tests/Support/TestItem.cs b/tests/Khaemhat.Data.Tests/Support/TestItem.cs new file mode 100644 index 0000000..8ec3f25 --- /dev/null +++ b/tests/Khaemhat.Data.Tests/Support/TestItem.cs @@ -0,0 +1,79 @@ +using Khaemhat.Core.Entities; +using Khaemhat.Data.Mapping; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.Tests.Support; + +internal enum TestKind +{ + Goods = 0, + Service = 1, + Kit = 2, +} + +/// An entity with one property of every logical type. +internal sealed class TestItem : Entity, IDemoMarkable +{ + public string Code { get; set; } = string.Empty; + + public string? Name { get; set; } + + public int Quantity { get; set; } + + public long Big { get; set; } + + public decimal Amount { get; set; } + + public bool Active { get; set; } + + public DateOnly? Day { get; set; } + + public DateTime? At { get; set; } + + public Guid? Uid { get; set; } + + public byte[]? Data { get; set; } + + public string? Payload { get; set; } + + public TestKind Kind { get; set; } + + public bool IsDemo { get; set; } +} + +internal sealed class TestItemMap : EntityMap +{ + public TestItemMap() + { + ToTable("test_items"); + Map(x => x.Code).MaxLength(20); + Map(x => x.Name).MaxLength(100); + Map(x => x.Payload).Json(); + } + + public static IReadOnlyList Schema() + { + var schema = new SchemaBuilder(); + schema.CreateTable("test_items", t => + { + t.Id(); + t.String("code", 20).NotNull(); + t.String("name", 100); + t.Int32("quantity").NotNull(); + t.Int64("big").NotNull(); + t.Decimal("amount").NotNull(); + t.Bool("active").NotNull(); + t.Date("day"); + t.DateTimeUtc("at"); + t.Guid("uid"); + t.Bytes("data"); + t.Json("payload"); + t.String("kind", 40).NotNull(); + t.DemoColumn(); + t.AuditColumns(); + t.Unique("ux_test_items_code", "code"); + t.Index("ix_test_items_at", "at"); + }); + return schema.Operations; + } +} diff --git a/tests/Khaemhat.Infrastructure.Tests/ProjectSmokeTests.cs b/tests/Khaemhat.Infrastructure.Tests/ProjectSmokeTests.cs deleted file mode 100644 index fc262f5..0000000 --- a/tests/Khaemhat.Infrastructure.Tests/ProjectSmokeTests.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace Khaemhat.Infrastructure.Tests; - -/// Keeps the test project runnable until the milestone that fills it (see docs/STATUS.md). -[TestClass] -public sealed class ProjectSmokeTests -{ - [TestMethod] - public void TestedAssembly_IsKhaemhat() => - StringAssert.StartsWith(typeof(ProjectSmokeTests).Assembly.GetName().Name, "Khaemhat."); -} diff --git a/tests/Khaemhat.Infrastructure.Tests/SecretProtectorTests.cs b/tests/Khaemhat.Infrastructure.Tests/SecretProtectorTests.cs new file mode 100644 index 0000000..c6e6c81 --- /dev/null +++ b/tests/Khaemhat.Infrastructure.Tests/SecretProtectorTests.cs @@ -0,0 +1,133 @@ +using Khaemhat.Core.Security; +using Khaemhat.Infrastructure.Security; + +namespace Khaemhat.Infrastructure.Tests; + +/// Encryption of secrets at rest (SPEC 12.9, issue M1 #12). +[TestClass] +public sealed class SecretProtectorTests +{ + private const string Purpose = "company:ACME:database"; + private const string Secret = "postgres://acme:S3cret!@db/khaemhat_acme"; + + [TestMethod] + public void Protect_ThenUnprotect_RoundTrips() + { + var protector = NewProtector(); + + var protectedValue = protector.Protect(Secret, Purpose); + + StringAssert.StartsWith(protectedValue, "khs1." + protector.Key.Id + "."); + Assert.DoesNotContain("S3cret", protectedValue); + Assert.AreEqual(Secret, protector.Unprotect(protectedValue, Purpose)); + Assert.AreEqual(string.Empty, protector.Unprotect(protector.Protect(string.Empty, Purpose), Purpose)); + } + + [TestMethod] + public void TheSameSecret_IsEncryptedDifferentlyEachTime() + { + var protector = NewProtector(); + + Assert.AreNotEqual(protector.Protect(Secret, Purpose), protector.Protect(Secret, Purpose)); + } + + [TestMethod] + public void KeyId_IsStable_ForTheSameKey() + { + var key = SecretKey.GenerateBase64(); + + Assert.AreEqual(SecretKey.FromBase64(key).Id, SecretKey.FromBase64(key).Id); + Assert.AreEqual(8, SecretKey.FromBase64(key).Id.Length); + Assert.AreNotEqual(SecretKey.FromBase64(key).Id, SecretKey.FromBase64(SecretKey.GenerateBase64()).Id); + Assert.AreEqual("SecretKey " + SecretKey.FromBase64(key).Id, SecretKey.FromBase64(key).ToString()); + } + + [TestMethod] + public void AnotherKey_IsRefused_NamingBothKeyIds() + { + var protectedValue = NewProtector().Protect(Secret, Purpose); + var other = NewProtector(); + + var error = Assert.ThrowsExactly(() => other.Unprotect(protectedValue, Purpose)); + + StringAssert.Contains(error.Message, SecretProtector.KeyIdOf(protectedValue)!); + StringAssert.Contains(error.Message, other.Key.Id); + Assert.DoesNotContain("S3cret", error.ToString()); + } + + [TestMethod] + public void AnotherPurpose_IsRefused() + { + var protector = NewProtector(); + var protectedValue = protector.Protect(Secret, Purpose); + + Assert.ThrowsExactly(() => protector.Unprotect(protectedValue, "company:EVIL:database")); + } + + [TestMethod] + [DataRow("")] + [DataRow("plain text")] + [DataRow("khs1.abc")] + [DataRow("khs2.00000000.AAAA")] + public void NotAProtectedValue_IsRefused(string value) + { + Assert.ThrowsExactly(() => NewProtector().Unprotect(value, Purpose)); + } + + [TestMethod] + public void DamagedValues_AreRefused() + { + var protector = NewProtector(); + var protectedValue = protector.Protect(Secret, Purpose); + var payload = protectedValue[(protectedValue.LastIndexOf('.') + 1)..]; + var prefix = protectedValue[..(protectedValue.LastIndexOf('.') + 1)]; + var damaged = payload[..20] + (payload[20] == 'A' ? 'B' : 'A') + payload[21..]; + + Assert.ThrowsExactly(() => protector.Unprotect(prefix + damaged, Purpose)); + Assert.ThrowsExactly(() => protector.Unprotect(prefix + payload[..10], Purpose)); + Assert.ThrowsExactly(() => protector.Unprotect(prefix + "not*base64", Purpose)); + } + + [TestMethod] + [DataRow("")] + [DataRow(" ")] + [DataRow("not base64!")] + [DataRow("c2hvcnQ=")] + public void WeakOrInvalidKeys_AreRefused_WithoutEchoingThem(string value) + { + var error = Assert.ThrowsExactly(() => SecretKey.FromBase64(value)); + + StringAssert.Contains(error.Message, "KHAEMHAT_SECRET_KEY"); + if (value.Trim().Length > 0) + { + Assert.DoesNotContain(value, error.Message); + } + } + + [TestMethod] + public void Rotation_ReencryptsWithTheNewKey() + { + var current = NewProtector(); + var next = NewProtector(); + var rotation = new SecretRotation(current, next); + var old = current.Protect(Secret, Purpose); + + Assert.IsTrue(rotation.NeedsRotation(old)); + var rotated = rotation.Rotate(old, Purpose); + + Assert.IsFalse(rotation.NeedsRotation(rotated)); + Assert.AreEqual(next.Key.Id, SecretProtector.KeyIdOf(rotated)); + Assert.AreEqual(Secret, next.Unprotect(rotated, Purpose)); + Assert.ThrowsExactly(() => current.Unprotect(rotated, Purpose)); + } + + [TestMethod] + public void KeyIdOf_ReadsOnlyProtectedValues() + { + Assert.IsNull(SecretProtector.KeyIdOf(null)); + Assert.IsNull(SecretProtector.KeyIdOf("plain")); + Assert.AreEqual("0a1b2c3d", SecretProtector.KeyIdOf("khs1.0a1b2c3d.AAAA")); + } + + private static SecretProtector NewProtector() => new(SecretKey.FromBase64(SecretKey.GenerateBase64())); +} diff --git a/tools/Khaemhat.DataModelTool/Khaemhat.DataModelTool.csproj b/tools/Khaemhat.DataModelTool/Khaemhat.DataModelTool.csproj new file mode 100644 index 0000000..6d9e0a8 --- /dev/null +++ b/tools/Khaemhat.DataModelTool/Khaemhat.DataModelTool.csproj @@ -0,0 +1,14 @@ + + + + Exe + Khaemhat.DataModelTool + + false + + + + + + + diff --git a/tools/Khaemhat.DataModelTool/Program.cs b/tools/Khaemhat.DataModelTool/Program.cs new file mode 100644 index 0000000..37ce22a --- /dev/null +++ b/tools/Khaemhat.DataModelTool/Program.cs @@ -0,0 +1,31 @@ +// Khaemhat.DataModelTool: writes the generated section of docs/DATA-MODEL.md from the migrations (SPEC §18). +// dotnet run --project tools/Khaemhat.DataModelTool -- [--check] [path/to/DATA-MODEL.md] +// With --check it changes nothing and exits with 1 when the document is out of date (used by CI and tests). +using System.Text; +using Khaemhat.Data.Migrations; + +var check = args.Contains("--check"); +var path = args.FirstOrDefault(a => !a.StartsWith("--", StringComparison.Ordinal)) ?? Path.Combine("docs", "DATA-MODEL.md"); +if (!File.Exists(path)) +{ + Console.Error.WriteLine($"DataModelTool: file non trovato: {path}"); + return 2; +} + +var current = File.ReadAllText(path, Encoding.UTF8); +var updated = DataModelDocument.Update(current, DataModelDocument.Render(MigrationCatalog.ForSystem(), MigrationCatalog.ForCompany())); +if (string.Equals(current, updated, StringComparison.Ordinal)) +{ + Console.WriteLine($"DataModelTool: {path} è aggiornato."); + return 0; +} + +if (check) +{ + Console.Error.WriteLine($"DataModelTool: {path} non è aggiornato: eseguire scripts/gen-datamodel.sh."); + return 1; +} + +File.WriteAllText(path, updated, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false)); +Console.WriteLine($"DataModelTool: {path} rigenerato."); +return 0; -- 2.54.1