diff --git a/.editorconfig b/.editorconfig index 4ca6da5..eff14a3 100644 --- a/.editorconfig +++ b/.editorconfig @@ -70,4 +70,12 @@ dotnet_diagnostic.IDE0005.severity = warning [tests/**.cs] # CA1707 (underscores in names): test names read as Method_Condition_Result. dotnet_diagnostic.CA1707.severity = none +# CA1861/CA1847/CA1862: tests pass literal arrays and write the string predicates the query translator must handle. +dotnet_diagnostic.CA1861.severity = none +dotnet_diagnostic.CA1847.severity = none +dotnet_diagnostic.CA1862.severity = none # MSTEST0037 and friends stay at their defaults. + +# Migration classes are named after their version, M__ (SPEC 5.3). +[**/Migrations/*.cs] +dotnet_diagnostic.CA1707.severity = none diff --git a/CHANGELOG.md b/CHANGELOG.md index 21813ec..a596fa9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,33 @@ un'immagine `gitea.encke-hake.ts.net/balbosoft/khaemhat:X.Y.Z`. ## [Unreleased] +## [0.1.0] - 2026-11-11 + +M1 — Strato dati. Nessuna funzione visibile nell'interfaccia: è il motore su cui si appoggiano tutte le milestone +successive. + +### Added +- Micro-ORM interno in `Khaemhat.Data` (ADR 0004, ADR 0018): mappe esplicite `EntityMap`, query tipizzate + `Query.For().Where(...)` con un sottoinsieme sicuro di espressioni, `SqlFragment` con valori sempre + parametrizzati, `IUnitOfWork` con transazioni, `IRepository` con colonne di audit e concorrenza + ottimistica su `row_version` (`ConcurrencyException`), tentativi automatici sugli errori transitori. +- Provider SQLite (tabelle `STRICT`, WAL, decimali esatti come interi scalati x10^6, LIKE senza distinzione di + maiuscole anche per le lettere accentate, scritture serializzate) e PostgreSQL 16+ (`numeric(19,6)`, `timestamptz`, + `jsonb`, `ILIKE`, lock di transazione, creazione ed eliminazione dei database aziendali, informazioni sul server). +- Configurazione dei database da URL `postgres://` e `sqlite://` (`DatabaseUrl`), con la password mai mostrata nei + messaggi. +- Migrazioni in C#: cataloghi di sistema e aziendale, `schema_version` con checksum in ogni database, una + migrazione per transazione sotto lock, rifiuto delle versioni sconosciute; prime tabelle di sistema + `sys_companies` e `sys_settings`. +- Registro delle aziende (`CompanyRegistry`) con la posizione del database cifrata e risoluzione + `DatabaseScope.System` / `DatabaseScope.Company(codice)` in `IUnitOfWorkFactory`. +- Cifratura dei segreti a riposo con AES-256-GCM e chiave derivata da `KHAEMHAT_SECRET_KEY` (`SecretProtector`, + formato `khs1`), con il mattone della rotazione delle chiavi. +- `scripts/new-migration.sh` e `scripts/gen-datamodel.sh` (con `tools/Khaemhat.DataModelTool`): la sezione + *Schema attuale* di `docs/DATA-MODEL.md` è generata dalle migrazioni e un test ne verifica l'aggiornamento. +- Test su SQLite e PostgreSQL (tipi, decimali, paginazione, upsert, LIKE, ordinamento temporale, CRUD, concorrenza, + transazioni, input ostile, migrazioni, registro, cifratura) e misure di riferimento delle prestazioni. + ## [0.0.1] - 2026-11-11 Prima versione: M0 — Bootstrap. Nessuna funzione gestionale ancora; c'è lo scheletro su cui costruirle. diff --git a/Khaemhat.sln b/Khaemhat.sln index 3d70c39..d1ee8aa 100644 --- a/Khaemhat.sln +++ b/Khaemhat.sln @@ -41,6 +41,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tools", "tools", "{07C2787E EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Khaemhat.LicenseTool", "tools\Khaemhat.LicenseTool\Khaemhat.LicenseTool.csproj", "{4317FFD0-9473-43FD-8543-F80EF9BA812A}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Khaemhat.DataModelTool", "tools\Khaemhat.DataModelTool\Khaemhat.DataModelTool.csproj", "{7CA26565-5E1A-46AA-A2F7-C38FBADF0E3F}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -243,6 +245,18 @@ Global {4317FFD0-9473-43FD-8543-F80EF9BA812A}.Release|x64.Build.0 = Release|Any CPU {4317FFD0-9473-43FD-8543-F80EF9BA812A}.Release|x86.ActiveCfg = Release|Any CPU {4317FFD0-9473-43FD-8543-F80EF9BA812A}.Release|x86.Build.0 = Release|Any CPU + {7CA26565-5E1A-46AA-A2F7-C38FBADF0E3F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7CA26565-5E1A-46AA-A2F7-C38FBADF0E3F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7CA26565-5E1A-46AA-A2F7-C38FBADF0E3F}.Debug|x64.ActiveCfg = Debug|Any CPU + {7CA26565-5E1A-46AA-A2F7-C38FBADF0E3F}.Debug|x64.Build.0 = Debug|Any CPU + {7CA26565-5E1A-46AA-A2F7-C38FBADF0E3F}.Debug|x86.ActiveCfg = Debug|Any CPU + {7CA26565-5E1A-46AA-A2F7-C38FBADF0E3F}.Debug|x86.Build.0 = Debug|Any CPU + {7CA26565-5E1A-46AA-A2F7-C38FBADF0E3F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7CA26565-5E1A-46AA-A2F7-C38FBADF0E3F}.Release|Any CPU.Build.0 = Release|Any CPU + {7CA26565-5E1A-46AA-A2F7-C38FBADF0E3F}.Release|x64.ActiveCfg = Release|Any CPU + {7CA26565-5E1A-46AA-A2F7-C38FBADF0E3F}.Release|x64.Build.0 = Release|Any CPU + {7CA26565-5E1A-46AA-A2F7-C38FBADF0E3F}.Release|x86.ActiveCfg = Release|Any CPU + {7CA26565-5E1A-46AA-A2F7-C38FBADF0E3F}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -264,5 +278,6 @@ Global {C5C8124F-3B12-43A9-BD0F-5AA5F7261699} = {0AB3BF05-4346-4AA6-1389-037BE0695223} {FDCAC352-BB40-4B5E-948A-2CDAA48B9E6A} = {0AB3BF05-4346-4AA6-1389-037BE0695223} {4317FFD0-9473-43FD-8543-F80EF9BA812A} = {07C2787E-EAC7-C090-1BA3-A61EC2A24D84} + {7CA26565-5E1A-46AA-A2F7-C38FBADF0E3F} = {07C2787E-EAC7-C090-1BA3-A61EC2A24D84} EndGlobalSection EndGlobal diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 1618f05..07a371a 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,6 +1,6 @@ # Architettura di Khaemhat -> Stato: bozza · Revisione 2026-11-11 · Fonte: SPEC §1, §2, §3, §4, §5, §7, §9, §13, §15, §19, §21; ADR 0009-0017 +> Stato: bozza · Revisione 2026-11-11 · Fonte: SPEC §1, §2, §3, §4, §5, §7, §9, §13, §15, §19, §21; ADR 0009-0018 Questo documento descrive l'architettura **prevista** del gestionale: livelli, progetti, regole di dipendenza, composizione dei moduli, gestione multi-database, ciclo di una richiesta e servizi trasversali. Il codice non esiste @@ -74,6 +74,7 @@ client degli aggiornamenti è `GiteaPackagesClient`. | `Khaemhat.Application` | casi d'uso, moduli (`IModule`), motore documenti e contabile, seeding, report | M2 → M12 | | `Khaemhat.Web` | host ASP.NET Core 10, Blazor Interactive Server, Minimal API, MCP, autenticazione | M0 → M13 | | `tools/Khaemhat.LicenseTool` | CLI per chiavi ed emissione licenze (chiave privata mai nel repository; non distribuito) | M2 | +| `tools/Khaemhat.DataModelTool` | rigenera la sezione generata di `docs/DATA-MODEL.md` dalle migrazioni (non distribuito, ADR 0018) | M1 | | `tests/*` | `Core`, `Data`, `Infrastructure`, `Fiscal.Italy`, `Application`, `Web` (MSTest) | ogni milestone | ## 3. Regole di dipendenza @@ -171,11 +172,11 @@ tabelle è in [DATA-MODEL.md](DATA-MODEL.md). | Tipo | Progetto | Responsabilità | |---|---|---| -| `IDbProvider` | Data | motore (`DbEngine`), dialetto, `SchemaBuilder`, creazione connessioni, `EnsureDatabaseExistsAsync`, `DropDatabaseAsync`, `GetServerInfoAsync` | -| `ISqlDialect` | Data | quoting, parametri, paginazione, `InsertReturningId`, `Upsert`, `CaseInsensitiveLike`, `AdvisoryLock`, conversioni `ColumnType` | +| `IDbProvider` | Data | motore (`DbEngine`), dialetto, generatore DDL (`Ddl`), connection string da `DatabaseUrl`, creazione connessioni, `EnsureDatabaseExistsAsync`, `DatabaseExistsAsync`, `DropDatabaseAsync`, `GetServerInfoAsync`, `IsTransient`, `EnterWriteScopeAsync` (ADR 0018) | +| `ISqlDialect` | Data | quoting, parametri, paginazione, `InsertReturningId`, `Upsert`, `CaseInsensitiveLike` e `AdvisoryLock` come `SqlFragment`, conversioni `ColumnType` | | `IUnitOfWorkFactory` | Data | `CreateAsync(DatabaseScope scope)` dove lo scope è `System` oppure `Company(code)` | | `IUnitOfWork` / `IRepository` | Data | transazione, repository tipizzati, `ExecuteAsync`/`QueryAsync`/`StreamAsync` su `SqlFragment` | -| `ICompanyRegistry` (`CompanyRegistry`) | Data | risolve `companyCode → (IDbProvider, connection string decifrata)` leggendo `sys_companies` | +| `ICompanyRegistry` (`CompanyRegistry`) | Data | registra le aziende e risolve `companyCode → DatabaseTarget` (provider + connection string) decifrando l'URL salvato in `sys_companies` | | `ICompanyContext` | Core (interfaccia), Web (`CompanyContextAccessor`) | azienda corrente: **per circuito Blazor** e **per richiesta API** (header `X-Company`) | | `SecretProtector` | Infrastructure | AES-256-GCM con chiave derivata da `KHAEMHAT_SECRET_KEY` per connection string e credenziali | @@ -440,11 +441,11 @@ stando fuori dal pacchetto fiscale: è la collocazione indicata da SPEC §21. | Area | Classi | |---|---| -| Contratti | `IDbProvider`, `ISqlDialect`, `ISchemaBuilder`, `IUnitOfWork`, `IUnitOfWorkFactory`, `IRepository<,>`, `Repository<,>` | -| Registro database | `ICompanyRegistry`, `CompanyRegistry`, `DatabaseScope` | +| Contratti | `IDbProvider`, `ISqlDialect`, `ISchemaBuilder`, `IUnitOfWork`, `IUnitOfWorkFactory`, `IRepository<,>`, `Repository<,>`, `DatabaseUrl`, `DatabaseTarget`, `DbProviderCatalog` | +| Registro database | `ICompanyRegistry`, `CompanyRegistry`, `IDatabaseLocator`, `DatabaseLocator`, `DatabaseScope`, `CompanyCodes` | | Mappe e query | `EntityMap`, `ColumnMap`, `ColumnType`, `MapRegistry`, `QueryBuilder`, `Query`, `PagedResult`, `SqlFragment`, `ExpressionTranslator` | -| Esecuzione | `DbExecutor`, `ReaderMapper`, `CommandCache`, `TransientRetryPolicy` | -| Migrazioni | `IMigration`, `MigrationAttribute`, `MigrationRunner`, `MigrationCatalog` (System/Company), `SchemaVersionStore`, `TableDefinition`, `ColumnDefinition`, `IndexDefinition`, `ForeignKeyDefinition`, `DdlGenerator` | +| Esecuzione | `UnitOfWork`, `UnitOfWorkFactory`, `DataAccessContext` (orologio, utente, tentativi), `DbExecutor`, `ReaderMapper`, `CommandCache`, `TransientRetryPolicy` | +| Migrazioni | `IMigration`, `MigrationAttribute`, `MigrationRunner`, `MigrationReport`, `MigrationCatalog` (`ForSystem`/`ForCompany`), `SchemaVersionStore`, `SchemaModel`, `DataModelDocument`, `TableDefinition`, `ColumnDefinition`, `IndexDefinition`, `ForeignKeyDefinition`, `DdlGenerator` | | SQLite | `SqliteProvider`, `SqliteDialect`, `SqliteSchemaBuilder`, `SqliteWriteLock`, `SqliteDecimalConverter` | | PostgreSQL | `PostgresProvider`, `PostgresDialect`, `PostgresSchemaBuilder`, `PostgresDatabaseCreator`, `PostgresAdvisoryLocks`, `PostgresPartitioning`, `PostgresServerInfo` | diff --git a/docs/DATA-MODEL.md b/docs/DATA-MODEL.md index 57e58d0..46d5aa7 100644 --- a/docs/DATA-MODEL.md +++ b/docs/DATA-MODEL.md @@ -1,12 +1,13 @@ # Modello dati -> Stato: bozza · Revisione 2026-11-10 · Fonte: SPEC §5, §7, §8, §9, §11, §12, §13, §14, §21 +> Stato: in corso · Revisione 2026-11-11 (M1) · Fonte: SPEC §5, §7, §8, §9, §11, §12, §13, §14, §21 -> **Versione pianificata.** A partire dalla milestone 1 questo file sarà **generato** da `scripts/gen-datamodel.sh` -> leggendo le mappe `XxxMap : EntityMap` e le migrazioni: non andrà più modificato a mano. Finché il codice non -> esiste, questa è la versione pianificata ricavata dalla specifica: nomi e colonne sono indicativi e verranno -> confermati (o corretti) dalla prima generazione. I nomi di tabella contrassegnati con **†** non compaiono -> letteralmente in SPEC: sono derivati dal nome dell'entità di SPEC §21 secondo la convenzione snake_case plurale. +> **Due parti.** La sezione [8. Schema attuale](#8-schema-attuale-generato) è **generata** da +> `scripts/gen-datamodel.sh` leggendo le migrazioni: non si modifica a mano e un test fallisce se non è aggiornata. +> Le sezioni 3-4 sono la **pianificazione** ricavata dalla specifica: nomi e colonne sono indicativi e vengono +> confermati (o corretti) dalla migrazione che crea ciascuna tabella, milestone dopo milestone. I nomi di tabella +> contrassegnati con **†** non compaiono letteralmente in SPEC: sono derivati dal nome dell'entità di SPEC §21 +> secondo la convenzione snake_case plurale. Documenti collegati: [ARCHITECTURE.md](ARCHITECTURE.md) · [MODULES.md](MODULES.md) · [API.md](API.md). @@ -31,24 +32,29 @@ Documenti collegati: [ARCHITECTURE.md](ARCHITECTURE.md) · [MODULES.md](MODULES. | `ColumnType` | PostgreSQL | SQLite | Note | |---|---|---|---| -| `String(n)` | `varchar(n)` | `TEXT` | lunghezza verificata dall'applicazione su SQLite | +| `String(n)` | `varchar(n)` | `TEXT` + `CHECK (length(col) <= n)` | la lunghezza è imposta dal database su entrambi i motori | | `Text` | `text` | `TEXT` | | -| `Int32` | `int` | `INTEGER` | | +| `Int32` | `integer` | `INTEGER` | | | `Int64` | `bigint` | `INTEGER` | chiavi primarie ed esterne | | `Decimal(19,6)` | `numeric(19,6)` | `INTEGER` scalato x10⁶ | vedi §1.3 | -| `Bool` | `boolean` | `INTEGER` 0/1 | | +| `Bool` | `boolean` | `INTEGER` + `CHECK (col IN (0, 1))` | | | `Date` | `date` | `TEXT` `yyyy-MM-dd` | `DateOnly` in C# | -| `DateTimeUtc` | `timestamptz` | `TEXT` ISO 8601 UTC | `DateTime` UTC; conversione a `Europe/Rome` solo in presentazione | +| `DateTimeUtc` | `timestamptz` | `TEXT` `yyyy-MM-ddTHH:mm:ss.ffffffZ` | `DateTime` UTC al microsecondo su entrambi i motori; conversione a `Europe/Rome` solo in presentazione | | `Guid` | `uuid` | `TEXT` | | | `Bytes` | `bytea` | `BLOB` | | | `Json` | `jsonb` | `TEXT` | diff di audit, payload dei canali, dati di log | +Le tabelle SQLite sono `STRICT`: un valore del tipo sbagliato è rifiutato invece di essere convertito. Le enumerazioni +sono memorizzate **per nome** (`String(40)`), così un riordino dei valori in C# non altera i dati. + ### 1.3 Decimali scalati su SQLite SQLite non ha un tipo decimale esatto: i valori `Decimal(19,6)` sono memorizzati come **interi moltiplicati per 10⁶** (`SqliteDecimalConverter`). Conseguenze per chi scrive query: - `ISqlDialect.ToDbValue` / `FromDbValue` convertono in scrittura e in lettura; i parametri passano già scalati; +- l'arrotondamento al sesto decimale è *half away from zero* (`1,2345675` diventa `1,234568`), come `numeric(19,6)` + su PostgreSQL; un valore oltre l'intervallo solleva `OverflowException` invece di essere troncato; - le **somme in SQL** (`SUM`, `GROUP BY`) operano sugli interi scalati e vengono **riscalate in lettura**: niente aritmetica in virgola mobile; - prodotti e divisioni fra due colonne decimali non vanno fatti in SQL su SQLite (il fattore di scala si @@ -59,8 +65,14 @@ SQLite non ha un tipo decimale esatto: i valori `Decimal(19,6)` sono memorizzati - Ogni entità ha una mappa `XxxMap : EntityMap` (tabella, colonne, chiave, indici, FK, conversioni); nessuna riflessione a runtime oltre la costruzione iniziale (`MapRegistry`). -- `QueryBuilder`: uguaglianza, confronto, `Contains`/`StartsWith`, `In`, `And`/`Or`/`Not`, `IsNull`; il resto come - `SqlFragment` parametrizzato. Mai concatenare input utente in SQL. +- `Query.For().Where(x => ...)`: uguaglianza, confronto, `== null`, `Contains`/`StartsWith`/`EndsWith` (LIKE + senza distinzione di maiuscole e accenti compresi, con `%` e `_` dell'utente protetti), `lista.Contains(x.Col)` + (`IN`), `&&`/`||`/`!`, `string.IsNullOrEmpty`; il resto come `SqlFragment` parametrizzato. Un'espressione fuori da + questo insieme solleva `NotSupportedException`, non viene valutata in memoria. Mai concatenare input utente in SQL: + in `SqlFragment.Of($"... {valore}")` ogni valore interpolato diventa un parametro. +- `IRepository` compila le colonne di audit, imposta `row_version = 1` all'inserimento e la incrementa a ogni + modifica; `IUnitOfWork.AcquireLockAsync(chiave)` prende un lock di transazione (`pg_advisory_xact_lock` o il lock di + scrittura di SQLite) per numeratori e chiusure. - SQLite: `journal_mode=WAL; foreign_keys=ON; busy_timeout=5000`; scritture serializzate da `SqliteWriteLock`. - PostgreSQL: pool separato per database (`MaxPoolSize` da impostazioni), `statement_timeout` per sessione, `pg_advisory_xact_lock` per numeratori, chiusure e scheduler. @@ -68,22 +80,24 @@ SQLite non ha un tipo decimale esatto: i valori `Decimal(19,6)` sono memorizzati ## 2. Migrazioni (SPEC §5.3) ```csharp -[Migration(20261013_001, "create_parties")] -public sealed class M20261013_001 : IMigration +[Migration(20261013_001, "create_parties")] // Target = MigrationTarget.Company se omesso +public sealed class M20261013_001_CreateParties : IMigration { - public void Up(ISchemaBuilder sb) { /* tabelle, colonne, indici, FK in forma neutra */ } - public string? UpSql(ISqlDialect d) => null; // SQL specifico del dialetto, se serve + public void Up(ISchemaBuilder schema) { /* tabelle, colonne, indici, FK in forma neutra */ } + public string? UpSql(ISqlDialect dialect) => null; // SQL specifico del dialetto, se serve } ``` | Regola | Dettaglio | |---|---| -| Due cataloghi | `MigrationCatalog.System` (database di sistema) e `MigrationCatalog.Company` (ogni azienda) | -| Versioni | `schema_version(module, version, name, applied_at, checksum)` in **ogni** database | -| Esecuzione | `MigrationRunner` applica in transazione, è idempotente, registra l'esito nella sezione di log *Migrazioni* | -| Downgrade | non supportato: se trova versioni applicate non note rifiuta di partire | +| Due cataloghi | `MigrationCatalog.ForSystem()` (database di sistema) e `MigrationCatalog.ForCompany(assembly dei moduli)` | +| Versioni | `yyyymmdd_nnn` (11 cifre, data valida, `nnn` da 001), uniche per modulo; `schema_version(module, version, name, applied_at, checksum)` in **ogni** database, `checksum` = SHA-256 delle istruzioni generate | +| Esecuzione | `MigrationRunner` applica **una migrazione per transazione** sotto lock (più istanze che partono insieme non si pestano i piedi), è idempotente e restituisce un `MigrationReport`; la registrazione nella sezione di log *Migrazioni* arriva con il log (M2) | +| Checksum diverso | una migrazione già applicata e poi modificata produce un avviso, non un errore: le migrazioni applicate non si cambiano, se ne scrive una nuova | +| Downgrade | non supportato: se trova versioni applicate non note rifiuta di partire (`MigrationException`) | | Avvio | il sistema è migrato all'avvio; le aziende all'avvio (impostazione) o dal pannello (SPEC §12.1) in modalità manutenzione | -| Nuova migrazione | `scripts/new-migration.sh` | +| Nuova migrazione | `scripts/new-migration.sh [cartella] [modulo]` | +| Documentazione | `scripts/gen-datamodel.sh` rigenera la [sezione 8](#8-schema-attuale-generato); `--check` verifica soltanto | | Test obbligatorio | tutte le migrazioni da zero su SQLite e PostgreSQL, `schema_version` coerente, riapplicazione senza effetti | ## 3. Database di sistema (`khaemhat_system`) @@ -325,9 +339,106 @@ qualunque modifica ──► audit_log - Obiettivi di prestazione di riferimento (SPEC §18): bilancio di verifica su 2 M righe < 10 s, pagina log con 1 M righe < 1 s per filtro. +### 6.1 Misure di riferimento dello strato dati (M1) + +`PerformanceBaselineTests` inserisce righe con tutti i tipi di colonna tramite `IRepository` in una sola transazione, +poi le legge tutte e legge una pagina filtrata con il conteggio totale. Con `KHAEMHAT_PERF=1` usa 100.000 righe, +altrimenti 2.000 (prova di fumo, sempre eseguita anche in CI). + +```bash +KHAEMHAT_PERF=1 scripts/test.sh --filter TestCategory=Performance --logger "console;verbosity=detailed" +``` + +| Data | Motore | Macchina | Inserimenti | Lettura di tutte le righe | Pagina filtrata + conteggio | +|---|---|---|---|---|---| +| 2026-11-11 | SQLite (file in memoria) | Ryzen 5 2600X, Windows 11, .NET 10.0.403 | 100.000 in 5,6 s (circa 17.800/s) | 0,9 s | 17 ms | + +Le misure servono a riconoscere un peggioramento, non come soglia: il test non fallisce sui tempi. Le misure su +PostgreSQL si aggiungono dal job `postgres` della CI alla chiusura della milestone. + ## 7. Segreti nel database Cifrati con AES-256-GCM e chiave derivata da `KHAEMHAT_SECRET_KEY` (`SecretProtector`): connection string delle aziende (`sys_companies`), password SMTP/PEC/S3/WebDAV (`sys_settings`), token degli intermediari e credenziali dei canali SDI (`sdi_channels`), credenziali dei canali e-commerce (`sales_channels`), segreti dei webhook. La rotazione avviene con `KHAEMHAT_SECRET_KEY_NEW` (SPEC §12.9). Gli hash (password, chiavi API) non sono cifrati ma irreversibili. + +## 8. Schema attuale (generato) + +Questa sezione è scritta da `scripts/gen-datamodel.sh` a partire dalle migrazioni presenti nel codice; il test +`DataModelDocumentTests` fallisce se non è aggiornata. Comprende `schema_version`, presente in ogni database. + + + +### Database di sistema + +Migrazioni applicate in ordine: + +- `core` 20261111001 `create_sys_companies` (M20261111_001_CreateSysCompanies) +- `core` 20261111002 `create_sys_settings` (M20261111_002_CreateSysSettings) + +#### `schema_version` + +| Colonna | Tipo | Null | Default | Note | +|---|---|---|---|---| +| `module` | String(50) | no | | chiave primaria | +| `version` | Int64 | no | | chiave primaria | +| `name` | String(200) | no | | | +| `applied_at` | DateTimeUtc | no | | | +| `checksum` | String(64) | no | | | + +Chiave primaria: `module`, `version`. + +#### `sys_companies` + +| Colonna | Tipo | Null | Default | Note | +|---|---|---|---|---| +| `id` | Int64 | no | | chiave primaria, identity | +| `code` | String(20) | no | | | +| `name` | String(200) | no | | | +| `engine` | String(20) | no | | | +| `connection_string` | Text | no | | | +| `status` | String(40) | no | `'Active'` | | +| `is_default` | Bool | no | `false` | | +| `notes` | Text | sì | | | +| `created_at` | DateTimeUtc | no | | | +| `created_by` | String(100) | sì | | | +| `updated_at` | DateTimeUtc | sì | | | +| `updated_by` | String(100) | sì | | | +| `row_version` | Int64 | no | `1` | | + +- indice unico `ux_sys_companies_code` su `code` + +#### `sys_settings` + +| Colonna | Tipo | Null | Default | Note | +|---|---|---|---|---| +| `id` | Int64 | no | | chiave primaria, identity | +| `key` | String(200) | no | | | +| `value` | Text | sì | | | +| `is_secret` | Bool | no | `false` | | +| `created_at` | DateTimeUtc | no | | | +| `created_by` | String(100) | sì | | | +| `updated_at` | DateTimeUtc | sì | | | +| `updated_by` | String(100) | sì | | | +| `row_version` | Int64 | no | `1` | | + +- indice unico `ux_sys_settings_key` su `key` + +### Database aziendale + +Nessuna migrazione: la tabella `schema_version` è l'unica presente. + +#### `schema_version` + +| Colonna | Tipo | Null | Default | Note | +|---|---|---|---|---| +| `module` | String(50) | no | | chiave primaria | +| `version` | Int64 | no | | chiave primaria | +| `name` | String(200) | no | | | +| `applied_at` | DateTimeUtc | no | | | +| `checksum` | String(64) | no | | | + +Chiave primaria: `module`, `version`. + + diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index d25cfff..5fe8bae 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -30,9 +30,20 @@ Se manca l'SDK 10 ci si ferma e lo si chiede al titolare (SPEC §23). Su Windows | `docker compose -f deploy/docker-compose.sqlite.yml up` | demo autonoma (con `KHAEMHAT_VERSION` per scegliere l'immagine) | | `scripts/publish-image.sh vX.Y.Z [--arm64]` | pubblicazione dalla macchina di sviluppo se il runner non c'è (ADR 0015) | | `scripts/publish-wiki.sh` | pubblica `docs/wiki/` nel wiki Gitea senza passare dal runner | +| `scripts/new-migration.sh [cartella] [modulo]` | crea `M__.cs` con il primo numero libero del giorno (SPEC §5.3) | +| `scripts/gen-datamodel.sh [--check]` | rigenera la sezione generata di `docs/DATA-MODEL.md` dalle migrazioni; `--check` verifica soltanto | +| `KHAEMHAT_PERF=1 scripts/test.sh --filter TestCategory=Performance` | misure di riferimento dello strato dati con 100.000 righe (`docs/DATA-MODEL.md` §6.1) | -Gli stub `new-migration.sh` (M1), `gen-datamodel.sh` (M1), `seed.sh` (M4) e `restore-system.sh` (M12, incluso -nell'immagine, ADR 0017) spiegano l'uso previsto ed escono con errore finché la loro milestone non li completa. +Gli stub `seed.sh` (M4) e `restore-system.sh` (M12, incluso nell'immagine, ADR 0017) spiegano l'uso previsto ed +escono con errore finché la loro milestone non li completa. + +#### Test sui due motori + +I test dello strato dati girano con `[DataRow("Sqlite")]` e `[DataRow("PostgreSql")]`. SQLite usa un database in +memoria condiviso, sempre disponibile. PostgreSQL usa `KHAEMHAT_TEST_PG` (URL +`postgres://utente:password@host:5432/postgres` di un ruolo con `CREATEDB`): ogni test crea un database +`khaemhat_t_` e lo elimina alla fine; senza la variabile i test PostgreSQL risultano *saltati*. In CI il +job `postgres` li esegue su PostgreSQL 16 e 17 sui tag e all'avvio manuale del workflow. ### 1.2 Visual Studio Code diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 3597f6d..0d630ec 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -240,11 +240,17 @@ passaggio TOTP (prevenzione della session fixation). - `SecretProtector` cifra con **AES-256-GCM** (`System.Security.Cryptography.AesGcm`) i segreti salvati: connection string delle aziende (`sys_companies`), password SMTP/PEC/S3/WebDAV, token degli intermediari SDI, credenziali dei canali e-commerce. -- La chiave AES è **derivata** da `KHAEMHAT_SECRET_KEY`. Proposta: HKDF-SHA256 (in-box, `HKDF`) con etichette di - contesto distinte per uso (`secrets`, `backup`, `totp`), così una chiave non viene riusata fra domini. -- Proposta di formato del testo cifrato: `v1` (versione/identificativo chiave) ‖ nonce 12 byte casuale ‖ - ciphertext ‖ tag 16 byte, in base64. Il nonce non si riusa mai con la stessa chiave. Come dato associato (AAD) - usare tabella+colonna+id, così un segreto copiato in un'altra riga non si decifra. +- La chiave AES è **derivata** da `KHAEMHAT_SECRET_KEY` con HKDF-SHA256 (in-box, `HKDF`) ed etichetta di contesto + `khaemhat/secrets/aes-256-gcm/v1`; altri usi (backup, TOTP) avranno etichette proprie, così una chiave non viene + riusata fra domini. L'identificativo della chiave è una seconda derivazione HKDF di 4 byte (8 cifre esadecimali): + identifica la chiave senza rivelarla. +- **Formato del testo cifrato (M1):** `khs1..`. + Il nonce è casuale e non si riusa mai con la stessa chiave. Il dato associato (AAD) è lo **scopo** del segreto + (per esempio `company:ACME:database` per la connessione di un'azienda): un valore copiato in un'altra riga o + usato per un altro scopo non si decifra. Un valore cifrato con un'altra chiave è rifiutato con un messaggio che + indica i due identificativi (`SecretProtectionException`), senza mai ripetere il segreto. +- Implementazione: `Khaemhat.Infrastructure.Security.SecretProtector` (con `SecretKey` e `SecretRotation`), + interfaccia `ISecretProtector` in `Khaemhat.Core`. - La chiave e i segreti in chiaro **non vanno mai** in log, messaggi d'errore, diagnostica, risposte API o file su disco (SPEC §1.5: nessun `appsettings..json` con segreti, nessun file temporaneo). - Le varianti `_FILE` delle variabili (Docker secrets) sono previste per `KHAEMHAT_SYSTEM_DB` e le credenziali del @@ -258,6 +264,9 @@ passaggio TOTP (prevenzione della session fixation). 3. L'operatore sostituisce `KHAEMHAT_SECRET_KEY` con la nuova e rimuove `KHAEMHAT_SECRET_KEY_NEW`. 4. Evento in `sys_security_audit`. +In M1 esiste il mattone `SecretRotation` (`NeedsRotation` legge l'identificativo della chiave, `Rotate` decifra e +ri-cifra); la procedura che percorre tutte le colonne cifrate arriva in M12. + Da decidere con ADR: trattamento dei **backup già cifrati** con la vecchia chiave (ri-cifratura o conservazione della vecchia chiave come chiave di sola decifratura identificata dalla versione nel formato). diff --git a/docs/STATUS.md b/docs/STATUS.md index c91ddc8..26346ea 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -3,25 +3,32 @@ > Punto di ripresa per ogni sessione. Aggiornare a fine sessione. **Ultimo aggiornamento:** 2026-11-11 -**Milestone corrente:** M0 — Bootstrap (via allo sviluppo dato dal titolare il 2026-11-11) -**Prossimo passo:** chiudere M0 (PR `feat/m0-bootstrap` → `main`, CI verde, tag `v0.0.1`, immagine e release), poi -M1 — Fondamenta dati, partendo dall'issue #7 (colonna *Da fare* del progetto *Khaemhat 1.0 — Sviluppo*). +**Milestone corrente:** M1 — Strato dati (in chiusura sul ramo `feat/7-strato-dati`) +**Prossimo passo:** chiudere M1 (PR verso `main`, CI verde compreso il job `postgres` avviato a mano sul ramo, tag +`v0.1.0`, immagine e release; misure PostgreSQL in `docs/DATA-MODEL.md` §6.1), poi M2 — Sistema e sicurezza, +partendo dall'issue #15. ## Fatto - Specifica normativa in `docs/SPEC.md`; `CLAUDE.md` con emendamenti E1-E6. -- ADR 0001-0011. +- ADR 0001-0018. - Mockup in `docs/mockups/` (22 in alta risoluzione + galleria, 88 miniature, `mockup.css`, indice). - Icone in `assets/`; deploy in `deploy/` (compose Unraid, `.env.example`, template Unraid). - Documentazione tecnica: `ARCHITECTURE.md`, `DATA-MODEL.md` (provvisorio, sarà generato), `MODULES.md`, `API.md`, `MCP.md`, `DEPLOY.md`, `SECURITY.md`, `DEVELOPMENT.md`. - Wiki completa in bozza in `docs/wiki/`, pubblicata nel wiki Gitea. -- **M0 (in chiusura):** soluzione `Khaemhat.sln` con i 10 progetti di SPEC §3 e 6 progetti di test; regole di build +- **M0 (chiusa, `v0.0.1`):** soluzione `Khaemhat.sln` con i 10 progetti di SPEC §3 e 6 progetti di test; regole di build (`Directory.Build.props/targets`, Central Package Management con l'allowlist, `.editorconfig`, warning come errori); test architetturali delle dipendenze e dei pacchetti; `Khaemhat.Web` con pagina "in costruzione", icone, `/health/live`, `/health/ready`, `/health` (ADR 0012); `docker/Dockerfile`, `entrypoint.sh` con `PUID`/`PGID` (ADR 0013), `postgres/init.sql`; `deploy/docker-compose.yml` e `docker-compose.sqlite.yml`; Gitea Actions `ci.yml`, `release.yml`, `wiki.yml` con controllo YAML; script di build, test, avvio, pubblicazione e stub; ADR 0012-0017. +- **M1 (in chiusura, `v0.1.0`):** micro-ORM in `Khaemhat.Data` (mappe, query tipizzate, `SqlFragment`, unit of work, + repository con audit e `row_version`), provider SQLite e PostgreSQL, `DatabaseUrl`, migrazioni con + `schema_version`, tabelle `sys_companies` e `sys_settings`, `CompanyRegistry` con URL cifrato, `SecretProtector` + (AES-256-GCM, formato `khs1`), `tools/Khaemhat.DataModelTool` e gli script `new-migration.sh` e + `gen-datamodel.sh`; ADR 0018. Test: 85 su SQLite verdi in locale, 21 su PostgreSQL eseguiti solo in CI (in locale + Docker non è disponibile). - Su Gitea: etichette, milestone M0-M14, 114 issue con checklist, progetti *Khaemhat 1.0 — Sviluppo* (M0-M13) e *Khaemhat 1.1+ — Fase 2* (M14). Numerazione: M0 #1-#6 e #114, M1 #7-#14, M2 #15-#24, M3 #25-#36, M4 #37-#46, M5 #47-#54, M6 #55-#61, M7 #62-#68, M8 #69-#73, M9 #74-#81, M10 #82-#87, M11 #88-#91, @@ -62,7 +69,7 @@ Emersi durante la stesura della documentazione. Si decidono con un ADR quando si 6. Tag e doppia build: `publish-image.sh` (Appendice D) crea il tag git, ma con `release.yml` sui tag `v*` l'immagine verrebbe costruita due volte. Proposta: lo script non crea tag, oppure `release.yml` salta le versioni già nel registro. **Deciso in M0: ADR 0015.** 7. Formato tag: tag git `vX.Y.Z`, tag immagine `X.Y.Z` (senza `v`). **Deciso in M0: ADR 0015.** 8. Icona versionata: `build.sh` deve produrre PNG, serve un rasterizzatore; proposta: generarla sulla macchina di sviluppo e committarla prima del rilascio. **Deciso in M0: ADR 0016.** -9. Rotazione della password PostgreSQL: le connection string delle aziende sono cifrate per intero in `sys_companies`; serve una procedura di aggiornamento. +9. Rotazione della password PostgreSQL: l'URL del database di ogni azienda è cifrato per intero in `sys_companies` (ADR 0018); serve una procedura di aggiornamento (M12). 10. Il mockup 085 mostra ancora `ghcr.io`: l'app mostrerà il registro Gitea (ADR 0009). 11. Non definiti: schermata di attivazione TOTP, nome dell'impostazione "migra aziende all'avvio", recupero password del super utente senza SMTP, chi modifica le impostazioni di backup cloud (scritto: super utente), flusso di ripristino da backup logico. diff --git a/docs/adr/0018-contratti-dello-strato-dati.md b/docs/adr/0018-contratti-dello-strato-dati.md new file mode 100644 index 0000000..cc3de92 --- /dev/null +++ b/docs/adr/0018-contratti-dello-strato-dati.md @@ -0,0 +1,44 @@ +# ADR 0018 - Contratti dello strato dati: scostamenti da SPEC §5.1 + +- **Stato:** accettata +- **Data:** 2026-11-11 +- **Modifica:** SPEC §5.1-§5.4, §3 (con ADR 0004); milestone M1, issue #7-#14 + +## Contesto + +SPEC §5.1 elenca firme indicative di `IDbProvider`, `ISqlDialect`, `ICompanyRegistry` e del motore di migrazione. +Implementandole su due motori sono emersi punti in cui la firma letterale costringeva a soluzioni peggiori +(stringhe SQL da ricomporre, chiamate non annullabili, segreti in chiaro in memoria più del necessario). Questo ADR +fissa le firme effettive. + +## Decisione + +1. **`CancellationToken` ovunque.** Tutti i metodi asincroni di `IDbProvider`, `IUnitOfWork`, `IRepository`, + `ICompanyRegistry` e `MigrationRunner` accettano un `CancellationToken` (default `default`). +2. **DDL fuori dal builder.** `IDbProvider` espone `DdlGenerator Ddl` invece di `ISchemaBuilder SchemaBuilder`: + le migrazioni scrivono su un `ISchemaBuilder` neutro che **registra** operazioni (`SchemaOperation`), e il + generatore del provider le traduce in istruzioni. Così la stessa registrazione serve per eseguire, calcolare il + checksum e generare `docs/DATA-MODEL.md`. Le classi dei provider mantengono i nomi di SPEC + (`SqliteSchemaBuilder`, `PostgresSchemaBuilder`) ma sono generatori DDL. +3. **Frammenti, non stringhe.** `ISqlDialect.CaseInsensitiveLike` e `AdvisoryLock` restituiscono `SqlFragment` + (testo + parametri), non `string`, perché il pattern e la chiave del lock sono valori e devono restare parametri. +4. **Membri aggiunti a `IDbProvider`:** `BuildConnectionString(DatabaseUrl)` (configurazione da URL `postgres://` o + `sqlite://`, mai stringhe ADO.NET nelle variabili), `DatabaseExistsAsync`, `IsTransient(Exception)` per i + tentativi automatici, `EnterWriteScopeAsync` (il `SqliteWriteLock` su SQLite, nulla su PostgreSQL). +5. **Registro aziende cifrato per intero.** `sys_companies.connection_string` contiene l'URL del database cifrato + con `SecretProtector` e scopo `company::database`: una riga copiata su un'altra azienda non si decifra. + La rotazione della password PostgreSQL resta un punto aperto (`docs/STATUS.md`, punto 9). +6. **Esito delle migrazioni al chiamante.** `MigrationRunner.RunAsync` restituisce un `MigrationReport` (applicate con + durata, già presenti, avvisi); la scrittura nella sezione di log *Migrazioni* arriva con il log in M2. Ogni + migrazione gira in una propria transazione sotto lock, con un nuovo controllo di `schema_version` dopo il lock. +7. **Enumerazioni per nome.** Le proprietà enum sono salvate come testo (`String(40)`), non come intero. +8. **Strumento di sviluppo.** `tools/Khaemhat.DataModelTool` (dipende solo da `Khaemhat.Data`, non distribuito) + rigenera la sezione generata di `docs/DATA-MODEL.md`; la regola di dipendenza è nel test architetturale. + +## Conseguenze + +- Il codice sopra lo strato dati usa `DatabaseScope`, `IUnitOfWorkFactory` e `SqlFragment`; nessun riferimento a + Npgsql o Microsoft.Data.Sqlite fuori dai provider (verificato da un test). +- Chi scrive una migrazione usa `scripts/new-migration.sh` e poi `scripts/gen-datamodel.sh`; un test fallisce se il + documento non è aggiornato. +- Le firme di SPEC §5.1 restano il riferimento per i nomi; dove divergono vale questo ADR. diff --git a/docs/wiki/Changelog.md b/docs/wiki/Changelog.md index 409a1de..19b1938 100644 --- a/docs/wiki/Changelog.md +++ b/docs/wiki/Changelog.md @@ -1,6 +1,6 @@ # Changelog -> **Stato:** completa per 0.0.1 · **Ultima revisione:** 2026-11-11 +> **Stato:** completa per 0.1.0 · **Ultima revisione:** 2026-11-11 Le novità di ogni versione sono nel file [`CHANGELOG.md`](https://gitea.encke-hake.ts.net/BalboSoft/Khaemhat/src/branch/main/CHANGELOG.md) del repository @@ -11,6 +11,7 @@ Le novità di ogni versione sono nel file | Versione | Data | Contenuto | |---|---|---| +| 0.1.0 | 2026-11-11 | M1 — Strato dati: accesso ai database SQLite e PostgreSQL, migrazioni dello schema, registro delle aziende, cifratura dei dati riservati. Nessuna funzione visibile nell'interfaccia. | | 0.0.1 | 2026-11-11 | M0 — Bootstrap: struttura del codice, immagine Docker con pagina "in costruzione" ed endpoint di salute, compose, CI e rilascio su Gitea Actions. Nessuna funzione gestionale ancora. | | *preparazione* | 2026-11-10 | Documentazione, specifica, mockup, wiki e pianificazione su Gitea. Nessun software. | diff --git a/docs/wiki/Modello-dati.md b/docs/wiki/Modello-dati.md index d3a704c..8809548 100644 --- a/docs/wiki/Modello-dati.md +++ b/docs/wiki/Modello-dati.md @@ -1,12 +1,13 @@ # Modello dati -> **Stato:** bozza · **Ultima revisione:** 2026-11-10 · **Versione software:** non ancora rilasciata (milestone M1 e seguenti) +> **Stato:** bozza · **Ultima revisione:** 2026-11-11 · **Versione software:** 0.1.0 (strato dati, M1); le tabelle funzionali arrivano con le milestone successive > **Specifica:** SPEC §5, §11, §13, §14 Come Khaemhat organizza i dati, in sintesi. L'elenco completo di tabelle e colonne è in -[docs/DATA-MODEL.md](https://gitea.encke-hake.ts.net/BalboSoft/Khaemhat/src/branch/main/docs/DATA-MODEL.md): oggi è -la versione **pianificata**; dalla milestone M1 sarà generato automaticamente dal codice, quindi sempre allineato al -programma. +[docs/DATA-MODEL.md](https://gitea.encke-hake.ts.net/BalboSoft/Khaemhat/src/branch/main/docs/DATA-MODEL.md). La sua +sezione *Schema attuale* è **generata automaticamente** dalle migrazioni del programma, quindi è sempre allineata a +ciò che il programma crea davvero nei database; le altre sezioni descrivono le tabelle **pianificate**, che compaiono +nello schema attuale man mano che le milestone le realizzano. ## Due tipi di database @@ -62,8 +63,11 @@ settimana). Quando una nuova versione cambia la struttura dei dati, Khaemhat applica da solo le **migrazioni**: il database di sistema all'avvio, le aziende all'avvio o dal pannello **Amministrazione → Istanza e database**, mettendo l'azienda in -manutenzione per il tempo necessario. Non è possibile tornare a una versione precedente dello schema: per questo prima -di ogni aggiornamento conviene un backup ([Aggiornamento](Aggiornamento), [Backup-e-ripristino](Backup-e-ripristino)). +manutenzione per il tempo necessario. Ogni migrazione è applicata +per intero o per niente, e più copie del programma avviate insieme non la applicano due volte. Non è possibile +tornare a una versione precedente dello schema: se il database contiene una migrazione che il programma non conosce +(per esempio dopo aver rimesso un'immagine più vecchia) Khaemhat si rifiuta di partire invece di rischiare i dati. +Per questo prima di ogni aggiornamento conviene un backup ([Aggiornamento](Aggiornamento), [Backup-e-ripristino](Backup-e-ripristino)). ## Pagine correlate diff --git a/scripts/gen-datamodel.sh b/scripts/gen-datamodel.sh index 870b608..719c4a9 100755 --- a/scripts/gen-datamodel.sh +++ b/scripts/gen-datamodel.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash -# Regenerates docs/DATA-MODEL.md from the entity maps (SPEC §18). -# scripts/gen-datamodel.sh -# Stub: completed in milestone M1 (SPEC §20). Until then it explains what it will do and exits with an error. +# Regenerates the generated section of docs/DATA-MODEL.md from the migrations (SPEC §18). +# scripts/gen-datamodel.sh rewrite the section +# scripts/gen-datamodel.sh --check only verify it is up to date (exit 1 if not) set -euo pipefail -echo "gen-datamodel.sh: non ancora disponibile, arriva con la milestone M1." >&2 -echo "Uso previsto: scripts/gen-datamodel.sh" >&2 -exit 1 +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$root" + +dotnet run --project tools/Khaemhat.DataModelTool -c Release -- "$@" docs/DATA-MODEL.md diff --git a/scripts/new-migration.sh b/scripts/new-migration.sh index c841cf2..daa9eca 100755 --- a/scripts/new-migration.sh +++ b/scripts/new-migration.sh @@ -1,9 +1,88 @@ #!/usr/bin/env bash # Creates a new numbered migration for the system or company schema (SPEC §5.3). -# scripts/new-migration.sh -# Stub: completed in milestone M1 (SPEC §20). Until then it explains what it will do and exits with an error. +# scripts/new-migration.sh [cartella] [modulo] +# Example: scripts/new-migration.sh system create_sys_users +# The version is today's date plus the next free number of the day across the repository (yyyymmdd_nnn); the +# class is M__ in the given folder (default: the system or company migrations folder). set -euo pipefail -echo "new-migration.sh: non ancora disponibile, arriva con la milestone M1." >&2 -echo "Uso previsto: scripts/new-migration.sh " >&2 -exit 1 +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$root" + +usage() { + echo "Uso: scripts/new-migration.sh [cartella] [modulo]" >&2 + exit 2 +} + +[ $# -ge 2 ] || usage +target="$1" +name="$2" +module="${4:-core}" +case "$target" in + system) dir="${3:-src/Khaemhat.Data/SystemDatabase/Migrations}"; attribute_target="MigrationTarget.System" ;; + company) dir="${3:-src/Khaemhat.Application/Migrations}"; attribute_target="MigrationTarget.Company" ;; + *) usage ;; +esac + +if ! [[ "$name" =~ ^[a-z][a-z0-9_]{0,62}$ ]]; then + echo "Il nome deve essere snake_case (lettere minuscole, cifre, _): $name" >&2 + exit 2 +fi + +if ! [[ "$module" =~ ^[a-z][a-z0-9_]{0,39}$ ]]; then + echo "Il modulo deve essere snake_case: $module" >&2 + exit 2 +fi + +if [[ "$dir" != src/* ]]; then + echo "La cartella deve stare sotto src/: $dir" >&2 + exit 2 +fi + +day="$(date +%Y%m%d)" +last="$(grep -rhoE "\[Migration\(${day}_[0-9]{3}" src --include='*.cs' 2>/dev/null | sed -E 's/.*_([0-9]{3})$/\1/' | sort -n | tail -1 || true)" +next=$(printf '%03d' $((10#${last:-0} + 1))) +pascal="$(echo "$name" | awk -F_ '{ for (i = 1; i <= NF; i++) printf "%s%s", toupper(substr($i, 1, 1)), substr($i, 2) }')" +class="M${day}_${next}_${pascal}" +namespace="$(echo "${dir#src/}" | tr '/' '.')" +file="$dir/$class.cs" + +mkdir -p "$dir" +if [ -e "$file" ]; then + echo "Esiste già: $file" >&2 + exit 1 +fi + +if [ "$attribute_target" = "MigrationTarget.Company" ]; then + target_argument="" +else + target_argument=", Target = $attribute_target" +fi + +if [ "$module" = "core" ]; then + module_argument="" +else + module_argument=", Module = \"$module\"" +fi + +cat > "$file" <TODO: what this migration changes and why (SPEC section). +[Migration(${day}_${next}, "$name"$target_argument$module_argument)] +public sealed class $class : IMigration +{ + /// + public void Up(ISchemaBuilder schema) + { + ArgumentNullException.ThrowIfNull(schema); + // schema.CreateTable("table_name", t => { t.Id(); t.AuditColumns(); }); + } +} +CS + +echo "Creata $file (versione ${day}_${next}, modulo $module)." +echo "Dopo averla scritta: aggiornare la mappa dell'entità, eseguire scripts/gen-datamodel.sh e i test." diff --git a/src/Khaemhat.Core/Common/ConcurrencyException.cs b/src/Khaemhat.Core/Common/ConcurrencyException.cs new file mode 100644 index 0000000..608a9c5 --- /dev/null +++ b/src/Khaemhat.Core/Common/ConcurrencyException.cs @@ -0,0 +1,51 @@ +namespace Khaemhat.Core.Common; + +/// +/// Thrown when a row changed (or was deleted) after it was read: the update found no row with the expected +/// row_version (SPEC §5.2). The interface reloads and asks the user; the API answers 409. +/// +public sealed class ConcurrencyException : Exception +{ + /// Creates the exception for a row of . + /// The table name. + /// The primary key of the row, as text. + /// The row_version the caller read. + public ConcurrencyException(string table, string key, long expectedVersion) + : base($"The row {key} of {table} was changed or deleted by someone else (expected version {expectedVersion}).") + { + Table = table; + Key = key; + ExpectedVersion = expectedVersion; + } + + /// Creates the exception with a message only. + /// The message. + public ConcurrencyException(string message) : base(message) + { + Table = string.Empty; + Key = string.Empty; + } + + /// Creates the exception with a message and an inner exception. + /// The message. + /// The cause. + public ConcurrencyException(string message, Exception innerException) : base(message, innerException) + { + Table = string.Empty; + Key = string.Empty; + } + + /// Creates the exception without details. + public ConcurrencyException() : this("The row was changed or deleted by someone else.") + { + } + + /// The table of the row. + public string Table { get; } + + /// The primary key of the row, as text. + public string Key { get; } + + /// The row_version the caller read. + public long ExpectedVersion { get; } +} diff --git a/src/Khaemhat.Core/Entities/Entity.cs b/src/Khaemhat.Core/Entities/Entity.cs new file mode 100644 index 0000000..ea29134 --- /dev/null +++ b/src/Khaemhat.Core/Entities/Entity.cs @@ -0,0 +1,27 @@ +namespace Khaemhat.Core.Entities; + +/// +/// Convenience base class for entities with an identity key and the audit columns. Entities may also implement +/// and directly. +/// +/// The key type. +public abstract class Entity : IEntity, IAuditable +{ + /// + public TKey Id { get; set; } = default!; + + /// + public DateTime CreatedAt { get; set; } + + /// + public string? CreatedBy { get; set; } + + /// + public DateTime? UpdatedAt { get; set; } + + /// + public string? UpdatedBy { get; set; } + + /// + public long RowVersion { get; set; } +} diff --git a/src/Khaemhat.Core/Entities/IAuditable.cs b/src/Khaemhat.Core/Entities/IAuditable.cs new file mode 100644 index 0000000..ca90418 --- /dev/null +++ b/src/Khaemhat.Core/Entities/IAuditable.cs @@ -0,0 +1,27 @@ +namespace Khaemhat.Core.Entities; + +/// +/// An entity with the audit columns of SPEC §5.2 (created_at, created_by, updated_at, +/// updated_by) and the row_version used for optimistic concurrency. The data layer fills them; code above +/// it only reads them. +/// +public interface IAuditable +{ + /// When the row was inserted, in UTC. + DateTime CreatedAt { get; set; } + + /// Who inserted the row (user name, or a system actor such as system), when known. + string? CreatedBy { get; set; } + + /// When the row was last updated, in UTC; if it never was. + DateTime? UpdatedAt { get; set; } + + /// Who last updated the row, when known. + string? UpdatedBy { get; set; } + + /// + /// Incremented by every update. An update succeeds only if the stored value still equals this one; otherwise the + /// data layer throws . + /// + long RowVersion { get; set; } +} diff --git a/src/Khaemhat.Core/Entities/IDemoMarkable.cs b/src/Khaemhat.Core/Entities/IDemoMarkable.cs new file mode 100644 index 0000000..8d485ac --- /dev/null +++ b/src/Khaemhat.Core/Entities/IDemoMarkable.cs @@ -0,0 +1,8 @@ +namespace Khaemhat.Core.Entities; + +/// An entity whose rows can be created by the demo seeder and removed with it (SPEC §14, column is_demo). +public interface IDemoMarkable +{ + /// for rows created by the demo data seeder. + bool IsDemo { get; set; } +} diff --git a/src/Khaemhat.Core/Entities/IEntity.cs b/src/Khaemhat.Core/Entities/IEntity.cs new file mode 100644 index 0000000..f4c14f0 --- /dev/null +++ b/src/Khaemhat.Core/Entities/IEntity.cs @@ -0,0 +1,9 @@ +namespace Khaemhat.Core.Entities; + +/// An entity with an identity of type , stored in the id column (SPEC §5.2). +/// The key type: for identity keys, which is the default for every table. +public interface IEntity +{ + /// The primary key. For identity keys it is the default value until the row is inserted. + TKey Id { get; set; } +} diff --git a/src/Khaemhat.Core/Instance/Company.cs b/src/Khaemhat.Core/Instance/Company.cs new file mode 100644 index 0000000..85a8287 --- /dev/null +++ b/src/Khaemhat.Core/Instance/Company.cs @@ -0,0 +1,31 @@ +using Khaemhat.Core.Entities; + +namespace Khaemhat.Core.Instance; + +/// +/// A company in the instance registry (sys_companies, SPEC §5.4): its code, name and the location of its own +/// database. The location is stored encrypted and is never shown or logged. +/// +public sealed class Company : Entity +{ + /// Company code: upper-case letters, digits and underscore, 1 to 20 characters (database khaemhat_code). + public string Code { get; set; } = string.Empty; + + /// Company name (ragione sociale). + public string Name { get; set; } = string.Empty; + + /// The database engine name: Sqlite or PostgreSql. + public string Engine { get; set; } = string.Empty; + + /// The database URL, encrypted with the instance key. + public string ConnectionString { get; set; } = string.Empty; + + /// The company state. + public CompanyStatus Status { get; set; } + + /// Whether this is the company opened by default. + public bool IsDefault { get; set; } + + /// Free notes for administrators. + public string? Notes { get; set; } +} diff --git a/src/Khaemhat.Core/Instance/CompanyStatus.cs b/src/Khaemhat.Core/Instance/CompanyStatus.cs new file mode 100644 index 0000000..731d207 --- /dev/null +++ b/src/Khaemhat.Core/Instance/CompanyStatus.cs @@ -0,0 +1,17 @@ +namespace Khaemhat.Core.Instance; + +/// State of a company database in the registry (SPEC §5.4, §12.1). +public enum CompanyStatus +{ + /// In use. + Active = 0, + + /// Maintenance in progress (migration, restore): nobody but administrators can open it. + Maintenance = 1, + + /// Readable only, for example an archive. + ReadOnly = 2, + + /// Kept but not available. + Disabled = 3, +} diff --git a/src/Khaemhat.Core/Instance/SystemSetting.cs b/src/Khaemhat.Core/Instance/SystemSetting.cs new file mode 100644 index 0000000..eb36b72 --- /dev/null +++ b/src/Khaemhat.Core/Instance/SystemSetting.cs @@ -0,0 +1,16 @@ +using Khaemhat.Core.Entities; + +namespace Khaemhat.Core.Instance; + +/// An instance setting (sys_settings). Secret values are stored encrypted (). +public sealed class SystemSetting : Entity +{ + /// The setting key, for example smtp.host. + public string Key { get; set; } = string.Empty; + + /// The value; encrypted when is set. + public string? Value { get; set; } + + /// Whether is a secret encrypted with the instance key. + public bool IsSecret { get; set; } +} diff --git a/src/Khaemhat.Core/Security/ISecretProtector.cs b/src/Khaemhat.Core/Security/ISecretProtector.cs new file mode 100644 index 0000000..c8fb346 --- /dev/null +++ b/src/Khaemhat.Core/Security/ISecretProtector.cs @@ -0,0 +1,22 @@ +namespace Khaemhat.Core.Security; + +/// +/// Encrypts secrets stored in the databases (company connection strings, SMTP/PEC/S3/WebDAV passwords, tokens) with +/// the instance key KHAEMHAT_SECRET_KEY (SPEC §5.4). The purpose binds a ciphertext to its use: a value +/// protected for one purpose cannot be read back for another. +/// +public interface ISecretProtector +{ + /// Encrypts ; the result is printable text safe to store in a column. + /// The secret. + /// What the secret is for, for example company:DEMO:database. + /// The protected value. + string Protect(string plaintext, string purpose); + + /// Decrypts a value produced by with the same key and purpose. + /// The protected value. + /// The purpose given to . + /// The secret. + /// The value was protected with another key or purpose, or is damaged. + string Unprotect(string protectedValue, string purpose); +} diff --git a/src/Khaemhat.Core/Security/SecretProtectionException.cs b/src/Khaemhat.Core/Security/SecretProtectionException.cs new file mode 100644 index 0000000..7ea3c01 --- /dev/null +++ b/src/Khaemhat.Core/Security/SecretProtectionException.cs @@ -0,0 +1,26 @@ +namespace Khaemhat.Core.Security; + +/// +/// A secret could not be decrypted: wrong instance key, wrong purpose or damaged value. The message never contains +/// the secret or the ciphertext. +/// +public sealed class SecretProtectionException : Exception +{ + /// Creates the exception. + public SecretProtectionException() : base("The secret cannot be decrypted.") + { + } + + /// Creates the exception with a message. + /// The message (never the secret). + public SecretProtectionException(string message) : base(message) + { + } + + /// Creates the exception with a message and a cause. + /// The message (never the secret). + /// The cause. + public SecretProtectionException(string message, Exception innerException) : base(message, innerException) + { + } +} diff --git a/src/Khaemhat.Data.PostgreSql/PostgresAdvisoryLocks.cs b/src/Khaemhat.Data.PostgreSql/PostgresAdvisoryLocks.cs new file mode 100644 index 0000000..7a4cf8d --- /dev/null +++ b/src/Khaemhat.Data.PostgreSql/PostgresAdvisoryLocks.cs @@ -0,0 +1,26 @@ +namespace Khaemhat.Data.PostgreSql; + +/// +/// Application locks on PostgreSQL (SPEC §5.2) for numerators, closings and the scheduler. The text key is hashed to +/// 64 bits by the server (hashtextextended); transaction locks are released by commit or rollback. +/// +public static class PostgresAdvisoryLocks +{ + /// Waits for the lock and holds it until the end of the transaction. + /// The lock key. + /// The statement. + public static SqlFragment TransactionLock(string key) + { + ArgumentException.ThrowIfNullOrEmpty(key); + return SqlFragment.Of($"SELECT pg_advisory_xact_lock(hashtextextended({key}, 0))"); + } + + /// Takes the lock if free, without waiting; the statement returns a boolean. + /// The lock key. + /// The statement. + public static SqlFragment TryTransactionLock(string key) + { + ArgumentException.ThrowIfNullOrEmpty(key); + return SqlFragment.Of($"SELECT pg_try_advisory_xact_lock(hashtextextended({key}, 0))"); + } +} diff --git a/src/Khaemhat.Data.PostgreSql/PostgresDatabaseCreator.cs b/src/Khaemhat.Data.PostgreSql/PostgresDatabaseCreator.cs new file mode 100644 index 0000000..f1782f1 --- /dev/null +++ b/src/Khaemhat.Data.PostgreSql/PostgresDatabaseCreator.cs @@ -0,0 +1,92 @@ +using System.Diagnostics.CodeAnalysis; +using Npgsql; + +namespace Khaemhat.Data.PostgreSql; + +/// +/// Creates and drops company databases on the server of KHAEMHAT_DB_SERVER (SPEC §5.4). It connects to the +/// maintenance database postgres with the same credentials. Database names are validated identifiers: they +/// cannot be parameters in DDL, so nothing else may reach the statement. +/// +public static class PostgresDatabaseCreator +{ + private const string MaintenanceDatabase = "postgres"; + + /// Creates the database of if it does not exist (UTF-8, from template0). + /// The connection string of the database to create. + /// Cancellation. + /// A task. + public static async Task EnsureExistsAsync(string connectionString, CancellationToken cancellationToken = default) + { + var name = DatabaseName(connectionString); + if (await ExistsAsync(connectionString, cancellationToken).ConfigureAwait(false)) + { + return; + } + + try + { + await ExecuteOnMaintenanceAsync( + connectionString, $"CREATE DATABASE \"{name}\" TEMPLATE template0 ENCODING 'UTF8'", cancellationToken).ConfigureAwait(false); + } + catch (PostgresException ex) when (ex.SqlState == PostgresErrorCodes.DuplicateDatabase) + { + // Created concurrently by another instance: the goal is reached. + } + } + + /// Drops the database, closing its connections (WITH (FORCE)). Used only by the registry and restores. + /// The connection string of the database to drop. + /// Cancellation. + /// A task. + public static async Task DropAsync(string connectionString, CancellationToken cancellationToken = default) + { + var name = DatabaseName(connectionString); + if (name == MaintenanceDatabase) + { + throw new InvalidOperationException("The maintenance database cannot be dropped."); + } + + using (var connection = new NpgsqlConnection(connectionString)) + { + NpgsqlConnection.ClearPool(connection); + } + + await ExecuteOnMaintenanceAsync(connectionString, $"DROP DATABASE IF EXISTS \"{name}\" WITH (FORCE)", cancellationToken).ConfigureAwait(false); + } + + /// Whether the database exists. + /// The connection string of the database. + /// Cancellation. + /// if it exists. + public static async Task ExistsAsync(string connectionString, CancellationToken cancellationToken = default) + { + var name = DatabaseName(connectionString); + await using var connection = new NpgsqlConnection(MaintenanceConnectionString(connectionString)); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + await using var command = new NpgsqlCommand("SELECT 1 FROM pg_database WHERE datname = @name", connection); + command.Parameters.AddWithValue("name", name); + return await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false) is not null; + } + + private static string DatabaseName(string connectionString) + { + var name = new NpgsqlConnectionStringBuilder(connectionString).Database ?? string.Empty; + return SqlNames.IsValidIdentifier(name) + ? name + : throw new ArgumentException("The database name must be lower-case letters, digits and underscore.", nameof(connectionString)); + } + + private static string MaintenanceConnectionString(string connectionString) => + new NpgsqlConnectionStringBuilder(connectionString) { Database = MaintenanceDatabase, Pooling = false }.ToString(); + + [SuppressMessage("Security", "CA2100:Review SQL queries for security vulnerabilities", + Justification = "The only variable part is a database name validated as a snake_case identifier.")] + private static async Task ExecuteOnMaintenanceAsync(string connectionString, string sql, CancellationToken cancellationToken) + { + await using var connection = new NpgsqlConnection(MaintenanceConnectionString(connectionString)); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + await using var command = new NpgsqlCommand(sql, connection); + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/Khaemhat.Data.PostgreSql/PostgresDialect.cs b/src/Khaemhat.Data.PostgreSql/PostgresDialect.cs new file mode 100644 index 0000000..e403c17 --- /dev/null +++ b/src/Khaemhat.Data.PostgreSql/PostgresDialect.cs @@ -0,0 +1,126 @@ +using System.Data.Common; +using System.Globalization; +using Khaemhat.Data.Schema; +using Npgsql; +using NpgsqlTypes; + +namespace Khaemhat.Data.PostgreSql; + +/// +/// The PostgreSQL dialect (SPEC §5.2): varchar(n)/text/integer/bigint/numeric(19,6)/boolean/date/timestamptz/uuid/ +/// bytea/jsonb, ILIKE for case-insensitive search and pg_advisory_xact_lock for application locks. +/// Parameters get an explicit driver type, so jsonb and timestamptz columns accept them without casts. +/// +public sealed class PostgresDialect : SqlDialectBase +{ + /// The shared instance. + public static PostgresDialect Instance { get; } = new(); + + /// + public override DbEngine Engine => DbEngine.PostgreSql; + + /// + public override string CurrentTimestampUtc => "now()"; + + /// + public override SqlFragment CaseInsensitiveLike(SqlFragment column, SqlFragment pattern) => + column + SqlFragment.Literal(" ILIKE ") + pattern + SqlFragment.Literal(" ESCAPE '\\'"); + + /// + public override SqlFragment AdvisoryLock(string key) => PostgresAdvisoryLocks.TransactionLock(key); + + /// + public override object ToDbValue(object? value, ColumnType type) + { + if (value is null or DBNull) + { + return DBNull.Value; + } + + return type switch + { + ColumnType.String or ColumnType.Text or ColumnType.Json => value as string ?? Convert.ToString(value, CultureInfo.InvariantCulture)!, + ColumnType.Int32 => value is Enum ? Convert.ToInt32(value, CultureInfo.InvariantCulture) : ConvertTo(value), + ColumnType.Int64 => value is Enum ? Convert.ToInt64(value, CultureInfo.InvariantCulture) : ConvertTo(value), + ColumnType.Decimal => ConvertTo(value), + ColumnType.Bool => ConvertTo(value), + ColumnType.Date => ToDate(value), + ColumnType.DateTimeUtc => ToUtc(value), + ColumnType.Guid => value is Guid g ? g : Guid.Parse((string)value, CultureInfo.InvariantCulture), + ColumnType.Bytes => (byte[])value, + _ => throw new ArgumentOutOfRangeException(nameof(type), type, null), + }; + } + + /// + public override object? FromDbValue(object? value, ColumnType type) + { + if (value is null or DBNull) + { + return null; + } + + return type switch + { + ColumnType.String or ColumnType.Text or ColumnType.Json => value as string ?? Convert.ToString(value, CultureInfo.InvariantCulture), + ColumnType.Int32 => ConvertTo(value), + ColumnType.Int64 => ConvertTo(value), + ColumnType.Decimal => ConvertTo(value), + ColumnType.Bool => ConvertTo(value), + ColumnType.Date => ToDate(value), + ColumnType.DateTimeUtc => ToUtc(value), + ColumnType.Guid => value is Guid g ? g : Guid.Parse(Convert.ToString(value, CultureInfo.InvariantCulture)!, CultureInfo.InvariantCulture), + ColumnType.Bytes => (byte[])value, + _ => throw new ArgumentOutOfRangeException(nameof(type), type, null), + }; + } + + /// + public override string SqlType(ColumnType type, int? length, int? scale) => type switch + { + ColumnType.String => length is null ? "text" : string.Create(CultureInfo.InvariantCulture, $"varchar({length})"), + ColumnType.Text => "text", + ColumnType.Int32 => "integer", + ColumnType.Int64 => "bigint", + ColumnType.Decimal => string.Create(CultureInfo.InvariantCulture, $"numeric({ColumnTypes.DecimalPrecision},{scale ?? ColumnTypes.DecimalScale})"), + ColumnType.Bool => "boolean", + ColumnType.Date => "date", + ColumnType.DateTimeUtc => "timestamptz", + ColumnType.Guid => "uuid", + ColumnType.Bytes => "bytea", + ColumnType.Json => "jsonb", + _ => throw new ArgumentOutOfRangeException(nameof(type), type, null), + }; + + /// + public override string Literal(object? value, ColumnType type) => value switch + { + bool b => b ? "TRUE" : "FALSE", + decimal d => d.ToString(CultureInfo.InvariantCulture), + _ => base.Literal(value, type), + }; + + /// + public override void ConfigureParameter(DbParameter parameter, object? value, ColumnType? type) + { + base.ConfigureParameter(parameter, value, type); + var logical = type ?? (value is null ? null : ColumnTypes.FromClrType(value.GetType())); + if (parameter is NpgsqlParameter npgsql && logical is not null) + { + npgsql.NpgsqlDbType = logical.Value switch + { + ColumnType.String or ColumnType.Text => NpgsqlDbType.Text, + ColumnType.Int32 => NpgsqlDbType.Integer, + ColumnType.Int64 => NpgsqlDbType.Bigint, + ColumnType.Decimal => NpgsqlDbType.Numeric, + ColumnType.Bool => NpgsqlDbType.Boolean, + ColumnType.Date => NpgsqlDbType.Date, + ColumnType.DateTimeUtc => NpgsqlDbType.TimestampTz, + ColumnType.Guid => NpgsqlDbType.Uuid, + ColumnType.Bytes => NpgsqlDbType.Bytea, + ColumnType.Json => NpgsqlDbType.Jsonb, + _ => npgsql.NpgsqlDbType, + }; + } + } +} diff --git a/src/Khaemhat.Data.PostgreSql/PostgresPartitioning.cs b/src/Khaemhat.Data.PostgreSql/PostgresPartitioning.cs new file mode 100644 index 0000000..7020756 --- /dev/null +++ b/src/Khaemhat.Data.PostgreSql/PostgresPartitioning.cs @@ -0,0 +1,46 @@ +using System.Globalization; + +namespace Khaemhat.Data.PostgreSql; + +/// +/// Yearly range partitioning for the large tables (SPEC §5.2: gl_entry_lines, inventory_entry_lines, +/// audit_log, sys_log). Prepared, not active: tables record their partition column with +/// TableBuilder.PartitionByYear; when one exceeds 50 million rows a migration (through +/// ISchemaBuilder.Sql) recreates it partitioned using these statements and moves the rows. +/// +public static class PostgresPartitioning +{ + /// The clause that makes a CREATE TABLE partitioned by range of . + /// The date or timestamp column. + /// The clause, starting with a space. + public static string PartitionClause(string column) => " PARTITION BY RANGE (" + PostgresDialect.Instance.Quote(column) + ")"; + + /// The name of the partition of a year, for example gl_entry_lines_2026. + /// The partitioned table. + /// The year. + /// The partition name. + public static string PartitionName(string table, int year) => table + "_" + year.ToString(CultureInfo.InvariantCulture); + + /// The statement creating the partition of one year if it does not exist. + /// The partitioned table. + /// The year. + /// The statement. + public static string CreateYearPartition(string table, int year) + { + ArgumentOutOfRangeException.ThrowIfLessThan(year, 1900); + ArgumentOutOfRangeException.ThrowIfGreaterThan(year, 9998); + var dialect = PostgresDialect.Instance; + return string.Create( + CultureInfo.InvariantCulture, + $"CREATE TABLE IF NOT EXISTS {dialect.Quote(PartitionName(table, year))} PARTITION OF {dialect.Quote(table)} FOR VALUES FROM ('{year:D4}-01-01') TO ('{year + 1:D4}-01-01')"); + } + + /// The statement creating the default partition, for rows outside the yearly ones. + /// The partitioned table. + /// The statement. + public static string CreateDefaultPartition(string table) + { + var dialect = PostgresDialect.Instance; + return $"CREATE TABLE IF NOT EXISTS {dialect.Quote(table + "_default")} PARTITION OF {dialect.Quote(table)} DEFAULT"; + } +} diff --git a/src/Khaemhat.Data.PostgreSql/PostgresProvider.cs b/src/Khaemhat.Data.PostgreSql/PostgresProvider.cs new file mode 100644 index 0000000..9f62c2d --- /dev/null +++ b/src/Khaemhat.Data.PostgreSql/PostgresProvider.cs @@ -0,0 +1,116 @@ +using System.Data.Common; +using System.Globalization; +using Khaemhat.Data.Schema; +using Npgsql; + +namespace Khaemhat.Data.PostgreSql; + +/// Settings of the PostgreSQL provider (SPEC §5.2), from the instance settings. +public sealed record PostgresProviderOptions +{ + /// Maximum pooled connections per database (each database has its own pool). + public int MaxPoolSize { get; init; } = 20; + + /// Server-side limit of a statement (statement_timeout of every session). + public TimeSpan StatementTimeout { get; init; } = TimeSpan.FromSeconds(60); + + /// The application name shown in pg_stat_activity. + public string ApplicationName { get; init; } = "khaemhat"; +} + +/// +/// The PostgreSQL engine (SPEC §1.3, §5.2), version 16 or later: one connection pool per database, a statement timeout +/// per session, databases created with CREATE DATABASE on the server (the role needs CREATEDB). +/// +/// The provider settings. +public sealed class PostgresProvider(PostgresProviderOptions options) : IDbProvider +{ + /// The oldest supported major version. + public const int MinimumMajorVersion = 16; + + /// Creates the provider with default settings. + public PostgresProvider() + : this(new PostgresProviderOptions()) + { + } + + /// + public DbEngine Engine => DbEngine.PostgreSql; + + /// + public ISqlDialect Dialect => PostgresDialect.Instance; + + /// + public DdlGenerator Ddl { get; } = new PostgresSchemaBuilder(); + + /// The settings. + public PostgresProviderOptions Options { get; } = options ?? throw new ArgumentNullException(nameof(options)); + + /// + public string BuildConnectionString(DatabaseUrl url) + { + ArgumentNullException.ThrowIfNull(url); + if (url.Engine != DbEngine.PostgreSql) + { + throw new ArgumentException("Not a PostgreSQL URL.", nameof(url)); + } + + var builder = new NpgsqlConnectionStringBuilder + { + Host = url.Host, + Port = url.Port ?? 5432, + Username = url.UserName, + Password = url.Password, + Database = url.Database.Length == 0 ? "postgres" : url.Database, + MaxPoolSize = Options.MaxPoolSize, + ApplicationName = Options.ApplicationName, + Options = string.Create(CultureInfo.InvariantCulture, $"-c statement_timeout={(long)Options.StatementTimeout.TotalMilliseconds}"), + }; + foreach (var (key, value) in url.Options) + { + try + { + builder[key] = value; + } + catch (ArgumentException) + { + throw new FormatException($"The PostgreSQL URL option '{key}' is not supported."); + } + } + + return builder.ToString(); + } + + /// + public DbConnection CreateConnection(string connectionString) => new NpgsqlConnection(connectionString); + + /// + public Task EnsureDatabaseExistsAsync(string connectionString, CancellationToken cancellationToken = default) => + PostgresDatabaseCreator.EnsureExistsAsync(connectionString, cancellationToken); + + /// + public Task DropDatabaseAsync(string connectionString, CancellationToken cancellationToken = default) => + PostgresDatabaseCreator.DropAsync(connectionString, cancellationToken); + + /// + public Task DatabaseExistsAsync(string connectionString, CancellationToken cancellationToken = default) => + PostgresDatabaseCreator.ExistsAsync(connectionString, cancellationToken); + + /// + public Task GetServerInfoAsync(string connectionString, CancellationToken cancellationToken = default) => + PostgresServerInfo.ReadAsync(connectionString, cancellationToken); + + /// + public bool IsTransient(Exception exception) => exception is NpgsqlException { IsTransient: true }; + + /// + public ValueTask EnterWriteScopeAsync(string connectionString, CancellationToken cancellationToken = default) => + ValueTask.FromResult(NoScope.Instance); + + private sealed class NoScope : IAsyncDisposable + { + public static NoScope Instance { get; } = new(); + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } +} diff --git a/src/Khaemhat.Data.PostgreSql/PostgresSchemaBuilder.cs b/src/Khaemhat.Data.PostgreSql/PostgresSchemaBuilder.cs new file mode 100644 index 0000000..380864e --- /dev/null +++ b/src/Khaemhat.Data.PostgreSql/PostgresSchemaBuilder.cs @@ -0,0 +1,18 @@ +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.PostgreSql; + +/// +/// DDL for PostgreSQL (SPEC §21 PostgresSchemaBuilder): identity keys are +/// bigint GENERATED BY DEFAULT AS IDENTITY, so restores and imports may still write explicit ids. +/// +public sealed class PostgresSchemaBuilder() : DdlGenerator(PostgresDialect.Instance) +{ + /// + protected override string IdentityPrimaryKey(ColumnDefinition column) + { + ArgumentNullException.ThrowIfNull(column); + var type = column.Type == ColumnType.Int32 ? "integer" : "bigint"; + return Dialect.Quote(column.Name) + " " + type + " GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY"; + } +} diff --git a/src/Khaemhat.Data.PostgreSql/PostgresServerInfo.cs b/src/Khaemhat.Data.PostgreSql/PostgresServerInfo.cs new file mode 100644 index 0000000..cd1ab27 --- /dev/null +++ b/src/Khaemhat.Data.PostgreSql/PostgresServerInfo.cs @@ -0,0 +1,59 @@ +using System.Globalization; +using Npgsql; + +namespace Khaemhat.Data.PostgreSql; + +/// Reads version, role, connections, size and extensions of a PostgreSQL database (SPEC §12.1). +public static class PostgresServerInfo +{ + private const string Query = """ + SELECT current_setting('server_version'), + current_setting('server_version_num')::int, + current_database(), + pg_database_size(current_database()), + current_user, + (SELECT rolcreatedb FROM pg_roles WHERE rolname = current_user), + (SELECT numbackends FROM pg_stat_database WHERE datname = current_database()), + current_setting('max_connections')::int, + (SELECT coalesce(string_agg(extname || ' ' || extversion, ',' ORDER BY extname), '') FROM pg_extension) + """; + + /// Reads the information. + /// The connection string of the database. + /// Cancellation. + /// The information, with a warning below version 16 and when the role cannot create databases. + public static async Task ReadAsync(string connectionString, CancellationToken cancellationToken = default) + { + await using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + await using var command = new NpgsqlCommand(Query, connection); + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + await reader.ReadAsync(cancellationToken).ConfigureAwait(false); + + var versionNumber = reader.GetInt32(1); + var canCreate = !reader.IsDBNull(5) && reader.GetBoolean(5); + var warnings = new List(); + if (versionNumber / 10000 < PostgresProvider.MinimumMajorVersion) + { + warnings.Add(string.Create( + CultureInfo.InvariantCulture, + $"PostgreSQL {reader.GetString(0)} is older than {PostgresProvider.MinimumMajorVersion}, the oldest supported version; upgrade it (17 is recommended).")); + } + + if (!canCreate) + { + warnings.Add($"The role {reader.GetString(4)} has no CREATEDB: new company databases must be created by an administrator."); + } + + var extensions = reader.GetString(8); + return new DbServerInfo(DbEngine.PostgreSql, reader.GetString(0), versionNumber, reader.GetString(2), reader.GetInt64(3)) + { + UserName = reader.GetString(4), + CanCreateDatabases = canCreate, + ActiveConnections = reader.IsDBNull(6) ? null : reader.GetInt32(6), + MaxConnections = reader.GetInt32(7), + Details = extensions.Length == 0 ? [] : extensions.Split(','), + Warnings = warnings, + }; + } +} diff --git a/src/Khaemhat.Data.Sqlite/SqliteDecimalConverter.cs b/src/Khaemhat.Data.Sqlite/SqliteDecimalConverter.cs new file mode 100644 index 0000000..f13bd5c --- /dev/null +++ b/src/Khaemhat.Data.Sqlite/SqliteDecimalConverter.cs @@ -0,0 +1,39 @@ +namespace Khaemhat.Data.Sqlite; + +/// +/// SQLite has no exact decimal type: Decimal(19,6) values are stored as integers multiplied by 10⁶ (SPEC §5.2). +/// Sums in SQL therefore stay exact integers and are scaled back when read. Values are rounded to 6 decimals half +/// away from zero, like PostgreSQL numeric(19,6). The usable range is ±9.2 x 10¹². +/// +public static class SqliteDecimalConverter +{ + /// The scale factor, 10⁶. + public const long Factor = 1_000_000; + + /// The largest absolute value that can be stored. + public static readonly decimal MaxValue = long.MaxValue / (decimal)Factor; + + /// Converts a decimal to its scaled integer. + /// The value. + /// The scaled integer. + /// The value is beyond ±9.2 x 10¹². + public static long ToScaled(decimal value) + { + var scaled = decimal.Round(value * Factor, 0, MidpointRounding.AwayFromZero); + return scaled is > long.MaxValue or < long.MinValue + ? throw new OverflowException($"The amount exceeds the SQLite decimal range of ±{MaxValue:N0}.") + : (long)scaled; + } + + /// Converts a scaled integer (or a scaled sum read as integer or real) back to a decimal. + /// The stored value. + /// The decimal. + public static decimal FromScaled(object value) => value switch + { + long l => (decimal)l / Factor, + int i => (decimal)i / Factor, + double d => (decimal)d / Factor, + decimal m => m / Factor, + _ => throw new InvalidCastException($"A {value.GetType().Name} is not a scaled SQLite decimal."), + }; +} diff --git a/src/Khaemhat.Data.Sqlite/SqliteDialect.cs b/src/Khaemhat.Data.Sqlite/SqliteDialect.cs new file mode 100644 index 0000000..69c8299 --- /dev/null +++ b/src/Khaemhat.Data.Sqlite/SqliteDialect.cs @@ -0,0 +1,99 @@ +using System.Globalization; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.Sqlite; + +/// +/// The SQLite dialect (SPEC §5.2): STRICT tables with TEXT, INTEGER and BLOB; decimals as +/// integers x10⁶, dates yyyy-MM-dd, timestamps ISO 8601 UTC with microseconds, booleans 0/1, GUIDs as text. +/// Case-insensitive search uses the kh_lower function registered on every connection, which lowers Unicode text +/// (SQLite's own LIKE and lower only fold ASCII). +/// +public sealed class SqliteDialect : SqlDialectBase +{ + /// The format of stored timestamps: fixed width, so text order is time order. + public const string TimestampFormat = "yyyy-MM-dd'T'HH:mm:ss.ffffff'Z'"; + + /// The name of the Unicode lower-case function registered on connections. + public const string LowerFunction = "kh_lower"; + + /// The shared instance. + public static SqliteDialect Instance { get; } = new(); + + /// + public override DbEngine Engine => DbEngine.Sqlite; + + /// + public override string CurrentTimestampUtc => "strftime('%Y-%m-%dT%H:%M:%f000Z', 'now')"; + + /// + public override SqlFragment CaseInsensitiveLike(SqlFragment column, SqlFragment pattern) => + SqlFragment.Literal(LowerFunction + "(") + column + SqlFragment.Literal(") LIKE " + LowerFunction + "(") + pattern + + SqlFragment.Literal(") ESCAPE '\\'"); + + /// + public override SqlFragment AdvisoryLock(string key) => SqlFragment.Empty; + + /// + public override object ToDbValue(object? value, ColumnType type) + { + if (value is null or DBNull) + { + return DBNull.Value; + } + + return type switch + { + ColumnType.String or ColumnType.Text or ColumnType.Json => value as string ?? Convert.ToString(value, CultureInfo.InvariantCulture)!, + ColumnType.Int32 or ColumnType.Int64 => value is Enum ? Convert.ToInt64(value, CultureInfo.InvariantCulture) : ConvertTo(value), + ColumnType.Decimal => SqliteDecimalConverter.ToScaled(ConvertTo(value)), + ColumnType.Bool => ConvertTo(value) ? 1L : 0L, + ColumnType.Date => ToDate(value).ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), + ColumnType.DateTimeUtc => ToUtc(value).ToString(TimestampFormat, CultureInfo.InvariantCulture), + ColumnType.Guid => (value is Guid g ? g : Guid.Parse((string)value, CultureInfo.InvariantCulture)).ToString("D"), + ColumnType.Bytes => (byte[])value, + _ => throw new ArgumentOutOfRangeException(nameof(type), type, null), + }; + } + + /// + public override object? FromDbValue(object? value, ColumnType type) + { + if (value is null or DBNull) + { + return null; + } + + return type switch + { + ColumnType.String or ColumnType.Text or ColumnType.Json => value as string ?? Convert.ToString(value, CultureInfo.InvariantCulture), + ColumnType.Int32 => ConvertTo(value), + ColumnType.Int64 => ConvertTo(value), + ColumnType.Decimal => SqliteDecimalConverter.FromScaled(value), + ColumnType.Bool => ConvertTo(value) != 0, + ColumnType.Date => ToDate(value), + ColumnType.DateTimeUtc => value is string s + ? DateTime.Parse(s, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal) + : ToUtc(value), + ColumnType.Guid => value is Guid g ? g : Guid.Parse((string)value, CultureInfo.InvariantCulture), + ColumnType.Bytes => (byte[])value, + _ => throw new ArgumentOutOfRangeException(nameof(type), type, null), + }; + } + + /// + public override string SqlType(ColumnType type, int? length, int? scale) => type switch + { + ColumnType.Int32 or ColumnType.Int64 or ColumnType.Decimal or ColumnType.Bool => "INTEGER", + ColumnType.Bytes => "BLOB", + _ => "TEXT", + }; + + /// + public override string Literal(object? value, ColumnType type) => value switch + { + bool b => b ? "1" : "0", + decimal d when type == ColumnType.Decimal => SqliteDecimalConverter.ToScaled(d).ToString(CultureInfo.InvariantCulture), + _ => base.Literal(value, type), + }; +} diff --git a/src/Khaemhat.Data.Sqlite/SqliteProvider.cs b/src/Khaemhat.Data.Sqlite/SqliteProvider.cs new file mode 100644 index 0000000..0632fd7 --- /dev/null +++ b/src/Khaemhat.Data.Sqlite/SqliteProvider.cs @@ -0,0 +1,145 @@ +using System.Data; +using System.Data.Common; +using System.Globalization; +using Khaemhat.Data.Schema; +using Microsoft.Data.Sqlite; + +namespace Khaemhat.Data.Sqlite; + +/// +/// The SQLite engine (SPEC §1.3, §5.2): tests, demo, trial and read-only archives. Every connection gets +/// foreign_keys=ON and busy_timeout=5000; file databases use journal_mode=WAL. Databases are files +/// under KHAEMHAT_DATA_DIR, or in-memory databases shared by the connections of the process (tests). +/// +public sealed class SqliteProvider : IDbProvider +{ + private const int BusyTimeoutMilliseconds = 5000; + + /// + public DbEngine Engine => DbEngine.Sqlite; + + /// + public ISqlDialect Dialect => SqliteDialect.Instance; + + /// + public DdlGenerator Ddl { get; } = new SqliteSchemaBuilder(); + + /// + public string BuildConnectionString(DatabaseUrl url) + { + ArgumentNullException.ThrowIfNull(url); + if (url.Engine != DbEngine.Sqlite) + { + throw new ArgumentException("Not a SQLite URL.", nameof(url)); + } + + var builder = url.IsInMemory + ? new SqliteConnectionStringBuilder { DataSource = url.Database, Mode = SqliteOpenMode.Memory, Cache = SqliteCacheMode.Shared } + : new SqliteConnectionStringBuilder { DataSource = Path.GetFullPath(url.Database), Mode = SqliteOpenMode.ReadWriteCreate }; + builder.Pooling = true; + builder.DefaultTimeout = 30; + return builder.ToString(); + } + + /// + public DbConnection CreateConnection(string connectionString) + { + var connection = new SqliteConnection(connectionString); + connection.CreateFunction(SqliteDialect.LowerFunction, static s => s?.ToLowerInvariant(), isDeterministic: true); + var isFile = IsFile(connectionString); + connection.StateChange += (sender, args) => + { + if (args.CurrentState == ConnectionState.Open && sender is SqliteConnection open) + { + ApplyPragmas(open, isFile); + } + }; + return connection; + } + + /// + public async Task EnsureDatabaseExistsAsync(string connectionString, CancellationToken cancellationToken = default) + { + if (!IsFile(connectionString)) + { + return; + } + + var path = new SqliteConnectionStringBuilder(connectionString).DataSource; + Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(path))!); + await using var connection = CreateConnection(connectionString); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + } + + /// + public Task DropDatabaseAsync(string connectionString, CancellationToken cancellationToken = default) + { + if (!IsFile(connectionString)) + { + return Task.CompletedTask; + } + + using (var connection = new SqliteConnection(connectionString)) + { + SqliteConnection.ClearPool(connection); + } + + var path = new SqliteConnectionStringBuilder(connectionString).DataSource; + foreach (var file in new[] { path, path + "-wal", path + "-shm", path + "-journal" }) + { + File.Delete(file); + } + + return Task.CompletedTask; + } + + /// + public Task DatabaseExistsAsync(string connectionString, CancellationToken cancellationToken = default) => + Task.FromResult(!IsFile(connectionString) || File.Exists(new SqliteConnectionStringBuilder(connectionString).DataSource)); + + /// + public async Task GetServerInfoAsync(string connectionString, CancellationToken cancellationToken = default) + { + await using var connection = CreateConnection(connectionString); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + var version = (string)(await ScalarAsync(connection, "SELECT sqlite_version()", cancellationToken).ConfigureAwait(false))!; + var pages = Convert.ToInt64(await ScalarAsync(connection, "PRAGMA page_count", cancellationToken).ConfigureAwait(false), CultureInfo.InvariantCulture); + var pageSize = Convert.ToInt64(await ScalarAsync(connection, "PRAGMA page_size", cancellationToken).ConfigureAwait(false), CultureInfo.InvariantCulture); + var journal = Convert.ToString(await ScalarAsync(connection, "PRAGMA journal_mode", cancellationToken).ConfigureAwait(false), CultureInfo.InvariantCulture); + var parts = version.Split('.').Select(p => int.Parse(p, CultureInfo.InvariantCulture)).Concat([0, 0, 0]).ToArray(); + var source = new SqliteConnectionStringBuilder(connectionString).DataSource; + return new DbServerInfo( + DbEngine.Sqlite, version, (parts[0] * 1_000_000) + (parts[1] * 1_000) + parts[2], + IsFile(connectionString) ? Path.GetFileNameWithoutExtension(source) : source, pages * pageSize) + { + Details = ["journal_mode=" + journal], + }; + } + + /// + public bool IsTransient(Exception exception) => + exception is SqliteException { SqliteErrorCode: 5 or 6 }; // SQLITE_BUSY, SQLITE_LOCKED + + /// + public ValueTask EnterWriteScopeAsync(string connectionString, CancellationToken cancellationToken = default) => + SqliteWriteLock.AcquireAsync(new SqliteConnectionStringBuilder(connectionString).DataSource, cancellationToken); + + private static bool IsFile(string connectionString) => + new SqliteConnectionStringBuilder(connectionString).Mode != SqliteOpenMode.Memory; + + private static void ApplyPragmas(SqliteConnection connection, bool isFile) + { + using var command = connection.CreateCommand(); + command.CommandText = isFile + ? $"PRAGMA journal_mode=WAL; PRAGMA foreign_keys=ON; PRAGMA busy_timeout={BusyTimeoutMilliseconds};" + : $"PRAGMA foreign_keys=ON; PRAGMA busy_timeout={BusyTimeoutMilliseconds};"; + command.ExecuteNonQuery(); + } + + private static async Task ScalarAsync(DbConnection connection, string sql, CancellationToken cancellationToken) + { + await using var command = connection.CreateCommand(); + command.CommandText = sql; + return await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/Khaemhat.Data.Sqlite/SqliteSchemaBuilder.cs b/src/Khaemhat.Data.Sqlite/SqliteSchemaBuilder.cs new file mode 100644 index 0000000..a490c0e --- /dev/null +++ b/src/Khaemhat.Data.Sqlite/SqliteSchemaBuilder.cs @@ -0,0 +1,34 @@ +using System.Globalization; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.Sqlite; + +/// +/// DDL for SQLite (SPEC §21 SqliteSchemaBuilder): STRICT tables, INTEGER PRIMARY KEY for identity +/// keys, and CHECK constraints that give text lengths and booleans the same limits as on PostgreSQL. +/// +public sealed class SqliteSchemaBuilder() : DdlGenerator(SqliteDialect.Instance) +{ + /// + protected override string TableOptions(TableDefinition table) => " STRICT"; + + /// + protected override string IdentityPrimaryKey(ColumnDefinition column) + { + ArgumentNullException.ThrowIfNull(column); + return Dialect.Quote(column.Name) + " INTEGER PRIMARY KEY"; + } + + /// + protected override string ColumnConstraints(ColumnDefinition column) + { + ArgumentNullException.ThrowIfNull(column); + var name = Dialect.Quote(column.Name); + return column.Type switch + { + ColumnType.String => string.Create(CultureInfo.InvariantCulture, $" CHECK (length({name}) <= {column.Length})"), + ColumnType.Bool => $" CHECK ({name} IN (0, 1))", + _ => string.Empty, + }; + } +} diff --git a/src/Khaemhat.Data.Sqlite/SqliteWriteLock.cs b/src/Khaemhat.Data.Sqlite/SqliteWriteLock.cs new file mode 100644 index 0000000..b768d5e --- /dev/null +++ b/src/Khaemhat.Data.Sqlite/SqliteWriteLock.cs @@ -0,0 +1,41 @@ +using System.Collections.Concurrent; + +namespace Khaemhat.Data.Sqlite; + +/// +/// Serializes the writers of one SQLite database inside the process (SPEC §5.2). SQLite allows a single writer; waiting +/// here is fairer and cheaper than retrying on SQLITE_BUSY. Other processes are kept out by +/// BEGIN IMMEDIATE and busy_timeout. Do not open a second writing unit of work on the same database while +/// holding one in the same flow: it would wait for itself. +/// +public static class SqliteWriteLock +{ + private static readonly ConcurrentDictionary Locks = new(StringComparer.OrdinalIgnoreCase); + + /// Waits for the write lock of a database. + /// The database key: full file path, or the in-memory name. + /// Cancellation. + /// The held lock; dispose it to release. + public static async ValueTask AcquireAsync(string database, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrEmpty(database); + var semaphore = Locks.GetOrAdd(database, static _ => new SemaphoreSlim(1, 1)); + await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + return new Releaser(semaphore); + } + + private sealed class Releaser(SemaphoreSlim semaphore) : IAsyncDisposable + { + private int _released; + + public ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _released, 1) == 0) + { + semaphore.Release(); + } + + return ValueTask.CompletedTask; + } + } +} diff --git a/src/Khaemhat.Data/Companies/CompanyRegistry.cs b/src/Khaemhat.Data/Companies/CompanyRegistry.cs new file mode 100644 index 0000000..19bcde9 --- /dev/null +++ b/src/Khaemhat.Data/Companies/CompanyRegistry.cs @@ -0,0 +1,200 @@ +using System.Collections.Concurrent; +using Khaemhat.Core.Instance; +using Khaemhat.Core.Security; +using Khaemhat.Data.Execution; +using Khaemhat.Data.Mapping; +using Khaemhat.Data.Querying; + +namespace Khaemhat.Data.Companies; + +/// A company to add to the registry. +/// The company code. +/// The company name. +/// Where its database is (printed redacted). +/// Whether it becomes the default company. +/// Notes for administrators. +public sealed record CompanyRegistration(string Code, string Name, DatabaseUrl Database, bool IsDefault = false, string? Notes = null); + +/// +/// The company registry in the system database (SPEC §5.4): it resolves company code → (provider, connection +/// string), decrypting the location stored in sys_companies. +/// +public interface ICompanyRegistry +{ + /// All companies, by code. + /// Cancellation. + /// The companies (their location stays encrypted). + Task> ListAsync(CancellationToken cancellationToken = default); + + /// A company by code. + /// The code. + /// Cancellation. + /// The company, or . + Task FindAsync(string code, CancellationToken cancellationToken = default); + + /// Adds a company whose database already exists or will be created by the caller. + /// The company. + /// Cancellation. + /// The stored company. + Task RegisterAsync(CompanyRegistration registration, CancellationToken cancellationToken = default); + + /// The database of a company. + /// The code. + /// Cancellation. + /// The target. + /// No company has that code. + Task ResolveAsync(string code, CancellationToken cancellationToken = default); + + /// Changes the state of a company. + /// The code. + /// The new state. + /// Cancellation. + /// A task. + Task SetStatusAsync(string code, CompanyStatus status, CancellationToken cancellationToken = default); +} + +/// +/// on sys_companies. The database URL is encrypted with +/// for the purpose company:<code>:database, so a row copied to another company +/// cannot be decrypted. Resolved targets are cached until the row changes through this registry. +/// +public sealed class CompanyRegistry : ICompanyRegistry +{ + private readonly DatabaseTarget _system; + private readonly DbProviderCatalog _providers; + private readonly ISecretProtector _protector; + private readonly MapRegistry _maps; + private readonly DataAccessContext _context; + private readonly ConcurrentDictionary _resolved = new(StringComparer.Ordinal); + + /// Creates the registry. + /// The system database. + /// The providers by engine. + /// Encrypts the database locations. + /// The entity maps, including . + /// Clock, user and retry policy. + public CompanyRegistry(DatabaseTarget system, DbProviderCatalog providers, ISecretProtector protector, MapRegistry maps, DataAccessContext context) + { + ArgumentNullException.ThrowIfNull(system); + ArgumentNullException.ThrowIfNull(providers); + ArgumentNullException.ThrowIfNull(protector); + ArgumentNullException.ThrowIfNull(maps); + ArgumentNullException.ThrowIfNull(context); + _system = system; + _providers = providers; + _protector = protector; + _maps = maps; + _context = context; + } + + /// The purpose used to encrypt the database location of a company. + /// The normalized code. + /// The purpose. + public static string Purpose(string code) => "company:" + code + ":database"; + + /// + public async Task> ListAsync(CancellationToken cancellationToken = default) + { + await using var unitOfWork = await OpenAsync(cancellationToken).ConfigureAwait(false); + var page = await unitOfWork.Repository().ListAsync(Query.For().OrderBy(c => c.Code), cancellationToken).ConfigureAwait(false); + return page.Items; + } + + /// + public async Task FindAsync(string code, CancellationToken cancellationToken = default) + { + var normalized = CompanyCodes.Normalize(code); + await using var unitOfWork = await OpenAsync(cancellationToken).ConfigureAwait(false); + return await unitOfWork.Repository().FindAsync(Query.For().Where(c => c.Code == normalized), cancellationToken) + .ConfigureAwait(false); + } + + /// + public async Task RegisterAsync(CompanyRegistration registration, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(registration); + ArgumentException.ThrowIfNullOrWhiteSpace(registration.Name); + var code = CompanyCodes.Normalize(registration.Code); + _ = _providers.Get(registration.Database.Engine); + + var company = new Company + { + Code = code, + Name = registration.Name.Trim(), + Engine = registration.Database.Engine.ToString(), + ConnectionString = _protector.Protect(registration.Database.ToUnredactedString(), Purpose(code)), + Status = CompanyStatus.Active, + IsDefault = registration.IsDefault, + Notes = registration.Notes, + }; + + await using var unitOfWork = await OpenAsync(cancellationToken).ConfigureAwait(false); + var companies = unitOfWork.Repository(); + await unitOfWork.InTransactionAsync( + async ct => + { + if (await companies.CountAsync(Query.For().Where(c => c.Code == code), ct).ConfigureAwait(false) > 0) + { + throw new InvalidOperationException($"A company with code {code} already exists."); + } + + if (company.IsDefault) + { + await unitOfWork.ExecuteAsync( + SqlFragment.Of($"UPDATE {SqlFragment.Identifier("sys_companies")} SET is_default = {false} WHERE is_default = {true}"), ct) + .ConfigureAwait(false); + } + + await companies.InsertAsync(company, ct).ConfigureAwait(false); + }, + cancellationToken).ConfigureAwait(false); + + _resolved.TryRemove(code, out _); + return company; + } + + /// + public async Task ResolveAsync(string code, CancellationToken cancellationToken = default) + { + var normalized = CompanyCodes.Normalize(code); + if (_resolved.TryGetValue(normalized, out var cached)) + { + return cached; + } + + var company = await FindAsync(normalized, cancellationToken).ConfigureAwait(false) + ?? throw new KeyNotFoundException($"No company has code {normalized}."); + var url = DatabaseUrl.Parse(_protector.Unprotect(company.ConnectionString, Purpose(normalized))); + var target = _providers.Resolve(url); + _resolved[normalized] = target; + return target; + } + + /// + public async Task SetStatusAsync(string code, CompanyStatus status, CancellationToken cancellationToken = default) + { + var normalized = CompanyCodes.Normalize(code); + await using var unitOfWork = await OpenAsync(cancellationToken).ConfigureAwait(false); + var companies = unitOfWork.Repository(); + var company = await companies.FindAsync(Query.For().Where(c => c.Code == normalized), cancellationToken).ConfigureAwait(false) + ?? throw new KeyNotFoundException($"No company has code {normalized}."); + company.Status = status; + await companies.UpdateAsync(company, cancellationToken).ConfigureAwait(false); + _resolved.TryRemove(normalized, out _); + } + + private Task OpenAsync(CancellationToken cancellationToken) => UnitOfWork.OpenAsync(_system, _maps, _context, cancellationToken); +} + +/// Finds the system database, or a company database through the registry. +/// The system database. +/// The company registry. +public sealed class DatabaseLocator(DatabaseTarget system, ICompanyRegistry companies) : IDatabaseLocator +{ + /// + public async ValueTask LocateAsync(DatabaseScope scope, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(scope); + return scope.IsSystem ? system : await companies.ResolveAsync(scope.CompanyCode!, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/Khaemhat.Data/CompanyCodes.cs b/src/Khaemhat.Data/CompanyCodes.cs new file mode 100644 index 0000000..8146bc0 --- /dev/null +++ b/src/Khaemhat.Data/CompanyCodes.cs @@ -0,0 +1,43 @@ +using System.Text.RegularExpressions; + +namespace Khaemhat.Data; + +/// +/// Rules for company codes and the names of their databases (SPEC §5.4): the code is 1 to 20 upper-case letters, +/// digits or underscores starting with a letter; the database is khaemhat_<code in lower case>. +/// +public static partial class CompanyCodes +{ + /// The prefix of every company database name. + public const string DatabasePrefix = "khaemhat_"; + + /// Whether (already normalized) is a valid company code. + /// The code. + /// if valid. + public static bool IsValid(string? code) => code is not null && CodePattern().IsMatch(code); + + /// Trims and upper-cases a code, then validates it. + /// The code as typed. + /// The normalized code. + /// The code is not valid. + public static string Normalize(string code) + { + ArgumentNullException.ThrowIfNull(code); + var normalized = code.Trim().ToUpperInvariant(); + if (!IsValid(normalized)) + { + throw new ArgumentException( + "A company code has 1 to 20 characters: letters, digits or underscore, starting with a letter.", nameof(code)); + } + + return normalized; + } + + /// The database name of a company, for example khaemhat_demo for DEMO. + /// The company code. + /// The database name. + public static string DatabaseName(string code) => DatabasePrefix + Normalize(code).ToLowerInvariant(); + + [GeneratedRegex("^[A-Z][A-Z0-9_]{0,19}$", RegexOptions.CultureInvariant)] + private static partial Regex CodePattern(); +} diff --git a/src/Khaemhat.Data/DatabaseScope.cs b/src/Khaemhat.Data/DatabaseScope.cs new file mode 100644 index 0000000..bfb2f78 --- /dev/null +++ b/src/Khaemhat.Data/DatabaseScope.cs @@ -0,0 +1,26 @@ +namespace Khaemhat.Data; + +/// +/// Which database a unit of work opens (SPEC §5.4): the instance system database or the database of one company. +/// +public sealed record DatabaseScope +{ + private DatabaseScope(string? companyCode) => CompanyCode = companyCode; + + /// The system database (KHAEMHAT_SYSTEM_DB). + public static DatabaseScope System { get; } = new((string?)null); + + /// The company code, or for the system database. + public string? CompanyCode { get; } + + /// Whether this is the system database. + public bool IsSystem => CompanyCode is null; + + /// The database of the company with the given code. + /// The company code (case-insensitive; stored upper-case). + /// The scope. + public static DatabaseScope Company(string code) => new(CompanyCodes.Normalize(code)); + + /// + public override string ToString() => CompanyCode is null ? "system" : "company " + CompanyCode; +} diff --git a/src/Khaemhat.Data/DatabaseTarget.cs b/src/Khaemhat.Data/DatabaseTarget.cs new file mode 100644 index 0000000..a4d5258 --- /dev/null +++ b/src/Khaemhat.Data/DatabaseTarget.cs @@ -0,0 +1,36 @@ +namespace Khaemhat.Data; + +/// +/// A database ready to be opened: its provider and driver connection string. shows only the +/// engine and the database name, never the connection string. +/// +public sealed class DatabaseTarget +{ + /// Creates a target. + /// The provider. + /// The driver connection string. + /// The database name, for messages. + public DatabaseTarget(IDbProvider provider, string connectionString, string databaseName) + { + ArgumentNullException.ThrowIfNull(provider); + ArgumentException.ThrowIfNullOrEmpty(connectionString); + Provider = provider; + ConnectionString = connectionString; + DatabaseName = databaseName ?? string.Empty; + } + + /// The provider. + public IDbProvider Provider { get; } + + /// The driver connection string. It contains credentials: never log it. + public string ConnectionString { get; } + + /// The database name. + public string DatabaseName { get; } + + /// The engine. + public DbEngine Engine => Provider.Engine; + + /// + public override string ToString() => Engine + " " + DatabaseName; +} diff --git a/src/Khaemhat.Data/DatabaseUrl.cs b/src/Khaemhat.Data/DatabaseUrl.cs new file mode 100644 index 0000000..d27d254 --- /dev/null +++ b/src/Khaemhat.Data/DatabaseUrl.cs @@ -0,0 +1,259 @@ +using System.Globalization; +using System.Text; + +namespace Khaemhat.Data; + +/// +/// A database location in URL form, as given by KHAEMHAT_SYSTEM_DB, KHAEMHAT_DB_SERVER and the company +/// registry (SPEC §6): +/// +/// postgres://user:password@host:5432/database?sslmode=require (also postgresql://; the database +/// may be empty for a server URL); +/// sqlite:///absolute/path/file.db (on Windows sqlite:///C:/path/file.db); +/// sqlite://memory/name, an in-memory database shared by the connections of the process (tests). +/// +/// never shows the password; parsing errors never echo the input. +/// +public sealed class DatabaseUrl +{ + private const string SqliteMemoryPrefix = "sqlite://memory/"; + + private DatabaseUrl( + DbEngine engine, string database, string? host, int? port, string? userName, string? password, + IReadOnlyDictionary options, bool isInMemory) + { + Engine = engine; + Database = database; + Host = host; + Port = port; + UserName = userName; + Password = password; + Options = options; + IsInMemory = isInMemory; + } + + /// The engine. + public DbEngine Engine { get; } + + /// + /// PostgreSQL: the database name (empty for a server URL). SQLite: the file path, or the name of the in-memory database. + /// + public string Database { get; } + + /// PostgreSQL host. + public string? Host { get; } + + /// PostgreSQL port, when given. + public int? Port { get; } + + /// PostgreSQL user. + public string? UserName { get; } + + /// PostgreSQL password. Never printed by . + public string? Password { get; } + + /// Query-string options, for example sslmode. + public IReadOnlyDictionary Options { get; } + + /// Whether this is an in-memory SQLite database. + public bool IsInMemory { get; } + + /// + /// The short database name: the PostgreSQL database, or the SQLite file name without extension (or the in-memory name). + /// + public string DatabaseName => Engine == DbEngine.PostgreSql || IsInMemory + ? Database + : Path.GetFileNameWithoutExtension(Database); + + /// Parses a database URL. + /// The URL. + /// The parsed URL. + /// The URL is not valid (the message does not repeat it). + public static DatabaseUrl Parse(string url) + { + ArgumentNullException.ThrowIfNull(url); + return TryParse(url, out var result, out var error) ? result : throw new FormatException(error); + } + + /// Tries to parse a database URL. + /// The URL. + /// The parsed URL. + /// if the URL is valid. + public static bool TryParse(string? url, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out DatabaseUrl? result) => + TryParse(url, out result, out _); + + /// + /// The same server with another database: on PostgreSQL the database name changes; on SQLite the file + /// <name>.db in the same directory (or another in-memory database). + /// + /// The database name, for example khaemhat_demo. + /// The new URL. + public DatabaseUrl WithDatabase(string databaseName) + { + ArgumentException.ThrowIfNullOrWhiteSpace(databaseName); + if (Engine == DbEngine.PostgreSql || IsInMemory) + { + return new DatabaseUrl(Engine, databaseName, Host, Port, UserName, Password, Options, IsInMemory); + } + + var directory = Path.GetDirectoryName(Database) ?? string.Empty; + var path = directory.Length == 0 ? databaseName + ".db" : directory.Replace('\\', '/') + "/" + databaseName + ".db"; + return new DatabaseUrl(Engine, path, Host, Port, UserName, Password, Options, IsInMemory); + } + + /// + /// The complete URL including the password, for storing it encrypted. Never log or display the result. + /// + /// The URL. + public string ToUnredactedString() => Format(includePassword: true); + + /// The URL with the password replaced by ***. + /// The redacted URL. + public override string ToString() => Format(includePassword: false); + + private string Format(bool includePassword) + { + if (Engine == DbEngine.Sqlite) + { + return IsInMemory ? SqliteMemoryPrefix + Database : "sqlite://" + (Database.StartsWith('/') ? Database : "/" + Database); + } + + var builder = new StringBuilder("postgres://"); + if (UserName is not null) + { + builder.Append(Uri.EscapeDataString(UserName)); + if (Password is not null) + { + builder.Append(':').Append(includePassword ? Uri.EscapeDataString(Password) : "***"); + } + + builder.Append('@'); + } + + builder.Append(Host); + if (Port is not null) + { + builder.Append(':').Append(Port.Value.ToString(CultureInfo.InvariantCulture)); + } + + builder.Append('/').Append(Uri.EscapeDataString(Database)); + var separator = '?'; + foreach (var (key, value) in Options) + { + builder.Append(separator).Append(Uri.EscapeDataString(key)).Append('=').Append(Uri.EscapeDataString(value)); + separator = '&'; + } + + return builder.ToString(); + } + + private static bool TryParse(string? url, out DatabaseUrl result, out string error) + { + result = null!; + url = url?.Trim(); + if (string.IsNullOrEmpty(url)) + { + error = "The database URL is empty."; + return false; + } + + if (url.StartsWith("sqlite:", StringComparison.OrdinalIgnoreCase)) + { + return TryParseSqlite(url, out result, out error); + } + + if (url.StartsWith("postgres://", StringComparison.OrdinalIgnoreCase) + || url.StartsWith("postgresql://", StringComparison.OrdinalIgnoreCase)) + { + return TryParsePostgres(url, out result, out error); + } + + error = "The database URL must start with postgres:// or sqlite://."; + return false; + } + + private static bool TryParseSqlite(string url, out DatabaseUrl result, out string error) + { + result = null!; + var empty = new Dictionary(StringComparer.OrdinalIgnoreCase); + if (url.StartsWith(SqliteMemoryPrefix, StringComparison.OrdinalIgnoreCase)) + { + var name = url[SqliteMemoryPrefix.Length..]; + if (name.Length == 0 || name.Any(c => !(char.IsAsciiLetterOrDigit(c) || c is '_' or '-'))) + { + error = "An in-memory SQLite URL is sqlite://memory/ with letters, digits, '-' or '_'."; + return false; + } + + result = new DatabaseUrl(DbEngine.Sqlite, name, null, null, null, null, empty, isInMemory: true); + error = string.Empty; + return true; + } + + if (!url.StartsWith("sqlite:///", StringComparison.OrdinalIgnoreCase)) + { + error = "A SQLite URL is sqlite:/// (three slashes)."; + return false; + } + + var path = url["sqlite://".Length..].Replace('\\', '/'); + // sqlite:///C:/data/x.db -> C:/data/x.db + if (path.Length >= 3 && path[0] == '/' && char.IsAsciiLetter(path[1]) && path[2] == ':') + { + path = path[1..]; + } + + if (path.Length < 2 || path.EndsWith('/')) + { + error = "The SQLite URL has no file name."; + return false; + } + + result = new DatabaseUrl(DbEngine.Sqlite, path, null, null, null, null, empty, isInMemory: false); + error = string.Empty; + return true; + } + + private static bool TryParsePostgres(string url, out DatabaseUrl result, out string error) + { + result = null!; + if (!Uri.TryCreate(url, UriKind.Absolute, out var uri) || string.IsNullOrEmpty(uri.Host)) + { + error = "The PostgreSQL URL is not valid: expected postgres://user:password@host:port/database."; + return false; + } + + string? user = null; + string? password = null; + if (uri.UserInfo.Length > 0) + { + var parts = uri.UserInfo.Split(':', 2); + user = Uri.UnescapeDataString(parts[0]); + password = parts.Length == 2 ? Uri.UnescapeDataString(parts[1]) : null; + } + + var database = Uri.UnescapeDataString(uri.AbsolutePath.Trim('/')); + if (database.Contains('/', StringComparison.Ordinal)) + { + error = "The PostgreSQL URL path must be a single database name."; + return false; + } + + var options = new Dictionary(StringComparer.OrdinalIgnoreCase); + var query = uri.Query.TrimStart('?'); + if (query.Length > 0) + { + foreach (var pair in query.Split('&', StringSplitOptions.RemoveEmptyEntries)) + { + var kv = pair.Split('=', 2); + options[Uri.UnescapeDataString(kv[0])] = kv.Length == 2 ? Uri.UnescapeDataString(kv[1]) : string.Empty; + } + } + + result = new DatabaseUrl( + DbEngine.PostgreSql, database, uri.Host, uri.IsDefaultPort || uri.Port < 0 ? null : uri.Port, user, password, + options, isInMemory: false); + error = string.Empty; + return true; + } +} diff --git a/src/Khaemhat.Data/DbEngine.cs b/src/Khaemhat.Data/DbEngine.cs new file mode 100644 index 0000000..893e158 --- /dev/null +++ b/src/Khaemhat.Data/DbEngine.cs @@ -0,0 +1,11 @@ +namespace Khaemhat.Data; + +/// The database engines Khaemhat supports (SPEC §1.3). +public enum DbEngine +{ + /// SQLite: tests, demo, trial and read-only archives. + Sqlite = 0, + + /// PostgreSQL 16 or later (17 recommended): real installations. + PostgreSql = 1, +} diff --git a/src/Khaemhat.Data/DbProviderCatalog.cs b/src/Khaemhat.Data/DbProviderCatalog.cs new file mode 100644 index 0000000..1d9a85d --- /dev/null +++ b/src/Khaemhat.Data/DbProviderCatalog.cs @@ -0,0 +1,37 @@ +namespace Khaemhat.Data; + +/// The providers available to the instance, by engine. The web host registers SQLite and PostgreSQL. +public sealed class DbProviderCatalog +{ + private readonly Dictionary _providers = []; + + /// Creates a catalog with the given providers. + /// The providers. + public DbProviderCatalog(params IEnumerable providers) + { + ArgumentNullException.ThrowIfNull(providers); + foreach (var provider in providers) + { + _providers[provider.Engine] = provider; + } + } + + /// The provider of an engine. + /// The engine. + /// The provider. + /// No provider is registered for the engine. + public IDbProvider Get(DbEngine engine) => + _providers.TryGetValue(engine, out var provider) + ? provider + : throw new InvalidOperationException($"No database provider is registered for {engine}."); + + /// The target for a database URL: its provider and connection string. + /// The URL. + /// The target. + public DatabaseTarget Resolve(DatabaseUrl url) + { + ArgumentNullException.ThrowIfNull(url); + var provider = Get(url.Engine); + return new DatabaseTarget(provider, provider.BuildConnectionString(url), url.DatabaseName); + } +} diff --git a/src/Khaemhat.Data/DbServerInfo.cs b/src/Khaemhat.Data/DbServerInfo.cs new file mode 100644 index 0000000..92cd119 --- /dev/null +++ b/src/Khaemhat.Data/DbServerInfo.cs @@ -0,0 +1,28 @@ +namespace Khaemhat.Data; + +/// Information about a database and its server, for the system status page (SPEC §12.1). +/// The engine. +/// The server version as text, for example 17.2 or 3.46.1. +/// The version as a number: PostgreSQL server_version_num (170002), SQLite (3046001). +/// The database name. +/// The database size in bytes, when known. +public sealed record DbServerInfo(DbEngine Engine, string Version, int VersionNumber, string Database, long? SizeBytes) +{ + /// The user the application connects as (PostgreSQL). + public string? UserName { get; init; } + + /// Whether that user may create databases (PostgreSQL CREATEDB). + public bool? CanCreateDatabases { get; init; } + + /// Open connections to this database (PostgreSQL). + public int? ActiveConnections { get; init; } + + /// The server connection limit (PostgreSQL max_connections). + public int? MaxConnections { get; init; } + + /// Installed extensions (PostgreSQL) or the journal mode (SQLite). + public IReadOnlyList Details { get; init; } = []; + + /// Problems to show to administrators, for example a PostgreSQL version below 16. + public IReadOnlyList Warnings { get; init; } = []; +} diff --git a/src/Khaemhat.Data/Execution/CommandCache.cs b/src/Khaemhat.Data/Execution/CommandCache.cs new file mode 100644 index 0000000..0c2b31d --- /dev/null +++ b/src/Khaemhat.Data/Execution/CommandCache.cs @@ -0,0 +1,90 @@ +using System.Collections.Concurrent; +using System.Globalization; +using Khaemhat.Data.Mapping; + +namespace Khaemhat.Data.Execution; + +/// The kinds of statements the repositories cache. +public enum CachedStatementKind +{ + /// INSERT, with RETURNING for identity keys. + Insert = 0, + + /// UPDATE … WHERE key = … [AND row_version = …]. + Update = 1, + + /// DELETE … WHERE key = …. + Delete = 2, +} + +/// A cached statement: SQL text whose parameter p{i} takes the value of [i]. +/// The SQL text. +/// The columns whose values fill the parameters, in order. +public sealed record CachedStatement(string Text, IReadOnlyList Parameters); + +/// +/// The SQL text of the repository statements, built once per entity, statement and dialect and reused (SPEC §21, +/// CommandCache). +/// +public static class CommandCache +{ + private static readonly ConcurrentDictionary<(Type Entity, CachedStatementKind Kind, DbEngine Engine), CachedStatement> Statements = new(); + + /// The statement of a kind for an entity map and dialect. + /// The entity map. + /// The statement kind. + /// The dialect. + /// The statement. + public static CachedStatement Get(IEntityMap map, CachedStatementKind kind, ISqlDialect dialect) + { + ArgumentNullException.ThrowIfNull(map); + ArgumentNullException.ThrowIfNull(dialect); + return Statements.GetOrAdd((map.EntityType, kind, dialect.Engine), _ => Build(map, kind, dialect)); + } + + private static CachedStatement Build(IEntityMap map, CachedStatementKind kind, ISqlDialect dialect) + { + string P(int index) => dialect.Parameter("p" + index.ToString(CultureInfo.InvariantCulture)); + switch (kind) + { + case CachedStatementKind.Insert: + { + var columns = map.Columns.Where(c => !(c.IsKey && c.IsIdentity)).ToList(); + var names = columns.Select(c => c.ColumnName).ToList(); + var placeholders = columns.Select((_, i) => P(i)).ToList(); + var text = map.Key.IsIdentity + ? dialect.InsertReturningId(map.Table, names, placeholders, map.Key.ColumnName) + : InsertOnly(dialect, map.Table, names, placeholders); + return new CachedStatement(text, columns); + } + + case CachedStatementKind.Update: + { + var set = map.Columns + .Where(c => !c.IsKey && c.Audit is not (Mapping.AuditRole.CreatedAt or Mapping.AuditRole.CreatedBy or Mapping.AuditRole.RowVersion)) + .ToList(); + var assignments = set.Select((c, i) => dialect.Quote(c.ColumnName) + " = " + P(i)).ToList(); + var parameters = new List(set) { map.Key }; + var where = dialect.Quote(map.Key.ColumnName) + " = " + P(set.Count); + if (map.RowVersion is { } version) + { + assignments.Add(dialect.Quote(version.ColumnName) + " = " + dialect.Quote(version.ColumnName) + " + 1"); + where += " AND " + dialect.Quote(version.ColumnName) + " = " + P(parameters.Count); + parameters.Add(version); + } + + var text = "UPDATE " + dialect.Quote(map.Table) + " SET " + string.Join(", ", assignments) + " WHERE " + where; + return new CachedStatement(text, parameters); + } + + case CachedStatementKind.Delete: + return new CachedStatement( + "DELETE FROM " + dialect.Quote(map.Table) + " WHERE " + dialect.Quote(map.Key.ColumnName) + " = " + P(0), [map.Key]); + default: + throw new ArgumentOutOfRangeException(nameof(kind), kind, null); + } + } + + private static string InsertOnly(ISqlDialect dialect, string table, IReadOnlyList columns, IReadOnlyList placeholders) => + "INSERT INTO " + dialect.Quote(table) + " (" + string.Join(", ", columns.Select(dialect.Quote)) + ") VALUES (" + string.Join(", ", placeholders) + ")"; +} diff --git a/src/Khaemhat.Data/Execution/DataAccessContext.cs b/src/Khaemhat.Data/Execution/DataAccessContext.cs new file mode 100644 index 0000000..255a0c2 --- /dev/null +++ b/src/Khaemhat.Data/Execution/DataAccessContext.cs @@ -0,0 +1,43 @@ +namespace Khaemhat.Data.Execution; + +/// +/// What the data layer needs from the running application: the clock for audit timestamps, the current user for +/// created_by/updated_by, and the retry policy. The web host builds it from ICurrentUser (M2). +/// +public sealed class DataAccessContext +{ + private readonly Func _currentUser; + + /// Creates a context. + /// The clock. + /// Returns the current user name, or . + /// The retry policy for transient errors outside transactions. + public DataAccessContext(TimeProvider clock, Func currentUser, TransientRetryPolicy? retryPolicy = null) + { + ArgumentNullException.ThrowIfNull(clock); + ArgumentNullException.ThrowIfNull(currentUser); + Clock = clock; + _currentUser = currentUser; + RetryPolicy = retryPolicy ?? TransientRetryPolicy.Default; + } + + /// System clock, no user, default retries. + public static DataAccessContext Default { get; } = new(TimeProvider.System, static () => null); + + /// The clock. + public TimeProvider Clock { get; } + + /// The retry policy. + public TransientRetryPolicy RetryPolicy { get; } + + /// The current user name, or . + public string? CurrentUser => _currentUser(); + + /// The current UTC time, truncated to microseconds (the precision of PostgreSQL and of SQLite storage). + public DateTime UtcNow => Truncate(Clock.GetUtcNow().UtcDateTime); + + /// Truncates a timestamp to microseconds. + /// The timestamp. + /// The truncated timestamp. + public static DateTime Truncate(DateTime value) => new(value.Ticks - (value.Ticks % 10), value.Kind); +} diff --git a/src/Khaemhat.Data/Execution/DbExecutor.cs b/src/Khaemhat.Data/Execution/DbExecutor.cs new file mode 100644 index 0000000..4805d5c --- /dev/null +++ b/src/Khaemhat.Data/Execution/DbExecutor.cs @@ -0,0 +1,51 @@ +using System.Data.Common; +using System.Diagnostics.CodeAnalysis; + +namespace Khaemhat.Data.Execution; + +/// +/// Creates driver commands from rendered SQL. This is the only place where command text is assigned: the text comes +/// from or from the , both built from code and mapped +/// names, and every value travels as a parameter. +/// +/// The dialect that configures parameters. +public sealed class DbExecutor(ISqlDialect dialect) +{ + /// Creates a command for a fragment. + /// The open connection. + /// The transaction, if any. + /// The fragment. + /// The command; the caller disposes it. + public DbCommand CreateCommand(DbConnection connection, DbTransaction? transaction, SqlFragment sql) + { + ArgumentNullException.ThrowIfNull(sql); + var rendered = sql.Render(dialect); + return CreateCommand(connection, transaction, rendered.Text, rendered.Parameters); + } + + /// Creates a command for rendered text and its parameters. + /// The open connection. + /// The transaction, if any. + /// SQL text produced by the data layer. + /// The parameters. + /// The command; the caller disposes it. + [SuppressMessage("Security", "CA2100:Review SQL queries for security vulnerabilities", + Justification = "Text comes only from SqlFragment rendering or the command cache; values are always parameters (SPEC §5.2).")] + public DbCommand CreateCommand(DbConnection connection, DbTransaction? transaction, string text, IEnumerable parameters) + { + ArgumentNullException.ThrowIfNull(connection); + ArgumentNullException.ThrowIfNull(parameters); + var command = connection.CreateCommand(); + command.Transaction = transaction; + command.CommandText = text; + foreach (var parameter in parameters) + { + var dbParameter = command.CreateParameter(); + dbParameter.ParameterName = parameter.Name; + dialect.ConfigureParameter(dbParameter, parameter.Value, parameter.Type); + command.Parameters.Add(dbParameter); + } + + return command; + } +} diff --git a/src/Khaemhat.Data/Execution/ReaderMapper.cs b/src/Khaemhat.Data/Execution/ReaderMapper.cs new file mode 100644 index 0000000..e3c09f8 --- /dev/null +++ b/src/Khaemhat.Data/Execution/ReaderMapper.cs @@ -0,0 +1,154 @@ +using System.Collections.Concurrent; +using System.Data.Common; +using System.Linq.Expressions; +using System.Reflection; +using Khaemhat.Data.Mapping; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.Execution; + +/// +/// Maps reader rows to objects (SPEC §21, ReaderMapper<T>). The plan for a type and a column layout is +/// built once and cached; rows are then mapped with compiled delegates. Supported targets: +/// +/// scalars (numbers, text, dates, GUIDs, enums, nullable or not): the first column; +/// entities with a registered map: columns matched by column name; +/// other classes and records: settable properties, or the parameters of the public constructor, matched by +/// snake_case name. +/// +/// Values pass through , so for example scaled SQLite decimals (sums included) +/// come back as . +/// +public static class ReaderMapper +{ + private static readonly ConcurrentDictionary<(Type Type, DbEngine Engine, string Layout), Delegate> Plans = new(); + + /// The row mapper for and the columns of . + /// The target type. + /// The reader, positioned before or on a row. + /// The dialect. + /// The entity maps. + /// A function mapping the current row. + public static Func For(DbDataReader reader, ISqlDialect dialect, MapRegistry maps) + { + ArgumentNullException.ThrowIfNull(reader); + ArgumentNullException.ThrowIfNull(dialect); + ArgumentNullException.ThrowIfNull(maps); + var columns = Enumerable.Range(0, reader.FieldCount).Select(reader.GetName).ToArray(); + var key = (typeof(T), dialect.Engine, string.Join('|', columns)); + return (Func)Plans.GetOrAdd(key, _ => Build(columns, dialect, maps)); + } + + private static Func Build(string[] columns, ISqlDialect dialect, MapRegistry maps) + { + var target = typeof(T); + if (IsScalar(target)) + { + var type = LogicalType(target); + return reader => (T)ClrConverter.ConvertTo(dialect.FromDbValue(reader.GetValue(0), type), target)!; + } + + if (maps.Find(target) is { } map) + { + return BuildEntity(columns, dialect, map); + } + + var constructor = target.GetConstructor(Type.EmptyTypes); + return constructor is not null ? BuildProperties(columns, dialect) : BuildConstructor(columns, dialect); + } + + private static Func BuildEntity(string[] columns, ISqlDialect dialect, IEntityMap map) + { + var slots = columns + .Select((name, ordinal) => (Ordinal: ordinal, Column: map.FindByColumn(name))) + .Where(s => s.Column is not null) + .Select(s => (s.Ordinal, Column: s.Column!)) + .ToArray(); + return reader => + { + var entity = map.CreateEntity(); + foreach (var (ordinal, column) in slots) + { + column.SetValue(entity, column.FromProvider(dialect.FromDbValue(reader.GetValue(ordinal), column.Type))); + } + + return (T)entity; + }; + } + + private static Func BuildProperties(string[] columns, ISqlDialect dialect) + { + var properties = typeof(T).GetProperties(BindingFlags.Public | BindingFlags.Instance) + .Where(p => p.CanWrite && p.GetSetMethod() is not null) + .ToDictionary(p => SqlNames.ToSnakeCase(p.Name), StringComparer.OrdinalIgnoreCase); + var create = Expression.Lambda>(Expression.New(typeof(T))).Compile(); + var slots = new List<(int Ordinal, ColumnType Type, Type Target, Action Set)>(); + for (var ordinal = 0; ordinal < columns.Length; ordinal++) + { + if (!properties.TryGetValue(columns[ordinal], out var property)) + { + continue; + } + + var instance = Expression.Parameter(typeof(T)); + var value = Expression.Parameter(typeof(object)); + var set = Expression.Lambda>( + Expression.Assign(Expression.Property(instance, property), Expression.Convert(value, property.PropertyType)), instance, value).Compile(); + slots.Add((ordinal, LogicalType(property.PropertyType), property.PropertyType, set)); + } + + return reader => + { + var item = create(); + foreach (var (ordinal, type, target, set) in slots) + { + set(item, ClrConverter.ConvertTo(dialect.FromDbValue(reader.GetValue(ordinal), type), target)); + } + + return item; + }; + } + + private static Func BuildConstructor(string[] columns, ISqlDialect dialect) + { + var constructor = typeof(T).GetConstructors().OrderByDescending(c => c.GetParameters().Length).FirstOrDefault() + ?? throw new InvalidOperationException($"{typeof(T).Name} has no public constructor to map rows to."); + var parameters = constructor.GetParameters(); + var ordinals = parameters.Select(p => + { + var index = Array.FindIndex(columns, c => string.Equals(c, SqlNames.ToSnakeCase(p.Name!), StringComparison.OrdinalIgnoreCase)); + return index >= 0 + ? index + : throw new InvalidOperationException($"{typeof(T).Name}: no column {SqlNames.ToSnakeCase(p.Name!)} for constructor parameter {p.Name}."); + }).ToArray(); + var types = parameters.Select(p => LogicalType(p.ParameterType)).ToArray(); + var arguments = Expression.Parameter(typeof(object?[])); + var create = Expression.Lambda>( + Expression.New(constructor, parameters.Select((p, i) => + (Expression)Expression.Convert(Expression.ArrayIndex(arguments, Expression.Constant(i)), p.ParameterType))), + arguments).Compile(); + + return reader => + { + var values = new object?[parameters.Length]; + for (var i = 0; i < parameters.Length; i++) + { + values[i] = ClrConverter.ConvertTo(dialect.FromDbValue(reader.GetValue(ordinals[i]), types[i]), parameters[i].ParameterType); + } + + return create(values); + }; + } + + private static bool IsScalar(Type type) + { + var underlying = Nullable.GetUnderlyingType(type) ?? type; + return underlying.IsEnum || ColumnTypes.FromClrType(underlying) is not null; + } + + private static ColumnType LogicalType(Type type) + { + var underlying = Nullable.GetUnderlyingType(type) ?? type; + return underlying.IsEnum ? ColumnType.Text : ColumnTypes.FromClrType(underlying) ?? ColumnType.Text; + } +} diff --git a/src/Khaemhat.Data/Execution/Repository.cs b/src/Khaemhat.Data/Execution/Repository.cs new file mode 100644 index 0000000..bece971 --- /dev/null +++ b/src/Khaemhat.Data/Execution/Repository.cs @@ -0,0 +1,140 @@ +using System.Globalization; +using Khaemhat.Core.Common; +using Khaemhat.Core.Entities; +using Khaemhat.Data.Mapping; +using Khaemhat.Data.Querying; + +namespace Khaemhat.Data.Execution; + +/// +/// The repository of SPEC §5.1 for one entity type in one unit of work. Insert, update and delete statements are +/// cached per dialect (); queries go through . +/// +/// The entity type. +/// The key type. +public sealed class Repository : IRepository + where T : class, IEntity, new() +{ + private readonly UnitOfWork _unitOfWork; + private readonly EntityMap _map; + private readonly QueryBuilder _builder; + + internal Repository(UnitOfWork unitOfWork, EntityMap map) + { + _unitOfWork = unitOfWork; + _map = map; + _builder = new QueryBuilder(map, unitOfWork.Dialect); + } + + /// + public Task GetAsync(TKey id, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(id); + return _unitOfWork.QuerySingleAsync(_builder.SelectByKey(id), cancellationToken); + } + + /// + public Task FindAsync(Query query, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(query); + return _unitOfWork.QuerySingleAsync(_builder.Select(query.Take(1)), cancellationToken); + } + + /// + public async Task> ListAsync(Query query, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(query); + var items = await _unitOfWork.QueryAsync(_builder.Select(query), cancellationToken).ConfigureAwait(false); + long total = query.Limit is null && query.Offset == 0 + ? items.Count + : await CountAsync(query, cancellationToken).ConfigureAwait(false); + return new PagedResult(items, total, query.Offset, query.Limit); + } + + /// + public async Task CountAsync(Query query, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(query); + return (int)await _unitOfWork.QuerySingleAsync(_builder.Count(query), cancellationToken).ConfigureAwait(false); + } + + /// + public async Task InsertAsync(T entity, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(entity); + if (entity is IAuditable audit) + { + audit.CreatedAt = _unitOfWork.Context.UtcNow; + audit.CreatedBy = _unitOfWork.Context.CurrentUser; + audit.UpdatedAt = null; + audit.UpdatedBy = null; + audit.RowVersion = 1; + } + + var statement = CommandCache.Get(_map, CachedStatementKind.Insert, _unitOfWork.Dialect); + var parameters = Parameters(statement, entity, overrides: null); + if (!_map.Key.IsIdentity) + { + await _unitOfWork.WriteAsync(statement.Text, parameters, (c, ct) => c.ExecuteNonQueryAsync(ct), cancellationToken).ConfigureAwait(false); + return entity.Id; + } + + var generated = await _unitOfWork.WriteAsync(statement.Text, parameters, (c, ct) => c.ExecuteScalarAsync(ct), cancellationToken) + .ConfigureAwait(false); + var key = (TKey)ClrConverter.ConvertTo(_unitOfWork.Dialect.FromDbValue(generated, _map.Key.Type), typeof(TKey))!; + entity.Id = key; + return key; + } + + /// + public async Task UpdateAsync(T entity, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(entity); + var statement = CommandCache.Get(_map, CachedStatementKind.Update, _unitOfWork.Dialect); + var now = _unitOfWork.Context.UtcNow; + var user = _unitOfWork.Context.CurrentUser; + var overrides = new Dictionary { [AuditRole.UpdatedAt] = now, [AuditRole.UpdatedBy] = user }; + var parameters = Parameters(statement, entity, overrides); + + var affected = await _unitOfWork.WriteAsync(statement.Text, parameters, (c, ct) => c.ExecuteNonQueryAsync(ct), cancellationToken) + .ConfigureAwait(false); + if (affected == 0) + { + var version = entity is IAuditable a ? a.RowVersion : 0; + throw new ConcurrencyException(_map.Table, Convert.ToString(entity.Id, CultureInfo.InvariantCulture) ?? string.Empty, version); + } + + if (entity is IAuditable audit) + { + audit.UpdatedAt = now; + audit.UpdatedBy = user; + audit.RowVersion++; + } + } + + /// + public async Task DeleteAsync(TKey id, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(id); + var statement = CommandCache.Get(_map, CachedStatementKind.Delete, _unitOfWork.Dialect); + var parameters = new[] { new SqlParameterValue("p0", _map.Key.ToProvider(id), _map.Key.Type) }; + var affected = await _unitOfWork.WriteAsync(statement.Text, parameters, (c, ct) => c.ExecuteNonQueryAsync(ct), cancellationToken) + .ConfigureAwait(false); + return affected > 0; + } + + private static List Parameters(CachedStatement statement, T entity, Dictionary? overrides) + { + var parameters = new List(statement.Parameters.Count); + for (var i = 0; i < statement.Parameters.Count; i++) + { + var column = statement.Parameters[i]; + var value = overrides is not null && column.Audit != AuditRole.None && overrides.TryGetValue(column.Audit, out var replaced) + ? replaced + : column.GetValue(entity); + parameters.Add(new SqlParameterValue("p" + i.ToString(CultureInfo.InvariantCulture), column.ToProvider(value), column.Type)); + } + + return parameters; + } +} diff --git a/src/Khaemhat.Data/Execution/TransientRetryPolicy.cs b/src/Khaemhat.Data/Execution/TransientRetryPolicy.cs new file mode 100644 index 0000000..8ffecd8 --- /dev/null +++ b/src/Khaemhat.Data/Execution/TransientRetryPolicy.cs @@ -0,0 +1,45 @@ +namespace Khaemhat.Data.Execution; + +/// +/// Retries an operation that failed with a transient error (database busy, connection lost). It is applied only +/// outside transactions: inside one, a retry could repeat half of the work, so the error goes to the caller. +/// +/// Total attempts, at least 1. +/// Delay before the second attempt; it doubles each time. +public sealed class TransientRetryPolicy(int maxAttempts, TimeSpan baseDelay) +{ + /// Three attempts, 100 ms then 200 ms apart. + public static TransientRetryPolicy Default { get; } = new(3, TimeSpan.FromMilliseconds(100)); + + /// No retries. + public static TransientRetryPolicy None { get; } = new(1, TimeSpan.Zero); + + /// Total attempts. + public int MaxAttempts { get; } = maxAttempts >= 1 ? maxAttempts : throw new ArgumentOutOfRangeException(nameof(maxAttempts)); + + /// Runs , retrying when says the error is transient. + /// The result type. + /// The operation. + /// Classifies exceptions. + /// Cancellation. + /// The result. + public async Task ExecuteAsync( + Func> operation, Func isTransient, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(operation); + ArgumentNullException.ThrowIfNull(isTransient); + var delay = baseDelay; + for (var attempt = 1; ; attempt++) + { + try + { + return await operation(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (attempt < MaxAttempts && !cancellationToken.IsCancellationRequested && isTransient(ex)) + { + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + delay += delay; + } + } + } +} diff --git a/src/Khaemhat.Data/Execution/UnitOfWork.cs b/src/Khaemhat.Data/Execution/UnitOfWork.cs new file mode 100644 index 0000000..2e5bedc --- /dev/null +++ b/src/Khaemhat.Data/Execution/UnitOfWork.cs @@ -0,0 +1,287 @@ +using System.Data.Common; +using Khaemhat.Core.Entities; +using Khaemhat.Data.Mapping; + +namespace Khaemhat.Data.Execution; + +/// +/// The unit of work of SPEC §5.1 over one open connection. Statements outside a transaction are retried on transient +/// errors; on SQLite every write enters the database write scope (the whole transaction, or the single statement). +/// +public sealed class UnitOfWork : IUnitOfWork +{ + private readonly Dictionary _repositories = []; + private IAsyncDisposable? _writeScope; + private bool _disposed; + + private UnitOfWork(DatabaseTarget target, DbConnection connection, MapRegistry maps, DataAccessContext context) + { + Target = target; + Connection = connection; + Maps = maps; + Context = context; + Executor = new DbExecutor(target.Provider.Dialect); + } + + /// The database. + public DatabaseTarget Target { get; } + + /// The provider. + public IDbProvider Provider => Target.Provider; + + /// + public DbConnection Connection { get; } + + /// + public DbTransaction? Transaction { get; private set; } + + /// + public ISqlDialect Dialect => Provider.Dialect; + + /// The entity maps. + public MapRegistry Maps { get; } + + /// Clock, user and retry policy. + public DataAccessContext Context { get; } + + /// Creates commands. + public DbExecutor Executor { get; } + + /// Opens a unit of work on a database. + /// The database. + /// The entity maps. + /// Clock, user and retry policy. + /// Cancellation. + /// The open unit of work. + public static async Task OpenAsync( + DatabaseTarget target, MapRegistry maps, DataAccessContext context, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(target); + ArgumentNullException.ThrowIfNull(maps); + ArgumentNullException.ThrowIfNull(context); + var connection = target.Provider.CreateConnection(target.ConnectionString); + try + { + await context.RetryPolicy.ExecuteAsync( + async ct => + { + await connection.OpenAsync(ct).ConfigureAwait(false); + return true; + }, + target.Provider.IsTransient, + cancellationToken).ConfigureAwait(false); + } + catch + { + await connection.DisposeAsync().ConfigureAwait(false); + throw; + } + + return new UnitOfWork(target, connection, maps, context); + } + + /// + public async Task BeginAsync(CancellationToken cancellationToken = default) + { + ObjectDisposedException.ThrowIf(_disposed, this); + if (Transaction is not null) + { + throw new InvalidOperationException("A transaction is already open on this unit of work."); + } + + _writeScope = await Provider.EnterWriteScopeAsync(Target.ConnectionString, cancellationToken).ConfigureAwait(false); + try + { + Transaction = await Connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + } + catch + { + await ReleaseWriteScopeAsync().ConfigureAwait(false); + throw; + } + } + + /// + public async Task CommitAsync(CancellationToken cancellationToken = default) + { + var transaction = Transaction ?? throw new InvalidOperationException("No transaction is open."); + try + { + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + } + finally + { + await EndTransactionAsync().ConfigureAwait(false); + } + } + + /// + public async Task RollbackAsync(CancellationToken cancellationToken = default) + { + var transaction = Transaction ?? throw new InvalidOperationException("No transaction is open."); + try + { + await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false); + } + finally + { + await EndTransactionAsync().ConfigureAwait(false); + } + } + + /// + public IRepository Repository() + where T : class, IEntity, new() + { + ObjectDisposedException.ThrowIf(_disposed, this); + if (!_repositories.TryGetValue(typeof(T), out var repository)) + { + repository = new Repository(this, Maps.Get()); + _repositories[typeof(T)] = repository; + } + + return (IRepository)repository; + } + + /// + public Task ExecuteAsync(SqlFragment sql, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(sql); + var rendered = sql.Render(Dialect); + return WriteAsync(rendered.Text, rendered.Parameters, (command, ct) => command.ExecuteNonQueryAsync(ct), cancellationToken); + } + + /// + public async Task QuerySingleAsync(SqlFragment sql, CancellationToken cancellationToken = default) + { + var rows = await ReadAsync(sql, single: true, cancellationToken).ConfigureAwait(false); + return rows.Count == 0 ? default : rows[0]; + } + + /// + public Task> QueryAsync(SqlFragment sql, CancellationToken cancellationToken = default) => + ReadAsync(sql, single: false, cancellationToken); + + /// + public async Task StreamAsync(SqlFragment sql, CancellationToken cancellationToken = default) + { + ObjectDisposedException.ThrowIf(_disposed, this); + var command = Executor.CreateCommand(Connection, Transaction, sql); + try + { + return await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + } + catch + { + await command.DisposeAsync().ConfigureAwait(false); + throw; + } + } + + /// + public async ValueTask DisposeAsync() + { + if (_disposed) + { + return; + } + + _disposed = true; + if (Transaction is not null) + { + try + { + await Transaction.RollbackAsync().ConfigureAwait(false); + } + catch (DbException) + { + // The connection may already be broken; disposing it releases the transaction anyway. + } + + await EndTransactionAsync().ConfigureAwait(false); + } + + await Connection.DisposeAsync().ConfigureAwait(false); + } + + /// + /// Runs a writing command: inside the transaction if there is one; otherwise inside the write scope, with retries + /// on transient errors. + /// + /// The result type. + /// The SQL text. + /// The parameters. + /// Runs the command. + /// Cancellation. + /// The result. + internal async Task WriteAsync( + string text, IEnumerable parameters, Func> run, CancellationToken cancellationToken) + { + ObjectDisposedException.ThrowIf(_disposed, this); + if (Transaction is not null) + { + await using var command = Executor.CreateCommand(Connection, Transaction, text, parameters); + return await run(command, cancellationToken).ConfigureAwait(false); + } + + var materialized = parameters.ToList(); + return await Context.RetryPolicy.ExecuteAsync( + async ct => + { + await using var scope = await Provider.EnterWriteScopeAsync(Target.ConnectionString, ct).ConfigureAwait(false); + await using var command = Executor.CreateCommand(Connection, null, text, materialized); + return await run(command, ct).ConfigureAwait(false); + }, + Provider.IsTransient, + cancellationToken).ConfigureAwait(false); + } + + private async Task> ReadAsync(SqlFragment sql, bool single, CancellationToken cancellationToken) + { + ObjectDisposedException.ThrowIf(_disposed, this); + ArgumentNullException.ThrowIfNull(sql); + + async Task> Run(CancellationToken ct) + { + await using var command = Executor.CreateCommand(Connection, Transaction, sql); + await using var reader = await command.ExecuteReaderAsync(ct).ConfigureAwait(false); + var rows = new List(); + Func? map = null; + while (await reader.ReadAsync(ct).ConfigureAwait(false)) + { + map ??= ReaderMapper.For(reader, Dialect, Maps); + rows.Add(map(reader)); + if (single) + { + break; + } + } + + return rows; + } + + return Transaction is not null + ? await Run(cancellationToken).ConfigureAwait(false) + : await Context.RetryPolicy.ExecuteAsync(Run, Provider.IsTransient, cancellationToken).ConfigureAwait(false); + } + + private async Task EndTransactionAsync() + { + if (Transaction is not null) + { + await Transaction.DisposeAsync().ConfigureAwait(false); + Transaction = null; + } + + await ReleaseWriteScopeAsync().ConfigureAwait(false); + } + + private async Task ReleaseWriteScopeAsync() + { + if (_writeScope is not null) + { + await _writeScope.DisposeAsync().ConfigureAwait(false); + _writeScope = null; + } + } +} diff --git a/src/Khaemhat.Data/Execution/UnitOfWorkFactory.cs b/src/Khaemhat.Data/Execution/UnitOfWorkFactory.cs new file mode 100644 index 0000000..87f1dd4 --- /dev/null +++ b/src/Khaemhat.Data/Execution/UnitOfWorkFactory.cs @@ -0,0 +1,18 @@ +using Khaemhat.Data.Mapping; + +namespace Khaemhat.Data.Execution; + +/// Opens s on the database of a scope (SPEC §5.1). +/// Finds the database of a scope. +/// The entity maps. +/// Clock, user and retry policy. +public sealed class UnitOfWorkFactory(IDatabaseLocator locator, MapRegistry maps, DataAccessContext context) : IUnitOfWorkFactory +{ + /// + public async Task CreateAsync(DatabaseScope scope, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(scope); + var target = await locator.LocateAsync(scope, cancellationToken).ConfigureAwait(false); + return await UnitOfWork.OpenAsync(target, maps, context, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/Khaemhat.Data/GlobalSuppressions.cs b/src/Khaemhat.Data/GlobalSuppressions.cs new file mode 100644 index 0000000..ce3c5b5 --- /dev/null +++ b/src/Khaemhat.Data/GlobalSuppressions.cs @@ -0,0 +1,5 @@ +using System.Diagnostics.CodeAnalysis; + +[assembly: SuppressMessage( + "Naming", "CA1720:Identifier contains type name", Scope = "namespaceanddescendants", Target = "~N:Khaemhat.Data.Schema", + Justification = "The logical column types are named as in SPEC §5.2 (String, Int32, Int64, Decimal, Guid): they are the vocabulary of migrations and docs.")] diff --git a/src/Khaemhat.Data/IDbProvider.cs b/src/Khaemhat.Data/IDbProvider.cs new file mode 100644 index 0000000..90fac46 --- /dev/null +++ b/src/Khaemhat.Data/IDbProvider.cs @@ -0,0 +1,68 @@ +using System.Data.Common; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data; + +/// +/// A database engine (SPEC §5.1): connections, DDL, creation and removal of databases, server information. Implemented +/// by SqliteProvider and PostgresProvider. +/// +public interface IDbProvider +{ + /// The engine. + DbEngine Engine { get; } + + /// The SQL dialect. + ISqlDialect Dialect { get; } + + /// The DDL generator used by migrations. + DdlGenerator Ddl { get; } + + /// The driver connection string for a database URL. + /// The database URL (its engine must be ). + /// The connection string. It contains credentials: never log it. + string BuildConnectionString(DatabaseUrl url); + + /// Creates a closed connection. Session settings (pragmas, timeouts) are applied when it opens. + /// The connection string. + /// The connection. + DbConnection CreateConnection(string connectionString); + + /// Creates the database if it does not exist (CREATE DATABASE, or the SQLite file). + /// The connection string of the database. + /// Cancellation. + /// A task. + Task EnsureDatabaseExistsAsync(string connectionString, CancellationToken cancellationToken = default); + + /// Drops the database. Only the company registry and the restore service call it. + /// The connection string of the database. + /// Cancellation. + /// A task. + Task DropDatabaseAsync(string connectionString, CancellationToken cancellationToken = default); + + /// Whether the database exists. + /// The connection string of the database. + /// Cancellation. + /// if it exists. + Task DatabaseExistsAsync(string connectionString, CancellationToken cancellationToken = default); + + /// Version, size, connections and warnings of the server and database (SPEC §12.1). + /// The connection string of the database. + /// Cancellation. + /// The information. + Task GetServerInfoAsync(string connectionString, CancellationToken cancellationToken = default); + + /// Whether an exception is transient (busy, connection lost) and the statement may be retried outside a transaction. + /// The exception. + /// if transient. + bool IsTransient(Exception exception); + + /// + /// Enters the write scope of a database: on SQLite it serializes writers of the process (SqliteWriteLock); + /// on PostgreSQL it does nothing. Dispose the result to leave. + /// + /// The connection string of the database. + /// Cancellation. + /// The scope. + ValueTask EnterWriteScopeAsync(string connectionString, CancellationToken cancellationToken = default); +} diff --git a/src/Khaemhat.Data/IRepository.cs b/src/Khaemhat.Data/IRepository.cs new file mode 100644 index 0000000..7a332f8 --- /dev/null +++ b/src/Khaemhat.Data/IRepository.cs @@ -0,0 +1,62 @@ +using Khaemhat.Core.Entities; +using Khaemhat.Data.Querying; + +namespace Khaemhat.Data; + +/// +/// Reads and writes the rows of one entity type in the database of a unit of work (SPEC §5.1). Updates check +/// row_version and throw on conflict. +/// +/// The entity type. +/// The key type. +public interface IRepository + where T : class, IEntity +{ + /// The row with the given key. + /// The key. + /// Cancellation. + /// The entity, or . + Task GetAsync(TKey id, CancellationToken cancellationToken = default); + + /// The first row of a query. + /// The query. + /// Cancellation. + /// The entity, or . + Task FindAsync(Query query, CancellationToken cancellationToken = default); + + /// The rows of a query, with the total count when it is paged. + /// The query. + /// Cancellation. + /// The page. + Task> ListAsync(Query query, CancellationToken cancellationToken = default); + + /// The number of rows matching a query. + /// The query. + /// Cancellation. + /// The count. + Task CountAsync(Query query, CancellationToken cancellationToken = default); + + /// + /// Inserts a row. Audit columns are filled, row_version starts at 1, an identity key is generated and set + /// on the entity. + /// + /// The entity. + /// Cancellation. + /// The key. + Task InsertAsync(T entity, CancellationToken cancellationToken = default); + + /// + /// Updates a row if its row_version is still the one on the entity, then increments it on the entity. + /// + /// The entity. + /// Cancellation. + /// A task. + /// The row changed or no longer exists. + Task UpdateAsync(T entity, CancellationToken cancellationToken = default); + + /// Deletes the row with the given key; nothing happens if it does not exist. + /// The key. + /// Cancellation. + /// if a row was deleted. + Task DeleteAsync(TKey id, CancellationToken cancellationToken = default); +} diff --git a/src/Khaemhat.Data/ISqlDialect.cs b/src/Khaemhat.Data/ISqlDialect.cs new file mode 100644 index 0000000..b789be2 --- /dev/null +++ b/src/Khaemhat.Data/ISqlDialect.cs @@ -0,0 +1,98 @@ +using System.Data.Common; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data; + +/// +/// What differs between SQL engines (SPEC §5.1): quoting, parameters, paging, insert/upsert forms, case-insensitive +/// search, advisory locks, value conversion and physical types. Everything above the providers uses only this +/// interface, so no code outside Khaemhat.Data.Sqlite and Khaemhat.Data.PostgreSql knows the engine. +/// +public interface ISqlDialect +{ + /// The engine. + DbEngine Engine { get; } + + /// The current UTC timestamp, in the same form the dialect stores . + string CurrentTimestampUtc { get; } + + /// Quotes a validated identifier. + /// The identifier (snake_case). + /// The quoted identifier. + string Quote(string identifier); + + /// The placeholder of a named parameter, for example @p0. + /// The parameter name without prefix. + /// The placeholder. + string Parameter(string name); + + /// The paging clause, appended after ORDER BY. + /// Rows to skip. + /// Rows to return. + /// The clause, starting with a space. + string Paging(int offset, int limit); + + /// An insert that returns the generated key. + /// The table. + /// The inserted columns. + /// The parameter placeholders, one per column. + /// The generated key column. + /// The statement. + string InsertReturningId(string table, IReadOnlyList columns, IReadOnlyList parameters, string keyColumn); + + /// An insert that updates the existing row on a conflict of . + /// The table. + /// The inserted columns. + /// The parameter placeholders, one per column. + /// The columns of the unique key. + /// The columns to update on conflict. + /// The statement. + string Upsert( + string table, IReadOnlyList columns, IReadOnlyList parameters, + IReadOnlyList conflictColumns, IReadOnlyList updateColumns); + + /// A case-insensitive LIKE with \ as escape character. + /// The column (or expression). + /// The pattern, normally a parameter. + /// The condition. + SqlFragment CaseInsensitiveLike(SqlFragment column, SqlFragment pattern); + + /// + /// A statement that takes a lock held until the end of the current transaction (numerators, closings, scheduler); + /// empty where writes are already serialized. + /// + /// The lock key, for example numerator:FV:2026. + /// The statement, possibly empty. + SqlFragment AdvisoryLock(string key); + + /// Converts a CLR value to what the driver stores for ; null becomes . + /// The value. + /// The logical type. + /// The value for the driver. + object ToDbValue(object? value, ColumnType type); + + /// Converts a value read from the driver to the canonical CLR type of (see ). + /// The value read. + /// The logical type. + /// The CLR value, or . + object? FromDbValue(object? value, ColumnType type); + + /// The physical type of a column. + /// The logical type. + /// The length for . + /// The scale for . + /// The SQL type. + string SqlType(ColumnType type, int? length, int? scale); + + /// A constant as SQL literal, for DDL defaults only (values in statements are always parameters). + /// The constant. + /// The logical type. + /// The literal. + string Literal(object? value, ColumnType type); + + /// Sets the value (converted with ) and the driver type of a parameter. + /// The parameter. + /// The CLR value. + /// The logical type, or to infer it from the value. + void ConfigureParameter(DbParameter parameter, object? value, ColumnType? type); +} diff --git a/src/Khaemhat.Data/IUnitOfWork.cs b/src/Khaemhat.Data/IUnitOfWork.cs new file mode 100644 index 0000000..a547fd0 --- /dev/null +++ b/src/Khaemhat.Data/IUnitOfWork.cs @@ -0,0 +1,93 @@ +using System.Data.Common; +using Khaemhat.Core.Entities; + +namespace Khaemhat.Data; + +/// +/// An open connection to one database, with an optional explicit transaction (SPEC §5.1). Application services open +/// one per operation, call for writes that must be atomic, and dispose it; a transaction not +/// committed is rolled back on dispose. +/// +public interface IUnitOfWork : IAsyncDisposable +{ + /// The open connection. + DbConnection Connection { get; } + + /// The current transaction, if any. + DbTransaction? Transaction { get; } + + /// The dialect of the database. + ISqlDialect Dialect { get; } + + /// Starts a transaction. On SQLite it also enters the write scope of the database. + /// Cancellation. + /// A task. + Task BeginAsync(CancellationToken cancellationToken = default); + + /// Commits the transaction. + /// Cancellation. + /// A task. + Task CommitAsync(CancellationToken cancellationToken = default); + + /// Rolls back the transaction. + /// Cancellation. + /// A task. + Task RollbackAsync(CancellationToken cancellationToken = default); + + /// The repository of an entity type. + /// The entity type, which needs a registered map. + /// The key type. + /// The repository. + IRepository Repository() + where T : class, IEntity, new(); + + /// Executes a statement that changes data or schema. + /// The statement. + /// Cancellation. + /// The number of affected rows. + Task ExecuteAsync(SqlFragment sql, CancellationToken cancellationToken = default); + + /// Reads the first row (or ), mapped to . + /// A scalar, a mapped entity, or a class/record whose members match the columns by snake_case name. + /// The query. + /// Cancellation. + /// The row, or . + Task QuerySingleAsync(SqlFragment sql, CancellationToken cancellationToken = default); + + /// Reads all rows, mapped to . + /// A scalar, a mapped entity, or a class/record whose members match the columns by snake_case name. + /// The query. + /// Cancellation. + /// The rows. + Task> QueryAsync(SqlFragment sql, CancellationToken cancellationToken = default); + + /// + /// Opens a reader for large results (exports, reports) without loading them in memory. Dispose the reader before + /// running anything else on this unit of work. Values are raw driver values: convert them with + /// . + /// + /// The query. + /// Cancellation. + /// The reader. + Task StreamAsync(SqlFragment sql, CancellationToken cancellationToken = default); +} + +/// Opens units of work on the system database or on a company database (SPEC §5.1). +public interface IUnitOfWorkFactory +{ + /// Opens a unit of work. + /// The system database, or a company. + /// Cancellation. + /// The open unit of work. + Task CreateAsync(DatabaseScope scope, CancellationToken cancellationToken = default); +} + +/// Finds the database of a scope: the system database, or a company through the registry. +public interface IDatabaseLocator +{ + /// The database of a scope. + /// The scope. + /// Cancellation. + /// The target. + ValueTask LocateAsync(DatabaseScope scope, CancellationToken cancellationToken = default); +} diff --git a/src/Khaemhat.Data/Mapping/ClrConverter.cs b/src/Khaemhat.Data/Mapping/ClrConverter.cs new file mode 100644 index 0000000..9b516af --- /dev/null +++ b/src/Khaemhat.Data/Mapping/ClrConverter.cs @@ -0,0 +1,63 @@ +using System.Globalization; + +namespace Khaemhat.Data.Mapping; + +/// +/// Converts the canonical values returned by (or raw driver values) to the CLR +/// type of a property, constructor parameter or scalar result. +/// +public static class ClrConverter +{ + /// Converts to . + /// The value, possibly or . + /// The target type, nullable or not. + /// The converted value; for a null input and a nullable target. + /// A null value for a non-nullable value type, or an impossible conversion. + public static object? ConvertTo(object? value, Type target) + { + ArgumentNullException.ThrowIfNull(target); + var underlying = Nullable.GetUnderlyingType(target); + if (value is null or DBNull) + { + return target.IsValueType && underlying is null + ? throw new InvalidCastException($"NULL cannot be assigned to {target.Name}.") + : null; + } + + var type = underlying ?? target; + if (type.IsInstanceOfType(value)) + { + return type == typeof(DateTime) && value is DateTime { Kind: not DateTimeKind.Utc } d ? DateTime.SpecifyKind(d, DateTimeKind.Utc) : value; + } + + if (type.IsEnum) + { + return value is string name ? Enum.Parse(type, name, ignoreCase: false) : Enum.ToObject(type, value); + } + + return type switch + { + _ when type == typeof(DateOnly) => value switch + { + DateTime dt => DateOnly.FromDateTime(dt), + string s => DateOnly.ParseExact(s, "yyyy-MM-dd", CultureInfo.InvariantCulture), + _ => throw Impossible(value, type), + }, + _ when type == typeof(DateTime) => value switch + { + DateTimeOffset o => o.UtcDateTime, + DateOnly d => d.ToDateTime(TimeOnly.MinValue, DateTimeKind.Utc), + string s => DateTime.Parse(s, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal), + _ => throw Impossible(value, type), + }, + _ when type == typeof(Guid) => value is string g ? Guid.Parse(g, CultureInfo.InvariantCulture) : throw Impossible(value, type), + _ when type == typeof(bool) && value is IConvertible => Convert.ToInt64(value, CultureInfo.InvariantCulture) != 0, + _ when type == typeof(string) => Convert.ToString(value, CultureInfo.InvariantCulture), + _ when value is IConvertible => Convert.ChangeType(value, type, CultureInfo.InvariantCulture), + _ => throw Impossible(value, type), + }; + } + + private static InvalidCastException Impossible(object value, Type type) => + new($"A {value.GetType().Name} value cannot be converted to {type.Name}."); +} diff --git a/src/Khaemhat.Data/Mapping/ColumnMap.cs b/src/Khaemhat.Data/Mapping/ColumnMap.cs new file mode 100644 index 0000000..bf4de94 --- /dev/null +++ b/src/Khaemhat.Data/Mapping/ColumnMap.cs @@ -0,0 +1,207 @@ +using System.Reflection; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.Mapping; + +/// The role of a column among the audit columns of SPEC §5.2. +public enum AuditRole +{ + /// Not an audit column. + None = 0, + + /// created_at. + CreatedAt = 1, + + /// created_by. + CreatedBy = 2, + + /// updated_at. + UpdatedAt = 3, + + /// updated_by. + UpdatedBy = 4, + + /// row_version. + RowVersion = 5, +} + +/// +/// How a property is stored: column name, logical type, nullability, key and audit role, plus compiled accessors. +/// Configured in the constructor of an , then frozen. +/// +public sealed class ColumnMap +{ + private readonly Func _getter; + private readonly Action _setter; + private bool _frozen; + private bool? _nullable; + + internal ColumnMap(PropertyInfo property, Func getter, Action setter) + { + PropertyName = property.Name; + PropertyType = property.PropertyType; + ValueType = Nullable.GetUnderlyingType(property.PropertyType) ?? property.PropertyType; + ColumnName = SqlNames.ToSnakeCase(property.Name); + _getter = getter; + _setter = setter; + } + + /// The C# property name. + public string PropertyName { get; } + + /// The declared property type. + public Type PropertyType { get; } + + /// The property type without . + public Type ValueType { get; } + + /// The column name (snake_case of the property name unless set with ). + public string ColumnName { get; private set; } + + /// The logical type. + public ColumnType Type { get; private set; } + + /// The maximum length, for . + public int? Length { get; private set; } + + /// The scale, for . + public int? Scale { get; private set; } + + /// Whether the column accepts NULL. + public bool IsNullable => _nullable ?? false; + + /// Whether this is the primary key. + public bool IsKey { get; internal set; } + + /// Whether the key is generated by the database. + public bool IsIdentity { get; internal set; } + + /// The audit role. + public AuditRole Audit { get; internal set; } + + /// The logical type as documented, for example String(20). + public string TypeDescription => ColumnTypes.Describe(Type, Length, Scale); + + internal bool HasExplicitType { get; private set; } + + internal bool HasExplicitNullability => _nullable is not null; + + /// Sets the column name. + /// The column name (snake_case). + /// This map. + public ColumnMap Named(string column) + { + EnsureNotFrozen(); + SqlNames.EnsureIdentifier(column); + ColumnName = column; + return this; + } + + /// Stores a string as with a maximum length (an enum by its name). + /// The maximum length. + /// This map. + public ColumnMap MaxLength(int length) + { + ArgumentOutOfRangeException.ThrowIfLessThan(length, 1); + return SetType(ColumnType.String, length, null); + } + + /// Stores a string as (the default for strings). + /// This map. + public ColumnMap Text() => SetType(ColumnType.Text, null, null); + + /// Stores a string as . + /// This map. + public ColumnMap Json() => SetType(ColumnType.Json, null, null); + + /// Sets the scale of a decimal (default 6). + /// Digits after the point, 0 to 6. + /// This map. + public ColumnMap Scaled(int scale) + { + ArgumentOutOfRangeException.ThrowIfNegative(scale); + ArgumentOutOfRangeException.ThrowIfGreaterThan(scale, ColumnTypes.DecimalScale); + return SetType(ColumnType.Decimal, null, scale); + } + + /// The column refuses NULL, whatever the property nullability says. + /// This map. + public ColumnMap Required() + { + EnsureNotFrozen(); + _nullable = false; + return this; + } + + /// The column accepts NULL. + /// This map. + public ColumnMap Optional() + { + EnsureNotFrozen(); + _nullable = true; + return this; + } + + /// Reads the property of an entity. + /// The entity. + /// The property value. + public object? GetValue(object entity) => _getter(entity); + + /// Writes the property of an entity. + /// The entity. + /// The value, already of the property type. + public void SetValue(object entity, object? value) + { + if (value is null && PropertyType.IsValueType && Nullable.GetUnderlyingType(PropertyType) is null) + { + throw new InvalidCastException($"Column {ColumnName} is NULL but property {PropertyName} cannot hold null."); + } + + _setter(entity, value); + } + + /// Converts a property value to the value given to the dialect (enums become their name). + /// The property value. + /// The value for . + public object? ToProvider(object? value) => value is Enum e && ValueType.IsEnum ? e.ToString() : value; + + /// Converts a canonical value from to the property type. + /// The canonical value. + /// The property value. + public object? FromProvider(object? value) => ClrConverter.ConvertTo(value, PropertyType); + + /// + public override string ToString() => ColumnName + " " + TypeDescription; + + internal void SetDefaults(ColumnType type, int? length, bool nullable) + { + if (!HasExplicitType) + { + Type = type; + Length = length; + Scale = type == ColumnType.Decimal ? ColumnTypes.DecimalScale : null; + } + + _nullable ??= nullable; + } + + internal void Freeze() => _frozen = true; + + private ColumnMap SetType(ColumnType type, int? length, int? scale) + { + EnsureNotFrozen(); + Type = type; + Length = length; + Scale = scale; + HasExplicitType = true; + return this; + } + + private void EnsureNotFrozen() + { + if (_frozen) + { + throw new InvalidOperationException("A column map cannot change after its entity map is registered."); + } + } +} diff --git a/src/Khaemhat.Data/Mapping/EntityMap.cs b/src/Khaemhat.Data/Mapping/EntityMap.cs new file mode 100644 index 0000000..adcfe5b --- /dev/null +++ b/src/Khaemhat.Data/Mapping/EntityMap.cs @@ -0,0 +1,353 @@ +using System.Linq.Expressions; +using System.Reflection; +using Khaemhat.Core.Entities; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.Mapping; + +/// A map from an entity type to its table (non-generic view used by the data layer). +public interface IEntityMap +{ + /// The entity type. + Type EntityType { get; } + + /// The table name. + string Table { get; } + + /// The primary key column. + ColumnMap Key { get; } + + /// All columns, key first, audit columns last. + IReadOnlyList Columns { get; } + + /// The row_version column, if the entity is . + ColumnMap? RowVersion { get; } + + /// Finds the column of a property. + /// The property name. + /// The column, or . + ColumnMap? FindByProperty(string propertyName); + + /// Finds a column by name. + /// The column name. + /// The column, or . + ColumnMap? FindByColumn(string columnName); + + /// Creates an empty entity. + /// The entity. + object CreateEntity(); + + /// Completes the map by convention and makes it read-only. Called by . + void Freeze(); +} + +/// +/// The map of an entity to its table (SPEC §5.2: XxxMap : EntityMap<Xxx>). The constructor names the table +/// and states only what differs from the conventions; every other public read/write property is mapped by convention: +/// +/// column name = snake_case of the property; +/// Id is the key, generated by the database when it is or ; +/// types from , strings as unless +/// is set, enums by name as String(40); +/// nullability from the C# declaration; +/// the audit columns of and is_demo of . +/// +/// Reflection is used only here, once; reading and writing use compiled accessors. +/// +/// The entity type. +public abstract class EntityMap : IEntityMap + where T : class, new() +{ + private const int EnumLength = 40; + private static readonly NullabilityInfoContext NullabilityContext = new(); + + private readonly Dictionary _configured = new(StringComparer.Ordinal); + private readonly HashSet _ignored = new(StringComparer.Ordinal); + private readonly object _gate = new(); + private string? _table; + private string _keyProperty = "Id"; + private bool? _keyIdentity; + private List? _columns; + private ColumnMap? _key; + private ColumnMap? _rowVersion; + + /// + public Type EntityType => typeof(T); + + /// + public string Table => _table ?? throw new InvalidOperationException($"{GetType().Name} does not call ToTable."); + + /// + public ColumnMap Key + { + get + { + Freeze(); + return _key!; + } + } + + /// + public IReadOnlyList Columns + { + get + { + Freeze(); + return _columns!; + } + } + + /// + public ColumnMap? RowVersion + { + get + { + Freeze(); + return _rowVersion; + } + } + + /// + public ColumnMap? FindByProperty(string propertyName) => Columns.FirstOrDefault(c => c.PropertyName == propertyName); + + /// + public ColumnMap? FindByColumn(string columnName) => Columns.FirstOrDefault(c => c.ColumnName == columnName); + + /// Creates an empty entity. + /// The entity. + public T Create() => new(); + + /// + object IEntityMap.CreateEntity() => new T(); + + /// The column of a property. + /// The property type. + /// The property, for example x => x.Code. + /// The column. + public ColumnMap Column(Expression> property) => + FindByProperty(PropertyOf(property).Name) ?? throw new InvalidOperationException($"{typeof(T).Name}.{PropertyOf(property).Name} is not mapped."); + + /// + public void Freeze() + { + if (_columns is not null) + { + return; + } + + lock (_gate) + { + if (_columns is null) + { + Build(); + } + } + } + + /// Sets the table name. + /// The table (snake_case; sys_ prefix in the system database). + protected void ToTable(string table) + { + SqlNames.EnsureIdentifier(table); + _table = table; + } + + /// Configures the column of a property. + /// The property type. + /// The property, for example x => x.Code. + /// The column map to configure. + protected ColumnMap Map(Expression> property) + { + if (_columns is not null) + { + throw new InvalidOperationException("The map is frozen."); + } + + var info = PropertyOf(property); + if (!_configured.TryGetValue(info.Name, out var column)) + { + column = CreateColumn(info); + _configured[info.Name] = column; + } + + return column; + } + + /// Sets a key other than Id. + /// The property type. + /// The key property. + /// Whether the database generates it. + protected void HasKey(Expression> property, bool identity) + { + _keyProperty = PropertyOf(property).Name; + _keyIdentity = identity; + } + + /// Excludes a property from the table. + /// The property type. + /// The property. + protected void Ignore(Expression> property) => _ignored.Add(PropertyOf(property).Name); + + private static PropertyInfo PropertyOf(Expression> property) + { + ArgumentNullException.ThrowIfNull(property); + var body = property.Body is UnaryExpression { NodeType: ExpressionType.Convert } convert ? convert.Operand : property.Body; + return body is MemberExpression { Member: PropertyInfo info } member && member.Expression == property.Parameters[0] + ? info + : throw new ArgumentException("Use a property of the entity, for example x => x.Code.", nameof(property)); + } + + private static ColumnMap CreateColumn(PropertyInfo property) + { + var entity = Expression.Parameter(typeof(object), "entity"); + var value = Expression.Parameter(typeof(object), "value"); + var typed = Expression.Convert(entity, typeof(T)); + var getter = Expression.Lambda>( + Expression.Convert(Expression.Property(typed, property), typeof(object)), entity).Compile(); + var setter = Expression.Lambda>( + Expression.Assign(Expression.Property(typed, property), Expression.Convert(value, property.PropertyType)), entity, value).Compile(); + return new ColumnMap(property, getter, setter); + } + + private void Build() + { + _ = Table; + var properties = typeof(T).GetProperties(BindingFlags.Public | BindingFlags.Instance) + .Where(p => p.CanRead && p.CanWrite && p.GetSetMethod() is not null && p.GetIndexParameters().Length == 0) + .ToList(); + var key = new List(1); + var regular = new List(); + var audit = new List(); + + foreach (var property in properties) + { + if (_ignored.Contains(property.Name)) + { + continue; + } + + var column = _configured.TryGetValue(property.Name, out var configured) ? configured : CreateColumn(property); + var role = AuditRoleOf(property.Name); + if (property.Name == _keyProperty) + { + column.IsKey = true; + column.IsIdentity = _keyIdentity ?? (column.ValueType == typeof(long) || column.ValueType == typeof(int)); + Infer(column, property, forceNotNull: true); + key.Add(column); + } + else if (role != AuditRole.None) + { + column.Audit = role; + ApplyAuditDefaults(column, role); + audit.Add(column); + } + else + { + if (property.Name == nameof(IDemoMarkable.IsDemo) && typeof(IDemoMarkable).IsAssignableFrom(typeof(T))) + { + column.SetDefaults(ColumnType.Bool, null, nullable: false); + } + + Infer(column, property, forceNotNull: false); + regular.Add(column); + } + } + + if (key.Count == 0) + { + throw new InvalidOperationException($"{typeof(T).Name} has no key property {_keyProperty}."); + } + + var unknown = _configured.Keys.Except(properties.Select(p => p.Name)).ToList(); + if (unknown.Count > 0) + { + throw new InvalidOperationException($"{typeof(T).Name}: cannot map {string.Join(", ", unknown)} (needs a public getter and setter)."); + } + + audit.Sort((a, b) => a.Audit.CompareTo(b.Audit)); + var columns = key.Concat(regular).Concat(audit).ToList(); + var duplicate = columns.GroupBy(c => c.ColumnName).FirstOrDefault(g => g.Count() > 1); + if (duplicate is not null) + { + throw new InvalidOperationException($"{typeof(T).Name}: column {duplicate.Key} is mapped twice."); + } + + foreach (var column in columns) + { + column.Freeze(); + } + + _key = key[0]; + _rowVersion = audit.Find(c => c.Audit == AuditRole.RowVersion); + _columns = columns; + } + + private static void Infer(ColumnMap column, PropertyInfo property, bool forceNotNull) + { + var nullable = !forceNotNull && IsNullable(property); + if (column.ValueType.IsEnum) + { + column.SetDefaults(ColumnType.String, EnumLength, nullable); + return; + } + + var type = ColumnTypes.FromClrType(column.ValueType) + ?? throw new InvalidOperationException( + $"{typeof(T).Name}.{property.Name}: type {column.ValueType.Name} has no column type; map or ignore it."); + column.SetDefaults(type, null, nullable); + if (column.Type == ColumnType.String && column.Length is null) + { + throw new InvalidOperationException($"{typeof(T).Name}.{property.Name}: a String column needs MaxLength."); + } + } + + private static bool IsNullable(PropertyInfo property) + { + if (property.PropertyType.IsValueType) + { + return Nullable.GetUnderlyingType(property.PropertyType) is not null; + } + + lock (NullabilityContext) + { + return NullabilityContext.Create(property).WriteState != NullabilityState.NotNull; + } + } + + private static AuditRole AuditRoleOf(string propertyName) + { + if (!typeof(IAuditable).IsAssignableFrom(typeof(T))) + { + return AuditRole.None; + } + + return propertyName switch + { + nameof(IAuditable.CreatedAt) => AuditRole.CreatedAt, + nameof(IAuditable.CreatedBy) => AuditRole.CreatedBy, + nameof(IAuditable.UpdatedAt) => AuditRole.UpdatedAt, + nameof(IAuditable.UpdatedBy) => AuditRole.UpdatedBy, + nameof(IAuditable.RowVersion) => AuditRole.RowVersion, + _ => AuditRole.None, + }; + } + + private static void ApplyAuditDefaults(ColumnMap column, AuditRole role) + { + switch (role) + { + case AuditRole.CreatedAt: + column.SetDefaults(ColumnType.DateTimeUtc, null, nullable: false); + break; + case AuditRole.UpdatedAt: + column.SetDefaults(ColumnType.DateTimeUtc, null, nullable: true); + break; + case AuditRole.CreatedBy or AuditRole.UpdatedBy: + column.SetDefaults(ColumnType.String, AuditColumnLengths.User, nullable: true); + break; + case AuditRole.RowVersion: + column.SetDefaults(ColumnType.Int64, null, nullable: false); + break; + } + } +} diff --git a/src/Khaemhat.Data/Mapping/MapRegistry.cs b/src/Khaemhat.Data/Mapping/MapRegistry.cs new file mode 100644 index 0000000..7b3e6cc --- /dev/null +++ b/src/Khaemhat.Data/Mapping/MapRegistry.cs @@ -0,0 +1,62 @@ +using System.Collections.Concurrent; +using System.Reflection; + +namespace Khaemhat.Data.Mapping; + +/// +/// The entity maps known to the instance (SPEC §5.2). Maps are completed and frozen when added; afterwards the data +/// layer only reads them. +/// +public sealed class MapRegistry +{ + private readonly ConcurrentDictionary _maps = new(); + + /// All registered maps. + public IReadOnlyCollection Maps => [.. _maps.Values.OrderBy(m => m.Table, StringComparer.Ordinal)]; + + /// Adds a map. + /// The map. + /// This registry. + /// The entity already has a map, or the map is not valid. + public MapRegistry Add(IEntityMap map) + { + ArgumentNullException.ThrowIfNull(map); + map.Freeze(); + if (!_maps.TryAdd(map.EntityType, map)) + { + throw new InvalidOperationException($"{map.EntityType.Name} has two maps."); + } + + return this; + } + + /// Adds every non-abstract with a public parameterless constructor found in an assembly. + /// The assembly. + /// This registry. + public MapRegistry AddFromAssembly(Assembly assembly) + { + ArgumentNullException.ThrowIfNull(assembly); + var types = assembly.GetTypes() + .Where(t => t is { IsClass: true, IsAbstract: false } && typeof(IEntityMap).IsAssignableFrom(t) && t.GetConstructor(Type.EmptyTypes) is not null) + .OrderBy(t => t.FullName, StringComparer.Ordinal); + foreach (var type in types) + { + Add((IEntityMap)Activator.CreateInstance(type)!); + } + + return this; + } + + /// The map of . + /// The entity type. + /// The map. + /// No map is registered. + public EntityMap Get() + where T : class, new() => + (EntityMap)(Find(typeof(T)) ?? throw new InvalidOperationException($"{typeof(T).Name} has no entity map.")); + + /// The map of a type, if any. + /// The entity type. + /// The map, or . + public IEntityMap? Find(Type entityType) => _maps.GetValueOrDefault(entityType); +} diff --git a/src/Khaemhat.Data/Migrations/DataModelDocument.cs b/src/Khaemhat.Data/Migrations/DataModelDocument.cs new file mode 100644 index 0000000..93e5ab5 --- /dev/null +++ b/src/Khaemhat.Data/Migrations/DataModelDocument.cs @@ -0,0 +1,141 @@ +using System.Globalization; +using System.Text; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.Migrations; + +/// +/// Renders the schema produced by the migrations as Markdown for docs/DATA-MODEL.md (SPEC §18), between two +/// markers, so the document can never drift from the code. Used by scripts/gen-datamodel.sh and checked by a +/// test. The text is Italian because it is documentation. +/// +public static class DataModelDocument +{ + /// The line that opens the generated section. + public const string BeginMarker = ""; + + /// The line that closes the generated section. + public const string EndMarker = ""; + + /// Renders both catalogues. + /// The system database migrations. + /// The company database migrations. + /// The Markdown, without the markers, ending with a newline. + public static string Render(MigrationCatalog system, MigrationCatalog company) + { + ArgumentNullException.ThrowIfNull(system); + ArgumentNullException.ThrowIfNull(company); + var text = new StringBuilder(); + RenderCatalog(text, "Database di sistema", system); + text.Append('\n'); + RenderCatalog(text, "Database aziendale", company); + return text.ToString(); + } + + /// Replaces the generated section of a document. + /// The current document. + /// The output of . + /// The updated document. + /// The markers are missing or out of order. + public static string Update(string document, string generated) + { + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(generated); + var begin = document.IndexOf(BeginMarker, StringComparison.Ordinal); + var end = document.IndexOf(EndMarker, StringComparison.Ordinal); + if (begin < 0 || end < begin) + { + throw new InvalidOperationException($"The document must contain '{BeginMarker}' followed by '{EndMarker}'."); + } + + var start = begin + BeginMarker.Length; + return document[..start] + "\n\n" + generated + "\n" + document[end..]; + } + + private static void RenderCatalog(StringBuilder text, string title, MigrationCatalog catalog) + { + var model = catalog.BuildModel(); + Line(text, $"### {title}"); + text.Append('\n'); + if (catalog.Migrations.Count == 0) + { + Line(text, "Nessuna migrazione: la tabella `schema_version` è l'unica presente."); + } + else + { + Line(text, "Migrazioni applicate in ordine:"); + text.Append('\n'); + foreach (var migration in catalog.Migrations) + { + Line(text, $"- `{migration.Module}` {migration.Version} `{migration.Name}` ({migration.Type.Name})"); + } + } + + foreach (var table in model.Tables.OrderBy(t => t.Name, StringComparer.Ordinal)) + { + text.Append('\n'); + RenderTable(text, table); + } + } + + private static void RenderTable(StringBuilder text, TableDefinition table) + { + Line(text, $"#### `{table.Name}`"); + text.Append('\n'); + Line(text, "| Colonna | Tipo | Null | Default | Note |"); + Line(text, "|---|---|---|---|---|"); + foreach (var column in table.Columns) + { + var notes = new List(); + if (table.PrimaryKey.Contains(column.Name)) + { + notes.Add("chiave primaria"); + } + + if (column.IsIdentity) + { + notes.Add("identity"); + } + + notes.AddRange(table.ForeignKeys + .Where(fk => fk.Columns.Contains(column.Name)) + .Select(fk => $"-> `{fk.ReferencedTable}` ({fk.OnDelete})")); + Line(text, $"| `{column.Name}` | {column.TypeDescription} | {(column.IsNullable ? "sì" : "no")} | {Default(column)} | {string.Join(", ", notes)} |"); + } + + if (table.PrimaryKey.Count > 1) + { + text.Append('\n'); + Line(text, $"Chiave primaria: {Columns(table.PrimaryKey)}."); + } + + if (table.Indexes.Count > 0) + { + text.Append('\n'); + foreach (var index in table.Indexes.OrderBy(i => i.Name, StringComparer.Ordinal)) + { + Line(text, $"- indice {(index.IsUnique ? "unico " : string.Empty)}`{index.Name}` su {Columns(index.Columns)}"); + } + } + + if (table.PartitionByYearColumn is { } partition) + { + text.Append('\n'); + Line(text, $"Partizionata per anno su `{partition}` (solo PostgreSQL)."); + } + } + + private static string Default(ColumnDefinition column) => column switch + { + { DefaultsToCurrentTimestamp: true } => "ora corrente (UTC)", + { DefaultValue: null } => string.Empty, + { DefaultValue: string s } => $"`'{s}'`", + { DefaultValue: bool b } => b ? "`true`" : "`false`", + { DefaultValue: IFormattable f } => $"`{f.ToString(null, CultureInfo.InvariantCulture)}`", + _ => $"`{column.DefaultValue}`", + }; + + private static string Columns(IEnumerable names) => string.Join(", ", names.Select(n => $"`{n}`")); + + private static void Line(StringBuilder text, string line) => text.Append(line).Append('\n'); +} diff --git a/src/Khaemhat.Data/Migrations/IMigration.cs b/src/Khaemhat.Data/Migrations/IMigration.cs new file mode 100644 index 0000000..8887b55 --- /dev/null +++ b/src/Khaemhat.Data/Migrations/IMigration.cs @@ -0,0 +1,56 @@ +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.Migrations; + +/// +/// A schema migration (SPEC §5.3). Describe the change in neutral form in ; add dialect-specific SQL in +/// only for what the neutral form cannot express. Once released, a migration never changes: write a +/// new one. Create the file with scripts/new-migration.sh. +/// +/// [Migration(20261111_002, "create_sys_settings", Target = MigrationTarget.System)] +/// public sealed class M20261111_002_CreateSysSettings : IMigration +/// { +/// public void Up(ISchemaBuilder schema) => schema.CreateTable("sys_settings", t => { … }); +/// } +/// +/// +public interface IMigration +{ + /// Describes the schema change. + /// The schema builder. + void Up(ISchemaBuilder schema); + + /// Dialect-specific SQL run after , or . + /// The dialect of the database being migrated. + /// Trusted SQL (several statements separated by semicolons are allowed), or . + string? UpSql(ISqlDialect dialect) => null; +} + +/// Which catalog a migration belongs to. +public enum MigrationTarget +{ + /// Every company database (khaemhat_<code>). + Company = 0, + + /// The system database (khaemhat_system). + System = 1, +} + +/// Marks an with its version and name. +/// The version yyyymmdd_nnn written as a number, for example 20261111_001. +/// The name in snake_case, for example create_sys_companies. +[AttributeUsage(AttributeTargets.Class, AllowMultiple = false, Inherited = false)] +public sealed class MigrationAttribute(long version, string name) : Attribute +{ + /// The version yyyymmddnnn. + public long Version { get; } = version; + + /// The name. + public string Name { get; } = name; + + /// The catalog (default ). + public MigrationTarget Target { get; init; } = MigrationTarget.Company; + + /// The module that owns the migration, as stored in schema_version.module (default core). + public string Module { get; init; } = "core"; +} diff --git a/src/Khaemhat.Data/Migrations/MigrationCatalog.cs b/src/Khaemhat.Data/Migrations/MigrationCatalog.cs new file mode 100644 index 0000000..227ba19 --- /dev/null +++ b/src/Khaemhat.Data/Migrations/MigrationCatalog.cs @@ -0,0 +1,127 @@ +using System.Globalization; +using System.Reflection; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.Migrations; + +/// A migration found in a catalog. +/// The version yyyymmddnnn. +/// The name. +/// The owning module. +/// The catalog. +/// The migration class. +public sealed record MigrationInfo(long Version, string Name, string Module, MigrationTarget Target, Type Type) +{ + /// Creates the migration. + /// The migration. + public IMigration Create() => (IMigration)Activator.CreateInstance(Type)!; + + /// Records the operations of the migration. + /// The operations of . + public IReadOnlyList Operations() + { + var builder = new SchemaBuilder(); + Create().Up(builder); + return builder.Operations; + } + + /// + public override string ToString() => string.Create(CultureInfo.InvariantCulture, $"{Module}/{Version} {Name}"); +} + +/// +/// The migrations of one database kind (SPEC §5.3): for the system database, +/// for every company. Built by scanning assemblies for . +/// +public sealed class MigrationCatalog +{ + private MigrationCatalog(MigrationTarget target, IReadOnlyList migrations) + { + Target = target; + Migrations = migrations; + } + + /// The catalog kind. + public MigrationTarget Target { get; } + + /// The migrations, ordered by version then module. + public IReadOnlyList Migrations { get; } + + /// The system catalog with the migrations of this assembly and of . + /// Further assemblies with system migrations (security, licences, logs...). + /// The catalog. + public static MigrationCatalog ForSystem(params Assembly[] assemblies) => + Discover(MigrationTarget.System, [typeof(MigrationCatalog).Assembly, .. assemblies]); + + /// The company catalog with the migrations of . + /// Assemblies with company migrations (the application modules). + /// The catalog. + public static MigrationCatalog ForCompany(params Assembly[] assemblies) => + Discover(MigrationTarget.Company, [typeof(MigrationCatalog).Assembly, .. assemblies]); + + /// Finds the migrations of in assemblies and validates them. + /// The catalog kind. + /// The assemblies. + /// The catalog. + /// A migration is not valid or a version is used twice in a module. + public static MigrationCatalog Discover(MigrationTarget target, IEnumerable assemblies) + { + ArgumentNullException.ThrowIfNull(assemblies); + var migrations = new List(); + foreach (var type in assemblies.Distinct().SelectMany(a => a.GetTypes())) + { + if (type.GetCustomAttribute() is not { } attribute || attribute.Target != target) + { + continue; + } + + if (!typeof(IMigration).IsAssignableFrom(type) || type.IsAbstract || type.GetConstructor(Type.EmptyTypes) is null) + { + throw new InvalidOperationException($"{type.Name} has [Migration] but is not a concrete IMigration with a parameterless constructor."); + } + + Validate(type, attribute); + migrations.Add(new MigrationInfo(attribute.Version, attribute.Name, attribute.Module, target, type)); + } + + var duplicate = migrations.GroupBy(m => (m.Module, m.Version)).FirstOrDefault(g => g.Count() > 1); + if (duplicate is not null) + { + throw new InvalidOperationException( + $"Version {duplicate.Key.Version} of module {duplicate.Key.Module} is used by {string.Join(", ", duplicate.Select(m => m.Type.Name))}."); + } + + return new MigrationCatalog( + target, [.. migrations.OrderBy(m => m.Version).ThenBy(m => m.Module, StringComparer.Ordinal)]); + } + + /// The schema produced by all migrations from an empty database, plus schema_version. + /// The schema model. + public SchemaModel BuildModel() + { + var model = new SchemaModel(); + model.Apply([SchemaVersionStore.CreateOperation]); + foreach (var migration in Migrations) + { + model.Apply(migration.Operations()); + } + + return model; + } + + private static void Validate(Type type, MigrationAttribute attribute) + { + var text = attribute.Version.ToString(CultureInfo.InvariantCulture); + if (text.Length != 11 + || !DateOnly.TryParseExact(text[..8], "yyyyMMdd", CultureInfo.InvariantCulture, DateTimeStyles.None, out _) + || text[8..] == "000") + { + throw new InvalidOperationException($"{type.Name}: the version must be yyyymmdd_nnn (for example 20261111_001), not {text}."); + } + + if (!SqlNames.IsValidIdentifier(attribute.Name) || !SqlNames.IsValidIdentifier(attribute.Module) || attribute.Module.Length > 50) + { + throw new InvalidOperationException($"{type.Name}: name and module must be snake_case."); + } + } +} diff --git a/src/Khaemhat.Data/Migrations/MigrationRunner.cs b/src/Khaemhat.Data/Migrations/MigrationRunner.cs new file mode 100644 index 0000000..941f80a --- /dev/null +++ b/src/Khaemhat.Data/Migrations/MigrationRunner.cs @@ -0,0 +1,155 @@ +using System.Diagnostics; +using System.Security.Cryptography; +using System.Text; +using Khaemhat.Data.Execution; +using Khaemhat.Data.Mapping; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.Migrations; + +/// The outcome of a migration run, for the log section Migrazioni (written by the caller). +/// The database name. +/// The migrations applied now, with their duration. +/// How many catalog migrations were already applied. +/// +/// Applied migrations whose statements differ from what the code generates now (a changed migration or DDL generator). +/// +public sealed record MigrationReport( + string Database, IReadOnlyList<(MigrationInfo Migration, TimeSpan Duration)> Applied, int AlreadyApplied, IReadOnlyList Warnings); + +/// A database cannot be migrated, for example because it is newer than the application. +public sealed class MigrationException : Exception +{ + /// Creates the exception. + public MigrationException() + { + } + + /// Creates the exception with a message. + /// The message. + public MigrationException(string message) : base(message) + { + } + + /// Creates the exception with a message and a cause. + /// The message. + /// The cause. + public MigrationException(string message, Exception innerException) : base(message, innerException) + { + } +} + +/// +/// Applies the pending migrations of a catalog to a database (SPEC §5.3): each in its own transaction together with its +/// schema_version row, under an application lock so that two instances never migrate at the same time. +/// Re-running does nothing. A database with applied versions unknown to the catalog (a downgrade) is refused. +/// +/// Clock and retry policy. +public sealed class MigrationRunner(DataAccessContext context) +{ + private const string LockKey = "khaemhat:migrations"; + private static readonly MapRegistry NoMaps = new(); + + /// Migrates a database. + /// The database. + /// The catalog for this kind of database. + /// Cancellation. + /// What was done. + /// The database has versions unknown to this application. + public async Task RunAsync(DatabaseTarget target, MigrationCatalog catalog, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(target); + ArgumentNullException.ThrowIfNull(catalog); + await using var unitOfWork = await UnitOfWork.OpenAsync(target, NoMaps, context, cancellationToken).ConfigureAwait(false); + + await unitOfWork.InTransactionAsync(ct => SchemaVersionStore.EnsureCreatedAsync(unitOfWork, ct), cancellationToken).ConfigureAwait(false); + var applied = await SchemaVersionStore.ReadAsync(unitOfWork, cancellationToken).ConfigureAwait(false); + + var known = catalog.Migrations.Select(m => (m.Module, m.Version)).ToHashSet(); + var unknown = applied.Where(a => !known.Contains((a.Module, a.Version))).ToList(); + if (unknown.Count > 0) + { + throw new MigrationException( + $"Database {target.DatabaseName} has migrations this version of Khaemhat does not know (" + + string.Join(", ", unknown.Select(u => $"{u.Module}/{u.Version} {u.Name}")) + + "): it was used by a newer version. Downgrades are not supported; restore a backup or install the newer version."); + } + + var appliedByKey = applied.ToDictionary(a => (a.Module, a.Version)); + var done = new List<(MigrationInfo, TimeSpan)>(); + var warnings = new List(); + var alreadyApplied = 0; + foreach (var migration in catalog.Migrations) + { + var statements = Statements(migration, unitOfWork.Provider); + var checksum = Checksum(statements); + if (appliedByKey.TryGetValue((migration.Module, migration.Version), out var row)) + { + alreadyApplied++; + if (!string.Equals(row.Checksum, checksum, StringComparison.Ordinal)) + { + warnings.Add($"{migration}: the statements differ from those applied on {row.AppliedAt:yyyy-MM-dd}."); + } + + continue; + } + + var watch = Stopwatch.StartNew(); + var appliedNow = false; + await unitOfWork.InTransactionAsync( + async ct => + { + await unitOfWork.AcquireLockAsync(LockKey, ct).ConfigureAwait(false); + if (await SchemaVersionStore.IsAppliedAsync(unitOfWork, migration.Module, migration.Version, ct).ConfigureAwait(false)) + { + return; // another instance applied it meanwhile + } + + foreach (var statement in statements) + { + await unitOfWork.ExecuteAsync(SqlFragment.Literal(statement), ct).ConfigureAwait(false); + } + + await SchemaVersionStore.RecordAsync( + unitOfWork, new AppliedMigration(migration.Module, migration.Version, migration.Name, context.UtcNow, checksum), ct) + .ConfigureAwait(false); + appliedNow = true; + }, + cancellationToken).ConfigureAwait(false); + + if (appliedNow) + { + done.Add((migration, watch.Elapsed)); + } + else + { + alreadyApplied++; + } + } + + return new MigrationReport(target.DatabaseName, done, alreadyApplied, warnings); + } + + /// The statements a migration runs on a provider. + /// The migration. + /// The provider. + /// The statements. + public static IReadOnlyList Statements(MigrationInfo migration, IDbProvider provider) + { + ArgumentNullException.ThrowIfNull(migration); + ArgumentNullException.ThrowIfNull(provider); + var instance = migration.Create(); + var builder = new SchemaBuilder(); + instance.Up(builder); + var statements = provider.Ddl.Generate(builder.Operations).ToList(); + if (instance.UpSql(provider.Dialect) is { Length: > 0 } sql) + { + statements.Add(sql); + } + + return statements; + } + + private static string Checksum(IEnumerable statements) => + Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(string.Join(";\n", statements)))); +} diff --git a/src/Khaemhat.Data/Migrations/SchemaVersionStore.cs b/src/Khaemhat.Data/Migrations/SchemaVersionStore.cs new file mode 100644 index 0000000..45f3090 --- /dev/null +++ b/src/Khaemhat.Data/Migrations/SchemaVersionStore.cs @@ -0,0 +1,95 @@ +using Khaemhat.Data.Execution; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.Migrations; + +/// A row of schema_version. +/// The owning module. +/// The version. +/// The name. +/// When it was applied (UTC). +/// SHA-256 of the statements that were run, in hexadecimal. +public sealed record AppliedMigration(string Module, long Version, string Name, DateTime AppliedAt, string Checksum); + +/// +/// The schema_version(module, version, name, applied_at, checksum) table present in every database (SPEC §5.3). +/// +public static class SchemaVersionStore +{ + /// The table name. + public const string Table = "schema_version"; + + /// The operation creating the table if it is missing. + public static CreateTableOperation CreateOperation { get; } = BuildCreateOperation(); + + /// Creates the table if it is missing. + /// The unit of work. + /// Cancellation. + /// A task. + public static async Task EnsureCreatedAsync(UnitOfWork unitOfWork, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(unitOfWork); + foreach (var statement in unitOfWork.Provider.Ddl.Generate([CreateOperation])) + { + await unitOfWork.ExecuteAsync(SqlFragment.Literal(statement), cancellationToken).ConfigureAwait(false); + } + } + + /// All applied migrations. + /// The unit of work. + /// Cancellation. + /// The rows, ordered by version. + public static Task> ReadAsync(UnitOfWork unitOfWork, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(unitOfWork); + return unitOfWork.QueryAsync( + SqlFragment.Literal("SELECT module, version, name, applied_at, checksum FROM ") + SqlFragment.Identifier(Table) + + SqlFragment.Literal(" ORDER BY version, module"), + cancellationToken); + } + + /// Whether a version is applied (read inside the migration transaction). + /// The unit of work. + /// The module. + /// The version. + /// Cancellation. + /// if applied. + public static async Task IsAppliedAsync(UnitOfWork unitOfWork, string module, long version, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(unitOfWork); + var count = await unitOfWork.QuerySingleAsync( + SqlFragment.Of($"SELECT COUNT(*) FROM {SqlFragment.Identifier(Table)} WHERE module = {module} AND version = {version}"), + cancellationToken).ConfigureAwait(false); + return count > 0; + } + + /// Records an applied migration. + /// The unit of work, inside the migration transaction. + /// The row. + /// Cancellation. + /// A task. + public static Task RecordAsync(UnitOfWork unitOfWork, AppliedMigration applied, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(unitOfWork); + ArgumentNullException.ThrowIfNull(applied); + return unitOfWork.ExecuteAsync( + SqlFragment.Of( + $"INSERT INTO {SqlFragment.Identifier(Table)} (module, version, name, applied_at, checksum) VALUES ({applied.Module}, {applied.Version}, {applied.Name}, {applied.AppliedAt}, {applied.Checksum})"), + cancellationToken); + } + + private static CreateTableOperation BuildCreateOperation() + { + var builder = new SchemaBuilder(); + builder.CreateTable(Table, t => + { + t.String("module", 50).NotNull(); + t.Int64("version").NotNull(); + t.String("name", 200).NotNull(); + t.DateTimeUtc("applied_at").NotNull(); + t.String("checksum", 64).NotNull(); + t.PrimaryKey("module", "version"); + }); + return ((CreateTableOperation)builder.Operations[0]) with { IfNotExists = true }; + } +} diff --git a/src/Khaemhat.Data/PagedResult.cs b/src/Khaemhat.Data/PagedResult.cs new file mode 100644 index 0000000..7172c25 --- /dev/null +++ b/src/Khaemhat.Data/PagedResult.cs @@ -0,0 +1,13 @@ +namespace Khaemhat.Data; + +/// A page of results with the total number of matching rows. +/// The item type. +/// The rows of the page. +/// All rows matching the filters. +/// Rows skipped. +/// Page size, or when not paged. +public sealed record PagedResult(IReadOnlyList Items, long TotalCount, int Offset, int? Limit) +{ + /// Whether more rows follow this page. + public bool HasMore => Offset + Items.Count < TotalCount; +} diff --git a/src/Khaemhat.Data/Querying/ExpressionTranslator.cs b/src/Khaemhat.Data/Querying/ExpressionTranslator.cs new file mode 100644 index 0000000..0f2368b --- /dev/null +++ b/src/Khaemhat.Data/Querying/ExpressionTranslator.cs @@ -0,0 +1,307 @@ +using System.Collections; +using System.Linq.Expressions; +using System.Reflection; +using Khaemhat.Data.Mapping; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.Querying; + +/// +/// Translates predicates of the safe subset of SPEC §5.2 into parameterized SQL. Every value — constants, captured +/// variables, list items — becomes a parameter; only mapped column names reach the SQL text. Anything outside the +/// subset throws : use a condition instead. +/// +/// The map of the queried entity. +/// The dialect, for the case-insensitive LIKE. +public sealed class ExpressionTranslator(IEntityMap map, ISqlDialect dialect) +{ + private static readonly SqlFragment True = SqlFragment.Literal("1 = 1"); + private static readonly SqlFragment False = SqlFragment.Literal("1 = 0"); + + /// Translates a predicate. + /// A lambda with one parameter of the entity type returning . + /// The condition. + public SqlFragment Translate(LambdaExpression predicate) + { + ArgumentNullException.ThrowIfNull(predicate); + return Condition(predicate.Body, predicate.Parameters[0]); + } + + /// The column of a property selector such as x => x.Name. + /// The selector. + /// The column. + public ColumnMap Column(LambdaExpression selector) + { + ArgumentNullException.ThrowIfNull(selector); + return TryColumn(selector.Body, selector.Parameters[0]) + ?? throw new NotSupportedException($"Only mapped properties of {map.EntityType.Name} can be used here: {selector}."); + } + + private SqlFragment Condition(Expression node, ParameterExpression entity) + { + switch (node) + { + case BinaryExpression { NodeType: ExpressionType.AndAlso } and: + return (Condition(and.Left, entity) + SqlFragment.Literal(" AND ") + Condition(and.Right, entity)).Parenthesize(); + case BinaryExpression { NodeType: ExpressionType.OrElse } or: + return (Condition(or.Left, entity) + SqlFragment.Literal(" OR ") + Condition(or.Right, entity)).Parenthesize(); + case UnaryExpression { NodeType: ExpressionType.Not } not: + return SqlFragment.Literal("NOT ") + Condition(not.Operand, entity).Parenthesize(); + case BinaryExpression binary when Operator(binary.NodeType) is not null: + return Comparison(binary, entity); + case MethodCallExpression call: + return Method(call, entity); + case ConstantExpression { Value: bool constant }: + return constant ? True : False; + default: + if (node.Type == typeof(bool) && TryColumn(node, entity) is { } flag) + { + return Name(flag) + SqlFragment.Literal(" = ") + Value(flag, true); + } + + if (node.Type == typeof(bool) && !References(node, entity)) + { + return (bool)Evaluate(node)! ? True : False; + } + + throw Unsupported(node); + } + } + + private SqlFragment Comparison(BinaryExpression binary, ParameterExpression entity) + { + var op = Operator(binary.NodeType)!; + var leftColumn = TryColumn(binary.Left, entity); + var rightColumn = TryColumn(binary.Right, entity); + + if (leftColumn is not null && rightColumn is not null) + { + return Name(leftColumn) + SqlFragment.Literal(" " + op + " ") + Name(rightColumn); + } + + if (leftColumn is null && rightColumn is not null) + { + // value op column -> column (mirrored op) value + (leftColumn, op) = (rightColumn, Mirror(op)); + return CompareWithValue(leftColumn, op, binary.Left, entity); + } + + return leftColumn is not null ? CompareWithValue(leftColumn, op, binary.Right, entity) : throw Unsupported(binary); + } + + private SqlFragment CompareWithValue(ColumnMap column, string op, Expression valueNode, ParameterExpression entity) + { + if (References(valueNode, entity)) + { + throw Unsupported(valueNode); + } + + var value = Evaluate(valueNode); + if (value is null) + { + return op switch + { + "=" => Name(column) + SqlFragment.Literal(" IS NULL"), + "<>" => Name(column) + SqlFragment.Literal(" IS NOT NULL"), + _ => throw new NotSupportedException($"Cannot compare {column.PropertyName} with null using {op}."), + }; + } + + return Name(column) + SqlFragment.Literal(" " + op + " ") + Value(column, value); + } + + private SqlFragment Method(MethodCallExpression call, ParameterExpression entity) + { + // column.Contains / StartsWith / EndsWith (value): case-insensitive LIKE + if (call.Method.DeclaringType == typeof(string) && call.Object is not null && call.Arguments.Count >= 1 + && call.Method.Name is nameof(string.Contains) or nameof(string.StartsWith) or nameof(string.EndsWith) + && TryColumn(call.Object, entity) is { } textColumn) + { + if (References(call.Arguments[0], entity)) + { + throw Unsupported(call); + } + + var text = Evaluate(call.Arguments[0]) as string ?? string.Empty; + var escaped = EscapeLike(text); + var pattern = call.Method.Name switch + { + nameof(string.StartsWith) => escaped + "%", + nameof(string.EndsWith) => "%" + escaped, + _ => "%" + escaped + "%", + }; + return Like(textColumn, pattern); + } + + // string.IsNullOrEmpty(column) + if (call.Method.DeclaringType == typeof(string) && call.Method.Name == nameof(string.IsNullOrEmpty) + && TryColumn(call.Arguments[0], entity) is { } emptyColumn) + { + return (Name(emptyColumn) + SqlFragment.Literal(" IS NULL OR ") + Name(emptyColumn) + SqlFragment.Literal(" = ") + + SqlFragment.Parameter(string.Empty, emptyColumn.Type)).Parenthesize(); + } + + // values.Contains(column) -> column IN (...) + Expression? collection = null; + Expression? item = null; + if (call.Method.Name == nameof(Enumerable.Contains)) + { + if (call.Object is null && call.Arguments.Count == 2) + { + (collection, item) = (call.Arguments[0], call.Arguments[1]); + } + else if (call.Object is not null && call.Arguments.Count == 1) + { + (collection, item) = (call.Object, call.Arguments[0]); + } + } + + if (collection is not null && item is not null && TryColumn(item, entity) is { } inColumn && !References(collection, entity)) + { + collection = UnwrapSpan(collection); + var values = (Evaluate(collection) as IEnumerable)?.Cast().ToList() ?? []; + if (values.Count == 0) + { + return False; + } + + var nonNull = values.Where(v => v is not null).Select(v => Value(inColumn, v!)).ToList(); + var condition = nonNull.Count == 0 + ? SqlFragment.Empty + : Name(inColumn) + SqlFragment.Literal(" IN (") + SqlFragment.Join(", ", nonNull) + SqlFragment.Literal(")"); + if (values.Count != nonNull.Count) + { + var isNull = Name(inColumn) + SqlFragment.Literal(" IS NULL"); + condition = condition.IsEmpty ? isNull : (condition + SqlFragment.Literal(" OR ") + isNull).Parenthesize(); + } + + return condition; + } + + if (call.Type == typeof(bool) && !References(call, entity)) + { + return (bool)Evaluate(call)! ? True : False; + } + + throw Unsupported(call); + } + + /// + /// C# 14 binds array.Contains(x) to MemoryExtensions.Contains(ReadOnlySpan<T>, T) through an + /// implicit conversion; spans cannot be evaluated by the expression interpreter, so read the array behind it. + /// + private static Expression UnwrapSpan(Expression node) + { + while (IsSpan(node.Type)) + { + switch (node) + { + case MethodCallExpression { Method.Name: "op_Implicit", Arguments.Count: 1 } conversion: + node = conversion.Arguments[0]; + break; + case UnaryExpression { NodeType: ExpressionType.Convert } convert: + node = convert.Operand; + break; + default: + return node; + } + } + + return node; + + static bool IsSpan(Type type) => + type.IsGenericType && (type.GetGenericTypeDefinition() == typeof(ReadOnlySpan<>) || type.GetGenericTypeDefinition() == typeof(Span<>)); + } + + private SqlFragment Like(ColumnMap column, string pattern) => + dialect.CaseInsensitiveLike(Name(column), SqlFragment.Parameter(pattern, ColumnType.Text)); + + private ColumnMap? TryColumn(Expression node, ParameterExpression entity) + { + while (node is UnaryExpression { NodeType: ExpressionType.Convert or ExpressionType.ConvertChecked } convert) + { + node = convert.Operand; + } + + return node is MemberExpression { Member: PropertyInfo property } member && member.Expression == entity + ? map.FindByProperty(property.Name) ?? throw new NotSupportedException($"{map.EntityType.Name}.{property.Name} is not mapped.") + : null; + } + + private static SqlFragment Name(ColumnMap column) => SqlFragment.Identifier(column.ColumnName); + + private static SqlFragment Value(ColumnMap column, object value) + { + if (column.ValueType.IsEnum && value is not Enum) + { + value = Enum.ToObject(column.ValueType, value); + } + + return SqlFragment.Parameter(column.ToProvider(value), column.Type); + } + + private static string? Operator(ExpressionType type) => type switch + { + ExpressionType.Equal => "=", + ExpressionType.NotEqual => "<>", + ExpressionType.LessThan => "<", + ExpressionType.LessThanOrEqual => "<=", + ExpressionType.GreaterThan => ">", + ExpressionType.GreaterThanOrEqual => ">=", + _ => null, + }; + + private static string Mirror(string op) => op switch + { + "<" => ">", + "<=" => ">=", + ">" => "<", + ">=" => "<=", + _ => op, + }; + + private static string EscapeLike(string text) => + text.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("%", "\\%", StringComparison.Ordinal).Replace("_", "\\_", StringComparison.Ordinal); + + private static bool References(Expression node, ParameterExpression entity) + { + var finder = new ParameterFinder(entity); + finder.Visit(node); + return finder.Found; + } + + private static object? Evaluate(Expression node) + { + switch (node) + { + case ConstantExpression constant: + return constant.Value; + case MemberExpression { Member: FieldInfo field } member when member.Expression is null or ConstantExpression: + return field.GetValue((member.Expression as ConstantExpression)?.Value); + case MemberExpression { Member: PropertyInfo property } member when member.Expression is null or ConstantExpression: + return property.GetValue((member.Expression as ConstantExpression)?.Value); + case MemberExpression { Member: FieldInfo field } member: + return field.GetValue(Evaluate(member.Expression!)); + case MemberExpression { Member: PropertyInfo property } member: + return property.GetValue(Evaluate(member.Expression!)); + case UnaryExpression { NodeType: ExpressionType.Convert } convert when convert.Type == typeof(object): + return Evaluate(convert.Operand); + default: + return Expression.Lambda>(Expression.Convert(node, typeof(object))).Compile(preferInterpretation: true)(); + } + } + + private NotSupportedException Unsupported(Expression node) => + new($"The expression '{node}' is outside the supported query subset for {map.EntityType.Name}; use a SqlFragment condition."); + + private sealed class ParameterFinder(ParameterExpression parameter) : ExpressionVisitor + { + public bool Found { get; private set; } + + protected override Expression VisitParameter(ParameterExpression node) + { + Found |= node == parameter; + return node; + } + } +} diff --git a/src/Khaemhat.Data/Querying/Query.cs b/src/Khaemhat.Data/Querying/Query.cs new file mode 100644 index 0000000..cfe1917 --- /dev/null +++ b/src/Khaemhat.Data/Querying/Query.cs @@ -0,0 +1,109 @@ +using System.Collections.Immutable; +using System.Linq.Expressions; + +namespace Khaemhat.Data.Querying; + +/// Starts queries: Query.For<Party>().Where(p => p.Name.Contains(text)).OrderBy(p => p.Name).Page(0, 50). +public static class Query +{ + /// A query over all rows of . + /// The entity type. + /// The query. + public static Query For() + where T : class => Query.All; +} + +/// +/// An immutable query over the table of (SPEC §5.2): filters in the safe expression subset +/// (equality, comparisons, Contains/StartsWith/EndsWith, In as list.Contains(x.Prop), +/// &&, ||, !, == null), parameterized conditions, ordering and +/// paging. Each method returns a new query. +/// +/// The entity type. +public sealed class Query + where T : class +{ + private Query( + ImmutableList filters, ImmutableList conditions, ImmutableList order, int offset, int? limit) + { + Filters = filters; + Conditions = conditions; + Order = order; + Offset = offset; + Limit = limit; + } + + internal static Query All { get; } = new([], [], [], 0, null); + + /// The expression filters, combined with AND. + public ImmutableList Filters { get; } + + /// The SQL conditions, combined with AND. + public ImmutableList Conditions { get; } + + /// The ordering, in priority order. + public ImmutableList Order { get; } + + /// Rows to skip. + public int Offset { get; } + + /// Rows to return at most, or for all. + public int? Limit { get; } + + /// Adds a filter. + /// The predicate, in the supported subset. + /// The new query. + public Query Where(Expression> predicate) + { + ArgumentNullException.ThrowIfNull(predicate); + return new(Filters.Add(predicate), Conditions, Order, Offset, Limit); + } + + /// Adds a parameterized SQL condition, for what expressions cannot say. + /// The condition, for example SqlFragment.Of($"{SqlFragment.Identifier("total")} > {min}"). + /// The new query. + public Query Where(SqlFragment condition) + { + ArgumentNullException.ThrowIfNull(condition); + return condition.IsEmpty ? this : new(Filters, Conditions.Add(condition), Order, Offset, Limit); + } + + /// Orders by a property, ascending; further calls add secondary orderings. + /// The property type. + /// The property. + /// The new query. + public Query OrderBy(Expression> property) => AddOrder(property, descending: false); + + /// Orders by a property, descending; further calls add secondary orderings. + /// The property type. + /// The property. + /// The new query. + public Query OrderByDescending(Expression> property) => AddOrder(property, descending: true); + + /// Returns at most rows after skipping . + /// Rows to skip. + /// Rows to return. + /// The new query. + public Query Page(int offset, int limit) + { + ArgumentOutOfRangeException.ThrowIfNegative(offset); + ArgumentOutOfRangeException.ThrowIfNegative(limit); + return new(Filters, Conditions, Order, offset, limit); + } + + /// Returns at most rows. + /// Rows to return. + /// The new query. + public Query Take(int limit) => Page(Offset, limit); + + private Query AddOrder(LambdaExpression property, bool descending) + { + ArgumentNullException.ThrowIfNull(property); + return new(Filters, Conditions, Order.Add(new QueryOrder(property, descending)), Offset, Limit); + } +} + +/// An ordering of a query. +/// The property selector. +/// Whether the order is descending. +public sealed record QueryOrder(LambdaExpression Property, bool Descending); diff --git a/src/Khaemhat.Data/Querying/QueryBuilder.cs b/src/Khaemhat.Data/Querying/QueryBuilder.cs new file mode 100644 index 0000000..c5c3e82 --- /dev/null +++ b/src/Khaemhat.Data/Querying/QueryBuilder.cs @@ -0,0 +1,75 @@ +using Khaemhat.Data.Mapping; + +namespace Khaemhat.Data.Querying; + +/// Turns a into parameterized SELECT and COUNT statements for one dialect. +/// The entity type. +/// The map of the entity. +/// The dialect. +public sealed class QueryBuilder(EntityMap map, ISqlDialect dialect) + where T : class, new() +{ + private readonly ExpressionTranslator _translator = new(map, dialect); + + /// The map. + public EntityMap Map { get; } = map; + + /// The list of mapped columns, in map order. + public SqlFragment ColumnList => SqlFragment.Join(", ", Map.Columns.Select(c => SqlFragment.Identifier(c.ColumnName))); + + /// The WHERE clause (with leading space), or empty when the query has no filters. + /// The query. + /// The clause. + public SqlFragment Where(Query query) + { + ArgumentNullException.ThrowIfNull(query); + var conditions = query.Filters.Select(_translator.Translate).Concat(query.Conditions.Select(c => c.Parenthesize())).ToList(); + return conditions.Count == 0 ? SqlFragment.Empty : SqlFragment.Literal(" WHERE ") + SqlFragment.Join(" AND ", conditions); + } + + /// + /// The SELECT statement with ordering and paging. A paged query without ordering is ordered by the key, + /// so pages are stable. + /// + /// The query. + /// The statement. + public SqlFragment Select(Query query) + { + ArgumentNullException.ThrowIfNull(query); + var sql = SqlFragment.Literal("SELECT ") + ColumnList + SqlFragment.Literal(" FROM ") + SqlFragment.Identifier(Map.Table) + Where(query); + var order = query.Order + .Select(o => SqlFragment.Identifier(_translator.Column(o.Property).ColumnName) + SqlFragment.Literal(o.Descending ? " DESC" : " ASC")) + .ToList(); + var paged = query.Limit is not null || query.Offset > 0; + if (order.Count == 0 && paged) + { + order.Add(SqlFragment.Identifier(Map.Key.ColumnName) + SqlFragment.Literal(" ASC")); + } + + if (order.Count > 0) + { + sql += SqlFragment.Literal(" ORDER BY ") + SqlFragment.Join(", ", order); + } + + if (paged) + { + sql += SqlFragment.Literal(dialect.Paging(query.Offset, query.Limit ?? int.MaxValue)); + } + + return sql; + } + + /// The COUNT(*) statement for the filters of the query (ordering and paging ignored). + /// The query. + /// The statement. + public SqlFragment Count(Query query) => + SqlFragment.Literal("SELECT COUNT(*) FROM ") + SqlFragment.Identifier(Map.Table) + Where(query); + + /// The SELECT of one row by key. + /// The key value. + /// The statement. + public SqlFragment SelectByKey(object key) => + SqlFragment.Literal("SELECT ") + ColumnList + SqlFragment.Literal(" FROM ") + SqlFragment.Identifier(Map.Table) + + SqlFragment.Literal(" WHERE ") + SqlFragment.Identifier(Map.Key.ColumnName) + SqlFragment.Literal(" = ") + + SqlFragment.Parameter(Map.Key.ToProvider(key), Map.Key.Type); +} diff --git a/src/Khaemhat.Data/Schema/ColumnType.cs b/src/Khaemhat.Data/Schema/ColumnType.cs new file mode 100644 index 0000000..b499418 --- /dev/null +++ b/src/Khaemhat.Data/Schema/ColumnType.cs @@ -0,0 +1,107 @@ +namespace Khaemhat.Data.Schema; + +/// +/// Logical column types (SPEC §5.2), mapped by each dialect to a physical type: +/// PostgreSQL varchar(n)/text/int/bigint/numeric(19,6)/boolean/date/timestamptz/uuid/bytea/jsonb; +/// SQLite TEXT/TEXT/INTEGER/INTEGER/INTEGER x10⁶/INTEGER 0-1/TEXT yyyy-MM-dd/TEXT ISO 8601 UTC/TEXT/BLOB/TEXT. +/// +public enum ColumnType +{ + /// Text with a maximum length n (). + String = 0, + + /// Text without a length limit (). + Text = 1, + + /// 32-bit integer (). + Int32 = 2, + + /// 64-bit integer (): primary and foreign keys. + Int64 = 3, + + /// Exact decimal with 19 digits, 6 after the point (): amounts and quantities. + Decimal = 4, + + /// Boolean (). + Bool = 5, + + /// Calendar date without time (). + Date = 6, + + /// Instant in UTC ( with ). + DateTimeUtc = 7, + + /// GUID (). + Guid = 8, + + /// Binary data (byte[]). + Bytes = 9, + + /// JSON document stored as text (). + Json = 10, +} + +/// Helpers on . +public static class ColumnTypes +{ + /// The fixed precision of . + public const int DecimalPrecision = 19; + + /// The default scale of . + public const int DecimalScale = 6; + + /// + /// The logical type for a CLR type, or when there is none (the value is then passed to the + /// driver as it is). maps to ; enums are not mapped here + /// (entity maps store them by name). + /// + /// The CLR type, nullable or not. + /// The logical type, if any. + public static ColumnType? FromClrType(Type type) + { + ArgumentNullException.ThrowIfNull(type); + type = Nullable.GetUnderlyingType(type) ?? type; + return type switch + { + _ when type == typeof(string) => ColumnType.Text, + _ when type == typeof(int) || type == typeof(short) || type == typeof(byte) => ColumnType.Int32, + _ when type == typeof(long) => ColumnType.Int64, + _ when type == typeof(decimal) => ColumnType.Decimal, + _ when type == typeof(bool) => ColumnType.Bool, + _ when type == typeof(DateOnly) => ColumnType.Date, + _ when type == typeof(DateTime) || type == typeof(DateTimeOffset) => ColumnType.DateTimeUtc, + _ when type == typeof(Guid) => ColumnType.Guid, + _ when type == typeof(byte[]) => ColumnType.Bytes, + _ => null, + }; + } + + /// The canonical CLR type the dialects return for a logical type. + /// The logical type. + /// The CLR type. + public static Type ClrType(ColumnType type) => type switch + { + ColumnType.String or ColumnType.Text or ColumnType.Json => typeof(string), + ColumnType.Int32 => typeof(int), + ColumnType.Int64 => typeof(long), + ColumnType.Decimal => typeof(decimal), + ColumnType.Bool => typeof(bool), + ColumnType.Date => typeof(DateOnly), + ColumnType.DateTimeUtc => typeof(DateTime), + ColumnType.Guid => typeof(Guid), + ColumnType.Bytes => typeof(byte[]), + _ => throw new ArgumentOutOfRangeException(nameof(type), type, null), + }; + + /// The logical type as written in the documentation, for example String(20) or Decimal(19,6). + /// The logical type. + /// The length for . + /// The scale for . + /// The description. + public static string Describe(ColumnType type, int? length, int? scale) => type switch + { + ColumnType.String => FormattableString.Invariant($"String({length})"), + ColumnType.Decimal => FormattableString.Invariant($"Decimal({DecimalPrecision},{scale ?? DecimalScale})"), + _ => type.ToString(), + }; +} diff --git a/src/Khaemhat.Data/Schema/DdlGenerator.cs b/src/Khaemhat.Data/Schema/DdlGenerator.cs new file mode 100644 index 0000000..9ce5a0b --- /dev/null +++ b/src/Khaemhat.Data/Schema/DdlGenerator.cs @@ -0,0 +1,144 @@ +using System.Text; + +namespace Khaemhat.Data.Schema; + +/// +/// Turns recorded s into DDL statements for one engine (SPEC §5.3, DdlGenerator). +/// The common SQL lives here; providers override what differs (identity columns, table options, constraints). +/// +/// The dialect of the engine. +public abstract class DdlGenerator(ISqlDialect dialect) +{ + /// The dialect. + public ISqlDialect Dialect { get; } = dialect ?? throw new ArgumentNullException(nameof(dialect)); + + /// Generates the statements for , in order. + /// The operations. + /// The statements, without trailing semicolons. + public IReadOnlyList Generate(IEnumerable operations) + { + ArgumentNullException.ThrowIfNull(operations); + var statements = new List(); + foreach (var operation in operations) + { + switch (operation) + { + case CreateTableOperation create: + statements.Add(CreateTable(create.Table, create.IfNotExists)); + statements.AddRange(create.Table.Indexes.Select(i => CreateIndex(i, create.IfNotExists))); + break; + case DropTableOperation drop: + statements.Add("DROP TABLE " + Dialect.Quote(drop.Table)); + break; + case AddColumnOperation add: + statements.Add("ALTER TABLE " + Dialect.Quote(add.Table) + " ADD COLUMN " + Column(add.Column, inlinePrimaryKey: false)); + break; + case CreateIndexOperation index: + statements.Add(CreateIndex(index.Index, ifNotExists: false)); + break; + case DropIndexOperation dropIndex: + statements.Add("DROP INDEX " + Dialect.Quote(dropIndex.Name)); + break; + case RawSqlOperation raw when raw.Sql(Dialect) is { Length: > 0 } sql: + statements.Add(sql); + break; + } + } + + return statements; + } + + /// The CREATE TABLE statement. + /// The table. + /// Whether to add IF NOT EXISTS. + /// The statement. + protected virtual string CreateTable(TableDefinition table, bool ifNotExists) + { + ArgumentNullException.ThrowIfNull(table); + var identityKey = table.PrimaryKey.Count == 1 && table.Column(table.PrimaryKey[0])?.IsIdentity == true; + var lines = table.Columns.Select(c => Column(c, inlinePrimaryKey: identityKey && c.Name == table.PrimaryKey[0])).ToList(); + if (!identityKey && table.PrimaryKey.Count > 0) + { + lines.Add("CONSTRAINT " + Dialect.Quote("pk_" + table.Name) + " PRIMARY KEY (" + string.Join(", ", table.PrimaryKey.Select(Dialect.Quote)) + ")"); + } + + foreach (var foreignKey in table.ForeignKeys) + { + lines.Add( + "CONSTRAINT " + Dialect.Quote(foreignKey.Name) + " FOREIGN KEY (" + string.Join(", ", foreignKey.Columns.Select(Dialect.Quote)) + + ") REFERENCES " + Dialect.Quote(foreignKey.ReferencedTable) + " (" + string.Join(", ", foreignKey.ReferencedColumns.Select(Dialect.Quote)) + + ") ON DELETE " + foreignKey.OnDelete switch + { + ReferentialAction.Cascade => "CASCADE", + ReferentialAction.SetNull => "SET NULL", + _ => "RESTRICT", + }); + } + + var builder = new StringBuilder("CREATE TABLE "); + if (ifNotExists) + { + builder.Append("IF NOT EXISTS "); + } + + builder.Append(Dialect.Quote(table.Name)).Append(" (\n ").AppendJoin(",\n ", lines).Append("\n)").Append(TableOptions(table)); + return builder.ToString(); + } + + /// Text appended after the closing parenthesis of CREATE TABLE (SQLite: STRICT). + /// The table. + /// The options, possibly empty. + protected virtual string TableOptions(TableDefinition table) => string.Empty; + + /// A column definition. + /// The column. + /// Whether the column is the identity primary key. + /// The SQL. + protected virtual string Column(ColumnDefinition column, bool inlinePrimaryKey) + { + ArgumentNullException.ThrowIfNull(column); + if (inlinePrimaryKey) + { + return IdentityPrimaryKey(column); + } + + var builder = new StringBuilder(Dialect.Quote(column.Name)).Append(' ').Append(Dialect.SqlType(column.Type, column.Length, column.Scale)); + if (!column.IsNullable) + { + builder.Append(" NOT NULL"); + } + + if (column.DefaultsToCurrentTimestamp) + { + builder.Append(" DEFAULT (").Append(Dialect.CurrentTimestampUtc).Append(')'); + } + else if (column.DefaultValue is not null) + { + builder.Append(" DEFAULT ").Append(Dialect.Literal(column.DefaultValue, column.Type)); + } + + builder.Append(ColumnConstraints(column)); + return builder.ToString(); + } + + /// Extra constraints after a column, for example SQLite checks for lengths and booleans. + /// The column. + /// The constraints, possibly empty. + protected virtual string ColumnConstraints(ColumnDefinition column) => string.Empty; + + /// The definition of an identity primary key column. + /// The column. + /// The SQL. + protected abstract string IdentityPrimaryKey(ColumnDefinition column); + + /// The CREATE INDEX statement. + /// The index. + /// Whether to add IF NOT EXISTS. + /// The statement. + protected virtual string CreateIndex(IndexDefinition index, bool ifNotExists) + { + ArgumentNullException.ThrowIfNull(index); + return "CREATE " + (index.IsUnique ? "UNIQUE " : string.Empty) + "INDEX " + (ifNotExists ? "IF NOT EXISTS " : string.Empty) + + Dialect.Quote(index.Name) + " ON " + Dialect.Quote(index.Table) + " (" + string.Join(", ", index.Columns.Select(Dialect.Quote)) + ")"; + } +} diff --git a/src/Khaemhat.Data/Schema/Definitions.cs b/src/Khaemhat.Data/Schema/Definitions.cs new file mode 100644 index 0000000..68efae2 --- /dev/null +++ b/src/Khaemhat.Data/Schema/Definitions.cs @@ -0,0 +1,136 @@ +namespace Khaemhat.Data.Schema; + +/// A column of a table, in engine-neutral form. +public sealed class ColumnDefinition +{ + /// Creates a column definition. + /// The column name (snake_case). + /// The logical type. + public ColumnDefinition(string name, ColumnType type) + { + SqlNames.EnsureIdentifier(name); + Name = name; + Type = type; + } + + /// The column name. + public string Name { get; } + + /// The logical type. + public ColumnType Type { get; } + + /// The maximum length, for . + public int? Length { get; init; } + + /// The scale, for (default 6). + public int? Scale { get; init; } + + /// Whether the column accepts NULL. + public bool IsNullable { get; init; } = true; + + /// Whether the column is an identity primary key (id). + public bool IsIdentity { get; init; } + + /// A constant default value written as an SQL literal, or . + public object? DefaultValue { get; init; } + + /// Whether the default is the current UTC timestamp. + public bool DefaultsToCurrentTimestamp { get; init; } + + /// The logical type as documented, for example String(20). + public string TypeDescription => ColumnTypes.Describe(Type, Length, Scale); +} + +/// An index. +/// The index name (for example ix_parties_name or ux_parties_code). +/// The table. +/// The indexed columns, in order. +/// Whether the index is unique. +public sealed record IndexDefinition(string Name, string Table, IReadOnlyList Columns, bool IsUnique); + +/// What happens to referencing rows when the referenced row is deleted. +public enum ReferentialAction +{ + /// The delete is refused while referencing rows exist (default). + Restrict = 0, + + /// Referencing rows are deleted too. + Cascade = 1, + + /// The referencing column is set to NULL. + SetNull = 2, +} + +/// A foreign key. +/// The constraint name (for example fk_party_addresses_party). +/// The referencing columns. +/// The referenced table. +/// The referenced columns (usually id). +/// The action on delete. +public sealed record ForeignKeyDefinition( + string Name, IReadOnlyList Columns, string ReferencedTable, IReadOnlyList ReferencedColumns, + ReferentialAction OnDelete = ReferentialAction.Restrict); + +/// A table, in engine-neutral form. +public sealed class TableDefinition +{ + private readonly List _columns = []; + private readonly List _indexes = []; + private readonly List _foreignKeys = []; + + /// Creates a table definition. + /// The table name (snake_case). + public TableDefinition(string name) + { + SqlNames.EnsureIdentifier(name); + Name = name; + } + + /// The table name. + public string Name { get; } + + /// The columns in order. + public IReadOnlyList Columns => _columns; + + /// The primary key columns (usually id). + public IReadOnlyList PrimaryKey { get; internal set; } = []; + + /// The indexes. + public IReadOnlyList Indexes => _indexes; + + /// The foreign keys. + public IReadOnlyList ForeignKeys => _foreignKeys; + + /// + /// The date column by whose year PostgreSQL may partition the table (SPEC §5.2, large tables); recorded now, + /// applied by a later migration with PostgresPartitioning. + /// + public string? PartitionByYearColumn { get; internal set; } + + /// Finds a column by name. + /// The column name. + /// The column, or . + public ColumnDefinition? Column(string name) => _columns.Find(c => c.Name == name); + + internal void AddColumn(ColumnDefinition column) + { + if (Column(column.Name) is not null) + { + throw new InvalidOperationException($"Column {column.Name} is defined twice in table {Name}."); + } + + _columns.Add(column); + } + + internal void ReplaceColumn(ColumnDefinition column) + { + var index = _columns.FindIndex(c => c.Name == column.Name); + _columns[index] = column; + } + + internal void AddIndex(IndexDefinition index) => _indexes.Add(index); + + internal void RemoveIndex(string name) => _indexes.RemoveAll(i => i.Name == name); + + internal void AddForeignKey(ForeignKeyDefinition foreignKey) => _foreignKeys.Add(foreignKey); +} diff --git a/src/Khaemhat.Data/Schema/ISchemaBuilder.cs b/src/Khaemhat.Data/Schema/ISchemaBuilder.cs new file mode 100644 index 0000000..bec308b --- /dev/null +++ b/src/Khaemhat.Data/Schema/ISchemaBuilder.cs @@ -0,0 +1,53 @@ +namespace Khaemhat.Data.Schema; + +/// +/// What a migration uses to describe schema changes in engine-neutral form (SPEC §5.3). The calls are recorded as +/// s; each provider's turns them into DDL for its engine. +/// +/// schema.CreateTable("sys_settings", t => +/// { +/// t.Id(); +/// t.String("key", 200).NotNull(); +/// t.Text("value"); +/// t.AuditColumns(); +/// t.Unique("ux_sys_settings_key", "key"); +/// }); +/// +/// +public interface ISchemaBuilder +{ + /// Creates a table. + /// The table name. + /// Defines columns, keys, indexes and foreign keys. + void CreateTable(string name, Action define); + + /// + /// Changes an existing table: adds columns and indexes. Foreign keys can only be declared when the table is + /// created (SQLite cannot add them later). + /// + /// The table name. + /// Adds columns and indexes. + void AlterTable(string name, Action define); + + /// Drops a table. + /// The table name. + void DropTable(string name); + + /// Creates an index. + /// The table. + /// The index name. + /// Whether it is unique. + /// The columns. + void CreateIndex(string table, string name, bool unique, params string[] columns); + + /// Drops an index. + /// The index name. + void DropIndex(string name); + + /// + /// Adds a dialect-specific statement at this point, for what the neutral operations cannot express. The function + /// returns for dialects where nothing is needed. + /// + /// Produces the trusted SQL for a dialect. + void Sql(Func sql); +} diff --git a/src/Khaemhat.Data/Schema/SchemaBuilder.cs b/src/Khaemhat.Data/Schema/SchemaBuilder.cs new file mode 100644 index 0000000..3d546ea --- /dev/null +++ b/src/Khaemhat.Data/Schema/SchemaBuilder.cs @@ -0,0 +1,53 @@ +namespace Khaemhat.Data.Schema; + +/// Records the calls of a migration as s. +public sealed class SchemaBuilder : ISchemaBuilder +{ + private readonly List _operations = []; + + /// The recorded operations, in order. + public IReadOnlyList Operations => _operations; + + /// + public void CreateTable(string name, Action define) + { + ArgumentNullException.ThrowIfNull(define); + var builder = new TableBuilder(name, alter: false); + define(builder); + _operations.AddRange(builder.Build()); + } + + /// + public void AlterTable(string name, Action define) + { + ArgumentNullException.ThrowIfNull(define); + var builder = new TableBuilder(name, alter: true); + define(builder); + _operations.AddRange(builder.Build()); + } + + /// + public void DropTable(string name) + { + SqlNames.EnsureIdentifier(name); + _operations.Add(new DropTableOperation(name)); + } + + /// + public void CreateIndex(string table, string name, bool unique, params string[] columns) => + AlterTable(table, t => _ = unique ? t.Unique(name, columns) : t.Index(name, columns)); + + /// + public void DropIndex(string name) + { + SqlNames.EnsureIdentifier(name); + _operations.Add(new DropIndexOperation(name)); + } + + /// + public void Sql(Func sql) + { + ArgumentNullException.ThrowIfNull(sql); + _operations.Add(new RawSqlOperation(sql)); + } +} diff --git a/src/Khaemhat.Data/Schema/SchemaModel.cs b/src/Khaemhat.Data/Schema/SchemaModel.cs new file mode 100644 index 0000000..808defd --- /dev/null +++ b/src/Khaemhat.Data/Schema/SchemaModel.cs @@ -0,0 +1,86 @@ +namespace Khaemhat.Data.Schema; + +/// +/// The schema that results from applying operations, kept in memory: used to document the data model +/// (scripts/gen-datamodel.sh) and to check that entity maps agree with the migrations. Dialect-specific +/// statements () are not modelled. +/// +public sealed class SchemaModel +{ + private readonly SortedDictionary _tables = new(StringComparer.Ordinal); + + /// The tables, by name. + public IReadOnlyCollection Tables => _tables.Values; + + /// Finds a table. + /// The table name. + /// The table, or . + public TableDefinition? Table(string name) => _tables.GetValueOrDefault(name); + + /// Applies operations in order. + /// The operations. + public void Apply(IEnumerable operations) + { + ArgumentNullException.ThrowIfNull(operations); + foreach (var operation in operations) + { + Apply(operation); + } + } + + private void Apply(SchemaOperation operation) + { + switch (operation) + { + case CreateTableOperation create when create.IfNotExists && _tables.ContainsKey(create.Table.Name): + break; + case CreateTableOperation create: + if (!_tables.TryAdd(create.Table.Name, Copy(create.Table))) + { + throw new InvalidOperationException($"Table {create.Table.Name} is created twice."); + } + + break; + case DropTableOperation drop: + _tables.Remove(drop.Table); + break; + case AddColumnOperation add: + Require(add.Table).AddColumn(add.Column); + break; + case CreateIndexOperation index: + Require(index.Index.Table).AddIndex(index.Index); + break; + case DropIndexOperation dropIndex: + foreach (var table in _tables.Values) + { + table.RemoveIndex(dropIndex.Name); + } + + break; + } + } + + private TableDefinition Require(string name) => + _tables.GetValueOrDefault(name) ?? throw new InvalidOperationException($"Table {name} does not exist."); + + private static TableDefinition Copy(TableDefinition source) + { + var copy = new TableDefinition(source.Name) { PrimaryKey = source.PrimaryKey, PartitionByYearColumn = source.PartitionByYearColumn }; + foreach (var column in source.Columns) + { + copy.AddColumn(column); + } + + foreach (var index in source.Indexes) + { + copy.AddIndex(index); + } + + foreach (var foreignKey in source.ForeignKeys) + { + copy.AddForeignKey(foreignKey); + } + + return copy; + } +} diff --git a/src/Khaemhat.Data/Schema/SchemaOperations.cs b/src/Khaemhat.Data/Schema/SchemaOperations.cs new file mode 100644 index 0000000..71eb310 --- /dev/null +++ b/src/Khaemhat.Data/Schema/SchemaOperations.cs @@ -0,0 +1,30 @@ +namespace Khaemhat.Data.Schema; + +/// A schema change recorded by . +public abstract record SchemaOperation; + +/// Creates a table. +/// The table. +/// Whether to skip the creation when the table exists (only for schema_version). +public sealed record CreateTableOperation(TableDefinition Table, bool IfNotExists = false) : SchemaOperation; + +/// Drops a table. +/// The table name. +public sealed record DropTableOperation(string Table) : SchemaOperation; + +/// Adds a column to an existing table. +/// The table name. +/// The column. +public sealed record AddColumnOperation(string Table, ColumnDefinition Column) : SchemaOperation; + +/// Creates an index. +/// The index. +public sealed record CreateIndexOperation(IndexDefinition Index) : SchemaOperation; + +/// Drops an index. +/// The index name. +public sealed record DropIndexOperation(string Name) : SchemaOperation; + +/// A dialect-specific statement. +/// Produces the SQL for a dialect, or . +public sealed record RawSqlOperation(Func Sql) : SchemaOperation; diff --git a/src/Khaemhat.Data/Schema/TableBuilder.cs b/src/Khaemhat.Data/Schema/TableBuilder.cs new file mode 100644 index 0000000..370ff84 --- /dev/null +++ b/src/Khaemhat.Data/Schema/TableBuilder.cs @@ -0,0 +1,347 @@ +namespace Khaemhat.Data.Schema; + +/// Defines the columns, keys, indexes and foreign keys of a table inside . +public sealed class TableBuilder +{ + private readonly List _columns = []; + private readonly List _indexes = []; + private readonly List _foreignKeys = []; + private readonly bool _alter; + private string[] _primaryKey = []; + private string? _partitionColumn; + + internal TableBuilder(string name, bool alter) + { + SqlNames.EnsureIdentifier(name); + Name = name; + _alter = alter; + } + + /// The table name. + public string Name { get; } + + /// The standard primary key: id, 64-bit identity. + /// This builder. + public TableBuilder Id() + { + var column = Add("id", ColumnType.Int64).NotNull(); + column.Identity = true; + _primaryKey = ["id"]; + return this; + } + + /// Adds a column of at most characters. + /// The column name. + /// The maximum length (1 to 10485760). + /// The column builder. + public ColumnBuilder String(string name, int length) + { + ArgumentOutOfRangeException.ThrowIfLessThan(length, 1); + ArgumentOutOfRangeException.ThrowIfGreaterThan(length, 10_485_760); + var column = Add(name, ColumnType.String); + column.Length = length; + return column; + } + + /// Adds a column. + /// The column name. + /// The column builder. + public ColumnBuilder Text(string name) => Add(name, ColumnType.Text); + + /// Adds an column. + /// The column name. + /// The column builder. + public ColumnBuilder Int32(string name) => Add(name, ColumnType.Int32); + + /// Adds an column (for example a foreign key party_id). + /// The column name. + /// The column builder. + public ColumnBuilder Int64(string name) => Add(name, ColumnType.Int64); + + /// Adds a column, precision 19. + /// The column name. + /// Digits after the point, 0 to 6. + /// The column builder. + public ColumnBuilder Decimal(string name, int scale = ColumnTypes.DecimalScale) + { + ArgumentOutOfRangeException.ThrowIfNegative(scale); + ArgumentOutOfRangeException.ThrowIfGreaterThan(scale, ColumnTypes.DecimalScale); + var column = Add(name, ColumnType.Decimal); + column.Scale = scale; + return column; + } + + /// Adds a column. + /// The column name. + /// The column builder. + public ColumnBuilder Bool(string name) => Add(name, ColumnType.Bool); + + /// Adds a column. + /// The column name. + /// The column builder. + public ColumnBuilder Date(string name) => Add(name, ColumnType.Date); + + /// Adds a column. + /// The column name. + /// The column builder. + public ColumnBuilder DateTimeUtc(string name) => Add(name, ColumnType.DateTimeUtc); + + /// Adds a column. + /// The column name. + /// The column builder. + public ColumnBuilder Guid(string name) => Add(name, ColumnType.Guid); + + /// Adds a column. + /// The column name. + /// The column builder. + public ColumnBuilder Bytes(string name) => Add(name, ColumnType.Bytes); + + /// Adds a column. + /// The column name. + /// The column builder. + public ColumnBuilder Json(string name) => Add(name, ColumnType.Json); + + /// + /// Adds the audit columns of SPEC §5.2: created_at, created_by, updated_at, updated_by, + /// row_version (default 1). + /// + /// This builder. + public TableBuilder AuditColumns() + { + DateTimeUtc("created_at").NotNull(); + String("created_by", AuditColumnLengths.User); + DateTimeUtc("updated_at"); + String("updated_by", AuditColumnLengths.User); + Int64("row_version").NotNull().Default(1L); + return this; + } + + /// Adds is_demo (boolean, default false) for rows created by the demo seeder (SPEC §14). + /// This builder. + public TableBuilder DemoColumn() + { + Bool("is_demo").NotNull().Default(false); + return this; + } + + /// Sets a primary key other than , for example a composite key. + /// The key columns. + /// This builder. + public TableBuilder PrimaryKey(params string[] columns) + { + EnsureCreate("A primary key"); + ArgumentNullException.ThrowIfNull(columns); + ArgumentOutOfRangeException.ThrowIfZero(columns.Length); + _primaryKey = columns; + return this; + } + + /// Adds a non-unique index. + /// The index name, for example ix_parties_name. + /// The columns. + /// This builder. + public TableBuilder Index(string name, params string[] columns) => AddIndex(name, false, columns); + + /// Adds a unique index. + /// The index name, for example ux_parties_code. + /// The columns. + /// This builder. + public TableBuilder Unique(string name, params string[] columns) => AddIndex(name, true, columns); + + /// Adds a foreign key from to the id of . + /// The constraint name, for example fk_party_addresses_party. + /// The referencing column. + /// The referenced table. + /// The action on delete. + /// This builder. + public TableBuilder ForeignKey(string name, string column, string referencedTable, ReferentialAction onDelete = ReferentialAction.Restrict) + { + EnsureCreate("A foreign key"); + SqlNames.EnsureIdentifier(name); + SqlNames.EnsureIdentifier(column); + SqlNames.EnsureIdentifier(referencedTable); + _foreignKeys.Add(new ForeignKeyDefinition(name, [column], referencedTable, ["id"], onDelete)); + return this; + } + + /// + /// Records that PostgreSQL may partition this table by the year of (SPEC §5.2). Nothing + /// changes now; a later migration activates it beyond 50 million rows. + /// + /// A or column. + /// This builder. + public TableBuilder PartitionByYear(string column) + { + SqlNames.EnsureIdentifier(column); + _partitionColumn = column; + return this; + } + + internal IEnumerable Build() + { + if (!_alter) + { + var table = new TableDefinition(Name) { PrimaryKey = _primaryKey, PartitionByYearColumn = _partitionColumn }; + foreach (var column in _columns) + { + table.AddColumn(column.ToDefinition()); + } + + foreach (var index in _indexes) + { + EnsureColumnsExist(table, index.Columns); + table.AddIndex(index); + } + + foreach (var foreignKey in _foreignKeys) + { + EnsureColumnsExist(table, foreignKey.Columns); + table.AddForeignKey(foreignKey); + } + + EnsureColumnsExist(table, table.PrimaryKey); + yield return new CreateTableOperation(table); + yield break; + } + + foreach (var column in _columns) + { + yield return new AddColumnOperation(Name, column.ToDefinition()); + } + + foreach (var index in _indexes) + { + yield return new CreateIndexOperation(index); + } + } + + private static void EnsureColumnsExist(TableDefinition table, IEnumerable columns) + { + foreach (var column in columns) + { + if (table.Column(column) is null) + { + throw new InvalidOperationException($"Table {table.Name} has no column {column}."); + } + } + } + + private ColumnBuilder Add(string name, ColumnType type) + { + if (_columns.Exists(c => c.Name == name)) + { + throw new InvalidOperationException($"Column {name} is defined twice in table {Name}."); + } + + var column = new ColumnBuilder(name, type); + _columns.Add(column); + return column; + } + + private TableBuilder AddIndex(string name, bool unique, string[] columns) + { + SqlNames.EnsureIdentifier(name); + ArgumentNullException.ThrowIfNull(columns); + ArgumentOutOfRangeException.ThrowIfZero(columns.Length); + foreach (var column in columns) + { + SqlNames.EnsureIdentifier(column); + } + + _indexes.Add(new IndexDefinition(name, Name, columns, unique)); + return this; + } + + private void EnsureCreate(string what) + { + if (_alter) + { + throw new NotSupportedException(what + " can only be declared when the table is created."); + } + } +} + +/// Configures a column inside . +public sealed class ColumnBuilder +{ + internal ColumnBuilder(string name, ColumnType type) + { + SqlNames.EnsureIdentifier(name); + Name = name; + Type = type; + } + + /// The column name. + public string Name { get; } + + /// The logical type. + public ColumnType Type { get; } + + internal int? Length { get; set; } + + internal int? Scale { get; set; } + + internal bool Identity { get; set; } + + private bool Nullable { get; set; } = true; + + private object? DefaultValue { get; set; } + + private bool DefaultTimestamp { get; set; } + + /// The column refuses NULL. + /// This builder. + public ColumnBuilder NotNull() + { + Nullable = false; + return this; + } + + /// The column accepts NULL (the default). + /// This builder. + public ColumnBuilder Null() + { + Nullable = true; + return this; + } + + /// A constant default value (string, number, boolean or decimal). + /// The value. + /// This builder. + public ColumnBuilder Default(object value) + { + ArgumentNullException.ThrowIfNull(value); + DefaultValue = value; + return this; + } + + /// The current UTC timestamp as default, for . + /// This builder. + public ColumnBuilder DefaultCurrentTimestamp() + { + if (Type != ColumnType.DateTimeUtc) + { + throw new InvalidOperationException("Only DateTimeUtc columns can default to the current timestamp."); + } + + DefaultTimestamp = true; + return this; + } + + internal ColumnDefinition ToDefinition() => new(Name, Type) + { + Length = Length, + Scale = Scale, + IsNullable = Nullable, + IsIdentity = Identity, + DefaultValue = DefaultValue, + DefaultsToCurrentTimestamp = DefaultTimestamp, + }; +} + +/// Lengths of the audit columns. +public static class AuditColumnLengths +{ + /// Length of created_by and updated_by. + public const int User = 100; +} diff --git a/src/Khaemhat.Data/SqlDialectBase.cs b/src/Khaemhat.Data/SqlDialectBase.cs new file mode 100644 index 0000000..2da8862 --- /dev/null +++ b/src/Khaemhat.Data/SqlDialectBase.cs @@ -0,0 +1,130 @@ +using System.Data.Common; +using System.Globalization; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data; + +/// +/// The parts of that SQLite (3.35 or later) and PostgreSQL write the same way: double-quoted +/// identifiers, @name parameters, LIMIT/OFFSET, RETURNING and ON CONFLICT … DO UPDATE. +/// +public abstract class SqlDialectBase : ISqlDialect +{ + /// + public abstract DbEngine Engine { get; } + + /// + public abstract string CurrentTimestampUtc { get; } + + /// + public string Quote(string identifier) + { + SqlNames.EnsureIdentifier(identifier); + return "\"" + identifier + "\""; + } + + /// + public string Parameter(string name) => "@" + name; + + /// + public string Paging(int offset, int limit) + { + ArgumentOutOfRangeException.ThrowIfNegative(offset); + ArgumentOutOfRangeException.ThrowIfNegative(limit); + return string.Create(CultureInfo.InvariantCulture, $" LIMIT {limit} OFFSET {offset}"); + } + + /// + public string InsertReturningId(string table, IReadOnlyList columns, IReadOnlyList parameters, string keyColumn) => + Insert(table, columns, parameters) + " RETURNING " + Quote(keyColumn); + + /// + public string Upsert( + string table, IReadOnlyList columns, IReadOnlyList parameters, + IReadOnlyList conflictColumns, IReadOnlyList updateColumns) + { + ArgumentNullException.ThrowIfNull(conflictColumns); + ArgumentNullException.ThrowIfNull(updateColumns); + var conflict = string.Join(", ", conflictColumns.Select(Quote)); + var action = updateColumns.Count == 0 + ? "DO NOTHING" + : "DO UPDATE SET " + string.Join(", ", updateColumns.Select(c => Quote(c) + " = excluded." + Quote(c))); + return Insert(table, columns, parameters) + " ON CONFLICT (" + conflict + ") " + action; + } + + /// + public abstract SqlFragment CaseInsensitiveLike(SqlFragment column, SqlFragment pattern); + + /// + public abstract SqlFragment AdvisoryLock(string key); + + /// + public abstract object ToDbValue(object? value, ColumnType type); + + /// + public abstract object? FromDbValue(object? value, ColumnType type); + + /// + public abstract string SqlType(ColumnType type, int? length, int? scale); + + /// + public virtual string Literal(object? value, ColumnType type) => value switch + { + null => "NULL", + string s => "'" + s.Replace("'", "''", StringComparison.Ordinal) + "'", + Enum e => "'" + e.ToString() + "'", + IFormattable f when type is ColumnType.Int32 or ColumnType.Int64 => f.ToString(null, CultureInfo.InvariantCulture), + _ => throw new NotSupportedException($"No SQL literal for a {value.GetType().Name} default of a {type} column."), + }; + + /// + public virtual void ConfigureParameter(DbParameter parameter, object? value, ColumnType? type) + { + ArgumentNullException.ThrowIfNull(parameter); + var logical = type ?? (value is null ? null : ColumnTypes.FromClrType(value.GetType())); + parameter.Value = logical is null ? value ?? DBNull.Value : ToDbValue(value, logical.Value); + } + + /// Converts a value to with invariant culture, as drivers may widen numbers. + /// The target type. + /// The value. + /// The converted value. + protected static T ConvertTo(object value) => (T)Convert.ChangeType(value, typeof(T), CultureInfo.InvariantCulture); + + /// Converts a or to UTC. An unspecified kind is taken as UTC. + /// The value. + /// The UTC instant. + protected static DateTime ToUtc(object value) => value switch + { + DateTime { Kind: DateTimeKind.Utc } d => d, + DateTime { Kind: DateTimeKind.Local } d => d.ToUniversalTime(), + DateTime d => DateTime.SpecifyKind(d, DateTimeKind.Utc), + DateTimeOffset o => o.UtcDateTime, + _ => throw new InvalidCastException($"A {value.GetType().Name} is not a timestamp."), + }; + + /// Converts a value to . + /// A , or yyyy-MM-dd text. + /// The date. + protected static DateOnly ToDate(object value) => value switch + { + DateOnly d => d, + DateTime d => DateOnly.FromDateTime(d), + string s => DateOnly.ParseExact(s, "yyyy-MM-dd", CultureInfo.InvariantCulture), + _ => throw new InvalidCastException($"A {value.GetType().Name} is not a date."), + }; + + private string Insert(string table, IReadOnlyList columns, IReadOnlyList parameters) + { + ArgumentNullException.ThrowIfNull(columns); + ArgumentNullException.ThrowIfNull(parameters); + if (columns.Count != parameters.Count) + { + throw new ArgumentException("One parameter per column is required.", nameof(parameters)); + } + + return columns.Count == 0 + ? "INSERT INTO " + Quote(table) + " DEFAULT VALUES" + : "INSERT INTO " + Quote(table) + " (" + string.Join(", ", columns.Select(Quote)) + ") VALUES (" + string.Join(", ", parameters) + ")"; + } +} diff --git a/src/Khaemhat.Data/SqlFragment.cs b/src/Khaemhat.Data/SqlFragment.cs new file mode 100644 index 0000000..b9a9a09 --- /dev/null +++ b/src/Khaemhat.Data/SqlFragment.cs @@ -0,0 +1,249 @@ +using System.Collections.Immutable; +using System.Globalization; +using System.Runtime.CompilerServices; +using System.Text; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data; + +/// +/// A piece of SQL whose values are always parameters (SPEC §5.2: never concatenate user input into SQL). +/// Build it with an interpolated string: the literal text is code written by us, every hole becomes a parameter. +/// +/// var sql = SqlFragment.Of($"SELECT name FROM parties WHERE code = {code} AND is_active = {true}"); +/// +/// A hole can also hold another (inserted as SQL, its parameters renumbered) or an +/// identifier made with (quoted by the dialect). The text is produced only by +/// , for one dialect. +/// +public sealed class SqlFragment +{ + private readonly ImmutableArray _parts; + + private SqlFragment(ImmutableArray parts) => _parts = parts; + + /// The empty fragment. + public static SqlFragment Empty { get; } = new([]); + + /// Whether the fragment contains nothing. + public bool IsEmpty => _parts.IsEmpty; + + /// The number of parameters. + public int ParameterCount => _parts.Count(p => p is ParameterPart); + + internal ImmutableArray Parts => _parts; + + internal static SqlFragment FromParts(ImmutableArray parts) => parts.IsEmpty ? Empty : new(parts); + + /// Builds a fragment from an interpolated string: holes become parameters. + /// The interpolated string. + /// The fragment. + public static SqlFragment Of(ref SqlInterpolatedStringHandler handler) => handler.ToFragment(); + + /// + /// A fragment of trusted SQL text written in the code (keywords, operators, DDL generated from definitions). + /// Never pass text that comes from users, files or the network: use parameters for values. + /// + /// The SQL text. + /// The fragment. + public static SqlFragment Literal(string trustedSql) + { + ArgumentNullException.ThrowIfNull(trustedSql); + return trustedSql.Length == 0 ? Empty : new([new TextPart(trustedSql)]); + } + + /// A table or column name, quoted by the dialect when rendered. + /// The identifier: lower-case letters, digits and underscore, optionally table.column. + /// The fragment. + public static SqlFragment Identifier(string name) + { + SqlNames.EnsureQualifiedIdentifier(name); + return new([new IdentifierPart(name)]); + } + + /// A single parameter, optionally with its logical type (needed for example for ). + /// The value. + /// The logical type, or to infer it from the CLR type. + /// The fragment. + public static SqlFragment Parameter(object? value, ColumnType? type = null) => new([new ParameterPart(value, type)]); + + /// Joins fragments with a trusted separator such as ", " or " AND ". + /// The separator text. + /// The fragments; empty ones are skipped. + /// The joined fragment. + public static SqlFragment Join(string separator, IEnumerable fragments) + { + ArgumentNullException.ThrowIfNull(fragments); + var builder = ImmutableArray.CreateBuilder(); + var first = true; + foreach (var fragment in fragments) + { + if (fragment.IsEmpty) + { + continue; + } + + if (!first && separator.Length > 0) + { + builder.Add(new TextPart(separator)); + } + + builder.AddRange(fragment._parts); + first = false; + } + + return new(builder.ToImmutable()); + } + + /// Concatenates two fragments. + /// The first fragment. + /// The second fragment. + /// The concatenation. + public static SqlFragment operator +(SqlFragment left, SqlFragment right) + { + ArgumentNullException.ThrowIfNull(left); + ArgumentNullException.ThrowIfNull(right); + return left.Append(right); + } + + /// Appends another fragment. + /// The fragment to append. + /// The concatenation. + public SqlFragment Append(SqlFragment other) + { + ArgumentNullException.ThrowIfNull(other); + if (other.IsEmpty) + { + return this; + } + + return IsEmpty ? other : new(_parts.AddRange(other._parts)); + } + + /// Wraps the fragment in parentheses. + /// The wrapped fragment. + public SqlFragment Parenthesize() => IsEmpty ? this : Literal("(") + this + Literal(")"); + + /// Produces the SQL text and the parameters for a dialect. Parameters are named p0, p1, … + /// The dialect. + /// The text and the parameters. + public RenderedSql Render(ISqlDialect dialect) + { + ArgumentNullException.ThrowIfNull(dialect); + var text = new StringBuilder(); + var parameters = new List(); + foreach (var part in _parts) + { + switch (part) + { + case TextPart t: + text.Append(t.Text); + break; + case IdentifierPart i: + text.Append(QuoteQualified(dialect, i.Name)); + break; + case ParameterPart p: + var name = "p" + parameters.Count.ToString(CultureInfo.InvariantCulture); + text.Append(dialect.Parameter(name)); + parameters.Add(new SqlParameterValue(name, p.Value, p.Type)); + break; + } + } + + return new RenderedSql(text.ToString(), parameters); + } + + /// A readable form for diagnostics: parameters appear as @p0, never with their values. + /// The text. + public override string ToString() + { + var text = new StringBuilder(); + var index = 0; + foreach (var part in _parts) + { + text.Append(part switch + { + TextPart t => t.Text, + IdentifierPart i => "\"" + i.Name + "\"", + _ => "@p" + (index++).ToString(CultureInfo.InvariantCulture), + }); + } + + return text.ToString(); + } + + private static string QuoteQualified(ISqlDialect dialect, string name) + { + var dot = name.IndexOf('.', StringComparison.Ordinal); + return dot < 0 ? dialect.Quote(name) : dialect.Quote(name[..dot]) + "." + dialect.Quote(name[(dot + 1)..]); + } + + internal abstract record SqlPart; + + internal sealed record TextPart(string Text) : SqlPart; + + internal sealed record IdentifierPart(string Name) : SqlPart; + + internal sealed record ParameterPart(object? Value, ColumnType? Type) : SqlPart; +} + +/// SQL text and parameters produced by . +/// The SQL text. +/// The parameters in order. +public sealed record RenderedSql(string Text, IReadOnlyList Parameters); + +/// A parameter of a rendered statement. +/// The name without prefix, for example p0. +/// The CLR value. +/// The logical type, when known. +public sealed record SqlParameterValue(string Name, object? Value, ColumnType? Type) +{ + /// + public override string ToString() => "@" + Name; +} + +/// +/// Interpolated string handler for : literal parts are kept as SQL text, holes become +/// parameters (or nested fragments). +/// +[InterpolatedStringHandler] +public ref struct SqlInterpolatedStringHandler +{ + private readonly ImmutableArray.Builder _parts; + + /// Called by the compiler. + /// Total length of the literal parts. + /// Number of holes. + public SqlInterpolatedStringHandler(int literalLength, int formattedCount) + { + _ = literalLength; + _parts = ImmutableArray.CreateBuilder((formattedCount * 2) + 1); + } + + /// Called by the compiler for literal text. + /// The literal. + public readonly void AppendLiteral(string value) + { + if (value.Length > 0) + { + _parts.Add(new SqlFragment.TextPart(value)); + } + } + + /// Called by the compiler for a hole: a parameter, or a nested fragment. + /// The value type. + /// The value. + public readonly void AppendFormatted(T value) + { + if (value is SqlFragment fragment) + { + _parts.AddRange(fragment.Parts); + } + else + { + _parts.Add(new SqlFragment.ParameterPart(value, null)); + } + } + + internal readonly SqlFragment ToFragment() => SqlFragment.FromParts(_parts.ToImmutable()); +} diff --git a/src/Khaemhat.Data/SqlNames.cs b/src/Khaemhat.Data/SqlNames.cs new file mode 100644 index 0000000..3cdbdb4 --- /dev/null +++ b/src/Khaemhat.Data/SqlNames.cs @@ -0,0 +1,82 @@ +using System.Text; +using System.Text.RegularExpressions; + +namespace Khaemhat.Data; + +/// +/// Naming rules for tables, columns, indexes and constraints (SPEC §5.2): snake_case, lower-case ASCII letters, +/// digits and underscore, at most 63 characters (the PostgreSQL limit). Identifiers are validated before being quoted, +/// so they can never carry SQL. +/// +public static partial class SqlNames +{ + /// Whether is a valid identifier. + /// The name. + /// if valid. + public static bool IsValidIdentifier(string? name) => name is not null && IdentifierPattern().IsMatch(name); + + /// Throws if is not a valid identifier. + /// The name. + /// Not a valid identifier. + public static void EnsureIdentifier(string name) + { + if (!IsValidIdentifier(name)) + { + throw new ArgumentException( + $"'{name}' is not a valid SQL identifier: use lower-case letters, digits and underscore (snake_case).", nameof(name)); + } + } + + /// Throws if is neither an identifier nor table.column. + /// The name. + public static void EnsureQualifiedIdentifier(string name) + { + ArgumentNullException.ThrowIfNull(name); + var dot = name.IndexOf('.', StringComparison.Ordinal); + if (dot < 0) + { + EnsureIdentifier(name); + return; + } + + EnsureIdentifier(name[..dot]); + EnsureIdentifier(name[(dot + 1)..]); + } + + /// + /// Converts a C# name to snake_case: RowVersion → row_version, VatID → vat_id, + /// HTTPCode → http_code, Address2 → address2. + /// + /// The C# name. + /// The snake_case name. + public static string ToSnakeCase(string name) + { + ArgumentException.ThrowIfNullOrEmpty(name); + var builder = new StringBuilder(name.Length + 8); + for (var i = 0; i < name.Length; i++) + { + var c = name[i]; + if (char.IsUpper(c)) + { + var previous = i > 0 ? name[i - 1] : '\0'; + var next = i + 1 < name.Length ? name[i + 1] : '\0'; + var startsWord = i > 0 && (char.IsLower(previous) || char.IsDigit(previous) || (char.IsUpper(previous) && char.IsLower(next))); + if (startsWord && previous != '_') + { + builder.Append('_'); + } + + builder.Append(char.ToLowerInvariant(c)); + } + else + { + builder.Append(c); + } + } + + return builder.ToString(); + } + + [GeneratedRegex("^[a-z_][a-z0-9_]{0,62}$", RegexOptions.CultureInvariant)] + private static partial Regex IdentifierPattern(); +} diff --git a/src/Khaemhat.Data/SystemDatabase/Migrations/M20261111_001_CreateSysCompanies.cs b/src/Khaemhat.Data/SystemDatabase/Migrations/M20261111_001_CreateSysCompanies.cs new file mode 100644 index 0000000..cc5ed88 --- /dev/null +++ b/src/Khaemhat.Data/SystemDatabase/Migrations/M20261111_001_CreateSysCompanies.cs @@ -0,0 +1,28 @@ +using Khaemhat.Data.Migrations; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.SystemDatabase.Migrations; + +/// The company registry (SPEC §5.4): one row per company database, its location stored encrypted. +[Migration(20261111_001, "create_sys_companies", Target = MigrationTarget.System)] +public sealed class M20261111_001_CreateSysCompanies : IMigration +{ + /// + public void Up(ISchemaBuilder schema) + { + ArgumentNullException.ThrowIfNull(schema); + schema.CreateTable("sys_companies", t => + { + t.Id(); + t.String("code", 20).NotNull(); + t.String("name", 200).NotNull(); + t.String("engine", 20).NotNull(); + t.Text("connection_string").NotNull(); + t.String("status", 40).NotNull().Default("Active"); + t.Bool("is_default").NotNull().Default(false); + t.Text("notes"); + t.AuditColumns(); + t.Unique("ux_sys_companies_code", "code"); + }); + } +} diff --git a/src/Khaemhat.Data/SystemDatabase/Migrations/M20261111_002_CreateSysSettings.cs b/src/Khaemhat.Data/SystemDatabase/Migrations/M20261111_002_CreateSysSettings.cs new file mode 100644 index 0000000..e0eea22 --- /dev/null +++ b/src/Khaemhat.Data/SystemDatabase/Migrations/M20261111_002_CreateSysSettings.cs @@ -0,0 +1,24 @@ +using Khaemhat.Data.Migrations; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.SystemDatabase.Migrations; + +/// Instance settings (SPEC §5.4): key/value pairs, secrets encrypted with the instance key. +[Migration(20261111_002, "create_sys_settings", Target = MigrationTarget.System)] +public sealed class M20261111_002_CreateSysSettings : IMigration +{ + /// + public void Up(ISchemaBuilder schema) + { + ArgumentNullException.ThrowIfNull(schema); + schema.CreateTable("sys_settings", t => + { + t.Id(); + t.String("key", 200).NotNull(); + t.Text("value"); + t.Bool("is_secret").NotNull().Default(false); + t.AuditColumns(); + t.Unique("ux_sys_settings_key", "key"); + }); + } +} diff --git a/src/Khaemhat.Data/SystemDatabase/SystemMaps.cs b/src/Khaemhat.Data/SystemDatabase/SystemMaps.cs new file mode 100644 index 0000000..6b7a734 --- /dev/null +++ b/src/Khaemhat.Data/SystemDatabase/SystemMaps.cs @@ -0,0 +1,29 @@ +using Khaemhat.Core.Instance; +using Khaemhat.Data.Mapping; + +namespace Khaemhat.Data.SystemDatabase; + +/// Map of to sys_companies. +public sealed class CompanyMap : EntityMap +{ + /// Creates the map. + public CompanyMap() + { + ToTable("sys_companies"); + Map(x => x.Code).MaxLength(20); + Map(x => x.Name).MaxLength(200); + Map(x => x.Engine).MaxLength(20); + Map(x => x.Status).MaxLength(40); + } +} + +/// Map of to sys_settings. +public sealed class SystemSettingMap : EntityMap +{ + /// Creates the map. + public SystemSettingMap() + { + ToTable("sys_settings"); + Map(x => x.Key).MaxLength(200); + } +} diff --git a/src/Khaemhat.Data/UnitOfWorkExtensions.cs b/src/Khaemhat.Data/UnitOfWorkExtensions.cs new file mode 100644 index 0000000..5550fa3 --- /dev/null +++ b/src/Khaemhat.Data/UnitOfWorkExtensions.cs @@ -0,0 +1,55 @@ +namespace Khaemhat.Data; + +/// Helpers on . +public static class UnitOfWorkExtensions +{ + /// + /// Takes the application lock until the end of the current transaction (numerators, closings, + /// scheduler): pg_advisory_xact_lock on PostgreSQL; on SQLite the transaction already holds the write lock. + /// + /// The unit of work, with an open transaction. + /// The lock key, for example numerator:FV:2026. + /// Cancellation. + /// A task completed when the lock is held. + public static async Task AcquireLockAsync(this IUnitOfWork unitOfWork, string key, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(unitOfWork); + ArgumentException.ThrowIfNullOrEmpty(key); + if (unitOfWork.Transaction is null) + { + throw new InvalidOperationException("An application lock is held until the end of a transaction: call BeginAsync first."); + } + + var statement = unitOfWork.Dialect.AdvisoryLock(key); + if (!statement.IsEmpty) + { + await unitOfWork.ExecuteAsync(statement, cancellationToken).ConfigureAwait(false); + } + } + + /// Runs in a transaction: commits if it completes, rolls back if it throws. + /// The unit of work. + /// The work. + /// Cancellation. + /// A task. + public static async Task InTransactionAsync(this IUnitOfWork unitOfWork, Func work, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(unitOfWork); + ArgumentNullException.ThrowIfNull(work); + await unitOfWork.BeginAsync(cancellationToken).ConfigureAwait(false); + try + { + await work(cancellationToken).ConfigureAwait(false); + await unitOfWork.CommitAsync(cancellationToken).ConfigureAwait(false); + } + catch + { + if (unitOfWork.Transaction is not null) + { + await unitOfWork.RollbackAsync(CancellationToken.None).ConfigureAwait(false); + } + + throw; + } + } +} diff --git a/src/Khaemhat.Infrastructure/Security/SecretProtector.cs b/src/Khaemhat.Infrastructure/Security/SecretProtector.cs new file mode 100644 index 0000000..663eac0 --- /dev/null +++ b/src/Khaemhat.Infrastructure/Security/SecretProtector.cs @@ -0,0 +1,189 @@ +using System.Security.Cryptography; +using System.Text; +using Khaemhat.Core.Security; + +namespace Khaemhat.Infrastructure.Security; + +/// +/// The instance key from KHAEMHAT_SECRET_KEY: base64 of at least 32 random bytes (SPEC §6). The encryption key +/// is derived with HKDF-SHA256, so the configured bytes are never used directly. The key is never printed. +/// +public sealed class SecretKey +{ + /// The minimum length of the configured key, in bytes. + public const int MinimumLength = 32; + + private static readonly byte[] EncryptionInfo = "khaemhat/secrets/aes-256-gcm/v1"u8.ToArray(); + private static readonly byte[] IdentifierInfo = "khaemhat/secrets/key-id/v1"u8.ToArray(); + + private SecretKey(byte[] encryptionKey, string id) + { + EncryptionKey = encryptionKey; + Id = id; + } + + /// A short public identifier of the key (8 hexadecimal digits), stored with each ciphertext. + public string Id { get; } + + internal byte[] EncryptionKey { get; } + + /// Reads a key in base64. + /// The configured value. + /// The key. + /// Not base64, or shorter than 32 bytes (the message never repeats the value). + public static SecretKey FromBase64(string base64) + { + if (string.IsNullOrWhiteSpace(base64)) + { + throw new FormatException("KHAEMHAT_SECRET_KEY is empty: set it to the base64 of at least 32 random bytes."); + } + + byte[] material; + try + { + material = Convert.FromBase64String(base64.Trim()); + } + catch (FormatException) + { + throw new FormatException("KHAEMHAT_SECRET_KEY is not valid base64."); + } + + if (material.Length < MinimumLength) + { + throw new FormatException($"KHAEMHAT_SECRET_KEY must be the base64 of at least {MinimumLength} bytes (for example: openssl rand -base64 32)."); + } + + var encryption = HKDF.DeriveKey(HashAlgorithmName.SHA256, material, 32, salt: null, info: EncryptionInfo); + var id = Convert.ToHexStringLower(HKDF.DeriveKey(HashAlgorithmName.SHA256, material, 4, salt: null, info: IdentifierInfo)); + CryptographicOperations.ZeroMemory(material); + return new SecretKey(encryption, id); + } + + /// Creates a new random key in base64, for setup instructions and tests. + /// The base64 text of 32 random bytes. + public static string GenerateBase64() => Convert.ToBase64String(RandomNumberGenerator.GetBytes(MinimumLength)); + + /// + public override string ToString() => "SecretKey " + Id; +} + +/// +/// with AES-256-GCM (SPEC §5.4). A protected value is +/// khs1.<key id>.<base64url(nonce | tag | ciphertext)>; the purpose is authenticated as associated +/// data, so a value cannot be moved to another purpose, and a value of another key is recognized by its key id. +/// +/// The instance key. +public sealed class SecretProtector(SecretKey key) : ISecretProtector +{ + private const string Prefix = "khs1"; + private const int NonceSize = 12; + private const int TagSize = 16; + + /// The key. + public SecretKey Key { get; } = key ?? throw new ArgumentNullException(nameof(key)); + + /// The key id stored in a protected value, or if it is not one. + /// The value. + /// The key id. + public static string? KeyIdOf(string? protectedValue) + { + var parts = protectedValue?.Split('.'); + return parts is [Prefix, var id, _] ? id : null; + } + + /// + public string Protect(string plaintext, string purpose) + { + ArgumentNullException.ThrowIfNull(plaintext); + ArgumentException.ThrowIfNullOrEmpty(purpose); + var plain = Encoding.UTF8.GetBytes(plaintext); + var buffer = new byte[NonceSize + TagSize + plain.Length]; + var nonce = buffer.AsSpan(0, NonceSize); + RandomNumberGenerator.Fill(nonce); + using (var aes = new AesGcm(Key.EncryptionKey, TagSize)) + { + aes.Encrypt(nonce, plain, buffer.AsSpan(NonceSize + TagSize), buffer.AsSpan(NonceSize, TagSize), Encoding.UTF8.GetBytes(purpose)); + } + + CryptographicOperations.ZeroMemory(plain); + return Prefix + "." + Key.Id + "." + Base64Url(buffer); + } + + /// + public string Unprotect(string protectedValue, string purpose) + { + ArgumentException.ThrowIfNullOrEmpty(purpose); + var parts = protectedValue?.Split('.'); + if (parts is not [Prefix, var id, var payload]) + { + throw new SecretProtectionException("The value is not a protected secret."); + } + + if (!string.Equals(id, Key.Id, StringComparison.Ordinal)) + { + throw new SecretProtectionException( + $"The secret was protected with another instance key ({id}, current {Key.Id}): check KHAEMHAT_SECRET_KEY or run the key rotation."); + } + + byte[] buffer; + try + { + buffer = FromBase64Url(payload); + } + catch (FormatException ex) + { + throw new SecretProtectionException("The protected secret is damaged.", ex); + } + + if (buffer.Length < NonceSize + TagSize) + { + throw new SecretProtectionException("The protected secret is damaged."); + } + + var plain = new byte[buffer.Length - NonceSize - TagSize]; + try + { + using var aes = new AesGcm(Key.EncryptionKey, TagSize); + aes.Decrypt( + buffer.AsSpan(0, NonceSize), buffer.AsSpan(NonceSize + TagSize), buffer.AsSpan(NonceSize, TagSize), plain, Encoding.UTF8.GetBytes(purpose)); + return Encoding.UTF8.GetString(plain); + } + catch (AuthenticationTagMismatchException ex) + { + throw new SecretProtectionException("The secret cannot be decrypted: damaged value or different purpose.", ex); + } + finally + { + CryptographicOperations.ZeroMemory(plain); + } + } + + private static string Base64Url(byte[] bytes) => Convert.ToBase64String(bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_'); + + private static byte[] FromBase64Url(string text) + { + var base64 = text.Replace('-', '+').Replace('_', '/'); + return Convert.FromBase64String(base64.PadRight(base64.Length + ((4 - (base64.Length % 4)) % 4), '=')); + } +} + +/// +/// Re-encryption of secrets from the current key to KHAEMHAT_SECRET_KEY_NEW (SPEC §12.9). This is the building +/// block; the procedure that walks every encrypted column of every database arrives in milestone M12. +/// +/// The protector with the current key. +/// The protector with the new key. +public sealed class SecretRotation(SecretProtector current, SecretProtector next) +{ + /// Whether a value still needs to be re-encrypted with the new key. + /// The protected value. + /// if it is not protected with the new key yet. + public bool NeedsRotation(string protectedValue) => SecretProtector.KeyIdOf(protectedValue) != next.Key.Id; + + /// Re-encrypts a value with the new key; values already rotated are returned unchanged. + /// The protected value. + /// Its purpose. + /// The value protected with the new key. + public string Rotate(string protectedValue, string purpose) => + NeedsRotation(protectedValue) ? next.Protect(current.Unprotect(protectedValue, purpose), purpose) : protectedValue; +} diff --git a/tests/Khaemhat.Core.Tests/Architecture/DependencyRulesTests.cs b/tests/Khaemhat.Core.Tests/Architecture/DependencyRulesTests.cs index 79e870d..747f364 100644 --- a/tests/Khaemhat.Core.Tests/Architecture/DependencyRulesTests.cs +++ b/tests/Khaemhat.Core.Tests/Architecture/DependencyRulesTests.cs @@ -22,6 +22,7 @@ public sealed class DependencyRulesTests ["Khaemhat.Application"] = ["Khaemhat.Core", "Khaemhat.Data", "Khaemhat.Infrastructure", "Khaemhat.Fiscal.Abstractions"], ["Khaemhat.Web"] = ["Khaemhat.Application", "Khaemhat.Data.Sqlite", "Khaemhat.Data.PostgreSql", "Khaemhat.Fiscal.Italy"], ["Khaemhat.LicenseTool"] = ["Khaemhat.Core", "Khaemhat.Infrastructure"], + ["Khaemhat.DataModelTool"] = ["Khaemhat.Data"], }; // The only packages, and where each may be referenced. diff --git a/tests/Khaemhat.Data.Tests/CompanyRegistryTests.cs b/tests/Khaemhat.Data.Tests/CompanyRegistryTests.cs new file mode 100644 index 0000000..a9921b4 --- /dev/null +++ b/tests/Khaemhat.Data.Tests/CompanyRegistryTests.cs @@ -0,0 +1,125 @@ +using Khaemhat.Core.Instance; +using Khaemhat.Core.Security; +using Khaemhat.Data.Companies; +using Khaemhat.Data.Execution; +using Khaemhat.Data.Migrations; +using Khaemhat.Data.Tests.Support; +using Khaemhat.Infrastructure.Security; + +namespace Khaemhat.Data.Tests; + +[TestClass] +public sealed class CompanyRegistryTests +{ + private const string CompanyPassword = "Company-Db-Secret-42"; + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task Register_StoresTheLocationEncrypted_AndResolvesIt(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + var registry = await CreateRegistryAsync(db, SecretKey.GenerateBase64()); + var location = DatabaseUrl.Parse($"postgres://acme:{CompanyPassword}@db:5432/khaemhat_acme"); + + var company = await registry.RegisterAsync(new CompanyRegistration("acme", "ACME S.r.l.", location, IsDefault: true)); + var target = await registry.ResolveAsync("ACME"); + + Assert.AreEqual("ACME", company.Code); + Assert.AreEqual("PostgreSql", company.Engine); + StringAssert.StartsWith(company.ConnectionString, "khs1."); + Assert.DoesNotContain(CompanyPassword, company.ConnectionString); + Assert.AreEqual(DbEngine.PostgreSql, target.Engine); + Assert.AreEqual("khaemhat_acme", target.DatabaseName); + StringAssert.Contains(target.ConnectionString, "Database=khaemhat_acme"); + Assert.AreEqual("PostgreSql khaemhat_acme", target.ToString()); + + await using var unitOfWork = await db.OpenAsync(); + var stored = await unitOfWork.QuerySingleAsync(SqlFragment.Literal("SELECT connection_string FROM sys_companies")); + Assert.DoesNotContain(CompanyPassword, stored!); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task Status_Default_AndDuplicates(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + var registry = await CreateRegistryAsync(db, SecretKey.GenerateBase64()); + var sqlite = DatabaseUrl.Parse("sqlite:///data/system.db"); + + await registry.RegisterAsync(new CompanyRegistration("ONE", "One", sqlite.WithDatabase("khaemhat_one"), IsDefault: true)); + await registry.RegisterAsync(new CompanyRegistration("TWO", "Two", sqlite.WithDatabase("khaemhat_two"), IsDefault: true)); + await registry.SetStatusAsync("one", CompanyStatus.ReadOnly); + + var companies = await registry.ListAsync(); + Assert.AreEqual("ONE,TWO", string.Join(",", companies.Select(c => c.Code))); + Assert.IsFalse(companies[0].IsDefault, "registering a new default clears the old one"); + Assert.IsTrue(companies[1].IsDefault); + Assert.AreEqual(CompanyStatus.ReadOnly, companies[0].Status); + Assert.AreEqual(2L, companies[0].RowVersion); + await Assert.ThrowsExactlyAsync( + () => registry.RegisterAsync(new CompanyRegistration("one", "Again", sqlite.WithDatabase("khaemhat_one")))); + await Assert.ThrowsExactlyAsync(() => registry.ResolveAsync("NOPE")); + } + + [TestMethod] + public async Task WrongInstanceKey_IsRejected_WithoutLeakingTheSecret() + { + await using var db = await TestDatabase.CreateAsync(TestDatabase.Sqlite); + var registry = await CreateRegistryAsync(db, SecretKey.GenerateBase64()); + await registry.RegisterAsync(new CompanyRegistration("ACME", "ACME", DatabaseUrl.Parse($"postgres://u:{CompanyPassword}@db/khaemhat_acme"))); + + var otherKey = await CreateRegistryAsync(db, SecretKey.GenerateBase64(), migrate: false); + var error = await Assert.ThrowsExactlyAsync(() => otherKey.ResolveAsync("ACME")); + + StringAssert.Contains(error.Message, "another instance key"); + Assert.DoesNotContain(CompanyPassword, error.ToString()); + } + + [TestMethod] + public async Task ACopiedRow_CannotBeReadForAnotherCompany() + { + await using var db = await TestDatabase.CreateAsync(TestDatabase.Sqlite); + var key = SecretKey.GenerateBase64(); + var registry = await CreateRegistryAsync(db, key); + await registry.RegisterAsync(new CompanyRegistration("ACME", "ACME", DatabaseUrl.Parse("sqlite:///data/khaemhat_acme.db"))); + await registry.RegisterAsync(new CompanyRegistration("EVIL", "Evil", DatabaseUrl.Parse("sqlite:///data/khaemhat_evil.db"))); + await using (var unitOfWork = await db.OpenAsync()) + { + await unitOfWork.ExecuteAsync(SqlFragment.Literal( + "UPDATE sys_companies SET connection_string = (SELECT connection_string FROM sys_companies WHERE code = 'ACME') WHERE code = 'EVIL'")); + } + + var fresh = await CreateRegistryAsync(db, key, migrate: false); + await Assert.ThrowsExactlyAsync(() => fresh.ResolveAsync("EVIL")); + } + + [TestMethod] + public async Task Locator_OpensTheSystemOrACompany() + { + await using var system = await TestDatabase.CreateAsync(TestDatabase.Sqlite); + await using var company = await TestDatabase.CreateAsync(TestDatabase.Sqlite); + var registry = await CreateRegistryAsync(system, SecretKey.GenerateBase64()); + await registry.RegisterAsync(new CompanyRegistration("DEMO", "Demo", company.Url)); + var factory = new UnitOfWorkFactory(new DatabaseLocator(system.Target, registry), system.Maps, system.Context); + + await using var systemWork = await factory.CreateAsync(DatabaseScope.System); + await using var companyWork = await factory.CreateAsync(DatabaseScope.Company("demo")); + + Assert.AreEqual(1L, await systemWork.QuerySingleAsync(SqlFragment.Literal("SELECT COUNT(*) FROM sys_companies"))); + Assert.AreEqual(0L, await companyWork.QuerySingleAsync( + SqlFragment.Literal("SELECT COUNT(*) FROM sqlite_master WHERE name = 'sys_companies'"))); + Assert.AreEqual("company DEMO", DatabaseScope.Company("demo").ToString()); + } + + private static async Task CreateRegistryAsync(TestDatabase db, string key, bool migrate = true) + { + if (migrate) + { + await new MigrationRunner(db.Context).RunAsync(db.Target, MigrationCatalog.ForSystem()); + } + + return new CompanyRegistry(db.Target, db.Providers, new SecretProtector(SecretKey.FromBase64(key)), db.Maps, db.Context); + } +} diff --git a/tests/Khaemhat.Data.Tests/DataModelDocumentTests.cs b/tests/Khaemhat.Data.Tests/DataModelDocumentTests.cs new file mode 100644 index 0000000..4da0e6b --- /dev/null +++ b/tests/Khaemhat.Data.Tests/DataModelDocumentTests.cs @@ -0,0 +1,48 @@ +using Khaemhat.Data.Migrations; + +namespace Khaemhat.Data.Tests; + +/// docs/DATA-MODEL.md never drifts from the migrations (SPEC §18). +[TestClass] +public sealed class DataModelDocumentTests +{ + [TestMethod] + public void GeneratedSection_IsUpToDate() + { + var path = Path.Combine(RepositoryRoot(), "docs", "DATA-MODEL.md"); + if (!File.Exists(path)) + { + Assert.Inconclusive("docs/ is not part of this checkout (the Docker build context excludes it)."); + } + + var current = File.ReadAllText(path).ReplaceLineEndings("\n"); + + var expected = DataModelDocument.Update(current, DataModelDocument.Render(MigrationCatalog.ForSystem(), MigrationCatalog.ForCompany())); + + Assert.AreEqual(expected, current, "docs/DATA-MODEL.md is out of date: run scripts/gen-datamodel.sh"); + } + + [TestMethod] + public void Update_ReplacesOnlyTheGeneratedSection() + { + var document = $"intro\n{DataModelDocument.BeginMarker}\nold\n{DataModelDocument.EndMarker}\nfooter\n"; + + var updated = DataModelDocument.Update(document, "new\n"); + + Assert.AreEqual($"intro\n{DataModelDocument.BeginMarker}\n\nnew\n\n{DataModelDocument.EndMarker}\nfooter\n", updated); + Assert.ThrowsExactly(() => DataModelDocument.Update("no markers", "x")); + } + + private static string RepositoryRoot() + { + for (var dir = new DirectoryInfo(AppContext.BaseDirectory); dir is not null; dir = dir.Parent) + { + if (File.Exists(Path.Combine(dir.FullName, "Khaemhat.sln"))) + { + return dir.FullName; + } + } + + throw new InvalidOperationException("Khaemhat.sln not found above " + AppContext.BaseDirectory); + } +} diff --git a/tests/Khaemhat.Data.Tests/DatabaseUrlTests.cs b/tests/Khaemhat.Data.Tests/DatabaseUrlTests.cs new file mode 100644 index 0000000..b0f461e --- /dev/null +++ b/tests/Khaemhat.Data.Tests/DatabaseUrlTests.cs @@ -0,0 +1,117 @@ +using Khaemhat.Data.PostgreSql; +using Khaemhat.Data.Sqlite; + +namespace Khaemhat.Data.Tests; + +[TestClass] +public sealed class DatabaseUrlTests +{ + [TestMethod] + public void Postgres_IsParsedWithEncodedPasswordAndOptions() + { + var url = DatabaseUrl.Parse("postgres://khaemhat:p%40ss%3Aword@khaemhat-db:5433/khaemhat_system?sslmode=require"); + + Assert.AreEqual(DbEngine.PostgreSql, url.Engine); + Assert.AreEqual("khaemhat-db", url.Host); + Assert.AreEqual(5433, url.Port); + Assert.AreEqual("khaemhat", url.UserName); + Assert.AreEqual("p@ss:word", url.Password); + Assert.AreEqual("khaemhat_system", url.Database); + Assert.AreEqual("require", url.Options["sslmode"]); + } + + [TestMethod] + public void Postgres_ServerUrl_HasEmptyDatabase() + { + var url = DatabaseUrl.Parse("postgresql://u:p@db/"); + + Assert.AreEqual(string.Empty, url.Database); + Assert.IsNull(url.Port); + Assert.AreEqual("khaemhat_demo", url.WithDatabase(CompanyCodes.DatabaseName("demo")).Database); + } + + [TestMethod] + public void ToString_NeverShowsThePassword() + { + var url = DatabaseUrl.Parse("postgres://khaemhat:VerySecret1@db:5432/khaemhat_system"); + + Assert.AreEqual("postgres://khaemhat:***@db:5432/khaemhat_system", url.ToString()); + Assert.DoesNotContain("VerySecret1", url.ToString()); + Assert.Contains("VerySecret1", url.ToUnredactedString()); + Assert.AreEqual(url.ToUnredactedString(), DatabaseUrl.Parse(url.ToUnredactedString()).ToUnredactedString()); + } + + [TestMethod] + public void InvalidUrl_ErrorDoesNotRepeatTheInput() + { + var error = Assert.ThrowsExactly(() => DatabaseUrl.Parse("mysql://root:TopSecret@db/x")); + + Assert.DoesNotContain("TopSecret", error.Message); + } + + [TestMethod] + [DataRow("sqlite:///data/system.db", "/data/system.db", "system")] + [DataRow("sqlite:///C:/Khaemhat/data/system.db", "C:/Khaemhat/data/system.db", "system")] + [DataRow("sqlite:///B:\\dev\\.data\\system.db", "B:/dev/.data/system.db", "system")] + public void Sqlite_FilePaths(string text, string path, string name) + { + var url = DatabaseUrl.Parse(text); + + Assert.AreEqual(DbEngine.Sqlite, url.Engine); + Assert.AreEqual(path, url.Database); + Assert.AreEqual(name, url.DatabaseName); + Assert.IsFalse(url.IsInMemory); + } + + [TestMethod] + public void Sqlite_CompanyDatabase_IsASiblingFile() + { + var url = DatabaseUrl.Parse("sqlite:///data/system.db").WithDatabase(CompanyCodes.DatabaseName("Demo")); + + Assert.AreEqual("/data/khaemhat_demo.db", url.Database); + Assert.AreEqual("sqlite:///data/khaemhat_demo.db", url.ToString()); + } + + [TestMethod] + public void Sqlite_InMemory() + { + var url = DatabaseUrl.Parse("sqlite://memory/tests_1"); + + Assert.IsTrue(url.IsInMemory); + Assert.AreEqual("tests_1", url.DatabaseName); + StringAssert.Contains(new SqliteProvider().BuildConnectionString(url), "Mode=Memory"); + } + + [TestMethod] + [DataRow("")] + [DataRow("sqlite://relative.db")] + [DataRow("sqlite:///")] + [DataRow("sqlite://memory/bad name")] + [DataRow("postgres://")] + [DataRow("postgres://u:p@host/a/b")] + public void Invalid_AreRejected(string text) => Assert.IsFalse(DatabaseUrl.TryParse(text, out _)); + + [TestMethod] + public void PostgresConnectionString_HasPoolAndStatementTimeout() + { + var provider = new PostgresProvider(new PostgresProviderOptions { MaxPoolSize = 7, StatementTimeout = TimeSpan.FromSeconds(30) }); + + var cs = provider.BuildConnectionString(DatabaseUrl.Parse("postgres://u:p@db/khaemhat_demo?sslmode=disable")); + + StringAssert.Contains(cs, "Maximum Pool Size=7"); + StringAssert.Contains(cs, "statement_timeout=30000"); + StringAssert.Contains(cs, "Database=khaemhat_demo"); + } + + [TestMethod] + [DataRow("DEMO", "khaemhat_demo")] + [DataRow(" acme_2 ", "khaemhat_acme_2")] + public void CompanyCodes_DatabaseNames(string code, string database) => Assert.AreEqual(database, CompanyCodes.DatabaseName(code)); + + [TestMethod] + [DataRow("")] + [DataRow("2ACME")] + [DataRow("AC-ME")] + [DataRow("ABCDEFGHIJKLMNOPQRSTU")] + public void CompanyCodes_InvalidAreRejected(string code) => Assert.ThrowsExactly(() => CompanyCodes.Normalize(code)); +} diff --git a/tests/Khaemhat.Data.Tests/DdlTests.cs b/tests/Khaemhat.Data.Tests/DdlTests.cs new file mode 100644 index 0000000..54ce6ef --- /dev/null +++ b/tests/Khaemhat.Data.Tests/DdlTests.cs @@ -0,0 +1,121 @@ +using Khaemhat.Data.Migrations; +using Khaemhat.Data.PostgreSql; +using Khaemhat.Data.Schema; +using Khaemhat.Data.Sqlite; +using Khaemhat.Data.SystemDatabase.Migrations; + +namespace Khaemhat.Data.Tests; + +[TestClass] +public sealed class DdlTests +{ + private static readonly MigrationInfo Companies = MigrationCatalog.ForSystem().Migrations.Single(m => m.Type == typeof(M20261111_001_CreateSysCompanies)); + + [TestMethod] + public void Sqlite_UsesStrictTables_IntegerKeys_AndChecks() + { + var sql = MigrationRunner.Statements(Companies, new SqliteProvider()); + + Assert.AreEqual( + """ + CREATE TABLE "sys_companies" ( + "id" INTEGER PRIMARY KEY, + "code" TEXT NOT NULL CHECK (length("code") <= 20), + "name" TEXT NOT NULL CHECK (length("name") <= 200), + "engine" TEXT NOT NULL CHECK (length("engine") <= 20), + "connection_string" TEXT NOT NULL, + "status" TEXT NOT NULL DEFAULT 'Active' CHECK (length("status") <= 40), + "is_default" INTEGER NOT NULL DEFAULT 0 CHECK ("is_default" IN (0, 1)), + "notes" TEXT, + "created_at" TEXT NOT NULL, + "created_by" TEXT CHECK (length("created_by") <= 100), + "updated_at" TEXT, + "updated_by" TEXT CHECK (length("updated_by") <= 100), + "row_version" INTEGER NOT NULL DEFAULT 1 + ) STRICT + """.ReplaceLineEndings("\n"), + sql[0]); + Assert.AreEqual("CREATE UNIQUE INDEX \"ux_sys_companies_code\" ON \"sys_companies\" (\"code\")", sql[1]); + } + + [TestMethod] + public void Postgres_UsesIdentity_AndNativeTypes() + { + var sql = MigrationRunner.Statements(Companies, new PostgresProvider()); + + StringAssert.Contains(sql[0], "\"id\" bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY"); + StringAssert.Contains(sql[0], "\"code\" varchar(20) NOT NULL"); + StringAssert.Contains(sql[0], "\"is_default\" boolean NOT NULL DEFAULT FALSE"); + StringAssert.Contains(sql[0], "\"created_at\" timestamptz NOT NULL"); + Assert.DoesNotContain("STRICT", sql[0]); + } + + [TestMethod] + public void AllTypes_MapAsSpecSection5_2Says() + { + var expected = new Dictionary + { + [ColumnType.String] = ("varchar(30)", "TEXT"), + [ColumnType.Text] = ("text", "TEXT"), + [ColumnType.Int32] = ("integer", "INTEGER"), + [ColumnType.Int64] = ("bigint", "INTEGER"), + [ColumnType.Decimal] = ("numeric(19,6)", "INTEGER"), + [ColumnType.Bool] = ("boolean", "INTEGER"), + [ColumnType.Date] = ("date", "TEXT"), + [ColumnType.DateTimeUtc] = ("timestamptz", "TEXT"), + [ColumnType.Guid] = ("uuid", "TEXT"), + [ColumnType.Bytes] = ("bytea", "BLOB"), + [ColumnType.Json] = ("jsonb", "TEXT"), + }; + + foreach (var (type, (postgres, sqlite)) in expected) + { + var length = type == ColumnType.String ? 30 : (int?)null; + Assert.AreEqual(postgres, PostgresDialect.Instance.SqlType(type, length, null), type.ToString()); + Assert.AreEqual(sqlite, SqliteDialect.Instance.SqlType(type, length, null), type.ToString()); + } + } + + [TestMethod] + public void ForeignKeys_CompositeKeys_AndAlterTable() + { + var schema = new SchemaBuilder(); + schema.CreateTable("parent_rows", t => t.Id()); + schema.CreateTable("child_rows", t => + { + t.Id(); + t.Int64("parent_row_id").NotNull(); + t.ForeignKey("fk_child_rows_parent", "parent_row_id", "parent_rows", ReferentialAction.Cascade); + }); + schema.AlterTable("child_rows", t => + { + t.Decimal("price", 2).NotNull().Default(0m); + t.Index("ix_child_rows_price", "price"); + }); + + var sql = new PostgresSchemaBuilder().Generate(schema.Operations); + + StringAssert.Contains(sql[1], "CONSTRAINT \"fk_child_rows_parent\" FOREIGN KEY (\"parent_row_id\") REFERENCES \"parent_rows\" (\"id\") ON DELETE CASCADE"); + Assert.AreEqual("ALTER TABLE \"child_rows\" ADD COLUMN \"price\" numeric(19,2) NOT NULL DEFAULT 0", sql[2]); + Assert.AreEqual("CREATE INDEX \"ix_child_rows_price\" ON \"child_rows\" (\"price\")", sql[3]); + Assert.ThrowsExactly(() => schema.AlterTable("child_rows", t => t.ForeignKey("fk_x", "id", "parent_rows"))); + } + + [TestMethod] + public void SchemaVersion_HasCompositeKey_AndIsCreatedIfMissing() + { + var sql = new SqliteSchemaBuilder().Generate([SchemaVersionStore.CreateOperation]); + + StringAssert.StartsWith(sql[0], "CREATE TABLE IF NOT EXISTS \"schema_version\""); + StringAssert.Contains(sql[0], "CONSTRAINT \"pk_schema_version\" PRIMARY KEY (\"module\", \"version\")"); + } + + [TestMethod] + public void Partitioning_IsPrepared() + { + Assert.AreEqual( + "CREATE TABLE IF NOT EXISTS \"gl_entry_lines_2026\" PARTITION OF \"gl_entry_lines\" FOR VALUES FROM ('2026-01-01') TO ('2027-01-01')", + PostgresPartitioning.CreateYearPartition("gl_entry_lines", 2026)); + Assert.AreEqual(" PARTITION BY RANGE (\"entry_date\")", PostgresPartitioning.PartitionClause("entry_date")); + } +} diff --git a/tests/Khaemhat.Data.Tests/DialectTests.cs b/tests/Khaemhat.Data.Tests/DialectTests.cs new file mode 100644 index 0000000..4b8e684 --- /dev/null +++ b/tests/Khaemhat.Data.Tests/DialectTests.cs @@ -0,0 +1,238 @@ +using Khaemhat.Data.Querying; +using Khaemhat.Data.Schema; +using Khaemhat.Data.Sqlite; +using Khaemhat.Data.Tests.Support; + +namespace Khaemhat.Data.Tests; + +/// The same behaviour on both engines: types, decimals, paging, upsert, LIKE, timestamps. +[TestClass] +public sealed class DialectTests +{ + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task EveryType_RoundTrips(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + await db.CreateTestTableAsync(); + var at = new DateTime(2026, 11, 11, 9, 30, 15, DateTimeKind.Utc).AddTicks(1234560); + var item = new TestItem + { + Code = "ALL", + Name = "Caffè «speciale» ✓", + Quantity = int.MaxValue, + Big = long.MinValue, + Amount = -123_456_789_012.345678m, + Active = true, + Day = new DateOnly(2026, 2, 28), + At = at, + Uid = Guid.Parse("7a1b0c3e-9d2f-4e5a-8b6c-1d2e3f4a5b6c"), + Data = [0, 1, 2, 254, 255], + Payload = "{\"a\": 1}", + Kind = TestKind.Kit, + IsDemo = true, + }; + + await using var unitOfWork = await db.OpenAsync(); + var items = unitOfWork.Repository(); + var id = await items.InsertAsync(item); + var read = await items.GetAsync(id); + + Assert.IsNotNull(read); + Assert.AreEqual(item.Name, read.Name); + Assert.AreEqual(int.MaxValue, read.Quantity); + Assert.AreEqual(long.MinValue, read.Big); + Assert.AreEqual(item.Amount, read.Amount); + Assert.IsTrue(read.Active); + Assert.AreEqual(item.Day, read.Day); + Assert.AreEqual(at, read.At); + Assert.AreEqual(DateTimeKind.Utc, read.At!.Value.Kind); + Assert.AreEqual(item.Uid, read.Uid); + CollectionAssert.AreEqual(item.Data, read.Data); + StringAssert.Contains(read.Payload, "\"a\""); + Assert.AreEqual(TestKind.Kit, read.Kind); + Assert.IsTrue(read.IsDemo); + Assert.AreEqual(TestDatabase.TestUser, read.CreatedBy); + Assert.AreEqual(1L, read.RowVersion); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task Decimals_AreRoundedToSixPlaces_AndSumsAreExact(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + await db.CreateTestTableAsync(); + await using var unitOfWork = await db.OpenAsync(); + var items = unitOfWork.Repository(); + + decimal[] amounts = [999_999_999_999.999999m, 0.000001m, 1.2345675m, -1.0000005m, 0.1m, 0.2m]; + await unitOfWork.InTransactionAsync(async ct => + { + for (var i = 0; i < amounts.Length; i++) + { + await items.InsertAsync(new TestItem { Code = "D" + i, Amount = amounts[i] }, ct); + } + }); + + var stored = (await items.ListAsync(Query.For().OrderBy(x => x.Id))).Items.Select(x => x.Amount).ToArray(); + CollectionAssert.AreEqual(new[] { 999_999_999_999.999999m, 0.000001m, 1.234568m, -1.000001m, 0.1m, 0.2m }, stored); + + var sum = await unitOfWork.QuerySingleAsync(SqlFragment.Literal("SELECT SUM(amount) FROM test_items")); + Assert.AreEqual(1_000_000_000_000.534567m, sum); + + var small = await unitOfWork.QuerySingleAsync(SqlFragment.Of($"SELECT SUM(amount) FROM test_items WHERE amount < {1m}")); + Assert.AreEqual(-0.7m, small); + } + + [TestMethod] + public void SqliteDecimalConverter_RoundsHalfAwayFromZero_AndRefusesOverflow() + { + Assert.AreEqual(1_234_568L, SqliteDecimalConverter.ToScaled(1.2345675m)); + Assert.AreEqual(-1_000_001L, SqliteDecimalConverter.ToScaled(-1.0000005m)); + Assert.AreEqual(1_000_000_000_000_000_000L, SqliteDecimalConverter.ToScaled(1_000_000_000_000m)); + Assert.AreEqual(123.456789m, SqliteDecimalConverter.FromScaled(123_456_789L)); + Assert.ThrowsExactly(() => SqliteDecimalConverter.ToScaled(10_000_000_000_000m)); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task Like_IsCaseInsensitive_UnicodeAware_AndEscaped(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + await db.CreateTestTableAsync(); + await using var unitOfWork = await db.OpenAsync(); + var items = unitOfWork.Repository(); + string[] names = ["Àlbero", "albero", "ALBERGO", "100% Bio", "a_b", "axb"]; + for (var i = 0; i < names.Length; i++) + { + await items.InsertAsync(new TestItem { Code = "L" + i, Name = names[i] }); + } + + async Task Names(Query query) => + [.. (await items.ListAsync(query.OrderBy(x => x.Code))).Items.Select(x => x.Name!)]; + + CollectionAssert.AreEqual(new[] { "Àlbero" }, await Names(Query.For().Where(x => x.Name!.Contains("àL")))); + CollectionAssert.AreEqual(new[] { "albero", "ALBERGO" }, await Names(Query.For().Where(x => x.Name!.StartsWith("alb")))); + CollectionAssert.AreEqual(new[] { "100% Bio" }, await Names(Query.For().Where(x => x.Name!.Contains("%")))); + CollectionAssert.AreEqual(new[] { "a_b" }, await Names(Query.For().Where(x => x.Name!.Contains("_")))); + CollectionAssert.AreEqual(new[] { "Àlbero", "albero" }, await Names(Query.For().Where(x => x.Name!.EndsWith("BERO")))); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task Timestamps_SortChronologically_AndCurrentTimestampIsUtc(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + await db.CreateTestTableAsync(); + await using var unitOfWork = await db.OpenAsync(); + var items = unitOfWork.Repository(); + var baseTime = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + TimeSpan[] offsets = [TimeSpan.FromHours(10), TimeSpan.FromMilliseconds(1), TimeSpan.FromDays(400), TimeSpan.FromTicks(10), TimeSpan.Zero]; + for (var i = 0; i < offsets.Length; i++) + { + await items.InsertAsync(new TestItem { Code = "T" + i, At = baseTime + offsets[i] }); + } + + var sorted = (await items.ListAsync(Query.For().OrderBy(x => x.At))).Items.Select(x => x.Code).ToArray(); + var after = (await items.ListAsync(Query.For().Where(x => x.At > baseTime).OrderBy(x => x.At))).Items.Select(x => x.Code).ToArray(); + var now = await unitOfWork.QuerySingleAsync(SqlFragment.Literal("SELECT " + db.Dialect.CurrentTimestampUtc)); + + CollectionAssert.AreEqual(new[] { "T4", "T3", "T1", "T0", "T2" }, sorted); + CollectionAssert.AreEqual(new[] { "T3", "T1", "T0", "T2" }, after); + Assert.AreEqual(DateTimeKind.Utc, now.Kind); + Assert.IsLessThan(TimeSpan.FromMinutes(1), (now - DateTime.UtcNow).Duration()); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task Paging_ReturnsStablePagesAndTotal(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + await db.CreateTestTableAsync(); + await using var unitOfWork = await db.OpenAsync(); + var items = unitOfWork.Repository(); + await unitOfWork.InTransactionAsync(async ct => + { + for (var i = 0; i < 25; i++) + { + await items.InsertAsync(new TestItem { Code = "P" + i.ToString("D2", System.Globalization.CultureInfo.InvariantCulture), Quantity = i }, ct); + } + }); + + var page = await items.ListAsync(Query.For().Where(x => x.Quantity >= 5).OrderBy(x => x.Code).Page(10, 4)); + + Assert.AreEqual(20, page.TotalCount); + CollectionAssert.AreEqual(new[] { "P15", "P16", "P17", "P18" }, page.Items.Select(x => x.Code).ToArray()); + Assert.IsTrue(page.HasMore); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task Upsert_InsertsThenUpdates(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + await using var unitOfWork = await db.OpenAsync(); + await unitOfWork.ExecuteAsync(SqlFragment.Literal(string.Join(";", db.Provider.Ddl.Generate(UpsertTable())))); + var text = db.Dialect.Upsert("kv", ["k", "v"], ["@p0", "@p1"], ["k"], ["v"]); + + foreach (var value in new[] { "first", "second" }) + { + await using var command = unitOfWork.Executor.CreateCommand( + unitOfWork.Connection, null, text, [new SqlParameterValue("p0", "key", ColumnType.Text), new SqlParameterValue("p1", value, ColumnType.Text)]); + await command.ExecuteNonQueryAsync(); + } + + Assert.AreEqual(1L, await unitOfWork.QuerySingleAsync(SqlFragment.Literal("SELECT COUNT(*) FROM kv"))); + Assert.AreEqual("second", await unitOfWork.QuerySingleAsync(SqlFragment.Literal("SELECT v FROM kv"))); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task StringLength_IsEnforcedOnBothEngines(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + await db.CreateTestTableAsync(); + await using var unitOfWork = await db.OpenAsync(); + + await Assert.ThrowsAsync( + () => unitOfWork.Repository().InsertAsync(new TestItem { Code = new string('x', 21) })); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task ServerInfo_IsReported(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + + var info = await db.Provider.GetServerInfoAsync(db.Target.ConnectionString); + + Assert.AreEqual(db.Provider.Engine, info.Engine); + Assert.IsGreaterThan(0, info.VersionNumber); + if (engine == TestDatabase.Postgres) + { + Assert.IsGreaterThanOrEqualTo(160000, info.VersionNumber, "CI runs PostgreSQL 16 and 17"); + Assert.AreEqual(db.Url.Database, info.Database); + Assert.IsEmpty(info.Warnings); + } + } + + private static IReadOnlyList UpsertTable() + { + var schema = new SchemaBuilder(); + schema.CreateTable("kv", t => + { + t.String("k", 50).NotNull(); + t.Text("v"); + t.PrimaryKey("k"); + }); + return schema.Operations; + } +} diff --git a/tests/Khaemhat.Data.Tests/Khaemhat.Data.Tests.csproj b/tests/Khaemhat.Data.Tests/Khaemhat.Data.Tests.csproj index 201710e..d595a2e 100644 --- a/tests/Khaemhat.Data.Tests/Khaemhat.Data.Tests.csproj +++ b/tests/Khaemhat.Data.Tests/Khaemhat.Data.Tests.csproj @@ -8,6 +8,7 @@ + diff --git a/tests/Khaemhat.Data.Tests/MappingTests.cs b/tests/Khaemhat.Data.Tests/MappingTests.cs new file mode 100644 index 0000000..8c6fc90 --- /dev/null +++ b/tests/Khaemhat.Data.Tests/MappingTests.cs @@ -0,0 +1,82 @@ +using Khaemhat.Core.Instance; +using Khaemhat.Data.Mapping; +using Khaemhat.Data.Schema; +using Khaemhat.Data.SystemDatabase; +using Khaemhat.Data.Tests.Support; + +namespace Khaemhat.Data.Tests; + +[TestClass] +public sealed class MappingTests +{ + [TestMethod] + [DataRow("RowVersion", "row_version")] + [DataRow("IsDemo", "is_demo")] + [DataRow("VatID", "vat_id")] + [DataRow("HTTPCode", "http_code")] + [DataRow("Address2", "address2")] + [DataRow("Id", "id")] + public void SnakeCase(string name, string expected) => Assert.AreEqual(expected, SqlNames.ToSnakeCase(name)); + + [TestMethod] + public void Conventions_KeyTypesNullabilityAndAudit() + { + var map = new TestItemMap(); + map.Freeze(); + + Assert.AreEqual("id", map.Key.ColumnName); + Assert.IsTrue(map.Key.IsIdentity); + Assert.AreEqual("id", map.Columns[0].ColumnName); + Assert.AreEqual("row_version", map.Columns[^1].ColumnName); + Assert.AreSame(map.Columns[^1], map.RowVersion); + + Assert.AreEqual("String(20)", map.Column(x => x.Code).TypeDescription); + Assert.IsFalse(map.Column(x => x.Code).IsNullable); + Assert.IsTrue(map.Column(x => x.Name).IsNullable); + Assert.AreEqual(ColumnType.Decimal, map.Column(x => x.Amount).Type); + Assert.AreEqual(ColumnType.Date, map.Column(x => x.Day).Type); + Assert.IsTrue(map.Column(x => x.Day).IsNullable); + Assert.AreEqual(ColumnType.Json, map.Column(x => x.Payload).Type); + Assert.AreEqual("String(40)", map.Column(x => x.Kind).TypeDescription); + Assert.AreEqual(ColumnType.Bool, map.Column(x => x.IsDemo).Type); + Assert.AreEqual(AuditRole.CreatedBy, map.Column(x => x.CreatedBy).Audit); + Assert.AreEqual("String(100)", map.Column(x => x.CreatedBy).TypeDescription); + } + + [TestMethod] + public void Enums_AreStoredByName() + { + var map = new CompanyMap(); + map.Freeze(); + var status = map.Column(x => x.Status); + + Assert.AreEqual("Maintenance", status.ToProvider(CompanyStatus.Maintenance)); + Assert.AreEqual(CompanyStatus.ReadOnly, status.FromProvider("ReadOnly")); + } + + [TestMethod] + public void Registry_RejectsTwoMapsForOneEntity() + { + var registry = new MapRegistry().Add(new CompanyMap()); + + Assert.ThrowsExactly(() => registry.Add(new CompanyMap())); + } + + [TestMethod] + public void Registry_FindsTheSystemMaps() + { + var registry = new MapRegistry().AddFromAssembly(typeof(CompanyMap).Assembly); + + CollectionAssert.AreEquivalent(new[] { "sys_companies", "sys_settings" }, registry.Maps.Select(m => m.Table).ToArray()); + } + + [TestMethod] + public void StringWithoutLength_IsText_AndFrozenMapsCannotChange() + { + var map = new SystemSettingMap(); + map.Freeze(); + + Assert.AreEqual(ColumnType.Text, map.Column(x => x.Value).Type); + Assert.ThrowsExactly(() => map.Column(x => x.Key).MaxLength(10)); + } +} diff --git a/tests/Khaemhat.Data.Tests/MigrationTests.cs b/tests/Khaemhat.Data.Tests/MigrationTests.cs new file mode 100644 index 0000000..364c2d6 --- /dev/null +++ b/tests/Khaemhat.Data.Tests/MigrationTests.cs @@ -0,0 +1,154 @@ +using Khaemhat.Data.Mapping; +using Khaemhat.Data.Migrations; +using Khaemhat.Data.Schema; +using Khaemhat.Data.SystemDatabase; +using Khaemhat.Data.Tests.Support; + +namespace Khaemhat.Data.Tests; + +[TestClass] +public sealed class MigrationTests +{ + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task SystemMigrations_FromZero_ThenReapplied_DoNothing(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + var catalog = MigrationCatalog.ForSystem(); + var runner = new MigrationRunner(db.Context); + + var first = await runner.RunAsync(db.Target, catalog); + var second = await runner.RunAsync(db.Target, catalog); + + Assert.HasCount(catalog.Migrations.Count, first.Applied); + Assert.AreEqual(0, first.AlreadyApplied); + Assert.IsEmpty(second.Applied); + Assert.AreEqual(catalog.Migrations.Count, second.AlreadyApplied); + Assert.IsEmpty(second.Warnings); + + await using var unitOfWork = await db.OpenAsync(); + var applied = await SchemaVersionStore.ReadAsync(unitOfWork); + CollectionAssert.AreEqual( + catalog.Migrations.Select(m => (m.Module, m.Version, m.Name)).ToArray(), + applied.Select(a => (a.Module, a.Version, a.Name)).ToArray()); + Assert.IsTrue(applied.All(a => a.Checksum.Length == 64)); + + // The tables work through their maps. + var settings = unitOfWork.Repository(); + await settings.InsertAsync(new Core.Instance.SystemSetting { Key = "smtp.host", Value = "mail.example.com" }); + Assert.AreEqual(1, await settings.CountAsync(Querying.Query.For())); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task UnknownAppliedVersion_IsRefused(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + var runner = new MigrationRunner(db.Context); + await runner.RunAsync(db.Target, MigrationCatalog.ForSystem()); + await using (var unitOfWork = await db.OpenAsync()) + { + await SchemaVersionStore.RecordAsync(unitOfWork, new AppliedMigration("core", 20991231_001, "from_the_future", DateTime.UtcNow, new string('0', 64))); + } + + var error = await Assert.ThrowsExactlyAsync(() => runner.RunAsync(db.Target, MigrationCatalog.ForSystem())); + + StringAssert.Contains(error.Message, "20991231001 from_the_future"); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task FailingMigration_LeavesNoTrace(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + var catalog = MigrationCatalog.Discover(MigrationTarget.Company, [typeof(MigrationTests).Assembly]); + var runner = new MigrationRunner(db.Context); + + await Assert.ThrowsAsync(() => runner.RunAsync(db.Target, catalog)); + + await using var unitOfWork = await db.OpenAsync(); + Assert.IsEmpty(await SchemaVersionStore.ReadAsync(unitOfWork)); + var exists = await unitOfWork.QueryAsync( + SqlFragment.Of($"SELECT COUNT(*) FROM test_failing")).ContinueWith(t => t.IsCompletedSuccessfully, TaskScheduler.Default); + Assert.IsFalse(exists, "the table of the failed migration was rolled back"); + } + + [TestMethod] + public void Catalog_IsOrdered_AndVersionsAreValid() + { + var catalog = MigrationCatalog.ForSystem(); + + Assert.IsGreaterThanOrEqualTo(2, catalog.Migrations.Count); + CollectionAssert.AreEqual(catalog.Migrations.OrderBy(m => m.Version).ToArray(), catalog.Migrations.ToArray()); + Assert.IsTrue(catalog.Migrations.All(m => m.Target == MigrationTarget.System)); + } + + [TestMethod] + public void SystemMaps_AgreeWithTheMigratedSchema() + { + var model = MigrationCatalog.ForSystem().BuildModel(); + var maps = new MapRegistry().AddFromAssembly(typeof(CompanyMap).Assembly); + var problems = new List(); + + foreach (var map in maps.Maps) + { + var table = model.Table(map.Table); + if (table is null) + { + problems.Add($"{map.Table}: no migration creates it"); + continue; + } + + foreach (var column in map.Columns) + { + var definition = table.Column(column.ColumnName); + if (definition is null) + { + problems.Add($"{map.Table}.{column.ColumnName}: missing in the schema"); + } + else if (definition.TypeDescription != column.TypeDescription || definition.IsNullable != column.IsNullable) + { + problems.Add( + $"{map.Table}.{column.ColumnName}: map {column.TypeDescription} null={column.IsNullable}, schema {definition.TypeDescription} null={definition.IsNullable}"); + } + } + + problems.AddRange(table.Columns.Where(c => map.FindByColumn(c.Name) is null).Select(c => $"{map.Table}.{c.Name}: not mapped")); + } + + Assert.IsEmpty(problems, string.Join("\n", problems)); + } + + [TestMethod] + public void InvalidMigrations_AreRejectedByTheCatalog() + { + Assert.ThrowsExactly( + () => MigrationCatalog.Discover(MigrationTarget.System, [typeof(MigrationTests).Assembly])); + } +} + +/// Creates a table, then fails: used by . +[Migration(20261111_001, "failing_on_purpose", Module = "tests")] +public sealed class M20261111_001_FailingOnPurpose : IMigration +{ + /// + public void Up(ISchemaBuilder schema) + { + ArgumentNullException.ThrowIfNull(schema); + schema.CreateTable("test_failing", t => t.Id()); + schema.Sql(_ => "SELECT * FROM table_that_does_not_exist"); + } +} + +/// A badly numbered system migration: used by . +[Migration(2026_001, "bad_version", Target = MigrationTarget.System, Module = "tests")] +public sealed class BadlyNumberedMigration : IMigration +{ + /// + public void Up(ISchemaBuilder schema) + { + } +} diff --git a/tests/Khaemhat.Data.Tests/PerformanceBaselineTests.cs b/tests/Khaemhat.Data.Tests/PerformanceBaselineTests.cs new file mode 100644 index 0000000..cb31f9a --- /dev/null +++ b/tests/Khaemhat.Data.Tests/PerformanceBaselineTests.cs @@ -0,0 +1,64 @@ +using System.Diagnostics; +using System.Globalization; +using Khaemhat.Data.Querying; +using Khaemhat.Data.Tests.Support; + +namespace Khaemhat.Data.Tests; + +/// +/// Reference numbers for the data layer (issue M1 #13): insert and read rows through the repository. With +/// KHAEMHAT_PERF=1 it uses 100,000 rows and prints the timings; otherwise 2,000 rows as a smoke test. No +/// threshold is asserted: the numbers are recorded in docs/DATA-MODEL.md. +/// +[TestClass] +public sealed class PerformanceBaselineTests +{ + public TestContext TestContext { get; set; } = null!; + + [TestMethod] + [TestCategory("Performance")] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task InsertAndQuery_Baseline(string engine) + { + var rows = Environment.GetEnvironmentVariable("KHAEMHAT_PERF") == "1" ? 100_000 : 2_000; + await using var db = await TestDatabase.CreateAsync(engine); + await db.CreateTestTableAsync(); + await using var unitOfWork = await db.OpenAsync(); + var items = unitOfWork.Repository(); + + var insert = Stopwatch.StartNew(); + await unitOfWork.InTransactionAsync(async ct => + { + for (var i = 0; i < rows; i++) + { + await items.InsertAsync( + new TestItem + { + Code = "K" + i.ToString(CultureInfo.InvariantCulture), + Name = "Item " + i.ToString(CultureInfo.InvariantCulture), + Quantity = i % 100, + Amount = i * 1.5m, + At = DateTime.UtcNow, + }, + ct); + } + }); + insert.Stop(); + + var query = Stopwatch.StartNew(); + var all = await items.ListAsync(Query.For()); + query.Stop(); + + var page = Stopwatch.StartNew(); + var filtered = await items.ListAsync(Query.For().Where(x => x.Quantity == 42).OrderBy(x => x.Code).Page(0, 50)); + page.Stop(); + + Assert.HasCount(rows, all.Items); + Assert.AreEqual(rows / 100, filtered.TotalCount); + TestContext.WriteLine(string.Create( + CultureInfo.InvariantCulture, + $"[perf] {engine}: {rows} inserts in one transaction {insert.ElapsedMilliseconds} ms ({rows * 1000.0 / Math.Max(1, insert.ElapsedMilliseconds):F0}/s); " + + $"read all {query.ElapsedMilliseconds} ms; filtered page+count {page.ElapsedMilliseconds} ms")); + } +} diff --git a/tests/Khaemhat.Data.Tests/ProjectSmokeTests.cs b/tests/Khaemhat.Data.Tests/ProjectSmokeTests.cs index b713c1a..b0ee2ce 100644 --- a/tests/Khaemhat.Data.Tests/ProjectSmokeTests.cs +++ b/tests/Khaemhat.Data.Tests/ProjectSmokeTests.cs @@ -1,10 +1,16 @@ namespace Khaemhat.Data.Tests; -/// Keeps the test project runnable until the milestone that fills it (see docs/STATUS.md). +/// Architecture rule of issue M1 #7: the contracts know no engine. [TestClass] public sealed class ProjectSmokeTests { [TestMethod] - public void TestedAssembly_IsKhaemhat() => - StringAssert.StartsWith(typeof(ProjectSmokeTests).Assembly.GetName().Name, "Khaemhat."); + public void DataContracts_DoNotReferenceAnyDriver() + { + var references = typeof(IDbProvider).Assembly.GetReferencedAssemblies().Select(a => a.Name).ToArray(); + + Assert.DoesNotContain("Npgsql", references); + Assert.DoesNotContain("Microsoft.Data.Sqlite", references); + Assert.Contains("Khaemhat.Core", references); + } } diff --git a/tests/Khaemhat.Data.Tests/QueryTranslationTests.cs b/tests/Khaemhat.Data.Tests/QueryTranslationTests.cs new file mode 100644 index 0000000..7561e99 --- /dev/null +++ b/tests/Khaemhat.Data.Tests/QueryTranslationTests.cs @@ -0,0 +1,112 @@ +using Khaemhat.Data.PostgreSql; +using Khaemhat.Data.Querying; +using Khaemhat.Data.Sqlite; +using Khaemhat.Data.Tests.Support; + +namespace Khaemhat.Data.Tests; + +[TestClass] +public sealed class QueryTranslationTests +{ + private static readonly TestItemMap Map = CreateMap(); + + [TestMethod] + public void Comparisons_AndLogic_UseParameters() + { + var min = 5; + var code = "A'1"; + var sql = Where(Query.For().Where(x => (x.Quantity >= min && x.Code != code) || !x.Active)); + + Assert.AreEqual(" WHERE ((\"quantity\" >= @p0 AND \"code\" <> @p1) OR NOT (\"active\" = @p2))", sql.Text); + CollectionAssert.AreEqual(new object?[] { 5, "A'1", true }, sql.Parameters.Select(p => p.Value).ToArray()); + } + + [TestMethod] + public void ValueOnTheLeft_IsMirrored() + { + var sql = Where(Query.For().Where(x => 10 < x.Quantity)); + + Assert.AreEqual(" WHERE \"quantity\" > @p0", sql.Text); + } + + [TestMethod] + public void Null_BecomesIsNull() + { + string? none = null; + var sql = Where(Query.For().Where(x => x.Name == none && x.Day != null)); + + Assert.AreEqual(" WHERE (\"name\" IS NULL AND \"day\" IS NOT NULL)", sql.Text); + Assert.IsEmpty(sql.Parameters); + } + + [TestMethod] + public void Enums_AreComparedByName() + { + var sql = Where(Query.For().Where(x => x.Kind == TestKind.Service)); + + Assert.AreEqual("Service", sql.Parameters[0].Value); + } + + [TestMethod] + public void In_FromListContains_AndEmptyListIsFalse() + { + var codes = new List { "A", "B" }; + var sql = Where(Query.For().Where(x => codes.Contains(x.Code))); + var none = Where(Query.For().Where(x => Array.Empty().Contains(x.Id))); + + Assert.AreEqual(" WHERE \"code\" IN (@p0, @p1)", sql.Text); + Assert.AreEqual(" WHERE 1 = 0", none.Text); + } + + [TestMethod] + public void Like_IsCaseInsensitiveAndEscaped_PerDialect() + { + var query = Query.For().Where(x => x.Name!.Contains("10%_off")); + + var sqlite = new QueryBuilder(Map, SqliteDialect.Instance).Where(query).Render(SqliteDialect.Instance); + var postgres = new QueryBuilder(Map, PostgresDialect.Instance).Where(query).Render(PostgresDialect.Instance); + + Assert.AreEqual(" WHERE kh_lower(\"name\") LIKE kh_lower(@p0) ESCAPE '\\'", sqlite.Text); + Assert.AreEqual(" WHERE \"name\" ILIKE @p0 ESCAPE '\\'", postgres.Text); + Assert.AreEqual("%10\\%\\_off%", sqlite.Parameters[0].Value); + } + + [TestMethod] + public void UnsupportedExpressions_AreRefused() + { + var query = Query.For().Where(x => x.Code.ToUpperInvariant() == "A"); + + Assert.ThrowsExactly(() => Where(query)); + } + + [TestMethod] + public void Select_OrdersAndPages_AndDefaultsToKeyOrder() + { + var builder = new QueryBuilder(Map, SqliteDialect.Instance); + + var ordered = builder.Select(Query.For().OrderByDescending(x => x.Amount).OrderBy(x => x.Code).Page(20, 10)).Render(SqliteDialect.Instance); + var paged = builder.Select(Query.For().Take(5)).Render(SqliteDialect.Instance); + + StringAssert.EndsWith(ordered.Text, " ORDER BY \"amount\" DESC, \"code\" ASC LIMIT 10 OFFSET 20"); + StringAssert.EndsWith(paged.Text, " ORDER BY \"id\" ASC LIMIT 5 OFFSET 0"); + } + + [TestMethod] + public void SqlConditions_AreParenthesizedAndCombined() + { + var max = 100m; + var sql = Where(Query.For().Where(x => x.Active).Where(SqlFragment.Of($"{SqlFragment.Identifier("amount")} < {max}"))); + + Assert.AreEqual(" WHERE \"active\" = @p0 AND (\"amount\" < @p1)", sql.Text); + } + + private static TestItemMap CreateMap() + { + var map = new TestItemMap(); + map.Freeze(); + return map; + } + + private static RenderedSql Where(Query query) => + new QueryBuilder(Map, SqliteDialect.Instance).Where(query).Render(SqliteDialect.Instance); +} diff --git a/tests/Khaemhat.Data.Tests/RepositoryTests.cs b/tests/Khaemhat.Data.Tests/RepositoryTests.cs new file mode 100644 index 0000000..89e7690 --- /dev/null +++ b/tests/Khaemhat.Data.Tests/RepositoryTests.cs @@ -0,0 +1,190 @@ +using Khaemhat.Core.Common; +using Khaemhat.Data.Querying; +using Khaemhat.Data.Tests.Support; + +namespace Khaemhat.Data.Tests; + +[TestClass] +public sealed class RepositoryTests +{ + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task Crud_WithAuditColumns(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + await db.CreateTestTableAsync(); + await using var unitOfWork = await db.OpenAsync(); + var items = unitOfWork.Repository(); + + var item = new TestItem { Code = "C1", Name = "First", Amount = 10.5m }; + var id = await items.InsertAsync(item); + Assert.IsGreaterThan(0L, id); + Assert.AreEqual(id, item.Id); + Assert.AreEqual(1L, item.RowVersion); + Assert.IsNull(item.UpdatedAt); + + item.Name = "Changed"; + await items.UpdateAsync(item); + Assert.AreEqual(2L, item.RowVersion); + Assert.IsNotNull(item.UpdatedAt); + + var read = await items.GetAsync(id); + Assert.AreEqual("Changed", read!.Name); + Assert.AreEqual(2L, read.RowVersion); + Assert.AreEqual(item.CreatedAt, read.CreatedAt); + Assert.AreEqual(item.UpdatedAt, read.UpdatedAt); + Assert.AreEqual(TestDatabase.TestUser, read.UpdatedBy); + + Assert.IsTrue(await items.DeleteAsync(id)); + Assert.IsFalse(await items.DeleteAsync(id)); + Assert.IsNull(await items.GetAsync(id)); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task Update_OfAStaleCopy_ThrowsConcurrencyException(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + await db.CreateTestTableAsync(); + long id; + await using (var setup = await db.OpenAsync()) + { + id = await setup.Repository().InsertAsync(new TestItem { Code = "CC", Name = "v1" }); + } + + await using var first = await db.OpenAsync(); + await using var second = await db.OpenAsync(); + var copyA = await first.Repository().GetAsync(id); + var copyB = await second.Repository().GetAsync(id); + + copyA!.Name = "by A"; + await first.Repository().UpdateAsync(copyA); + + copyB!.Name = "by B"; + var conflict = await Assert.ThrowsExactlyAsync(() => second.Repository().UpdateAsync(copyB)); + Assert.AreEqual("test_items", conflict.Table); + Assert.AreEqual(1L, conflict.ExpectedVersion); + Assert.AreEqual("by A", (await second.Repository().GetAsync(id))!.Name); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task Update_OfADeletedRow_ThrowsConcurrencyException(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + await db.CreateTestTableAsync(); + await using var unitOfWork = await db.OpenAsync(); + var items = unitOfWork.Repository(); + var item = new TestItem { Code = "GONE" }; + await items.InsertAsync(item); + await items.DeleteAsync(item.Id); + + await Assert.ThrowsExactlyAsync(() => items.UpdateAsync(item)); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task Transactions_CommitAndRollback(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + await db.CreateTestTableAsync(); + await using var unitOfWork = await db.OpenAsync(); + var items = unitOfWork.Repository(); + + await unitOfWork.BeginAsync(); + await items.InsertAsync(new TestItem { Code = "ROLLED" }); + await unitOfWork.RollbackAsync(); + + await Assert.ThrowsExactlyAsync( + () => unitOfWork.InTransactionAsync(async ct => + { + await items.InsertAsync(new TestItem { Code = "FAILED" }, ct); + throw new InvalidOperationException("business rule"); + })); + + await unitOfWork.InTransactionAsync(async ct => + { + await unitOfWork.AcquireLockAsync("numerator:FV:2026", ct); + await items.InsertAsync(new TestItem { Code = "KEPT" }, ct); + }); + + var codes = (await items.ListAsync(Query.For())).Items.Select(x => x.Code).ToArray(); + CollectionAssert.AreEqual(new[] { "KEPT" }, codes); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task HostileInput_IsStoredAndFound_AsData(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + await db.CreateTestTableAsync(); + await using var unitOfWork = await db.OpenAsync(); + var items = unitOfWork.Repository(); + const string hostile = "x'); DROP TABLE test_items; --"; + + await items.InsertAsync(new TestItem { Code = "H1", Name = hostile }); + var found = await items.FindAsync(Query.For().Where(x => x.Name == hostile)); + var like = await items.CountAsync(Query.For().Where(x => x.Name!.Contains("'); DROP"))); + var raw = await unitOfWork.QueryAsync(SqlFragment.Of($"SELECT code FROM test_items WHERE name = {hostile}")); + + Assert.AreEqual("H1", found?.Code); + Assert.AreEqual(1, like); + CollectionAssert.AreEqual(new[] { "H1" }, raw.ToArray()); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task Queries_MapToRecordsClassesAndScalars(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + await db.CreateTestTableAsync(); + await using var unitOfWork = await db.OpenAsync(); + var items = unitOfWork.Repository(); + await items.InsertAsync(new TestItem { Code = "R1", Amount = 1.25m, Kind = TestKind.Service }); + await items.InsertAsync(new TestItem { Code = "R2", Amount = 2.5m, Kind = TestKind.Service }); + await items.InsertAsync(new TestItem { Code = "R3", Amount = 4m, Kind = TestKind.Goods }); + + var totals = await unitOfWork.QueryAsync( + SqlFragment.Literal("SELECT kind, SUM(amount) AS total_amount, COUNT(*) AS item_count FROM test_items GROUP BY kind ORDER BY kind")); + var rows = await unitOfWork.QueryAsync(SqlFragment.Literal("SELECT code, amount FROM test_items ORDER BY code")); + var kinds = await unitOfWork.QueryAsync(SqlFragment.Literal("SELECT DISTINCT kind FROM test_items ORDER BY kind")); + var ids = await unitOfWork.QueryAsync(SqlFragment.Literal("SELECT id FROM test_items ORDER BY id")); + var inList = await items.CountAsync(Query.For().Where(x => ids.Contains(x.Id) && x.Kind == TestKind.Service)); + + CollectionAssert.AreEqual( + new[] { new KindTotal(TestKind.Goods, 4m, 1), new KindTotal(TestKind.Service, 3.75m, 2) }, totals.ToArray()); + Assert.AreEqual(2.5m, rows[1].Amount); + CollectionAssert.AreEqual(new[] { TestKind.Goods, TestKind.Service }, kinds.ToArray()); + Assert.AreEqual(2, inList); + } + + [TestMethod] + [DataRow(TestDatabase.Sqlite)] + [DataRow(TestDatabase.Postgres)] + public async Task Stream_ReadsWithoutMapping(string engine) + { + await using var db = await TestDatabase.CreateAsync(engine); + await db.CreateTestTableAsync(); + await using var unitOfWork = await db.OpenAsync(); + await unitOfWork.Repository().InsertAsync(new TestItem { Code = "S1", Amount = 7.000001m }); + + await using var reader = await unitOfWork.StreamAsync(SqlFragment.Literal("SELECT amount FROM test_items")); + Assert.IsTrue(await reader.ReadAsync()); + Assert.AreEqual(7.000001m, db.Dialect.FromDbValue(reader.GetValue(0), Schema.ColumnType.Decimal)); + } + + private sealed record KindTotal(TestKind Kind, decimal TotalAmount, long ItemCount); + + private sealed class CodeRow + { + public string Code { get; set; } = string.Empty; + + public decimal Amount { get; set; } + } +} diff --git a/tests/Khaemhat.Data.Tests/SqlFragmentTests.cs b/tests/Khaemhat.Data.Tests/SqlFragmentTests.cs new file mode 100644 index 0000000..fd5611a --- /dev/null +++ b/tests/Khaemhat.Data.Tests/SqlFragmentTests.cs @@ -0,0 +1,70 @@ +using Khaemhat.Data.PostgreSql; +using Khaemhat.Data.Schema; +using Khaemhat.Data.Sqlite; + +namespace Khaemhat.Data.Tests; + +[TestClass] +public sealed class SqlFragmentTests +{ + [TestMethod] + public void Holes_BecomeParameters_AndNeverReachTheText() + { + const string hostile = "x'); DROP TABLE parties; --"; + var fragment = SqlFragment.Of($"SELECT * FROM parties WHERE name = {hostile} AND id > {42L}"); + + var rendered = fragment.Render(SqliteDialect.Instance); + + Assert.AreEqual("SELECT * FROM parties WHERE name = @p0 AND id > @p1", rendered.Text); + Assert.HasCount(2, rendered.Parameters); + Assert.AreEqual(hostile, rendered.Parameters[0].Value); + Assert.AreEqual(42L, rendered.Parameters[1].Value); + Assert.DoesNotContain("DROP", rendered.Text); + } + + [TestMethod] + public void NestedFragments_AreRenumbered() + { + var inner = SqlFragment.Of($"code = {"A"}"); + var outer = SqlFragment.Of($"SELECT 1 WHERE {inner} OR name = {"B"}"); + + var rendered = outer.Render(PostgresDialect.Instance); + + Assert.AreEqual("SELECT 1 WHERE code = @p0 OR name = @p1", rendered.Text); + CollectionAssert.AreEqual(new object?[] { "A", "B" }, rendered.Parameters.Select(p => p.Value).ToArray()); + } + + [TestMethod] + public void Identifiers_AreValidatedAndQuoted() + { + var fragment = SqlFragment.Of($"SELECT {SqlFragment.Identifier("parties.vat_number")} FROM {SqlFragment.Identifier("parties")}"); + + Assert.AreEqual("SELECT \"parties\".\"vat_number\" FROM \"parties\"", fragment.Render(SqliteDialect.Instance).Text); + Assert.ThrowsExactly(() => SqlFragment.Identifier("parties; DROP TABLE x")); + Assert.ThrowsExactly(() => SqlFragment.Identifier("Name")); + } + + [TestMethod] + public void ToString_ShowsPlaceholdersButNotValues() + { + var fragment = SqlFragment.Of($"UPDATE sys_settings SET value = {"s3cr3t-password"}"); + + Assert.AreEqual("UPDATE sys_settings SET value = @p0", fragment.ToString()); + } + + [TestMethod] + public void TypedParameter_KeepsItsType() + { + var fragment = SqlFragment.Of($"INSERT INTO t (data) VALUES ({SqlFragment.Parameter("{}", ColumnType.Json)})"); + + Assert.AreEqual(ColumnType.Json, fragment.Render(PostgresDialect.Instance).Parameters[0].Type); + } + + [TestMethod] + public void Join_SkipsEmptyFragments() + { + var joined = SqlFragment.Join(" AND ", [SqlFragment.Literal("a = 1"), SqlFragment.Empty, SqlFragment.Of($"b = {2}")]); + + Assert.AreEqual("a = 1 AND b = @p0", joined.Render(SqliteDialect.Instance).Text); + } +} diff --git a/tests/Khaemhat.Data.Tests/Support/TestDatabase.cs b/tests/Khaemhat.Data.Tests/Support/TestDatabase.cs new file mode 100644 index 0000000..b4f89cf --- /dev/null +++ b/tests/Khaemhat.Data.Tests/Support/TestDatabase.cs @@ -0,0 +1,93 @@ +using System.Data.Common; +using Khaemhat.Data.Execution; +using Khaemhat.Data.Mapping; +using Khaemhat.Data.PostgreSql; +using Khaemhat.Data.Sqlite; +using Khaemhat.Data.SystemDatabase; + +namespace Khaemhat.Data.Tests.Support; + +/// +/// A fresh, empty database for one test: SQLite in memory, or a temporary PostgreSQL database on the server of +/// KHAEMHAT_TEST_PG (dropped at the end). PostgreSQL tests are inconclusive (skipped) when the variable is unset. +/// +internal sealed class TestDatabase : IAsyncDisposable +{ + public const string Sqlite = "Sqlite"; + public const string Postgres = "PostgreSql"; + public const string TestUser = "tester"; + + private readonly DbConnection? _keeper; + + private TestDatabase(DatabaseUrl url, DatabaseTarget target, DbProviderCatalog providers, DbConnection? keeper) + { + Url = url; + Target = target; + Providers = providers; + _keeper = keeper; + } + + public static string? PostgresServer => Environment.GetEnvironmentVariable("KHAEMHAT_TEST_PG") is { Length: > 0 } value ? value : null; + + public DatabaseUrl Url { get; } + + public DatabaseTarget Target { get; } + + public DbProviderCatalog Providers { get; } + + public IDbProvider Provider => Target.Provider; + + public ISqlDialect Dialect => Provider.Dialect; + + public MapRegistry Maps { get; } = new MapRegistry().AddFromAssembly(typeof(CompanyMap).Assembly).Add(new TestItemMap()); + + public DataAccessContext Context { get; } = new(TimeProvider.System, () => TestUser); + + public static DbProviderCatalog AllProviders() => new(new SqliteProvider(), new PostgresProvider()); + + public static async Task CreateAsync(string engine) + { + var providers = AllProviders(); + if (engine == Sqlite) + { + var url = DatabaseUrl.Parse("sqlite://memory/t" + Guid.NewGuid().ToString("N")); + var target = providers.Resolve(url); + var keeper = target.Provider.CreateConnection(target.ConnectionString); + await keeper.OpenAsync(); + return new TestDatabase(url, target, providers, keeper); + } + + if (PostgresServer is null) + { + Assert.Inconclusive("KHAEMHAT_TEST_PG is not set: PostgreSQL test skipped."); + } + + var pgUrl = DatabaseUrl.Parse(PostgresServer!).WithDatabase("khaemhat_t_" + Guid.NewGuid().ToString("N")[..12]); + var pgTarget = providers.Resolve(pgUrl); + await pgTarget.Provider.EnsureDatabaseExistsAsync(pgTarget.ConnectionString); + return new TestDatabase(pgUrl, pgTarget, providers, keeper: null); + } + + public Task OpenAsync() => UnitOfWork.OpenAsync(Target, Maps, Context); + + /// Creates the test_items table used by the repository and dialect tests. + public async Task CreateTestTableAsync() + { + await using var unitOfWork = await OpenAsync(); + foreach (var statement in Provider.Ddl.Generate(TestItemMap.Schema())) + { + await unitOfWork.ExecuteAsync(SqlFragment.Literal(statement)); + } + } + + public async ValueTask DisposeAsync() + { + if (_keeper is not null) + { + await _keeper.DisposeAsync(); + return; + } + + await Provider.DropDatabaseAsync(Target.ConnectionString); + } +} diff --git a/tests/Khaemhat.Data.Tests/Support/TestItem.cs b/tests/Khaemhat.Data.Tests/Support/TestItem.cs new file mode 100644 index 0000000..8ec3f25 --- /dev/null +++ b/tests/Khaemhat.Data.Tests/Support/TestItem.cs @@ -0,0 +1,79 @@ +using Khaemhat.Core.Entities; +using Khaemhat.Data.Mapping; +using Khaemhat.Data.Schema; + +namespace Khaemhat.Data.Tests.Support; + +internal enum TestKind +{ + Goods = 0, + Service = 1, + Kit = 2, +} + +/// An entity with one property of every logical type. +internal sealed class TestItem : Entity, IDemoMarkable +{ + public string Code { get; set; } = string.Empty; + + public string? Name { get; set; } + + public int Quantity { get; set; } + + public long Big { get; set; } + + public decimal Amount { get; set; } + + public bool Active { get; set; } + + public DateOnly? Day { get; set; } + + public DateTime? At { get; set; } + + public Guid? Uid { get; set; } + + public byte[]? Data { get; set; } + + public string? Payload { get; set; } + + public TestKind Kind { get; set; } + + public bool IsDemo { get; set; } +} + +internal sealed class TestItemMap : EntityMap +{ + public TestItemMap() + { + ToTable("test_items"); + Map(x => x.Code).MaxLength(20); + Map(x => x.Name).MaxLength(100); + Map(x => x.Payload).Json(); + } + + public static IReadOnlyList Schema() + { + var schema = new SchemaBuilder(); + schema.CreateTable("test_items", t => + { + t.Id(); + t.String("code", 20).NotNull(); + t.String("name", 100); + t.Int32("quantity").NotNull(); + t.Int64("big").NotNull(); + t.Decimal("amount").NotNull(); + t.Bool("active").NotNull(); + t.Date("day"); + t.DateTimeUtc("at"); + t.Guid("uid"); + t.Bytes("data"); + t.Json("payload"); + t.String("kind", 40).NotNull(); + t.DemoColumn(); + t.AuditColumns(); + t.Unique("ux_test_items_code", "code"); + t.Index("ix_test_items_at", "at"); + }); + return schema.Operations; + } +} diff --git a/tests/Khaemhat.Infrastructure.Tests/ProjectSmokeTests.cs b/tests/Khaemhat.Infrastructure.Tests/ProjectSmokeTests.cs deleted file mode 100644 index fc262f5..0000000 --- a/tests/Khaemhat.Infrastructure.Tests/ProjectSmokeTests.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace Khaemhat.Infrastructure.Tests; - -/// Keeps the test project runnable until the milestone that fills it (see docs/STATUS.md). -[TestClass] -public sealed class ProjectSmokeTests -{ - [TestMethod] - public void TestedAssembly_IsKhaemhat() => - StringAssert.StartsWith(typeof(ProjectSmokeTests).Assembly.GetName().Name, "Khaemhat."); -} diff --git a/tests/Khaemhat.Infrastructure.Tests/SecretProtectorTests.cs b/tests/Khaemhat.Infrastructure.Tests/SecretProtectorTests.cs new file mode 100644 index 0000000..c6e6c81 --- /dev/null +++ b/tests/Khaemhat.Infrastructure.Tests/SecretProtectorTests.cs @@ -0,0 +1,133 @@ +using Khaemhat.Core.Security; +using Khaemhat.Infrastructure.Security; + +namespace Khaemhat.Infrastructure.Tests; + +/// Encryption of secrets at rest (SPEC 12.9, issue M1 #12). +[TestClass] +public sealed class SecretProtectorTests +{ + private const string Purpose = "company:ACME:database"; + private const string Secret = "postgres://acme:S3cret!@db/khaemhat_acme"; + + [TestMethod] + public void Protect_ThenUnprotect_RoundTrips() + { + var protector = NewProtector(); + + var protectedValue = protector.Protect(Secret, Purpose); + + StringAssert.StartsWith(protectedValue, "khs1." + protector.Key.Id + "."); + Assert.DoesNotContain("S3cret", protectedValue); + Assert.AreEqual(Secret, protector.Unprotect(protectedValue, Purpose)); + Assert.AreEqual(string.Empty, protector.Unprotect(protector.Protect(string.Empty, Purpose), Purpose)); + } + + [TestMethod] + public void TheSameSecret_IsEncryptedDifferentlyEachTime() + { + var protector = NewProtector(); + + Assert.AreNotEqual(protector.Protect(Secret, Purpose), protector.Protect(Secret, Purpose)); + } + + [TestMethod] + public void KeyId_IsStable_ForTheSameKey() + { + var key = SecretKey.GenerateBase64(); + + Assert.AreEqual(SecretKey.FromBase64(key).Id, SecretKey.FromBase64(key).Id); + Assert.AreEqual(8, SecretKey.FromBase64(key).Id.Length); + Assert.AreNotEqual(SecretKey.FromBase64(key).Id, SecretKey.FromBase64(SecretKey.GenerateBase64()).Id); + Assert.AreEqual("SecretKey " + SecretKey.FromBase64(key).Id, SecretKey.FromBase64(key).ToString()); + } + + [TestMethod] + public void AnotherKey_IsRefused_NamingBothKeyIds() + { + var protectedValue = NewProtector().Protect(Secret, Purpose); + var other = NewProtector(); + + var error = Assert.ThrowsExactly(() => other.Unprotect(protectedValue, Purpose)); + + StringAssert.Contains(error.Message, SecretProtector.KeyIdOf(protectedValue)!); + StringAssert.Contains(error.Message, other.Key.Id); + Assert.DoesNotContain("S3cret", error.ToString()); + } + + [TestMethod] + public void AnotherPurpose_IsRefused() + { + var protector = NewProtector(); + var protectedValue = protector.Protect(Secret, Purpose); + + Assert.ThrowsExactly(() => protector.Unprotect(protectedValue, "company:EVIL:database")); + } + + [TestMethod] + [DataRow("")] + [DataRow("plain text")] + [DataRow("khs1.abc")] + [DataRow("khs2.00000000.AAAA")] + public void NotAProtectedValue_IsRefused(string value) + { + Assert.ThrowsExactly(() => NewProtector().Unprotect(value, Purpose)); + } + + [TestMethod] + public void DamagedValues_AreRefused() + { + var protector = NewProtector(); + var protectedValue = protector.Protect(Secret, Purpose); + var payload = protectedValue[(protectedValue.LastIndexOf('.') + 1)..]; + var prefix = protectedValue[..(protectedValue.LastIndexOf('.') + 1)]; + var damaged = payload[..20] + (payload[20] == 'A' ? 'B' : 'A') + payload[21..]; + + Assert.ThrowsExactly(() => protector.Unprotect(prefix + damaged, Purpose)); + Assert.ThrowsExactly(() => protector.Unprotect(prefix + payload[..10], Purpose)); + Assert.ThrowsExactly(() => protector.Unprotect(prefix + "not*base64", Purpose)); + } + + [TestMethod] + [DataRow("")] + [DataRow(" ")] + [DataRow("not base64!")] + [DataRow("c2hvcnQ=")] + public void WeakOrInvalidKeys_AreRefused_WithoutEchoingThem(string value) + { + var error = Assert.ThrowsExactly(() => SecretKey.FromBase64(value)); + + StringAssert.Contains(error.Message, "KHAEMHAT_SECRET_KEY"); + if (value.Trim().Length > 0) + { + Assert.DoesNotContain(value, error.Message); + } + } + + [TestMethod] + public void Rotation_ReencryptsWithTheNewKey() + { + var current = NewProtector(); + var next = NewProtector(); + var rotation = new SecretRotation(current, next); + var old = current.Protect(Secret, Purpose); + + Assert.IsTrue(rotation.NeedsRotation(old)); + var rotated = rotation.Rotate(old, Purpose); + + Assert.IsFalse(rotation.NeedsRotation(rotated)); + Assert.AreEqual(next.Key.Id, SecretProtector.KeyIdOf(rotated)); + Assert.AreEqual(Secret, next.Unprotect(rotated, Purpose)); + Assert.ThrowsExactly(() => current.Unprotect(rotated, Purpose)); + } + + [TestMethod] + public void KeyIdOf_ReadsOnlyProtectedValues() + { + Assert.IsNull(SecretProtector.KeyIdOf(null)); + Assert.IsNull(SecretProtector.KeyIdOf("plain")); + Assert.AreEqual("0a1b2c3d", SecretProtector.KeyIdOf("khs1.0a1b2c3d.AAAA")); + } + + private static SecretProtector NewProtector() => new(SecretKey.FromBase64(SecretKey.GenerateBase64())); +} diff --git a/tools/Khaemhat.DataModelTool/Khaemhat.DataModelTool.csproj b/tools/Khaemhat.DataModelTool/Khaemhat.DataModelTool.csproj new file mode 100644 index 0000000..6d9e0a8 --- /dev/null +++ b/tools/Khaemhat.DataModelTool/Khaemhat.DataModelTool.csproj @@ -0,0 +1,14 @@ + + + + Exe + Khaemhat.DataModelTool + + false + + + + + + + diff --git a/tools/Khaemhat.DataModelTool/Program.cs b/tools/Khaemhat.DataModelTool/Program.cs new file mode 100644 index 0000000..37ce22a --- /dev/null +++ b/tools/Khaemhat.DataModelTool/Program.cs @@ -0,0 +1,31 @@ +// Khaemhat.DataModelTool: writes the generated section of docs/DATA-MODEL.md from the migrations (SPEC §18). +// dotnet run --project tools/Khaemhat.DataModelTool -- [--check] [path/to/DATA-MODEL.md] +// With --check it changes nothing and exits with 1 when the document is out of date (used by CI and tests). +using System.Text; +using Khaemhat.Data.Migrations; + +var check = args.Contains("--check"); +var path = args.FirstOrDefault(a => !a.StartsWith("--", StringComparison.Ordinal)) ?? Path.Combine("docs", "DATA-MODEL.md"); +if (!File.Exists(path)) +{ + Console.Error.WriteLine($"DataModelTool: file non trovato: {path}"); + return 2; +} + +var current = File.ReadAllText(path, Encoding.UTF8); +var updated = DataModelDocument.Update(current, DataModelDocument.Render(MigrationCatalog.ForSystem(), MigrationCatalog.ForCompany())); +if (string.Equals(current, updated, StringComparison.Ordinal)) +{ + Console.WriteLine($"DataModelTool: {path} è aggiornato."); + return 0; +} + +if (check) +{ + Console.Error.WriteLine($"DataModelTool: {path} non è aggiornato: eseguire scripts/gen-datamodel.sh."); + return 1; +} + +File.WriteAllText(path, updated, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false)); +Console.WriteLine($"DataModelTool: {path} rigenerato."); +return 0;