diff --git a/Encelado/CHANGELOG.md b/Encelado/CHANGELOG.md index f59cb71..31fee17 100644 --- a/Encelado/CHANGELOG.md +++ b/Encelado/CHANGELOG.md @@ -2,6 +2,19 @@ Formato: una voce per sessione di lavoro, con data. Le voci più recenti in alto. +## 2026-09-23 — 5.0, Fasi 0-1: post-mortem degli ordini pendenti, registro degli ordini, orfane adottate + +- **Diagnosi verificata** sul codice e sul conto demo via API: il lookup per `referenceId` fallisce perché il server registra un riferimento nullo per gli ordini v2; 21 gambe singole del bot fra il 16 e il 21/9, chiuse a mano il 21/9; i primi due ordini hanno impegnato tutta l'equity come margine e da lì il server ha ridotto ogni ordine a 2 000 USD di margine. `docs/PIANO_5.0.md`, `docs/POSTMORTEM_ordini_pendenti.md`, domande D-26…D-37. +- **Registro persistente degli ordini** (`OrderTracker`, `data/state/pending_orders.json`, `orders.jsonl`): scritto prima di ogni invio, risolto per `orderId`, per riferimento e per posizione comparsa; ricaricato all'avvio e risolto prima di ogni decisione. Nessun esito inventato: `Unknown` resta pendente (ADR-0009). +- **Stati `PendingA`/`PendingB`**: una gamba senza esito non è più un rifiuto; alla risoluzione parte la gamba B (ridimensionata sulle unità eseguite di A) oppure la gamba A viene richiusa se il segnale è decaduto. +- **Classificazione delle posizioni** (`basket` / `orfana-bot` / `esterna`) a ogni riconciliazione; le orfane del bot vengono adottate e chiuse; contatori «in attesa · orfane · esterne» e P&L aperto **del conto** in dashboard; avviso «posizioni non riconciliate». +- **Movimenti di cassa** riconosciuti e scritti nel ledger; picco di equity e drawdown al netto (`EquityTracker`). +- **Bonifica** (`--headless --bonifica`, comando `bonifica`): elenco delle orfane con conferma per posizione, rapporto in `reports/bonifica_YYYYMMDD.csv`. +- `IBroker.LookupOrderByIdAsync` e `CancelOrderAsync`; `EtoroBroker.OpenAsync` legge l'esito per `orderId` e riconosce l'esecuzione dalla posizione; parser dell'esito v2 e v1. +- Il bandit **propone e non applica** più il preset (D-30). +- `BasketEngine` spezzato in sei file parziali. +- Test nuovi (m)-(q) e altri 13: 190 test verdi. + ## 2026-09-16 (pomeriggio) — 4.0.0: solo Correlation Baskets su eToro, bot autonomo, interfaccia nuova - **Rimossi** i motori precedenti: Binance, Alpaca, cTrader/proba, SQLite, GBDT, RL, TA-Lib, indicatori, backtest a coppie, pagine e test relativi (ADR-0004). Nessun pacchetto NuGet nell'applicazione. diff --git a/Encelado/docs/ARCHITECTURE.md b/Encelado/docs/ARCHITECTURE.md index d21359b..f98c139 100644 --- a/Encelado/docs/ARCHITECTURE.md +++ b/Encelado/docs/ARCHITECTURE.md @@ -1,6 +1,6 @@ # Architettura di Encelado -Aggiornato: 2026-09-16 (Fase 0 della modifica "Correlation Baskets" su eToro). +Aggiornato: 2026-09-23 (Fase 1 del piano 5.0: registro ordini, stati Pending, classificazione delle posizioni). ## 1. Che cosa c'era prima della modifica (ricognizione) @@ -75,8 +75,11 @@ Decisione presa (vedi `docs/QUESTIONS.md`, D-09): il motore cTrader resta nel re src/Encelado.Core/Broker/ IBroker, modelli (Instrument, QuoteSnapshot, AccountSnapshot, BrokerPosition, OrderRequest, OrderOutcome), PaperBroker (simulatore sopra un feed reale), RateLimiter src/Encelado.Core/Baskets/ matematica e logica pura, senza I/O: SyntheticCross (derivazione automatica del cross e dei segni), PipMath, BasketMath (rendimenti log, ATR, EWMA vol, ρ_W/ρ_20, z-score, semiperiodo OLS, forza di trend), - SymbolSeries (barre + quote + qualità dati), BasketDecider (entrate/uscite/averaging di §5), CostGate, VolParitySizing, BasketExecutor (protocollo leg-risk), - BasketPosition (macchina a stati), BasketStrategyConfig (strategy.json, preset), ExecutionMode (Paper | Demo | Live) + SymbolSeries (barre + quote + qualità dati), BasketDecider (entrate/uscite/averaging di §5), CostGate, VolParitySizing, BasketExecutor (protocollo leg-risk con il registro), + BasketPosition (macchina a stati, con PendingA/PendingB), PendingEntry (ingresso in sospeso), BasketStrategyConfig (strategy.json, preset), ExecutionMode (Paper | Demo | Live), + OrderTracker (registro persistente degli ordini, risoluzione per orderId / riferimento / posizioni — ADR-0009), PositionClassifier (basket | orfana-bot | esterna), + EquityTracker (picco al netto dei movimenti di cassa) +src/Encelado.Core/Baskets/History/ OrderRecord (riga di orders.jsonl); dalla Fase 7 PositionRecord, PeriodStats, HistoryBuilder src/Encelado.Core/Baskets/Data/ BidAskBar + CSV, TickToBars (tick MT5 → M15) src/Encelado.Core/Baskets/Learning/ livelli 0-3: CalibrationTables, OnlineLogistic (SGD+L2, standardizzazione rolling), SmallMlp (16 ReLU, Adam, early stopping, gradient check), ThompsonBandit (Beta per preset × terzile di vol), VolForecast (EWMA vs HAR-RV, PSI), LearningFeatures (28 feature del ledger), ModelEvaluator (walk-forward, fold purgati, bootstrap, attivazione) @@ -84,8 +87,10 @@ src/Encelado.Core/Baskets/Backtest/ BasketBacktest (event-driven su barre M15 b src/Encelado.Core/News/ parser puri: CalendarParser (JSON/XML FairEconomy), RssParser (XmlReader), SentimentLexicon, SentimentEngine (finestre 1h/4h/24h con decadimento) src/Encelado.Core/Ml/, Statistics/ la statistica condivisa rimasta: Classification (AUC, Brier, log-loss, calibrazione), Pbo (CSCV), Performance (Sharpe, PSR, DSR, drawdown, momenti), Ols, Normal src/Encelado.Etoro/ EtoroOptions, EtoroHttp (HttpClient, x-api-key/x-user-key/x-request-id, limitatore per classe di quota, 429 con Retry-After, scarto orologio dall'header Date), EtoroBroker : IBroker -src/Encelado.Bot/Baskets/ BasketEngine (ciclo di decisione a thread singolo, polling quote, barre locali, esecuzione diretta, equity stop, kill-switch, file STOP, riconciliazione), - Ledger (decisions.jsonl append-only, baskets.csv, rotazione mensile, scritture atomiche), Feeds (calendario + RSS con cache su disco, robots.txt, backoff), +src/Encelado.Bot/Baskets/ BasketEngine in file parziali: BasketEngine.cs (ciclo a thread singolo, polling quote, barre locali, decisioni, esecuzione), .Pending.cs (registro ordini: risoluzione e + ripresa degli ingressi in sospeso), .Reconcile.cs (conto, classificazione delle posizioni, adozione delle orfane, movimenti di cassa, equity stop, kill-switch, file STOP), + .State.cs (baskets_state.json), .Commands.cs (comandi, bonifica), .Snapshot.cs (snapshot per finestra e console), + Ledger (decisions.jsonl e orders.jsonl append-only, baskets.csv, firme delle decisioni, rotazione mensile, scritture atomiche), Feeds (calendario + RSS con cache su disco, robots.txt, backoff), LearningState (modello in ombra, bandit, ciclo settimanale, knowledge/), HeadlessRunner (--headless) src/Encelado.Bot/Configuration/ BotConfig (etoro, run, ui, logging), ConfigLoader (JsonDocument, avvisi sulle sezioni di versioni precedenti), ConfigDefaults, ConfigWriter, EtoroKeyStore (DPAPI) src/Encelado.Bot/Engine/ IEngine, BotSupervisor (ciclo di vita, snapshot, feed di attività), BotSnapshot @@ -122,7 +127,11 @@ Le decisioni avvengono su un solo thread; l'I/O è asincrono; l'unico gate umano ``` Idle ──(segnale + cancelli)──► Entering ──(A e B eseguite)──► Open ──(add)──► Adding ──► Open - ▲ │ (B rifiutata/timeout → chiudi A, leg_risk_unwind, basket disattivato 1 h) + ▲ ▲ │ (B rifiutata → chiudi A, leg_risk_unwind, basket disattivato 1 h) + │ │ │ (A senza esito oltre legTimeoutSec) ──► PendingA ──(A eseguita, segnale valido)──► Entering + │ │ │ │ (A rifiutata → Idle; A eseguita e segnale decaduto → chiudi A → Idle) + │ │ │ (A eseguita, B senza esito) ──► PendingB ──(B eseguita)──► Open + │ └───────────────────────────────────────────────────────────────── (B rifiutata → chiudi A → Idle) │ ▼ └────────── Closed ◄──── Exiting ◄──(TP | z_out | stop | time-stop | manuale | forzata)── Open │ (una gamba non chiude dopo 3 tentativi) @@ -130,9 +139,11 @@ Idle ──(segnale + cancelli)──► Entering ──(A e B eseguite)── Error (blocco nuove entrate finché non risolto) ``` +Negli stati `PendingA`/`PendingB` il basket non viene valutato e non manda ordini; il registro degli ordini (`OrderTracker`) chiede l'esito al server e, quando manca, lo ricostruisce dalla posizione comparsa sul conto. Gli ingressi in sospeso sopravvivono a un riavvio (`pendingEntries` in `baskets_state.json`) e vengono risolti all'avvio prima di qualsiasi decisione. + ### 2.4 Interfacce -- `IBroker`: `Environment`, `GetInstrumentsAsync`, `GetQuotesAsync(ids)`, `GetCandlesAsync(id, interval, count)`, `GetAccountAsync`, `GetPositionsAsync`, `OpenAsync(OrderRequest)`, `LookupOrderAsync`, `CloseAsync(positionId, instrumentId)`, `UpdateStopsAsync(positionId, sl, tp)`, `GetCostAsync(OrderRequest)`, `GetClosedTradesAsync`, `ClockSkew`. +- `IBroker`: `Environment`, `GetInstrumentsAsync`, `GetQuotesAsync(ids)`, `GetCandlesAsync(id, interval, count)`, `GetAccountAsync`, `GetPositionsAsync`, `OpenAsync(OrderRequest)` (esito per `orderId`, poi per posizione comparsa; mai un esito inventato), `LookupOrderAsync(clientRef)`, `LookupOrderByIdAsync(orderId)`, `CancelOrderAsync(orderId)`, `CloseAsync(positionId, instrumentId)`, `UpdateStopsAsync(positionId, sl, tp)`, `GetCostAsync(OrderRequest)`, `GetClosedTradesAsync`, `ClockSkew`. - `IContextProvider` (Bot): calendario, notizie e sentiment per basket (`FeedContextProvider`; `EmptyContextProvider` nei test). - `IModel`: `Predict(features)`, `Update(features, label)`, JSON, implementato da `OnlineLogistic` e `SmallMlp`. - `IEngine` (Bot): `RunAsync`, `CloseAllAsync`, `ExecuteAsync(EngineCommand)` con `Close`, `KillSwitch`, `SetPreset`, `ResetEquityStop(motivazione)`, `Snapshot()`. @@ -144,5 +155,5 @@ Idle ──(segnale + cancelli)──► Entering ──(A e B eseguite)── - Quote di mercato (`/api/v2/market-data/rates`) in batch fino a 1000 strumenti per chiamata: un polling ogni 3 s costa 20 richieste/min sulla quota condivisa di 120/min. - Quota ordini: 20 richieste/min (demo e reale separate). Un basket costa 2 aperture + 2 chiusure. - Le quote di `rates` sono senza markup; il costo effettivo (markup + spread di mercato + overnight) arriva da `POST /trading/info/{demo/}costs` (20/min dedicate). Il cost gate somma i due. -- Ordini: `POST /api/v2/trading/execution/{demo/}orders` (asincrono: esito con `orders:lookup` per `referenceId` = `x-request-id`); `sellShort` e leva > 1 richiedono `stopLossRate`. Chiusura: `POST /api/v1/trading/execution/{demo/}market-close-orders/positions/{id}`. +- Ordini: `POST /api/v2/trading/execution/{demo/}orders` (asincrono: esito con `orders:lookup?orderId=`; il server **non** registra l'`x-request-id` come `referenceId`, verificato il 2026-09-23); `sellShort` e leva > 1 richiedono `stopLossRate`. Il server può **ridurre** un ordine invece di rifiutarlo (2 000 USD di margine a margine esaurito, 2026-09-16): le unità eseguite si leggono dalla risposta, mai date per scontate. Chiusura: `POST /api/v1/trading/execution/{demo/}market-close-orders/positions/{id}`. Cancellazione: `DELETE /api/v2/trading/execution/{demo/}orders/{id}`. - Esposizione minima per posizione: 1000 USD (`minPositionExposure`); leva ammessa 1-30 (majors) e 1-20 (minors). Il conto reale dell'utente vale 193,18 USD: con i limiti di rischio della strategia il reale non è praticabile oggi (vedi QUESTIONS D-05). diff --git a/Encelado/docs/DATA_SOURCES.md b/Encelado/docs/DATA_SOURCES.md index f3a31d7..56570b5 100644 --- a/Encelado/docs/DATA_SOURCES.md +++ b/Encelado/docs/DATA_SOURCES.md @@ -15,6 +15,20 @@ Aggiornato: 2026-09-16. Ogni fonte è stata verificata alla data indicata; se un Qualità (`data/market/data_quality.csv`, generato da `backtest ticks`, e `reports/data_quality.csv` dal bot): buchi > 1 h nei giorni feriali, salti > 2 % fra barre, duplicati. Una barra sospetta sospende le decisioni sul basket coinvolto per quella barra. +### 1.1 Ordini e posizioni (rotte verificate il 2026-09-23 sull'OpenAPI v1.379.0 e con chiamate reali sul conto demo) + +| Rotta | Uso | Note verificate | +|---|---|---| +| `POST api/v2/trading/execution/{demo/}orders` | invio dell'ordine (`action open`, `transaction buy/sellShort`, `orderType mkt`, `units`, `leverage`, `stopLossRate`) | risponde 200 con `orderId`; il server lavora l'ordine in modo asincrono. **Non registra l'`x-request-id` come riferimento**: la lettura per `orderId` di un ordine del bot mostra `referenceID = 00000000-0000-0000-0000-000000000000`. Quota 20/min condivisa con chiusure e cancellazioni. | +| `GET api/v2/trading/info/{demo/}orders:lookup?orderId=` | esito dell'ordine, con le posizioni prodotte (`positionExecutions[].positionId`, `openingData.avgPrice`, `units`, `executionTime`, `fees`) | è la **chiave** usata dal bot. `requestedUnits`/`requestedAmount` possono differire dalle unità inviate: il 2026-09-16 il server ha ridotto gli ordini a 2 000 USD di margine (`frozenAmount 2000`, unità a sei decimali ricalcolate). Quota 60/min condivisa con `close-orders/{id}` e `orders/{id}`. | +| `GET api/v2/trading/info/{demo/}orders:lookup?referenceId=` | ripiego quando la risposta al `POST` è andata persa | 404 per gli ordini v2 del bot (vedi sopra). | +| `GET api/v1/trading/info/{demo/}orders/{orderId}` | ripiego per `orderId` con la risposta v1 (`statusID`, `errorCode`, `positions[] {positionID, rate, units, occurred, isOpen}`) | verificato con l'ordine 381739181. | +| `DELETE api/v2/trading/execution/{demo/}orders/{orderId}` | cancellazione di un ordine non ancora eseguito (kill-switch) | 200 = richiesta accettata, non annullamento avvenuto: confermare con il lookup (7 o 9 = annullato, 6 = in corso). Idempotente su ordini già chiusi. | +| `GET api/v1/trading/info/{demo/}pnl` | conto e posizioni in una chiamata: `clientPortfolio.credit`, `bonusCredit`, `unrealizedPnL`, `positions[] {positionID, instrumentID, isBuy, units, openRate, openDateTime, amount (margine), leverage, unrealizedPnL.pnL, totalFees}` | `equity = credit + bonus + Σ amount + unrealized`; `available = credit + bonus`; `usedMargin = Σ amount`. Il conto demo **non compare** in `api/v1/balances` (solo i conti reali). | +| `GET api/v1/trading/info/trade/{demo/}history?minDate=…&page=…&pageSize=200` | posizioni chiuse: `positionId`, `orderId`, `openRate`, `closeRate`, `openTime`, `closeTime`, `netProfit`, `fees`, `investment` | `netProfit` **non** include `fees`. Fonte del realizzato della scheda Storico e della distinzione fra chiusure e movimenti di cassa. | + +**Stati dell'ordine** (`status.id` / `statusID`): 1 Received, 2 Placed, 3 Filled, 4 Rejected, 5 PartiallyFilled, 6 PendingCancel, 7 Canceled, 8 Expired, 9 CanceledPartiallyFilled, 10 RejectedPartiallyFilled, 11 WaitingForMarket, 12 PendingTriggeredRate. Il bot tratta 3 e 5 come eseguito, 4, 7, 8, 9, 10 come rifiutato/annullato, 1, 2, 6, 11, 12 come in corso; in assenza di risposta lo stato è `Unknown` e l'ordine resta nel registro. Esiste anche `POST api/v3/trading/execution/{demo/}orders` (202, stessa semantica, `settlementType` obbligatorio): non usato, annotato per il futuro. + ## 2. Calendario economico | Fonte | URL | Formato | Aggiornamento | Note | @@ -60,10 +74,12 @@ data/news/news_YYYYMM.jsonl {hash,published,source,title,summary,li data/cache/.xml|json ultimo corpo buono di ogni feed data/ledger/decisions.jsonl vedi docs/LEDGER_SCHEMA.md (rotazione mensile in decisions_YYYYMM.jsonl) data/ledger/baskets.csv vedi docs/LEDGER_SCHEMA.md -data/state/baskets_state.json posizioni aperte, picco di equity, blocchi (per ripartire dopo un riavvio) +data/ledger/orders.jsonl una riga per ordine inviato e per cambio di stato (5.0) +data/state/baskets_state.json posizioni aperte, ingressi in attesa, picco di equity al netto dei movimenti di cassa, blocchi +data/state/pending_orders.json il registro degli ordini (5.0) data/state/paper_state.json il conto del simulatore (solo Paper) data/models/*.json modelli (livelli 1-3) e stato del bandit knowledge/*.csv, *.md calibrazione, proposte, registri, insight settimanali -reports/*.csv qualità dati, falsificazione +reports/*.csv qualità dati, falsificazione, bonifica_YYYYMMDD (5.0) logs/encelado.log log applicativo (;) ``` diff --git a/Encelado/docs/GLOSSARY.md b/Encelado/docs/GLOSSARY.md index 483c605..be7cd7b 100644 --- a/Encelado/docs/GLOSSARY.md +++ b/Encelado/docs/GLOSSARY.md @@ -13,7 +13,13 @@ | **Cost gate** | Il rifiuto di un ingresso se il TP non copre almeno `costMultiple` volte il costo stimato (spread reale + markup + commissioni + overnight atteso), o se lo spread è più del doppio della mediana delle ultime 24 ore. | | **Break-even** | Il costo in pip oltre il quale il P&L medio lordo di un basket diventa negativo: se è vicino a zero, il segnale non ha contenuto. | | **Vol-parity sizing** | Le unità di ogni gamba sono inversamente proporzionali alla sua volatilità (ATR), così le due gambe contribuiscono allo stesso rischio; il rischio totale è `riskPerBasketPct` dell'equity alla distanza dello stop. | -| **Leg-risk** | Il rischio di restare con una sola gamba: se la seconda non viene eseguita entro `legTimeoutSec`, la prima viene chiusa subito (`leg_risk_unwind`). | +| **Leg-risk** | Il rischio di restare con una sola gamba: se la seconda viene rifiutata, la prima viene chiusa subito (`leg_risk_unwind`); se la seconda è senza esito, il basket aspetta (`PendingB`) finché il registro degli ordini non sa. | +| **Registro degli ordini** | `OrderTracker` e il file `data/state/pending_orders.json`: ogni ordine inviato, scritto prima della chiamata e seguito finché il server non dice eseguito, rifiutato o annullato, o finché la posizione non compare sul conto. | +| **PendingA / PendingB** | Stati del basket con una gamba senza esito: nessun nuovo ordine, valutazione sospesa, ripresa alla risoluzione. | +| **Orfana-bot** | Una posizione sul conto aperta dal bot (registro o firma nel ledger) che non appartiene a nessun basket: adottata e chiusa. | +| **Esterna** | Una posizione sul conto senza la firma del bot: segnalata, contata, mai toccata. | +| **Movimento di cassa** | Deposito, prelievo o accredito virtuale: un salto del saldo che nessuna chiusura spiega. Escluso dal P&L, dal picco e dal drawdown. | +| **Bonifica** | La pulizia una tantum delle orfane con conferma per posizione (`--bonifica`). | | **Equity stop** | Chiusura di tutto e blocco a un drawdown del 9 % dal picco; riparte solo con un reset motivato. | | **Kill-switch** | Chiusura immediata di tutto e blocco delle nuove entrate: pulsante, comando o file `STOP`. | | **Paper / Demo / Live** | Simulatore locale / conto demo eToro / conto reale. Il bot opera da solo in tutte e tre (D-20). | diff --git a/Encelado/docs/KNOWN_ISSUES.md b/Encelado/docs/KNOWN_ISSUES.md index 91a2319..c3dbb94 100644 --- a/Encelado/docs/KNOWN_ISSUES.md +++ b/Encelado/docs/KNOWN_ISSUES.md @@ -1,6 +1,6 @@ # Problemi noti e limiti -Aggiornato: 2026-09-16. Una voce per limite, con lo stato. Quando un limite viene rimosso, la voce si sposta nel `CHANGELOG.md`. +Aggiornato: 2026-09-23. Una voce per limite, con lo stato. Quando un limite viene rimosso, la voce si sposta nel `CHANGELOG.md`. ## Strategia @@ -15,7 +15,9 @@ Aggiornato: 2026-09-16. Una voce per limite, con lo stato. Quando un limite vien - L'endpoint delle candele non pagina: al massimo ~10 giorni di M15. Lo storico dipende dai tick forniti dall'utente. - L'API demo mostra spread di mercato di 0,1-0,7 pip senza markup e un overnight di 0,91 USD/giorno per 10 000 EURUSD. Se l'esecuzione reale applica uno spread diverso, lo si vedrà dallo slippage scritto nel ledger a ogni ingresso. - Il campo dei costi si chiama `value` (non `amount`, come si era scritto in prima battuta): corretto il 2026-09-16 pomeriggio; le righe del ledger della mattina hanno `markupA/B = 0` e `overnight` nullo per questo motivo. -- Il conto reale dell'utente vale 193,18 USD: con l'esposizione minima di 1000 USD per gamba il Live non è praticabile a prescindere dai cancelli. +- Il conto reale dell'utente vale 224,90 USD (2026-09-23): con l'esposizione minima di 1000 USD per gamba il Live non è praticabile a prescindere dai cancelli. +- **Il server non registra l'`x-request-id` come `referenceId`** degli ordini v2: il lookup per riferimento risponde 404 anche per ordini eseguiti. Dalla 5.0 l'esito si legge per `orderId`; il riferimento resta solo come ripiego (ADR-0009). +- **A margine esaurito il server riduce l'ordine** a un importo fisso (2 000 USD di margine osservati il 16-21/9) invece di rifiutarlo; la regola non è documentata nell'OpenAPI. Il bot registra `unita_richieste` e `unita_eseguite` in `orders.jsonl` e dimensiona la gamba B sulle unità eseguite di A; i limiti di margine della Fase 2 evitano di arrivarci. ## Feed @@ -27,11 +29,14 @@ Aggiornato: 2026-09-16. Una voce per limite, con lo stato. Quando un limite vien - **Una sola istanza** per cartella di lavoro: non c'è un lock; due bot sullo stesso conto si contendono le posizioni. Documentato nel runbook, non imposto dal codice. - Le posizioni salvate in `baskets_state.json` da una modalità diversa non vengono riprese (si riparte dalla riconciliazione del conto). +- Il ledger delle sessioni Demo del 16-21/9 (le 65 righe `segnale_ingresso`/`rifiuto` del post-mortem) non è su questa macchina; l'analisi si basa sullo storico del conto letto via API (D-36). +- La finestra WPF mostra i contatori nuovi (in attesa, orfane, esterne, P&L del conto) nei riquadri esistenti; la scheda Storico, la bonifica con pulsante e la navigazione a sinistra arrivano con la web UI (Fasi 6-7, dopo D-28). In attesa, la bonifica si lancia da headless (`--bonifica`). +- Un movimento di cassa viene riconosciuto dal salto del saldo (oltre 10 USD e 0,25 %): un accredito piccolo sotto quella soglia passa per rumore; un prelievo che coincide con una chiusura viene distinto solo se lo storico del conto risponde. - Il ciclo settimanale gira solo mentre il bot è acceso la domenica dopo le 10 UTC (o al primo avvio dopo sette giorni). - La finestra e l'headless usano lo stesso log e lo stesso ledger: se si avviano insieme le righe si mescolano. ## Codice -- `BasketEngine.cs` è un file unico di ~1900 righe: funziona, ma un intervento vi costa più di quanto dovrebbe. Da spezzare (quote poller, riconciliazione, snapshot) in una sessione dedicata. +- `BasketEngine` è spezzato in sei file parziali dalla 5.0; il file principale resta di ~900 righe (loop, decisioni, esecuzione). - I test dell'interfaccia rendono le pagine in memoria (`UiRenderTests`, con `ENCELADO_RENDER_DIR`), non il comportamento della finestra vera (dialoghi, timer). - Il test (l) copre i blocchi nel decisore, non la simulazione completa dell'equity stop nel motore live; quella è coperta dal backtest (`EquityStops` in `BacktestResult`) e dal ledger. diff --git a/Encelado/docs/LEDGER_SCHEMA.md b/Encelado/docs/LEDGER_SCHEMA.md index c6b15c6..323601a 100644 --- a/Encelado/docs/LEDGER_SCHEMA.md +++ b/Encelado/docs/LEDGER_SCHEMA.md @@ -16,7 +16,7 @@ Una riga per **ogni** valutazione di ogni basket alla chiusura di ogni barra M15 | `cross` | testo | cross sintetico (`EURCHF`) | | `mode` (`Paper` | `Demo` | `Live`; i file scritti prima del 2026-09-16 pomeriggio portano i nomi precedenti `DemoApprove`/`DemoAuto`) | testo | `Paper`, `Demo`, `Live` | | `preset` | testo | `CONSERVATIVE`, `MODERATE`, `AGGRESSIVE` | -| `evento` | testo | `skip`, `segnale_ingresso`, `ingresso`, `rifiuto`, `leg_risk_unwind`, `posizione`, `segnale_aggiunta`, `aggiunta`, `segnale_uscita`, `uscita`, `correzione` | +| `evento` | testo | `skip`, `segnale_ingresso`, `ingresso`, `rifiuto`, `leg_risk_unwind`, `posizione`, `segnale_aggiunta`, `aggiunta`, `segnale_uscita`, `uscita`, `correzione`; dalla 5.0 anche le righe di evento (senza feature, solo `ts`, `run_id`, `evento`, `basket_id`, campi propri e `motivazione`): `pending` (gamba senza esito, con `leg`, `client_ref`, `order_id`), `pending_risolto` (con `esito`, `fonte`, `position_id`), `orfana_adottata` e `orfana_chiusa` (con `position_id`, `strumento`, `pnl`, `exit_reason`), `movimento_di_cassa` (con `importo`, `saldo_prima`, `saldo_dopo`, `chiusure_nel_frattempo`, `cassa_cumulata`), `kill_switch_avviato` | | `decision` | testo | `Skip`, `Enter`, `Add`, `Exit`, `Hold` | | `buy_cross` | bool | verso deciso (compra il cross = compra entrambe le gambe nei cinque basket) | | `z`, `z_in_eff` | numero | z-score del cross e soglia effettiva (scalata dalla vol prevista) | @@ -48,7 +48,38 @@ Una riga per basket chiuso. `label = 1` se `pnl_net_usd > 0`, altrimenti 0: è l basket_id;run_id;basket;mode;preset;opened_utc;closed_utc;buy_cross;entry_z;exit_z;pnl_gross_usd;pnl_net_usd;pips_gross;cost_pips;cost_usd;slippage_pips;adds;bars_held;exit_reason;equity_at_entry;p_ml_at_entry;label;durata_min;motivazione ``` -`pips_gross` è la somma dei pip delle due gambe ai prezzi di esecuzione (la colonna "Pips" della UI), `cost_pips` il costo stimato all'ingresso, `slippage_pips` la differenza fra quotazione vista e prezzo eseguito sommata sulle gambe, `exit_reason` uno dei codici sopra più `manual`, `closed_by_broker`, `leg_closed_by_broker`, `end_of_data`. +`pips_gross` è la somma dei pip delle due gambe ai prezzi di esecuzione (la colonna "Pips" della UI), `cost_pips` il costo stimato all'ingresso, `slippage_pips` la differenza fra quotazione vista e prezzo eseguito sommata sulle gambe, `exit_reason` uno dei codici sopra più `manual`, `closed_by_broker`, `leg_closed_by_broker`, `end_of_data`, e dalla 5.0 `leg_risk_unwind` (gamba A eseguita in ritardo e richiusa), `orphan_closed` (gamba orfana del bot adottata e chiusa alla riconciliazione), `bonifica_orfana` (chiusa dalla bonifica con conferma), `kill_switch`/`equity_stop` anche per le gambe singole. Le righe di una gamba singola hanno `entry_z`, `exit_z`, `pips_gross` e `cost_*` vuoti e `buy_cross` = verso della gamba. + +## `data/ledger/orders.jsonl` (dalla 5.0) + +Una riga per **ogni ordine inviato** e per **ogni cambio del suo stato** (append-only): la prima riga di un `client_ref` dice cosa è stato chiesto, l'ultima come è finita. Scritta dal registro degli ordini (`OrderTracker`) attraverso il ledger. + +| Campo | Significato | +|---|---| +| `ts`, `run_id`, `mode` | come in `decisions.jsonl` | +| `basket`, `basket_id` | slot (`EURUSD/USDCHF`) e istanza (`B2026…-EURUSDUSDCHF`) | +| `strumento`, `instrument_id`, `verso` | la gamba; `verso` = `long`/`short` dell'ordine (per una chiusura è il verso opposto alla posizione) | +| `leg` | `A`, `B`, `Add`, `Close`, `Unwind` | +| `unita_richieste`, `unita_eseguite` | differiscono quando il server riduce l'ordine (osservato il 2026-09-16) | +| `prezzo_richiesto`, `prezzo_eseguito`, `slippage_pip` | quotazione vista all'invio, prezzo del server, differenza in pip con il segno del costo | +| `stato`, `stato_id` | l'ultima parola del server (`Submitted`, `Received`, `Placed`, `Filled`, `Rejected`, …, `Unknown` quando non ha risposto), con l'id numerico di eToro | +| `esito` | `Pending`, `Filled`, `Rejected`, `Cancelled` | +| `order_id`, `position_id`, `client_ref` | le tre chiavi | +| `fee` | commissioni riportate dal server all'esecuzione | +| `evento` | `inviato`, `stato`, `risolto` | +| `motivazione` | la motivazione della decisione o l'errore del server | + +## `data/state/pending_orders.json` (dalla 5.0) + +Il registro degli ordini: `savedUtc` e l'array `orders` con gli stessi campi di `orders.jsonl` più `checks`, `lastCheckUtc`, `source` (`venue`, `lookup`, `lookup-v1`, `positions`). Contiene tutti gli ordini senza esito e quelli risolti nelle ultime 48 ore (servono a riconoscere una posizione come propria). Scritto **prima** di ogni chiamata HTTP e a ogni cambio di stato, con `.tmp` + `File.Move`. Un file illeggibile viene messo da parte come `pending_orders.json.illeggibile-`. In modalità Paper il file è `pending_orders_paper.json`. + +## `data/state/baskets_state.json` (campi aggiunti dalla 5.0) + +`peakNetEquity` (picco dell'equity al netto dei movimenti di cassa), `cumulativeCashFlow`, `lastBalance` e `lastBalanceUtc` (per riconoscere un deposito avvenuto a bot spento), `pendingEntries` (un elemento per basket in `PendingA`/`PendingB`: `name`, `state`, `basketId` e il piano `pending` con unità, TP, stop, riferimenti cliente e la gamba A eseguita). `peakEquity` resta per compatibilità e vale `peakNetEquity + cumulativeCashFlow`. + +## `reports/bonifica_YYYYMMDD.csv` (dalla 5.0) + +Una riga per orfana chiusa dalla bonifica: `ts;position_id;strumento;verso;unita;aperta_utc;pnl_realizzato;basket;motivazione`. ## `results/trials.csv` diff --git a/Encelado/docs/RISK_RULES.md b/Encelado/docs/RISK_RULES.md index 9fab982..540be81 100644 --- a/Encelado/docs/RISK_RULES.md +++ b/Encelado/docs/RISK_RULES.md @@ -24,13 +24,17 @@ Tutte le regole di §10 della specifica, con il valore di fabbrica, dove sta e c | API in errore | 5 letture consecutive fallite → niente nuove entrate finché non risponde | codice | nessuno | | Quotazione vecchia | > 15 s → niente nuove entrate | codice (`BasketEngine.MaxQuoteAgeSeconds`) | nessuno | | Qualità dati | buco > 2 h feriale o salto > 8 σ → decisioni sospese su quella barra | codice | nessuno | -| Leg-risk | seconda gamba non eseguita entro `legTimeoutSec` (5 s) → chiudi subito la prima, basket in pausa 1 h | `strategy.json` → `legTimeoutSec` (pausa: codice) | operatore (timeout) | -| Gamba orfana | una gamba sparisce dal conto → l'altra viene chiusa alla riconciliazione successiva | codice | nessuno | +| Leg-risk | seconda gamba **rifiutata** → chiudi subito la prima (`leg_risk_unwind`), basket in pausa 1 h. Seconda gamba **senza esito** entro `legTimeoutSec` (5 s) → basket in `PendingB`: il registro degli ordini continua a chiedere; eseguita → basket aperto; rifiutata → prima gamba richiusa | `strategy.json` → `legTimeoutSec` (pausa: codice) | operatore (timeout) | +| Ordine dall'esito ignoto | mai abbandonato: registrato in `data/state/pending_orders.json` **prima** dell'invio; esito chiesto per `orderId` (ogni 2 s nel primo minuto, poi ogni 10 s, poi ogni minuto) e riconosciuto anche dalla posizione comparsa sul conto (stesso strumento e verso, entro 90 s); una gamba A senza esito porta il basket in `PendingA` (nessun nuovo ordine su quel basket); all'avvio i pendenti si risolvono prima di qualsiasi decisione | codice (ADR-0009) | nessuno | +| Gamba A eseguita in ritardo | segnale ancora valido e nessun blocco → gamba B (ridimensionata sulle unità eseguite di A); altrimenti chiusura immediata di A (`leg_risk_unwind`) | codice | nessuno | +| Gamba orfana del bot | una gamba di un basket sparisce dal conto → l'altra viene chiusa alla riconciliazione successiva. Una posizione che porta la firma del bot (id nel registro, oppure strumento + verso + orario entro 90 s da una riga `segnale_ingresso`/`rifiuto`/`ingresso`/`pending` del ledger) ma non appartiene a nessun basket è `orfana-bot`: **adottata e chiusa** (tre tentativi, poi entrate bloccate con avviso). Contatore «orfane» in dashboard, rosso se > 0 | codice; `--bonifica` all'avvio la elenca e chiede conferma per ognuna | operatore (bonifica) | +| Movimenti di cassa | un salto del saldo non spiegato dalle chiusure (oltre 10 USD e 0,25 %) è un deposito o un prelievo: scritto nel ledger come `movimento_di_cassa`, escluso dal P&L, dal picco di equity e dal drawdown | codice (`EquityTracker`) | nessuno | +| Posizioni non riconciliate | se il P&L aperto del conto e la somma delle posizioni non tornano (oltre 5 USD e 1 %) per più di 60 s, o un basket ha una gamba che il conto non mostra: avviso «posizioni non riconciliate» (banner giallo, riga di stato) | codice | nessuno | | Chiusura incompleta | una gamba non chiude dopo 3 tentativi → stato `Error`, entrate bloccate, allarme | codice | nessuno; si risolve a mano sul conto e con la riconciliazione | | Kill-switch | pulsante con conferma; file `STOP` in `Documenti\Encelado` (controllato ogni 5 s) | codice | operatore; il reset richiede di rimuovere il file e una motivazione | -| Posizioni sconosciute sul conto | segnalate una volta nel log, **mai toccate** | codice | nessuno | +| Posizioni esterne | posizioni senza la firma del bot: segnalate una volta nel log, contate in dashboard, **mai toccate** (dalla Fase 3: chiuse dal kill-switch solo con `risk.closeForeignOnKill` o con la spunta esplicita) | codice | operatore | | Chiavi API | solo `%LOCALAPPDATA%\Encelado\etoro.dat` (DPAPI) o `ETORO_API_KEY`/`ETORO_USER_KEY`; mai nel repo (`.gitignore`: `*.local.json`, `.env`) | codice | operatore | | Ambiente visibile | badge `PAPER/DEMO/LIVE` nella barra, nel log e nel ledger (`mode`) | codice | nessuno | | Controlli all'avvio | chiavi (profilo), orologio, strumenti e limiti, conto, riconciliazione, calendario | codice | nessuno; se falliscono il bot resta in sola lettura o non parte | | Averaging | `Off` in live; `AddOnce` ammesso in paper; moltiplicatore di lotto 1,0 | `strategy.json` → `averagingMode`, `lotMultiplier` (max 1,5, solo backtest) | operatore | -| Parametri cambiati dal bot | mai. Le proposte vanno in `knowledge/proposals.csv` e passano dal forward test | codice | operatore | +| Parametri cambiati dal bot | mai. Le proposte vanno in `knowledge/proposals.csv` e passano dal forward test. Dalla 5.0 anche il bandit **propone soltanto**: fino alla 4.0.0 applicava il preset da solo in Paper e Demo (D-30) | codice | operatore | diff --git a/Encelado/docs/RUNBOOK.md b/Encelado/docs/RUNBOOK.md index febcadd..c7e3947 100644 --- a/Encelado/docs/RUNBOOK.md +++ b/Encelado/docs/RUNBOOK.md @@ -1,6 +1,6 @@ # Runbook -Aggiornato: 2026-09-16. Come si avvia, si ferma, si sblocca e si ripara il bot. I file dell'operatore stanno in `Documenti\Encelado\`; le chiavi in `%LOCALAPPDATA%\Encelado\etoro.dat`. +Aggiornato: 2026-09-23 (5.0, Fase 1). Come si avvia, si ferma, si sblocca e si ripara il bot. I file dell'operatore stanno in `Documenti\Encelado\`; le chiavi in `%LOCALAPPDATA%\Encelado\etoro.dat`. ## Prima volta @@ -27,7 +27,7 @@ In ogni modalità il bot apre e chiude da solo (decisione D-20). Il badge in alt Encelado.exe --headless [--minutes 240] [--confirm-live "CONFERMO LIVE"] ``` -Log sulla console e nel file; una riga di stato ogni `run.statusSeconds`. Comandi da tastiera: `status`, `close `, `kill`, `preset `, `reset `, `stop`. Variabile `ENCELADO_EXECUTION_MODE` per forzare la modalità senza toccare il file. +Log sulla console e nel file; una riga di stato ogni `run.statusSeconds`. Comandi da tastiera: `status`, `close `, `kill`, `preset `, `reset `, `bonifica`, `stop`. Argomento `--bonifica`: parte senza chiudere le orfane e le propone una per una. Variabile `ENCELADO_EXECUTION_MODE` per forzare la modalità senza toccare il file. **Una sola istanza per cartella di lavoro**: due bot sullo stesso conto e sullo stesso ledger si contendono le posizioni. Prima di aprire la finestra mentre gira l'headless, fermalo. @@ -46,9 +46,19 @@ Quando l'equity scende del 9 % dal picco (`equityStopPct`) il bot chiude tutto e La perdita giornaliera del 3 % (`dailyLossPct`) blocca solo le nuove entrate fino alla mezzanotte UTC e non richiede reset. +## Ordini senza esito + +eToro lavora gli ordini in modo asincrono e a volte non risponde al lookup. Il bot non dimentica mai un ordine: ogni invio è scritto in `data/state/pending_orders.json` **prima** della chiamata, l'esito viene chiesto per `orderId` e, se il server non lo trova, ricostruito dalla posizione comparsa sul conto. Un basket con una gamba senza esito compare in dashboard come «attesa gamba A/B» e nel contatore «in attesa»: non manda altri ordini finché il registro non lo risolve. Alla risoluzione: gamba A eseguita e segnale ancora valido → parte la gamba B; segnale decaduto → la gamba A viene richiusa subito (`leg_risk_unwind`); rifiutata → il basket torna libero. Se un ordine resta senza esito per più di dieci minuti il log lo ripete ogni dieci minuti: guarda `orders.jsonl` (ultima riga di quel `client_ref`) e, se serve, la posizione su eToro; non c'è niente da fare a mano finché la gamba non compare sul conto, e quando compare il bot la gestisce. All'avvio i pendenti del run precedente vengono risolti prima di qualsiasi decisione. + ## Riconciliazione -Ogni 20 secondi il bot rilegge conto e posizioni. Una gamba sparita dal conto (chiusa a mano, stop nativo) fa chiudere l'altra; una posizione sconosciuta viene segnalata e ignorata; una chiusura incompleta dopo tre tentativi mette il basket in stato `Error` e blocca le nuove entrate (banner giallo) finché non è risolta sul conto: chiudi la gamba a mano su eToro, la riconciliazione successiva la vede e sblocca. +Ogni 20 secondi il bot rilegge conto e posizioni e **classifica ogni posizione**: `basket` (gamba nota), `orfana-bot` (aperta dal bot ma senza basket: id nel registro degli ordini, oppure strumento, verso e orario coerenti con una decisione del ledger entro 90 s), `esterna` (tutto il resto). Una gamba di basket sparita dal conto (chiusa a mano, stop nativo) fa chiudere l'altra; un'orfana-bot viene **adottata e chiusa** (riga `orfana_adottata` e `orfana_chiusa` nel ledger, riga in `baskets.csv` con `exit_reason = orphan_closed`; dopo tre tentativi falliti le entrate si bloccano e il banner lo dice: chiudila a mano su eToro); un'esterna viene segnalata una volta, contata e mai toccata. Una chiusura incompleta dopo tre tentativi mette il basket in stato `Error` e blocca le nuove entrate finché non è risolta sul conto. Se il P&L aperto del conto e quello delle posizioni non tornano per più di un minuto compare «posizioni non riconciliate»: di solito è un'esecuzione in corso; se persiste, confronta `pending_orders.json` con le posizioni su eToro. + +Un deposito o un prelievo sul conto (anche l'accredito di fondi virtuali del demo) viene riconosciuto dal salto del saldo non spiegato dalle chiusure e scritto nel ledger come `movimento_di_cassa`: non è P&L, non muove il picco di equity né il drawdown. + +## Bonifica delle gambe orfane + +Una tantum, dopo un'anomalia: avvia il bot con `--headless --bonifica`. Il motore parte **senza** chiudere le orfane da solo, le elenca con P&L e motivo della classificazione insieme alle posizioni esterne, e per ogni orfana chiede `chiudere? [s/N]`. Ogni chiusura confermata scrive una riga in `baskets.csv` (`exit_reason = bonifica_orfana`, P&L dallo storico) e in `reports/bonifica_YYYYMMDD.csv`. Alla fine il bot torna a chiudere le orfane da solo. Lo stesso comando si lancia dalla console con `bonifica`. Al 2026-09-23 il conto demo è piatto: non c'è niente da bonificare. ## Errori API @@ -69,8 +79,8 @@ Calendario e notizie sono in cache su disco (`data/cache`) e vengono riletti ogn | Cosa | Dove | |---|---| | log | `Documenti\Encelado\logs\encelado.log` (CSV `;`) | -| ledger | `data\ledger\decisions.jsonl`, `data\ledger\baskets.csv` | -| stato | `data\state\baskets_state.json` (ripreso all'avvio) | +| ledger | `data\ledger\decisions.jsonl`, `data\ledger\baskets.csv`, `data\ledger\orders.jsonl` | +| stato | `data\state\baskets_state.json` (ripreso all'avvio), `data\state\pending_orders.json` (registro degli ordini) | | barre | `data\market\candles__M15.csv` | | modelli | `data\models\` | | conoscenza | `knowledge\` | diff --git a/Encelado/docs/STATE.md b/Encelado/docs/STATE.md index ca97703..7c00e64 100644 --- a/Encelado/docs/STATE.md +++ b/Encelado/docs/STATE.md @@ -1,34 +1,27 @@ # Stato del lavoro -Aggiornato: 2026-09-16 (fine della seconda sessione, rilascio 4.0.0). +Aggiornato: 2026-09-23 (sessione 5.0, Fasi 0-1 concluse). ## Fase in corso -**Forward test in Demo.** Il codice copre le fasi 0-7 della specifica; la strategia è in esercizio autonomo sul conto demo di eToro per accumulare basket nel ledger. Il backtest è negativo (`docs/STRATEGY.md`): il Demo misura, non guadagna. +**Piano 5.0, Fase 2** (`docs/PIANO_5.0.md`). Le Fasi 0 e 1 sono committate: post-mortem, registro degli ordini, stati `PendingA`/`PendingB`, classificazione e chiusura delle orfane, picco al netto dei movimenti di cassa, bonifica. Il forward test in Demo **non è ripartito**: la regola finale del piano dice che nessuna nuova funzione va in Demo finché un ordine dall'esito ignoto può restare sul conto senza padrone; la Fase 1 chiude quella falla, ma le Fasi 2 (margine) e 3 (kill-switch reale) vanno finite prima di riaccendere il bot, perché senza limiti di margine il primo basket può ancora impegnare tutta l'equity. -## Fatto nell'ultima sessione (2026-09-16, pomeriggio) +## Fatto nell'ultima sessione (2026-09-23) -- **Rework completo del codice**: rimossi Binance, Alpaca, cTrader/proba, SQLite, GBDT, RL, TA-Lib, indicatori e backtest a coppie (ADR-0004). Restano Core (basket, broker, notizie, statistica), Etoro, Bot, strumento di ricerca. Nessun pacchetto NuGet nell'applicazione. Test da 322 a 172, tutti verdi. -- **Niente approvazioni manuali** (decisione dell'utente, D-20, ADR-0005): modalità `Paper` / `Demo` (default) / `Live`; coda delle approvazioni rimossa; il Live conserva `run.allowLive` e la frase `CONFERMO LIVE`. -- **Interfaccia rifatta**: barra in alto con tre schede (Dashboard, Log, Impostazioni), stato, ambiente, ora nel fuso scelto, AVVIA; dashboard con i cinque numeri, la tabella dei basket, tre riquadri di contesto e l'attività. Tema nuovo. Test di rendering in PNG (`UiRenderTests`). -- **Fuso orario** selezionabile (`ui.timeZone`, default `computer`, elenco dei fusi di Windows in Impostazioni, `ENCELADO_TIME_ZONE`). -- **Bug corretto**: l'endpoint dei costi di eToro usa il campo `value`; markup e overnight risultavano 0 (D-24). Overnight osservato 0,9 pip/gamba/giorno. -- **Apprendimento collegato al motore**: `LearningState` (logistica in ombra, MLP challenger, bandit, ciclo settimanale, `knowledge/`), previsione di volatilità per basket, feature dal ledger. Standardizzatore adattato all'insieme di addestramento prima del fit dell'MLP (difetto trovato dal test sul cerchio). -- **Backtest completato**: test di falsificazione 5 (segnale invertito) e scenario di costi `api`; `docs/STRATEGY.md` con i numeri e il verdetto negativo. -- Documenti: `STRATEGY.md`, `ML_AND_LEARNING.md`, `RUNBOOK.md`, `GLOSSARY.md`, `KNOWN_ISSUES.md`, ADR-0004, ADR-0005; aggiornati `ARCHITECTURE.md`, `RISK_RULES.md`, `QUESTIONS.md` (D-17…D-25), `DATA_SOURCES.md`, `LEDGER_SCHEMA.md`, `CLAUDE.md`, catena di rilascio. -- Sessione di test autonoma in Demo dalle 12:56 alle 16:56 UTC (4 ore, `--headless`, preset Moderate): collegamento stabile, nessun errore, **31 segnali (|z| ≥ 2) tutti rifiutati dal solo cancello di correlazione** (ρ_W fra +0,14 e −0,42 contro la soglia −0,6), 0 basket aperti, 112 righe nel ledger delle decisioni. Il cancello ρ ≤ −0,6 è la prima cosa da misurare sul ledger nelle prossime settimane prima di proporre qualsiasi cambiamento. +- **Fase 0**: diagnosi verificata sul codice e sul conto demo via API (sola lettura): `referenceID` nullo sugli ordini v2 (ecco il 404), 21 gambe orfane del 16-21/9 chiuse a mano il 21/9, primi due ordini a 100 % del margine, poi ordini ridotti dal server a 2 000 USD di margine. Endpoint per `orderId` e cancellazione verificati (D-26, D-27). `docs/PIANO_5.0.md`, `docs/POSTMORTEM_ordini_pendenti.md`, `docs/QUESTIONS.md` D-26…D-37. +- **Fase 1**: `OrderTracker` + `pending_orders.json` + `orders.jsonl`; `EtoroBroker.OpenAsync` per `orderId` con riconoscimento dalla posizione; `BasketExecutor` con `ResumeAfterAAsync`/`CompleteAfterBAsync`/`UnwindLegAsync`; stati `PendingA`/`PendingB` persistiti; `PositionClassifier` e chiusura delle orfane; `EquityTracker`; contatori e P&L del conto nello snapshot e in dashboard; `--bonifica`; bandit senza applicazione automatica; `BasketEngine` in sei file parziali; ADR-0009; 18 test nuovi, 190 verdi. ## Prossimi passi -1. Lasciare girare il Demo per settimane; leggere `data/ledger/baskets.csv` e `knowledge/insights_*.md` prima di toccare qualsiasi parametro. -2. Se il ledger mostra che ρ_W ≤ −0,6 non si verifica mai, proporre in `proposals.csv` una soglia diversa **con** una pre-registrazione, non cambiarla a mano. -3. Spezzare `BasketEngine.cs` (~1900 righe) in quote poller, riconciliazione, snapshot. -4. Aggiungere un lock di istanza (un solo bot per cartella di lavoro). -5. Valutare una fonte per SNB e RBNZ che non sia Google News. +1. **Fase 2 — margine** (§10): sezione `risk` in `strategy.json` (`maxMarginUsePct` 40, `maxMarginPerBasketPct` 12, `marginBufferPct` 25, `closeForeignOnKill` false), sizing = min(rischio, margine) con `sizing_bound` nel ledger, ricontrollo di `available` prima di B, ordine dei segnali per |z|, margin guard (1,5 blocca, 1,2 chiude il peggiore). Test (y), (z). +2. **Fase 3 — kill-switch e ripristino** (§9): cancellazione dei pendenti, chiusura di basket + orfane, esterne opzionali, verifica di piattezza, `Halted-Residuo`; procedura di ripristino in cinque passi. Test (v)-(x). +3. Poi Fasi 4-5 (recupero dopo inattività, Telegram) e, **dopo la risposta a D-28/D-29**, le Fasi 6-9 (Engine/Server, web UI, Docker, Unraid, skill). +4. Riaccendere il Demo solo dopo la Fase 3, per 24 ore di verifica: contatore «orfane» a 0, `orders.jsonl` senza `Unknown` irrisolti. ## Problemi aperti -- Backtest negativo: la strategia non regge i costi (`docs/STRATEGY.md`, `docs/KNOWN_ISSUES.md`). -- Il conto reale vale 193,18 USD: il Live non è praticabile a prescindere. -- Google News blocca le ricerche RSS via robots.txt; RBA risponde 403 a intermittenza; Fed 404 a tratti. -- Il file di configurazione dell'utente porta ancora `allowDemoAuto` (avviso all'avvio; il ripristino dei valori di fabbrica lo toglie). +- Backtest negativo: la strategia non regge i costi (`docs/STRATEGY.md`). +- Domande in attesa: D-28 (ritiro WPF), D-29 (registry), D-36 (ledger delle sessioni 16-21/9), D-37 (chiusura manuale del 21/9). +- Il conto reale vale 224,90 USD: il Live non è praticabile. +- Google News blocca le ricerche RSS; RBA 403 a intermittenza; Fed 404 a tratti. +- `PROMPT.md` (la specifica 5.0) e `Modifiche.txt` sono nella radice del repository e non tracciati: decidere se spostarli in `docs/`. diff --git a/Encelado/docs/adr/ADR-0009-registro-ordini-e-orfane.md b/Encelado/docs/adr/ADR-0009-registro-ordini-e-orfane.md new file mode 100644 index 0000000..d4ed2f5 --- /dev/null +++ b/Encelado/docs/adr/ADR-0009-registro-ordini-e-orfane.md @@ -0,0 +1,31 @@ +# ADR-0009 — Registro persistente degli ordini e adozione delle gambe orfane + +Data: 2026-09-23. Stato: accettata (Fase 1 del piano 5.0, `docs/PIANO_5.0.md`; post-mortem in `docs/POSTMORTEM_ordini_pendenti.md`). + +## Contesto + +Fra il 16 e il 21 settembre 2026 il bot ha lasciato sul conto demo 21 gambe singole senza copertura. L'esito di ogni ordine era cercato con la chiave sbagliata (`referenceId`, che eToro non registra per gli ordini v2), l'ordine senza esito veniva dichiarato «non eseguito» e dimenticato, e la posizione che ne nasceva era «sconosciuta» e per regola intoccabile. Tre difetti che, insieme, hanno trasformato una regola di prudenza («non toccare ciò che non è tuo») in un accumulo di rischio scoperto. + +## Decisione + +1. **Ogni ordine entra in un registro persistente prima della chiamata HTTP** (`OrderTracker`, `data/state/pending_orders.json`, scrittura atomica). Il registro tiene riferimento cliente, `orderId`, strumento, verso, unità richieste ed eseguite, basket, gamba, orario, ultimo stato del server, esito e posizione. All'avvio viene ricaricato e ogni ordine senza esito viene risolto **prima** di qualsiasi decisione. +2. **La chiave dell'esito è l'`orderId`** (`orders:lookup?orderId=`, ripiego `api/v1/trading/info/{demo/}orders/{id}`). Il riferimento cliente resta nell'intestazione per l'idempotenza e serve solo quando la risposta al `POST` è andata persa. Quando il server non ha traccia sotto nessuna chiave, una posizione dello stesso strumento e verso comparsa entro 90 s dall'invio **è** l'esecuzione (le unità possono differire: il server può ridurre l'ordine). +3. **Nessun esito sintetico.** `OrderOutcome.Status` riporta la parola del server, oppure `Unknown`. Un ordine senza esito allo scadere del timeout della gamba porta il basket in `PendingA` o `PendingB`: nessun nuovo ordine su quel basket, il registro continua a chiedere (ogni 2 s nel primo minuto, poi ogni 10 s, poi ogni minuto), e alla risoluzione l'ingresso viene completato (gamba B, ridimensionata sulle unità eseguite di A), oppure annullato con la chiusura immediata della gamba eseguita se il segnale è decaduto o il bot è bloccato. +4. **Ogni posizione del conto viene classificata** a ogni riconciliazione (`PositionClassifier`): `basket` (gamba nota), `orfana-bot` (id nel registro, oppure strumento + verso + orario entro 90 s coerenti con una riga `segnale_ingresso`/`rifiuto`/`ingresso`/`pending` del ledger), `esterna` (tutto il resto). Le orfane-bot vengono **adottate e chiuse** (tre tentativi, poi blocco delle entrate con avviso); le esterne restano intoccate. Il contatore in dashboard distingue basket aperti, ingressi in attesa, orfane ed esterne. +5. **Il picco di equity è al netto dei movimenti di cassa** (`EquityTracker`): un salto del saldo non spiegato dalle chiusure è un deposito o un prelievo, viene scritto nel ledger come `movimento_di_cassa` e non muove né il picco né il drawdown. +6. **`orders.jsonl`** (append-only) riceve una riga a ogni invio e a ogni cambio di stato: è la fonte della scheda Storico → Ordini. + +## Alternative scartate + +- *Interrogare in parallelo `orderId` e `referenceId`*: la quota dei lookup (60/min, condivisa con l'esito delle chiusure) non regge due richieste ogni 400 ms per gamba, e il riferimento non è registrato dal server. +- *Adottare le orfane solo con unità ± 1 %*: il server ha ridotto gli ordini a 2 000 USD di margine, le unità non sono un identificatore. La finestra temporale sullo strumento e sul verso lo è, sul conto demo dove nient'altro opera così. +- *Ricomporre il basket con la gamba mancante quando si adotta un'orfana*: rifiutato (default della domanda D-35/§5.4): la gamba è vecchia di un tempo ignoto, il segnale che l'ha generata non c'è più; si chiude. + +## Conseguenze + +- `IBroker` ha due metodi in più (`LookupOrderByIdAsync`, `CancelOrderAsync`); ogni implementazione, anche quelle di prova, li fornisce. +- `BasketExecutor` accetta un `OrderTracker` e pubblica `ResumeAfterAAsync`, `CompleteAfterBAsync`, `UnwindLegAsync`, `ClosePositionAsync`. +- `BasketEngine` è spezzato in file parziali (loop e decisioni; registro; riconciliazione; stato; comandi; snapshot). +- Il bandit **non applica** più il preset da solo in Demo: propone e basta (D-30). +- Nuovi test (m)-(q) in `tests/Encelado.Tests/ExecutionTests.cs`. +- La bonifica delle orfane esistenti è un comando (`--bonifica` in headless, `bonifica` da console): sul conto demo del 2026-09-23 non c'è niente da bonificare (chiusura manuale del 21/9). diff --git a/Encelado/src/Encelado.Bot/Baskets/BasketEngine.Commands.cs b/Encelado/src/Encelado.Bot/Baskets/BasketEngine.Commands.cs new file mode 100644 index 0000000..68d4bed --- /dev/null +++ b/Encelado/src/Encelado.Bot/Baskets/BasketEngine.Commands.cs @@ -0,0 +1,209 @@ +using System.Globalization; +using System.Text; +using Encelado.Bot.Engine; +using Encelado.Bot.Logging; +using Encelado.Core.Baskets; +using Encelado.Core.Broker; + +namespace Encelado.Bot.Baskets; + +/// Commands from the window or the console, executed on the engine's own thread. +public sealed partial class BasketEngine +{ + public Task ExecuteAsync(EngineCommand command, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(command); + TaskCompletionSource tcs = new(TaskCreationOptions.RunContinuationsAsynchronously); + _commands.Enqueue((command, tcs)); + return tcs.Task.WaitAsync(TimeSpan.FromSeconds(120), ct); + } + + private async Task DrainCommandsAsync(CancellationToken ct) + { + while (_commands.TryDequeue(out (EngineCommand Command, TaskCompletionSource Done) item)) + { + CommandResult result; + try + { + result = await RunCommandAsync(item.Command, ct).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + Log.Error($"comando {item.Command.Kind} fallito", ex); + result = new CommandResult(false, ex.Message); + } + + item.Done.TrySetResult(result); + } + } + + private async Task RunCommandAsync(EngineCommand c, CancellationToken ct) + { + switch (c.Kind) + { + case EngineCommandKind.Close: + { + BasketSlot? slot = _slots.FirstOrDefault(s => s.Name.Equals(c.Argument, StringComparison.OrdinalIgnoreCase)); + if (slot is null) + { + return new CommandResult(false, $"basket {c.Argument} sconosciuto"); + } + + if (slot.Position is null) + { + return new CommandResult(false, $"{slot.Name} non ha un basket aperto"); + } + + BasketContext ctx = await BuildContextAsync(slot, DateTime.UtcNow, false, ct).ConfigureAwait(false); + await ExecuteExitAsync(slot, ctx, null, c.Reason.Length > 0 ? c.Reason : "chiusura manuale", "manual", ct).ConfigureAwait(false); + return new CommandResult(slot.Position is null, slot.Position is null ? $"{slot.Name} chiuso" : $"chiusura di {slot.Name} non completata"); + } + + case EngineCommandKind.KillSwitch: + await KillAsync(c.Reason.Length > 0 ? c.Reason : "comando", ct).ConfigureAwait(false); + return new CommandResult(true, "kill-switch eseguito: tutto chiuso, nuove entrate bloccate"); + + case EngineCommandKind.SetPreset: + if (!BasketPresets.TryParse(c.Argument, out PresetName preset)) + { + return new CommandResult(false, $"preset {c.Argument} sconosciuto"); + } + + _decider.SetPreset(preset); + Log.Info($"preset cambiato in {preset.ToString().ToUpperInvariant()} ({c.Reason}); i basket aperti non vengono toccati"); + _ledger.Correction(_runId, string.Empty, $"preset → {preset} ({c.Reason})"); + return new CommandResult(true, $"preset {preset.ToString().ToUpperInvariant()} attivo"); + + case EngineCommandKind.ResetEquityStop: + if (!_equityStopped && !_killSwitched) + { + return new CommandResult(false, "nessun blocco attivo"); + } + + if (c.Reason.Trim().Length < 10) + { + return new CommandResult(false, "serve una motivazione scritta (almeno dieci caratteri)"); + } + + if (File.Exists(_stopFile)) + { + return new CommandResult(false, $"rimuovi prima il file {_stopFile}"); + } + + _equityStopped = false; + _killSwitched = false; + _haltReason = null; + _equity.ResetPeak(_account.Equity); + _ledger.Correction(_runId, string.Empty, $"reset del blocco: {c.Reason}"); + Log.Warn($"blocco rimosso dall'operatore: {c.Reason}. Nuovo picco di equity {_equity.PeakEquity:F2}"); + SaveState(); + return new CommandResult(true, "blocco rimosso; il picco di equity riparte da adesso"); + + case EngineCommandKind.Bonifica: + return await BonificaAsync(c.Argument, ct).ConfigureAwait(false); + + default: + return new CommandResult(false, $"comando {c.Kind} non supportato"); + } + } + + // ----------------------------------------------------------------------- + // Bonifica (§5.5 of the 5.0 plan) + // ----------------------------------------------------------------------- + + /// + /// list: the orphans and the foreign positions right now, after a fresh + /// reconciliation. close:<id>: closes that orphan, records it in + /// baskets.csv (exit_reason = bonifica_orfana) and in + /// reports/bonifica_YYYYMMDD.csv. done: orphans are closed on their own from now on. + /// + private async Task BonificaAsync(string argument, CancellationToken ct) + { + string arg = (argument ?? string.Empty).Trim(); + if (arg.Length == 0 || arg.Equals("list", StringComparison.OrdinalIgnoreCase)) + { + await ReconcileAsync(ct).ConfigureAwait(false); + List rows = [.. _classified + .Where(static c => c.Origin != PositionOrigin.Basket) + .Select(c => new PositionInfo(c.Position.PositionId, SymbolOf(c.Position.InstrumentId), c.Position.IsBuy, c.Position.Units, c.Position.OpenedUtc, c.Position.UnrealizedPnl, + c.Origin == PositionOrigin.OrphanBot ? "orfana-bot" : "esterna", c.Basket, c.Reason))]; + int orphans = rows.Count(static r => r.Origin == "orfana-bot"); + return new CommandResult(true, $"{orphans} gambe orfane del bot, {rows.Count - orphans} posizioni esterne") { Payload = rows }; + } + + if (arg.Equals("done", StringComparison.OrdinalIgnoreCase)) + { + OrphanPolicy = OrphanPolicy.Close; + Log.Info("bonifica conclusa: da ora le gambe orfane vengono chiuse alla riconciliazione"); + return new CommandResult(true, "bonifica conclusa"); + } + + if (arg.StartsWith("close:", StringComparison.OrdinalIgnoreCase) && long.TryParse(arg[6..], NumberStyles.Integer, CultureInfo.InvariantCulture, out long positionId)) + { + ClassifiedPosition? c = _classified.FirstOrDefault(x => x.Position.PositionId == positionId); + if (c is null) + { + return new CommandResult(false, $"posizione {positionId} non trovata nell'ultima riconciliazione"); + } + + if (c.Origin != PositionOrigin.OrphanBot) + { + return new CommandResult(false, $"posizione {positionId} è {(c.Origin == PositionOrigin.Basket ? "una gamba di un basket" : "esterna")}: la bonifica chiude solo le orfane del bot"); + } + + double pnlBefore = c.Position.UnrealizedPnl; + _orphanAttempts.Remove(positionId); + bool closed = await CloseOrphanAsync(c, "bonifica_orfana", "chiusa dalla bonifica con conferma dell'operatore", ct).ConfigureAwait(false); + if (closed) + { + double realized = await RealizedOfAsync(positionId, pnlBefore, ct).ConfigureAwait(false); + AppendBonificaReport(c, realized); + _classified.Remove(c); + _orphanCount = Math.Max(0, _orphanCount - 1); + } + + return new CommandResult(closed, closed ? $"posizione {positionId} chiusa" : $"posizione {positionId} non chiusa: vedi il log"); + } + + return new CommandResult(false, "argomenti: list | close: | done"); + } + + /// The realised result from the venue's history, falling back to the last unrealised value seen. + private async Task RealizedOfAsync(long positionId, double fallback, CancellationToken ct) + { + try + { + IReadOnlyList closed = await _broker.GetClosedTradesAsync(DateTime.UtcNow.AddDays(-30), ct).ConfigureAwait(false); + ClosedTrade? t = closed.FirstOrDefault(x => x.PositionId == positionId); + return t is null ? fallback : t.NetProfit - t.Fees; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + Log.Warn($"storico non letto per la posizione {positionId}: {ex.Message}"); + return fallback; + } + } + + private void AppendBonificaReport(ClassifiedPosition c, double realized) + { + try + { + Directory.CreateDirectory(_config.Run.ReportsPath); + string path = Path.Combine(_config.Run.ReportsPath, $"bonifica_{DateTime.UtcNow:yyyyMMdd}.csv"); + bool isNew = !File.Exists(path); + using StreamWriter w = new(path, append: true, new UTF8Encoding(false)); + if (isNew) + { + w.WriteLine("ts;position_id;strumento;verso;unita;aperta_utc;pnl_realizzato;basket;motivazione"); + } + + BrokerPosition p = c.Position; + w.WriteLine(string.Create(CultureInfo.InvariantCulture, + $"{DateTime.UtcNow:O};{p.PositionId};{SymbolOf(p.InstrumentId)};{(p.IsBuy ? "long" : "short")};{p.Units:0.######};{p.OpenedUtc:O};{realized:0.00};{c.Basket};{c.Reason.Replace(';', ',')}")); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warn($"rapporto di bonifica non scritto: {ex.Message}"); + } + } +} diff --git a/Encelado/src/Encelado.Bot/Baskets/BasketEngine.Pending.cs b/Encelado/src/Encelado.Bot/Baskets/BasketEngine.Pending.cs new file mode 100644 index 0000000..07efee1 --- /dev/null +++ b/Encelado/src/Encelado.Bot/Baskets/BasketEngine.Pending.cs @@ -0,0 +1,249 @@ +using System.Globalization; +using Encelado.Bot.Logging; +using Encelado.Core.Baskets; +using Encelado.Core.Broker; + +namespace Encelado.Bot.Baskets; + +/// +/// The order register at work (§5.1-5.3 of the 5.0 plan): every second the pending +/// orders that are due get asked about; a resolution moves the basket that was waiting +/// (PendingA/PendingB) forward — leg B, the open basket, or the unwind of +/// leg A when the signal is gone — and a fill nobody was waiting for becomes an orphan +/// the reconciliation closes. +/// +public sealed partial class BasketEngine +{ + private DateTime _lastPendingWarnUtc; + private DateTime _lastPruneUtc; + + private async Task ResolvePendingOrdersAsync(CancellationToken ct, bool force = false) + { + DateTime now = DateTime.UtcNow; + if (now - _lastPruneUtc > TimeSpan.FromHours(1)) + { + _lastPruneUtc = now; + _tracker.Prune(now); + } + + if (_tracker.PendingCount == 0) + { + return; + } + + List<(TrackedOrder Order, OrderOutcome Outcome)> resolved; + try + { + if (force) + { + // At startup every pending order is due, whatever its last check said. + foreach (TrackedOrder o in _tracker.Pending) + { + o.LastCheckUtc = default; + } + } + + resolved = await _tracker.ResolveAsync(_broker, now, id => _knownPositions.Contains(id), ct).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + Log.Warn($"registro ordini: verifica non riuscita ({ex.Message})"); + return; + } + + foreach ((TrackedOrder order, OrderOutcome outcome) in resolved) + { + try + { + await OnOrderResolvedAsync(order, outcome, ct).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + Log.Error($"registro ordini: gestione dell'esito di {order.Describe()} fallita", ex); + } + } + + // Orders still unknown after ten minutes are said once every ten minutes: they are + // not forgotten, and a person should know. + List stale = [.. _tracker.Pending.Where(o => o.Age(now) > TimeSpan.FromMinutes(10))]; + if (stale.Count > 0 && now - _lastPendingWarnUtc > TimeSpan.FromMinutes(10)) + { + _lastPendingWarnUtc = now; + foreach (TrackedOrder o in stale) + { + Log.Warn(string.Create(CultureInfo.InvariantCulture, $"registro ordini: {o.Describe()} senza esito da {o.Age(now).TotalMinutes:0} minuti ({o.Checks} verifiche); il basket {o.Basket} resta in attesa")); + } + } + } + + private async Task OnOrderResolvedAsync(TrackedOrder order, OrderOutcome outcome, CancellationToken ct) + { + Log.Info($"registro ordini: {order.Describe()} — risolto ({outcome.Source})"); + _ledger.Note(_runId, "pending_risolto", order.BasketId, order.Describe(), w => + { + w.WriteString("basket", order.Basket); + w.WriteString("leg", order.Leg.ToString()); + w.WriteString("client_ref", order.ClientRef); + w.WriteNumber("order_id", order.OrderId); + w.WriteNumber("position_id", order.PositionId); + w.WriteString("esito", order.Resolution.ToString()); + w.WriteString("fonte", order.Source); + }); + + BasketSlot? slot = _slots.FirstOrDefault(s => s.Name.Equals(order.Basket, StringComparison.OrdinalIgnoreCase)); + switch (order.Leg) + { + case OrderLeg.A when slot is { State: BasketState.PendingA, Pending: { } plan } && plan.ClientRefA == order.ClientRef: + await HandleLegAResolvedAsync(slot, plan, outcome, ct).ConfigureAwait(false); + break; + case OrderLeg.B when slot is { State: BasketState.PendingB, Pending: { } plan } && plan.ClientRefB == order.ClientRef: + await HandleLegBResolvedAsync(slot, plan, outcome, ct).ConfigureAwait(false); + break; + default: + if (outcome.Filled && outcome.PositionId > 0 && !_knownPositions.Contains(outcome.PositionId)) + { + Log.Warn($"registro ordini: {order.Describe()} è stato eseguito ma nessun basket lo aspettava: la posizione {outcome.PositionId} è una gamba orfana del bot e verrà chiusa alla riconciliazione"); + _lastReconcileUtc = DateTime.MinValue; + } + + break; + } + } + + /// Leg A resolved late. Filled and the signal still valid: leg B. Filled and the signal gone: undo A. Rejected: back to idle. + private async Task HandleLegAResolvedAsync(BasketSlot slot, PendingEntry plan, OrderOutcome outcome, CancellationToken ct) + { + slot.Busy = true; + try + { + if (!outcome.Filled) + { + slot.Pending = null; + slot.PositionBasketId = string.Empty; + Transition(slot, BasketState.Idle); + slot.Intent = $"NON APERTO — gamba A {outcome.Status}: {outcome.Error}"; + Log.Warn($"[{slot.Name}] {slot.Intent}"); + _ledger.Note(_runId, "rifiuto", plan.BasketId, $"gamba A risolta come {outcome.Status}: {outcome.Error}", w => w.WriteString("basket", slot.Name)); + SaveState(); + return; + } + + if (outcome.PositionId > 0) + { + _knownPositions.Add(outcome.PositionId); + } + + if (!slot.A.HasQuote || !slot.B.HasQuote) + { + // No quotes: neither B nor an unwind can be priced. Keep the leg registered + // as ours and try again on the next resolution pass. + Log.Warn($"[{slot.Name}] gamba A eseguita ma senza quotazioni: riprovo al prossimo ciclo"); + plan.LegA = LegFromOutcome(slot, plan, outcome); + SaveState(); + return; + } + + BasketContext ctx = await BuildContextAsync(slot, DateTime.UtcNow, isBarClose: true, ct).ConfigureAwait(false); + BasketDecision d = _decider.Evaluate(ctx with { OpenBaskets = Math.Max(0, ctx.OpenBaskets - 1) }); + bool stillValid = d.Kind == DecisionKind.Enter && d.BuyCross == plan.BuyCross && !_killSwitched && !_equityStopped && _entriesBlocked is null; + if (stillValid) + { + Log.Info($"[{slot.Name}] gamba A eseguita in ritardo e segnale ancora valido (z {d.Evaluation.Z:+0.00;-0.00}): invio la gamba B"); + Transition(slot, BasketState.Entering); + EntryOutcome o = await _executor.ResumeAfterAAsync(ctx, plan, outcome, ct).ConfigureAwait(false); + ApplyEntryOutcome(slot, ctx, d, o, plan.BasketId); + return; + } + + string why = _killSwitched ? "kill-switch attivo" : _equityStopped ? "equity stop attivo" : _entriesBlocked ?? $"segnale decaduto ({d.Motivazione})"; + Log.Warn($"[{slot.Name}] gamba A eseguita in ritardo ma {why}: la richiudo subito (leg_risk_unwind)"); + BasketLeg legA = LegFromOutcome(slot, plan, outcome); + CloseOutcome undo = await _executor.UnwindLegAsync(legA, slot.Name, plan.BasketId, "leg_risk_unwind: " + why, ct).ConfigureAwait(false); + _ledger.Note(_runId, "leg_risk_unwind", plan.BasketId, $"gamba A eseguita in ritardo, {why}; {(undo.Closed ? "richiusa" : "NON richiusa: " + undo.Error)}", w => + { + w.WriteString("basket", slot.Name); + w.WriteNumber("position_id", legA.PositionId); + w.WriteNumber("pnl", undo.RealizedPnl); + }); + if (undo.Closed) + { + _todayRealized += undo.RealizedPnl; + _knownPositions.Remove(legA.PositionId); + RecordLoneLegClose(slot.Name, plan.BasketId, legA, undo, "leg_risk_unwind", why); + slot.Pending = null; + slot.PositionBasketId = string.Empty; + Transition(slot, BasketState.Idle); + slot.DisabledUntilUtc = DateTime.UtcNow.AddHours(1); + slot.DisabledReason = "gamba A eseguita in ritardo e richiusa"; + slot.Intent = $"richiusa la gamba A ({why})"; + } + else + { + plan.LegA = legA; + Transition(slot, BasketState.Error); + _entriesBlocked = $"gamba orfana su {slot.Name}: chiusura non riuscita ({undo.Error})"; + slot.Intent = $"ERRORE — gamba A non richiusa: {undo.Error}"; + } + + SaveState(); + } + finally + { + slot.Busy = false; + } + } + + /// Leg B resolved late: the basket is open, or leg A must go. + private async Task HandleLegBResolvedAsync(BasketSlot slot, PendingEntry plan, OrderOutcome outcome, CancellationToken ct) + { + slot.Busy = true; + try + { + if (!slot.A.HasQuote || !slot.B.HasQuote) + { + Log.Warn($"[{slot.Name}] gamba B risolta ma senza quotazioni: riprovo al prossimo ciclo"); + return; + } + + BasketContext ctx = await BuildContextAsync(slot, DateTime.UtcNow, isBarClose: false, ct).ConfigureAwait(false); + EntryOutcome o = await _executor.CompleteAfterBAsync(ctx, plan, outcome, ct).ConfigureAwait(false); + if (!o.Ok && plan.LegA is { } legA && o.Unwound) + { + _knownPositions.Remove(legA.PositionId); + } + + ApplyEntryOutcome(slot, ctx, null, o, plan.BasketId); + } + finally + { + slot.Busy = false; + } + } + + private static BasketLeg LegFromOutcome(BasketSlot slot, PendingEntry plan, OrderOutcome o) + { + (bool buyA, _) = slot.Cross.Legs(plan.BuyCross); + return new BasketLeg + { + Symbol = slot.A.Symbol, + InstrumentId = slot.A.Instrument.Id, + IsBuy = buyA, + Units = o.Units > 0 ? o.Units : plan.UnitsA, + EntryPrice = o.FillRate > 0 ? o.FillRate : plan.QuoteA, + PositionId = o.PositionId, + ClientRef = plan.ClientRefA, + OpenedUtc = o.TimeUtc == default ? DateTime.UtcNow : o.TimeUtc, + EntryFeesUsd = o.Fees, + }; + } + + /// A lone leg closed (unwind, orphan): one row in baskets.csv so the realised result is in the ledger. + private void RecordLoneLegClose(string basket, string basketId, BasketLeg leg, CloseOutcome close, string exitReason, string why) + { + double pnl = close.RealizedPnl; + BasketOutcomeRow row = new(basketId, _runId, basket, ModeLabel, PresetLabel, leg.OpenedUtc, close.TimeUtc == default ? DateTime.UtcNow : close.TimeUtc, + leg.IsBuy, double.NaN, double.NaN, pnl, pnl, double.NaN, double.NaN, double.NaN, double.NaN, 0, 0, exitReason, _account.Equity, double.NaN, + string.Create(CultureInfo.InvariantCulture, $"gamba singola {(leg.IsBuy ? "long" : "short")} {leg.Units:0.##} {leg.Symbol} @ {leg.EntryPrice} chiusa @ {close.CloseRate}: {why}")); + _ledger.Basket(row); + } +} diff --git a/Encelado/src/Encelado.Bot/Baskets/BasketEngine.Reconcile.cs b/Encelado/src/Encelado.Bot/Baskets/BasketEngine.Reconcile.cs new file mode 100644 index 0000000..756aa23 --- /dev/null +++ b/Encelado/src/Encelado.Bot/Baskets/BasketEngine.Reconcile.cs @@ -0,0 +1,477 @@ +using System.Globalization; +using System.Text; +using Encelado.Bot.Logging; +using Encelado.Core.Baskets; +using Encelado.Core.Broker; + +namespace Encelado.Bot.Baskets; + +/// +/// Account, reconciliation and safety. Every twenty seconds the account and the +/// positions are re-read; each position is classified as basket, orfana-bot +/// or esterna (§5.4 of the 5.0 plan); orphans are adopted and closed; cash +/// movements are told apart from trading results (§5.7); the engine's picture and the +/// account's are compared and a lasting disagreement is reported. +/// +public sealed partial class BasketEngine +{ + /// The account and the engine may disagree for this long (fills in flight) before it is a problem. + private static readonly TimeSpan UnreconciledGrace = TimeSpan.FromSeconds(60); + + private readonly Dictionary _orphanAttempts = []; + private readonly HashSet _orphanReported = []; + private HashSet _lastPositionIds = []; + private List _classified = []; + private List _signatures = []; + private DateTime _signaturesUtc; + private DateTime? _unreconciledSince; + private string _unreconciledReason = string.Empty; + private int _orphanCount; + private int _foreignCount; + + /// Reads the account; feeds the equity tracker with the realised result of the positions closed since the last reading. + private async Task RefreshAccountAsync(double closedNetSinceLast, CancellationToken ct) + { + try + { + _account = await _broker.GetAccountAsync(ct).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + Log.Warn($"conto non letto: {ex.Message}"); + return; + } + + CashMovement? movement = _equity.Observe(DateTime.UtcNow, _account.Balance, _account.Equity, closedNetSinceLast); + if (movement is not null) + { + Log.Warn($"MOVIMENTO DI CASSA: {movement.Motivazione}"); + _ledger.Note(_runId, "movimento_di_cassa", string.Empty, movement.Motivazione, w => + { + w.WriteNumber("importo", Math.Round(movement.Amount, 2)); + w.WriteNumber("saldo_prima", Math.Round(movement.BalanceBefore, 2)); + w.WriteNumber("saldo_dopo", Math.Round(movement.BalanceAfter, 2)); + w.WriteNumber("chiusure_nel_frattempo", Math.Round(movement.ClosedNetInBetween, 2)); + w.WriteNumber("cassa_cumulata", Math.Round(_equity.CumulativeCashFlow, 2)); + }); + SaveState(); + } + } + + /// + /// At startup the saved balance is compared with the account's: the trades closed + /// while the bot was off explain part of the difference, a cash movement the rest. + /// + private async Task StartupEquityAsync(CancellationToken ct) + { + double closedNet = 0; + if (double.IsFinite(_equity.LastBalance) && _equity.LastObservedUtc != default) + { + try + { + IReadOnlyList closed = await _broker.GetClosedTradesAsync(_equity.LastObservedUtc.AddDays(-1), ct).ConfigureAwait(false); + closedNet = closed.Where(c => c.ClosedUtc > _equity.LastObservedUtc).Sum(static c => c.NetProfit - c.Fees); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + Log.Warn($"storico delle chiusure non letto all'avvio: {ex.Message}; un eventuale movimento di cassa verrà stimato senza le chiusure"); + } + } + + await RefreshAccountAsync(closedNet, ct).ConfigureAwait(false); + } + + /// The realised net result of positions that vanished since the last reconciliation, from the venue's history. + private async Task ClosedNetAsync(IReadOnlyCollection vanished, CancellationToken ct) + { + if (vanished.Count == 0) + { + return 0; + } + + try + { + IReadOnlyList closed = await _broker.GetClosedTradesAsync(DateTime.UtcNow.AddDays(-3), ct).ConfigureAwait(false); + return closed.Where(c => vanished.Contains(c.PositionId)).Sum(static c => c.NetProfit - c.Fees); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + Log.Warn($"storico delle chiusure non letto: {ex.Message}"); + return 0; + } + } + + /// + /// Positions on the venue against the local state: a leg that vanished closes its + /// sibling; every position is classified; orphans of ours are adopted and closed; + /// strangers are reported once and left alone; the two pictures are compared. + /// + private async Task ReconcileAsync(CancellationToken ct) + { + IReadOnlyList positions; + try + { + positions = await _broker.GetPositionsAsync(ct).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + Log.Warn($"riconciliazione non riuscita: {ex.Message}"); + return; + } + + HashSet onVenue = [.. positions.Select(static p => p.PositionId)]; + List vanished = [.. _lastPositionIds.Where(id => !onVenue.Contains(id))]; + _lastPositionIds = onVenue; + await RefreshAccountAsync(await ClosedNetAsync(vanished, ct).ConfigureAwait(false), ct).ConfigureAwait(false); + + // 1. The baskets the engine holds. + foreach (BasketSlot slot in _slots) + { + if (slot.Position is not { } p || slot.Busy) + { + continue; + } + + bool aAlive = p.A.AllPositionIds.Any(onVenue.Contains); + bool bAlive = p.B.AllPositionIds.Any(onVenue.Contains); + if (aAlive && bAlive) + { + // Refresh accrued overnight from the venue's own fee field when it has one. + double fees = 0; + foreach (BrokerPosition bp in positions) + { + if (p.A.AllPositionIds.Contains(bp.PositionId) || p.B.AllPositionIds.Contains(bp.PositionId)) + { + fees += bp.Fees; + } + } + + if (fees > 0) + { + p.AccruedFeesUsd = Math.Max(0, fees - p.A.EntryFeesUsd - p.B.EntryFeesUsd); + } + + continue; + } + + if (!aAlive && !bAlive) + { + Log.Warn($"[{slot.Name}] entrambe le gambe sono sparite dal conto (chiuse dal broker o a mano): registro la chiusura"); + await RecordExternalCloseAsync(slot, p, "chiuso dal broker", ct).ConfigureAwait(false); + continue; + } + + string alive = aAlive ? p.A.Symbol : p.B.Symbol; + string gone = aAlive ? p.B.Symbol : p.A.Symbol; + Log.Warn($"[{slot.Name}] la gamba {gone} non è più sul conto (stop nativo o chiusura manuale): chiudo subito {alive} (leg_risk)"); + if (slot.A.HasQuote && slot.B.HasQuote) + { + BasketContext ctx = await BuildContextAsync(slot, DateTime.UtcNow, false, ct).ConfigureAwait(false); + await ExecuteExitAsync(slot, ctx, null, $"gamba {gone} chiusa dal broker", "leg_closed_by_broker", ct).ConfigureAwait(false); + } + } + + // A pending entry whose leg A vanished from the account (native stop, manual close): nothing left to finish. + foreach (BasketSlot slot in _slots) + { + if (slot.State == BasketState.PendingB && slot.Pending?.LegA is { PositionId: > 0 } legA && !onVenue.Contains(legA.PositionId) && !slot.Busy) + { + Log.Warn($"[{slot.Name}] la gamba A in attesa della B è sparita dal conto: annullo l'ingresso; se la B verrà eseguita, sarà un'orfana e verrà chiusa"); + _knownPositions.Remove(legA.PositionId); + slot.Pending = null; + slot.PositionBasketId = string.Empty; + Transition(slot, BasketState.Idle); + SaveState(); + } + } + + // 2. Whose is every position on the account. + Dictionary basketLegs = []; + foreach (BasketSlot slot in _slots) + { + foreach (long id in slot.LegPositionIds()) + { + basketLegs[id] = slot.Name; + } + } + + // An order still pending in the register may already be a position: that position + // is the basket's, not an orphan, until the register says otherwise. + foreach (TrackedOrder pending in _tracker.Pending) + { + BrokerPosition? match = OrderTracker.Match(pending, positions, id => basketLegs.ContainsKey(id)); + if (match is not null) + { + basketLegs[match.PositionId] = pending.Basket; + } + } + + Dictionary tracked = []; + foreach ((long id, string basket) in _tracker.OpenedPositions()) + { + if (!basketLegs.ContainsKey(id)) + { + tracked[id] = basket; + } + } + + _classified = PositionClassifier.Classify(positions, basketLegs, tracked, Signatures(), SymbolOf); + List orphans = [.. _classified.Where(static c => c.Origin == PositionOrigin.OrphanBot)]; + List foreign = [.. _classified.Where(static c => c.Origin == PositionOrigin.Foreign)]; + _orphanCount = orphans.Count; + _foreignCount = foreign.Count; + + foreach (ClassifiedPosition c in orphans) + { + BrokerPosition bp = c.Position; + if (_orphanReported.Add(bp.PositionId)) + { + Log.Warn(string.Create(CultureInfo.InvariantCulture, + $"GAMBA ORFANA del bot: posizione {bp.PositionId} su {SymbolOf(bp.InstrumentId)} ({(bp.IsBuy ? "long" : "short")} {bp.Units:0.##} @ {bp.OpenRate}, P&L {bp.UnrealizedPnl:+0.00;-0.00}) — {c.Reason}; {(OrphanPolicy == OrphanPolicy.Close ? "la adotto e la chiudo" : "in attesa della bonifica")}")); + _ledger.Note(_runId, "orfana_adottata", string.Empty, c.Reason, w => + { + w.WriteString("basket", c.Basket); + w.WriteNumber("position_id", bp.PositionId); + w.WriteString("strumento", SymbolOf(bp.InstrumentId)); + w.WriteString("verso", bp.IsBuy ? "long" : "short"); + w.WriteNumber("unita", bp.Units); + w.WriteString("aperta_utc", bp.OpenedUtc.ToString("O", CultureInfo.InvariantCulture)); + w.WriteNumber("pnl_aperto", Math.Round(bp.UnrealizedPnl, 2)); + }); + } + + if (OrphanPolicy == OrphanPolicy.Close) + { + await CloseOrphanAsync(c, "orphan_closed", "adottata alla riconciliazione e chiusa", ct).ConfigureAwait(false); + } + } + + foreach (ClassifiedPosition c in foreign) + { + BrokerPosition bp = c.Position; + if (_foreignPositions.Add(bp.PositionId)) + { + Log.Warn(string.Create(CultureInfo.InvariantCulture, $"posizione {bp.PositionId} su {SymbolOf(bp.InstrumentId)} ({(bp.IsBuy ? "long" : "short")} {bp.Units:0.##} @ {bp.OpenRate}) è esterna: {c.Reason}; la lascio com'è")); + } + } + + // 3. Do the two pictures agree? The account's unrealised result against the sum of + // what the positions say, and every position accounted for. + double sum = positions.Sum(static p => p.UnrealizedPnl); + double tolerance = Math.Max(5, Math.Abs(_account.UnrealizedPnl) * 0.01); + string? problem = null; + if (Math.Abs(_account.UnrealizedPnl - sum) > tolerance) + { + problem = string.Create(CultureInfo.InvariantCulture, $"P&L aperto del conto {_account.UnrealizedPnl:+0.00;-0.00} contro {sum:+0.00;-0.00} dalle posizioni"); + } + else if (_slots.Any(s => s.Position is { } p && !p.A.AllPositionIds.Any(onVenue.Contains))) + { + problem = "un basket aperto ha una gamba che il conto non mostra"; + } + + if (problem is null) + { + if (_unreconciledSince is not null) + { + Log.Info("posizioni riconciliate"); + } + + _unreconciledSince = null; + _unreconciledReason = string.Empty; + } + else + { + _unreconciledSince ??= DateTime.UtcNow; + _unreconciledReason = problem; + if (DateTime.UtcNow - _unreconciledSince.Value > UnreconciledGrace && (DateTime.UtcNow - _unreconciledSince.Value).TotalSeconds % 300 < ReconcileSeconds) + { + Log.Warn($"posizioni non riconciliate da {(DateTime.UtcNow - _unreconciledSince.Value).TotalSeconds:0} s: {problem}"); + } + } + } + + /// Whether the engine and the account have disagreed longer than the grace period. + private bool IsUnreconciled => _unreconciledSince is { } since && DateTime.UtcNow - since > UnreconciledGrace; + + /// The bot's entry signatures of the last three days: the ledger's decisions plus the order register. + private List Signatures() + { + DateTime now = DateTime.UtcNow; + if (now - _signaturesUtc > TimeSpan.FromMinutes(5)) + { + _signaturesUtc = now; + try + { + _signatures = _ledger.ReadEntrySignatures(now.AddDays(-3)); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warn($"firme del ledger non lette: {ex.Message}"); + } + } + + List all = new(_signatures); + foreach (TrackedOrder o in _tracker.All) + { + if (o.Leg is OrderLeg.A or OrderLeg.B or OrderLeg.Add) + { + all.Add(new EntrySignature(o.SentUtc, o.InstrumentId, o.Symbol, o.IsBuy, o.RequestedUnits, o.Basket, "registro ordini")); + } + } + + return all; + } + + /// Closes an orphan of ours, three attempts across reconciliations; records the realised result. + private async Task CloseOrphanAsync(ClassifiedPosition c, string exitReason, string why, CancellationToken ct) + { + BrokerPosition bp = c.Position; + int attempts = _orphanAttempts.GetValueOrDefault(bp.PositionId); + if (attempts >= 3) + { + if (_entriesBlocked is null) + { + _entriesBlocked = $"gamba orfana {bp.PositionId} non chiudibile dopo tre tentativi: chiudila a mano su eToro"; + Log.Error(_entriesBlocked, null); + } + + return false; + } + + _orphanAttempts[bp.PositionId] = attempts + 1; + string symbol = SymbolOf(bp.InstrumentId); + CloseOutcome close = await _executor.ClosePositionAsync(bp, symbol, c.Basket, $"{exitReason}: {why}", ct).ConfigureAwait(false); + if (!close.Closed) + { + Log.Warn($"gamba orfana {bp.PositionId} su {symbol}: chiusura non riuscita al tentativo {attempts + 1} ({close.Error})"); + return false; + } + + _todayRealized += close.RealizedPnl; + _orphanAttempts.Remove(bp.PositionId); + _knownPositions.Remove(bp.PositionId); + Log.Warn(string.Create(CultureInfo.InvariantCulture, $"gamba orfana {bp.PositionId} su {symbol} chiusa @ {close.CloseRate}: {close.RealizedPnl:+0.00;-0.00} USD")); + _ledger.Note(_runId, "orfana_chiusa", string.Empty, $"{why}: {close.RealizedPnl.ToString("+0.00;-0.00", CultureInfo.InvariantCulture)} USD", w => + { + w.WriteString("basket", c.Basket); + w.WriteNumber("position_id", bp.PositionId); + w.WriteString("strumento", symbol); + w.WriteNumber("pnl", Math.Round(close.RealizedPnl, 2)); + w.WriteString("exit_reason", exitReason); + }); + BasketLeg leg = new() + { + Symbol = symbol, + InstrumentId = bp.InstrumentId, + IsBuy = bp.IsBuy, + Units = bp.Units, + EntryPrice = bp.OpenRate, + PositionId = bp.PositionId, + OpenedUtc = bp.OpenedUtc, + }; + RecordLoneLegClose(c.Basket.Length > 0 ? c.Basket : symbol, string.Empty, leg, close, exitReason, why); + _lastReconcileUtc = DateTime.MinValue; // re-read the account soon: the balance moved + return true; + } + + private async Task RecordExternalCloseAsync(BasketSlot slot, BasketPosition p, string reason, CancellationToken ct) + { + double pnl = 0; + try + { + IReadOnlyList closed = await _broker.GetClosedTradesAsync(p.OpenedUtc.AddDays(-1), ct).ConfigureAwait(false); + HashSet ids = [.. p.A.AllPositionIds.Concat(p.B.AllPositionIds)]; + pnl = closed.Where(c => ids.Contains(c.PositionId)).Sum(static c => c.NetProfit - c.Fees); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + Log.Warn($"[{slot.Name}] storico chiusure non letto: {ex.Message}"); + } + + BasketOutcomeRow row = new(slot.PositionBasketId, _runId, slot.Name, ModeLabel, PresetLabel, p.OpenedUtc, DateTime.UtcNow, p.BuyCross, p.EntryZ, slot.LastEvaluation.Z, + pnl, pnl, double.NaN, p.EntryCostPips, double.NaN, double.NaN, p.Adds, p.BarsHeld, "closed_by_broker", p.EquityAtEntry, slot.PMl, reason); + _ledger.Basket(row); + _todayRealized += pnl; + foreach (long id in p.A.AllPositionIds.Concat(p.B.AllPositionIds)) + { + _knownPositions.Remove(id); + } + + slot.Position = null; + slot.PositionBasketId = string.Empty; + slot.State = BasketState.Idle; + slot.Intent = $"chiuso dal broker: {pnl:+0.00;-0.00} USD"; + SaveState(); + } + + private async Task CheckEquityStopAsync(CancellationToken ct) + { + if (_equityStopped || _equity.PeakNetEquity <= 0) + { + return; + } + + double dd = _equity.Drawdown(_account.Equity); + if (dd >= _strategy.EquityStopPct / 100.0) + { + _equityStopped = true; + _haltReason = string.Create(CultureInfo.InvariantCulture, $"equity {_account.Equity:F2} a {dd:P2} dal picco {_equity.PeakEquity:F2} (soglia {_strategy.EquityStopPct:0.#} %)"); + Log.Error($"EQUITY STOP: {_haltReason}. Chiudo tutto e mi blocco: serve un reset manuale con motivazione.", null); + _ledger.Correction(_runId, string.Empty, $"equity stop: {_haltReason}"); + await CloseAllAsync("equity stop", ct).ConfigureAwait(false); + SaveState(); + } + } + + private async Task CheckStopFileAsync(CancellationToken ct) + { + bool present = File.Exists(_stopFile); + if (present && !_killSwitched) + { + Log.Warn("file STOP trovato: KILL-SWITCH"); + await KillAsync("file STOP", ct).ConfigureAwait(false); + } + } + + private async Task KillAsync(string reason, CancellationToken ct) + { + _killSwitched = true; + _haltReason = $"kill-switch ({reason})"; + _ledger.Note(_runId, "kill_switch_avviato", string.Empty, $"kill-switch: {reason}"); + await CloseAllAsync(_haltReason, ct).ConfigureAwait(false); + SaveState(); + } + + /// Closes every basket and every orphan of ours; foreign positions are left alone. + public async Task CloseAllAsync(string reason, CancellationToken ct) + { + string code = reason.Contains("equity", StringComparison.OrdinalIgnoreCase) ? "equity_stop" : "kill_switch"; + foreach (BasketSlot slot in _slots) + { + if (slot.Position is not null && slot.A.HasQuote && slot.B.HasQuote) + { + BasketContext ctx = await BuildContextAsync(slot, DateTime.UtcNow, false, ct).ConfigureAwait(false); + await ExecuteExitAsync(slot, ctx, null, reason, code, ct).ConfigureAwait(false); + } + + if (slot.State.IsPending() && slot.Pending?.LegA is { PositionId: > 0 } legA) + { + CloseOutcome undo = await _executor.UnwindLegAsync(legA, slot.Name, slot.Pending.BasketId, reason, ct).ConfigureAwait(false); + if (undo.Closed) + { + _todayRealized += undo.RealizedPnl; + _knownPositions.Remove(legA.PositionId); + RecordLoneLegClose(slot.Name, slot.Pending.BasketId, legA, undo, code, reason); + slot.Pending = null; + slot.PositionBasketId = string.Empty; + Transition(slot, BasketState.Idle); + } + } + } + + foreach (ClassifiedPosition c in _classified.Where(static c => c.Origin == PositionOrigin.OrphanBot).ToList()) + { + await CloseOrphanAsync(c, code, reason, ct).ConfigureAwait(false); + } + } +} diff --git a/Encelado/src/Encelado.Bot/Baskets/BasketEngine.Snapshot.cs b/Encelado/src/Encelado.Bot/Baskets/BasketEngine.Snapshot.cs new file mode 100644 index 0000000..19a8948 --- /dev/null +++ b/Encelado/src/Encelado.Bot/Baskets/BasketEngine.Snapshot.cs @@ -0,0 +1,141 @@ +using System.Globalization; +using System.Reflection; +using System.Text.Json; +using Encelado.Bot.Configuration; +using Encelado.Bot.Engine; +using Encelado.Bot.Logging; +using Encelado.Core.Baskets; +using Encelado.Etoro; + +namespace Encelado.Bot.Baskets; + +/// The picture the window and the console render. +public sealed partial class BasketEngine +{ + public BotSnapshot Snapshot(BotState state, string? error, DateTime? startedUtc, EventRow[] events) + { + ArgumentNullException.ThrowIfNull(events); + double equity = _account.Equity; + double dd = _equity.Drawdown(equity); + double openPnl = 0; + List rows = new(_slots.Count); + DateTime now = DateTime.UtcNow; + + foreach (BasketSlot s in _slots) + { + BasketEvaluation e = s.LastEvaluation; + double pnl = s.Position is not null && double.IsFinite(e.PnlOpenUsd) ? e.PnlOpenUsd : 0; + openPnl += pnl; + int legs = s.Position is { } p ? 2 + (2 * p.Adds) : s.Pending?.LegA is not null ? 1 : 0; + string nextEvent = s.Features.NextEventLabel; + rows.Add(new BasketRow(s.Name, s.Definition.A, s.Definition.B, s.Cross.Symbol, s.State.ToString(), legs, pnl, + equity > 0 ? pnl / equity : 0, double.IsFinite(e.PipsOpen) ? e.PipsOpen : 0, s.Position?.TpPips ?? _decider.Preset.TpPips, + e.RhoW, e.RhoShort, e.Z, e.CostPips, s.PMl, _learning.Active, nextEvent, s.Enabled && s.DisabledUntilUtc <= now, + s.DisabledUntilUtc > now ? $"in pausa fino alle {s.DisabledUntilUtc:HH:mm} UTC: {s.DisabledReason}" : s.DisabledReason, + s.Intent, s.Position?.EntryZ ?? s.Pending?.EntryZ ?? 0, s.Position?.BarsHeld ?? 0, s.Position?.Adds ?? 0, s.Position is not null, e.HalfLife)); + } + + List quotes = new(_series.Count); + foreach (SymbolSeries s in _series.Values) + { + quotes.Add(new QuoteRow(s.Symbol, s.HasQuote ? s.Quote.Bid : 0, s.HasQuote ? s.Quote.Ask : 0, s.SpreadPips, s.HasQuote ? s.Quote.TimeUtc : default, + s.QuoteSeenUtc == default ? -1 : (now - s.QuoteSeenUtc).TotalSeconds)); + } + + return new BotSnapshot + { + State = state, + Error = error, + StartedAtUtc = startedUtc, + Uptime = startedUtc is { } st ? now - st : TimeSpan.Zero, + Mode = _mode.Badge(), + EnvironmentKind = _mode.Kind(), + ExecutionMode = _mode.ToString(), + Endpoint = _feed.Endpoint, + Preset = PresetLabel, + StrategyVersion = $"v{Assembly.GetExecutingAssembly().GetName().Version?.ToString(3) ?? "?"} · strategia {_configHash} · run {_runId}", + ApiState = _apiState, + ApiLatencyMs = _apiLatencyMs, + ClockSkewSeconds = _feed.ClockSkew.TotalSeconds, + Equity = equity, + Balance = _account.Balance, + AvailableBalance = _account.Available, + PeakEquity = _equity.PeakEquity, + DrawdownPct = dd, + EquityStopPct = _strategy.EquityStopPct / 100.0, + DailyLossPct = _strategy.DailyLossPct / 100.0, + TodayPnl = _todayRealized, + TodayPnlPct = _dayStartEquity > 0 ? _todayRealized / _dayStartEquity : 0, + OpenPnl = openPnl, + OpenPnlPct = equity > 0 ? openPnl / equity : 0, + OpenBaskets = _slots.Count(static s => s.Position is not null), + MaxBaskets = _decider.Preset.MaxBaskets, + PendingBaskets = _slots.Count(static s => s.State.IsPending()), + PendingOrders = _tracker.PendingCount, + OrphanLegs = _orphanCount, + ForeignPositions = _foreignCount, + AccountOpenPnl = _account.UnrealizedPnl, + UsedMargin = _account.UsedMargin, + CumulativeCashFlow = _equity.CumulativeCashFlow, + Unreconciled = IsUnreconciled, + UnreconciledReason = _unreconciledReason, + Halted = _killSwitched || _equityStopped, + HaltReason = _haltReason, + EquityStopped = _equityStopped, + KillSwitched = _killSwitched, + EntriesBlockedReason = _entriesBlocked, + Counters = string.Create(CultureInfo.InvariantCulture, $"quote/min {_feed.QuotaUsed(EtoroQuota.MarketData)}/110 · ordini/min {_feed.QuotaUsed(EtoroQuota.Trading)}/18 · esiti/min {_feed.QuotaUsed(EtoroQuota.Lookup)}/55 · ultima quotazione {(_lastQuoteUtc == default ? "—" : (now - _lastQuoteUtc).TotalSeconds.ToString("0") + " s fa")}"), + Events = events, + Baskets = rows, + Quotes = quotes, + Context = ContextWithLearning(now), + }; + } + + private ContextRow ContextWithLearning(DateTime now) + { + ContextRow row = _context.Row(now); + BasketSlot? first = _slots.FirstOrDefault(static s => s.Enabled && s.Vol.Count > 0); + string vol = first is null ? "in attesa di barre" : $"{first.Name}: {first.Vol.Describe()}"; + return row with { VolForecast = vol, MlState = _learning.Describe(), BanditProposal = _learning.BanditText }; + } + + /// The picture while the engine is not running. + public static BotSnapshot IdleSnapshot(BotConfig config, BotState state, string? error, EventRow[] events) + { + ArgumentNullException.ThrowIfNull(config); + ExecutionMode mode = config.Run.Mode; + List rows = []; + try + { + BasketStrategyConfig strategy = File.Exists(config.Run.StrategyPath) + ? BasketStrategyConfig.Load(config.Run.StrategyPath, out _) + : BasketStrategyConfig.ParseText(BasketStrategyConfig.DefaultJson, out _); + foreach (BasketDefinition d in strategy.Baskets) + { + string cross = SyntheticCross.TryDerive(d.A, d.B, out SyntheticCross? c) ? c!.Symbol : "?"; + rows.Add(new BasketRow(d.Name, d.A, d.B, cross, "fermo", 0, 0, 0, 0, strategy.Effective().TpPips, double.NaN, double.NaN, double.NaN, double.NaN, double.NaN, false, "—", + d.Enabled, d.Enabled ? string.Empty : "disattivato in strategy.json", "il bot è fermo", 0, 0, 0, false, double.NaN)); + } + } + catch (Exception ex) when (ex is IOException or InvalidOperationException or JsonException) + { + Log.Warn($"strategy.json non leggibile: {ex.Message}"); + } + + return new BotSnapshot + { + State = state, + Error = error, + Mode = mode.Badge(), + EnvironmentKind = mode.Kind(), + ExecutionMode = mode.ToString(), + Endpoint = config.Etoro.BaseUrl, + Preset = "—", + ApiState = "fermo", + Events = events, + Baskets = rows, + Context = new ContextRow([], [], "—", "—", "—", "—", "—"), + }; + } +} diff --git a/Encelado/src/Encelado.Bot/Baskets/BasketEngine.State.cs b/Encelado/src/Encelado.Bot/Baskets/BasketEngine.State.cs new file mode 100644 index 0000000..e25243b --- /dev/null +++ b/Encelado/src/Encelado.Bot/Baskets/BasketEngine.State.cs @@ -0,0 +1,204 @@ +using System.Globalization; +using System.Text.Json; +using Encelado.Bot.Logging; +using Encelado.Core.Baskets; + +namespace Encelado.Bot.Baskets; + +/// +/// The state on disk (data/state/baskets_state.json): open baskets, entries in +/// flight, the equity tracker, the session's counters and the halts. Written atomically +/// after every change that matters; read at startup before the reconciliation. +/// +public sealed partial class BasketEngine +{ + private void SaveState() + { + try + { + using MemoryStream ms = new(); + using (Utf8JsonWriter w = new(ms, new JsonWriterOptions { Indented = true })) + { + w.WriteStartObject(); + w.WriteString("savedUtc", DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture)); + w.WriteString("runId", _runId); + w.WriteString("mode", ModeLabel); + w.WriteNumber("peakEquity", _equity.PeakEquity); + w.WriteNumber("peakNetEquity", _equity.PeakNetEquity); + w.WriteNumber("cumulativeCashFlow", _equity.CumulativeCashFlow); + w.WriteNumber("lastBalance", double.IsFinite(_equity.LastBalance) ? _equity.LastBalance : 0); + w.WriteString("lastBalanceUtc", _equity.LastObservedUtc == default ? string.Empty : _equity.LastObservedUtc.ToString("O", CultureInfo.InvariantCulture)); + w.WriteString("sessionDate", _sessionDate.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)); + w.WriteNumber("dayStartEquity", _dayStartEquity); + w.WriteNumber("todayRealized", _todayRealized); + w.WriteBoolean("killSwitched", _killSwitched); + w.WriteBoolean("equityStopped", _equityStopped); + w.WriteString("haltReason", _haltReason ?? string.Empty); + w.WriteStartArray("baskets"); + foreach (BasketSlot s in _slots) + { + if (s.Position is not { } p) + { + continue; + } + + w.WriteStartObject(); + w.WriteString("name", s.Name); + w.WriteString("basketId", s.PositionBasketId); + w.WriteBoolean("buyCross", p.BuyCross); + w.WriteString("openedUtc", p.OpenedUtc.ToString("O", CultureInfo.InvariantCulture)); + w.WriteNumber("entryZ", p.EntryZ); + w.WriteNumber("lastAddZ", p.LastAddZ); + w.WriteNumber("adds", p.Adds); + w.WriteNumber("barsHeld", p.BarsHeld); + w.WriteNumber("entryCostPips", double.IsFinite(p.EntryCostPips) ? p.EntryCostPips : 0); + w.WriteNumber("tpPips", p.TpPips); + w.WriteNumber("maxLossUsd", p.MaxLossUsd); + w.WriteNumber("equityAtEntry", p.EquityAtEntry); + w.WriteNumber("accruedFeesUsd", p.AccruedFeesUsd); + w.WriteString("entryMotivazione", p.EntryMotivazione); + BasketLeg.Write(w, "a", p.A); + BasketLeg.Write(w, "b", p.B); + w.WriteEndObject(); + } + + w.WriteEndArray(); + + w.WriteStartArray("pendingEntries"); + foreach (BasketSlot s in _slots) + { + if (!s.State.IsPending() || s.Pending is not { } plan) + { + continue; + } + + w.WriteStartObject(); + w.WriteString("name", s.Name); + w.WriteString("state", s.State.ToString()); + w.WriteString("basketId", s.PositionBasketId); + plan.Write(w); + w.WriteEndObject(); + } + + w.WriteEndArray(); + w.WriteEndObject(); + } + + File.WriteAllBytes(_statePath + ".tmp", ms.ToArray()); + File.Move(_statePath + ".tmp", _statePath, overwrite: true); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warn($"stato non salvato: {ex.Message}"); + } + } + + private void LoadState() + { + if (!File.Exists(_statePath)) + { + return; + } + + try + { + using JsonDocument doc = JsonDocument.Parse(File.ReadAllBytes(_statePath)); + JsonElement root = doc.RootElement; + string savedMode = root.TryGetProperty("mode", out JsonElement m) ? m.GetString() ?? string.Empty : string.Empty; + if (!savedMode.Equals(ModeLabel, StringComparison.OrdinalIgnoreCase) && savedMode.Length > 0) + { + Log.Warn($"stato salvato in modalità {savedMode}, ora {ModeLabel}: le posizioni salvate non vengono riprese (riconciliazione dal conto)"); + return; + } + + double peakNet = root.TryGetProperty("peakNetEquity", out JsonElement pn) ? pn.GetDouble() : root.TryGetProperty("peakEquity", out JsonElement pe) ? pe.GetDouble() : 0; + double cash = root.TryGetProperty("cumulativeCashFlow", out JsonElement cf) ? cf.GetDouble() : 0; + double lastBalance = root.TryGetProperty("lastBalance", out JsonElement lb) && lb.GetDouble() > 0 ? lb.GetDouble() : double.NaN; + DateTime lastBalanceUtc = root.TryGetProperty("lastBalanceUtc", out JsonElement lbu) && DateTime.TryParse(lbu.GetString(), CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal, out DateTime t) ? t : default; + _equity.Restore(peakNet, cash, lastBalance, lastBalanceUtc); + + _killSwitched = root.TryGetProperty("killSwitched", out JsonElement ks) && ks.GetBoolean(); + _equityStopped = root.TryGetProperty("equityStopped", out JsonElement es) && es.GetBoolean(); + _haltReason = root.TryGetProperty("haltReason", out JsonElement hr) && hr.GetString() is { Length: > 0 } h ? h : null; + if (root.TryGetProperty("sessionDate", out JsonElement sd) && DateOnly.TryParse(sd.GetString(), CultureInfo.InvariantCulture, out DateOnly day) && day == DateOnly.FromDateTime(DateTime.UtcNow)) + { + _dayStartEquity = root.TryGetProperty("dayStartEquity", out JsonElement dse) ? dse.GetDouble() : 0; + _todayRealized = root.TryGetProperty("todayRealized", out JsonElement tr) ? tr.GetDouble() : 0; + } + + if (_equityStopped) + { + Log.Warn($"equity stop ancora attivo dal run precedente: {_haltReason}"); + } + + if (root.TryGetProperty("baskets", out JsonElement arr)) + { + foreach (JsonElement e in arr.EnumerateArray()) + { + string name = e.GetProperty("name").GetString() ?? string.Empty; + BasketSlot? slot = _slots.FirstOrDefault(s => s.Name == name); + if (slot is null) + { + continue; + } + + BasketPosition p = new() + { + BasketId = e.GetProperty("basketId").GetString() ?? string.Empty, + Name = name, + BuyCross = e.GetProperty("buyCross").GetBoolean(), + A = BasketLeg.Read(e.GetProperty("a")), + B = BasketLeg.Read(e.GetProperty("b")), + OpenedUtc = DateTime.Parse(e.GetProperty("openedUtc").GetString()!, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal), + EntryZ = e.GetProperty("entryZ").GetDouble(), + LastAddZ = e.GetProperty("lastAddZ").GetDouble(), + Adds = e.GetProperty("adds").GetInt32(), + BarsHeld = e.GetProperty("barsHeld").GetInt32(), + EntryCostPips = e.GetProperty("entryCostPips").GetDouble(), + TpPips = e.GetProperty("tpPips").GetDouble(), + MaxLossUsd = e.GetProperty("maxLossUsd").GetDouble(), + EquityAtEntry = e.GetProperty("equityAtEntry").GetDouble(), + AccruedFeesUsd = e.GetProperty("accruedFeesUsd").GetDouble(), + EntryMotivazione = e.GetProperty("entryMotivazione").GetString() ?? string.Empty, + }; + slot.Position = p; + slot.PositionBasketId = p.BasketId; + slot.State = BasketState.Open; + slot.Intent = "ripreso dallo stato salvato: " + p.Describe(); + _knownPositions.UnionWith(p.A.AllPositionIds); + _knownPositions.UnionWith(p.B.AllPositionIds); + Log.Info($"[{name}] basket ripreso dallo stato salvato: {p.Describe()}"); + } + } + + if (root.TryGetProperty("pendingEntries", out JsonElement pendings)) + { + foreach (JsonElement e in pendings.EnumerateArray()) + { + string name = e.GetProperty("name").GetString() ?? string.Empty; + BasketSlot? slot = _slots.FirstOrDefault(s => s.Name == name); + if (slot is null || slot.Position is not null || !e.TryGetProperty("pending", out JsonElement pe2)) + { + continue; + } + + PendingEntry plan = PendingEntry.Read(pe2); + slot.Pending = plan; + slot.PositionBasketId = e.TryGetProperty("basketId", out JsonElement bid) ? bid.GetString() ?? plan.BasketId : plan.BasketId; + slot.State = Enum.TryParse(e.GetProperty("state").GetString(), out BasketState st) && st.IsPending() ? st : BasketState.PendingA; + if (plan.LegA is { PositionId: > 0 } legA) + { + _knownPositions.Add(legA.PositionId); + } + + slot.Intent = $"ripreso dallo stato salvato: in attesa della gamba {(slot.State == BasketState.PendingA ? "A" : "B")}"; + Log.Warn($"[{name}] ingresso in sospeso ripreso dallo stato salvato ({slot.State}, basket {plan.BasketId}): il registro degli ordini lo risolve prima di ogni decisione"); + } + } + } + catch (Exception ex) when (ex is IOException or JsonException or KeyNotFoundException or InvalidOperationException or FormatException) + { + Log.Warn($"stato salvato non leggibile ({ex.Message}): riparto dalla riconciliazione con il conto"); + } + } +} diff --git a/Encelado/src/Encelado.Bot/Baskets/BasketEngine.cs b/Encelado/src/Encelado.Bot/Baskets/BasketEngine.cs index a2563af..e40b12f 100644 --- a/Encelado/src/Encelado.Bot/Baskets/BasketEngine.cs +++ b/Encelado/src/Encelado.Bot/Baskets/BasketEngine.cs @@ -1,7 +1,6 @@ using System.Collections.Concurrent; using System.Diagnostics; using System.Globalization; -using System.Reflection; using System.Text; using System.Text.Json; using Encelado.Bot.Configuration; @@ -9,12 +8,23 @@ using Encelado.Bot.Engine; using Encelado.Bot.Logging; using Encelado.Core.Baskets; using Encelado.Core.Baskets.Data; +using Encelado.Core.Baskets.History; using Encelado.Core.Baskets.Learning; using Encelado.Core.Broker; using Encelado.Etoro; namespace Encelado.Bot.Baskets; +/// What the engine does with a position that carries the bot's signature but belongs to no basket. +public enum OrphanPolicy +{ + /// Adopt it and close it at once (the default, §5.4 of the 5.0 plan). + Close = 0, + + /// Report it and wait for the bonifica to decide (the --bonifica start). + Report, +} + /// /// The live engine of the Correlation Baskets module: one broker, eight instruments, /// five baskets, one decision thread. @@ -22,12 +32,18 @@ namespace Encelado.Bot.Baskets; /// Quotes arrive by polling (one request for every instrument every few seconds) and /// build the M15 bars locally; on a bar close each basket is evaluated by the same /// the backtest uses; between bar closes only the exits of -/// open baskets are watched. Entries and take-profits go through the approval queue in -/// the approve modes; protective stops never wait for anyone. Every evaluation lands in -/// the ledger before its outcome is known. +/// open baskets are watched. Every evaluation lands in the ledger before its outcome is +/// known. Every order lands in the order register before it is sent, and a basket whose +/// leg has no outcome waits (PendingA/PendingB) instead of forgetting it. +/// +/// +/// The class is split in partial files: the loop and the decisions here; the order +/// register in BasketEngine.Pending.cs; account, reconciliation and safety in +/// BasketEngine.Reconcile.cs; the state on disk in BasketEngine.State.cs; +/// commands in BasketEngine.Commands.cs; the snapshot in BasketEngine.Snapshot.cs. /// /// -public sealed class BasketEngine : IEngine +public sealed partial class BasketEngine : IEngine { /// Bars pulled at startup: the 30-day volatility average needs 2880 M15 bars and the windows need room after it. private const int WarmupBars = 3000; @@ -50,6 +66,8 @@ public sealed class BasketEngine : IEngine private readonly PaperBroker? _paper; private readonly BasketDecider _decider; private readonly BasketExecutor _executor; + private readonly OrderTracker _tracker; + private readonly EquityTracker _equity = new(); private readonly Ledger _ledger; private readonly LearningState _learning; private readonly List _volHistory = []; @@ -69,7 +87,6 @@ public sealed class BasketEngine : IEngine private readonly HashSet _knownPositions = []; private AccountSnapshot _account = new(DateTime.MinValue, "USD", 0, 0, 0, 0, 0); - private double _peakEquity; private DateOnly _sessionDate; private double _dayStartEquity; private double _todayRealized; @@ -107,6 +124,9 @@ public sealed class BasketEngine : IEngine public BasketPosition? Position { get; set; } + /// The entry in flight while the state is PendingA or PendingB. + public PendingEntry? Pending { get; set; } + public BasketDecision? LastDecision { get; set; } public BasketEvaluation LastEvaluation { get; set; } = new(); @@ -134,6 +154,23 @@ public sealed class BasketEngine : IEngine public double[]? LastFeatures { get; set; } public double LastLogX { get; set; } = double.NaN; + + /// Position ids this slot accounts for: both legs of the open basket, or leg A of a pending entry. + public IEnumerable LegPositionIds() + { + if (Position is { } p) + { + foreach (long id in p.A.AllPositionIds.Concat(p.B.AllPositionIds)) + { + yield return id; + } + } + + if (Pending?.LegA is { PositionId: > 0 } leg) + { + yield return leg.PositionId; + } + } } public BasketEngine(BotConfig config, bool startConfirmed, IContextProvider? context = null) @@ -162,6 +199,7 @@ public sealed class BasketEngine : IEngine _strategy.Validate(); _strategy.InvertSignal = false; // research-only flag, never honoured by the bot _configHash = _strategy.Hash(); + _runId = string.Create(CultureInfo.InvariantCulture, $"{DateTime.UtcNow:yyyyMMdd-HHmmss}-{Guid.NewGuid().ToString("N")[..6]}"); EtoroKeyStore.Resolve(config, out string origin); Log.Info($"chiavi eToro: {origin}"); @@ -205,24 +243,30 @@ public sealed class BasketEngine : IEngine _broker = _feed; } - _decider = new BasketDecider(_strategy); - _executor = new BasketExecutor(_broker, _strategy, MidOf, static m => Log.Warn(m)); _ledger = new Ledger(Path.Combine(_dataDir, "ledger")); + _tracker = new OrderTracker(Path.Combine(_dataDir, "state", _mode == ExecutionMode.Paper ? "pending_orders_paper.json" : "pending_orders.json")); + _tracker.Changed += (order, evento) => _ledger.Order(OrderRecord.From(order, _runId, evento)); + _decider = new BasketDecider(_strategy); + _executor = new BasketExecutor(_broker, _strategy, MidOf, static m => Log.Warn(m), _tracker, _mode.ToString()); _learning = new LearningState(_dataDir, config.Run.KnowledgePath, _ledger, _strategy); _context = context ?? new FeedContextProvider(_dataDir, config.Etoro.UserAgent); - _runId = string.Create(CultureInfo.InvariantCulture, $"{DateTime.UtcNow:yyyyMMdd-HHmmss}-{Guid.NewGuid().ToString("N")[..6]}"); } public string Endpoint => _feed.Endpoint; public string RunId => _runId; + /// Set before the start: Report for the --bonifica session, Close otherwise. + public OrphanPolicy OrphanPolicy { get; set; } = OrphanPolicy.Close; + private string ModeLabel => _mode.ToString(); private string PresetLabel => _decider.Preset.Label; private double? MidOf(string symbol) => _series.TryGetValue(symbol, out SymbolSeries? s) && s.HasQuote ? s.Mid : null; + private string SymbolOf(long instrumentId) => _seriesById.TryGetValue(instrumentId, out SymbolSeries? s) ? s.Symbol : instrumentId.ToString(CultureInfo.InvariantCulture); + // ----------------------------------------------------------------------- // Lifecycle // ----------------------------------------------------------------------- @@ -234,17 +278,27 @@ public sealed class BasketEngine : IEngine await StartupChecksAsync(ct).ConfigureAwait(false); await LoadInstrumentsAsync(ct).ConfigureAwait(false); await WarmupAsync(ct).ConfigureAwait(false); - await RefreshAccountAsync(ct).ConfigureAwait(false); LoadState(); + await StartupEquityAsync(ct).ConfigureAwait(false); + + // The order register first: an order sent by the previous run and never answered + // must be settled before any decision, or the reconciliation would call its + // position an orphan. + int pending = _tracker.PendingCount; + if (pending > 0) + { + Log.Warn($"registro ordini: {pending} ordine/i senza esito dal run precedente: li risolvo prima di qualsiasi decisione"); + await ResolvePendingOrdersAsync(ct, force: true).ConfigureAwait(false); + } + await ReconcileAsync(ct).ConfigureAwait(false); _sessionDate = DateOnly.FromDateTime(DateTime.UtcNow); if (_dayStartEquity <= 0) { - _dayStartEquity = _account.Equity; + _dayStartEquity = _equity.NetEquity(_account.Equity); } - _peakEquity = Math.Max(_peakEquity, _account.Equity); SaveState(); try @@ -452,6 +506,10 @@ public sealed class BasketEngine : IEngine await DrainCommandsAsync(ct).ConfigureAwait(false); RollSessionIfNeeded(now); + // Orders without an outcome come first: their resolution changes what the + // reconciliation and the decisions below see. + await ResolvePendingOrdersAsync(ct).ConfigureAwait(false); + if (now - _lastStopCheckUtc >= TimeSpan.FromSeconds(5)) { _lastStopCheckUtc = now; @@ -467,7 +525,6 @@ public sealed class BasketEngine : IEngine if (now - _lastReconcileUtc >= reconcile) { _lastReconcileUtc = now; - await RefreshAccountAsync(ct).ConfigureAwait(false); await ReconcileAsync(ct).ConfigureAwait(false); await CheckEquityStopAsync(ct).ConfigureAwait(false); SaveState(); @@ -664,8 +721,13 @@ public sealed class BasketEngine : IEngine private async Task EvaluateAsync(BasketSlot slot, DateTime now, bool isBarClose, CancellationToken ct) { - if (slot.Busy || slot.State is BasketState.Entering or BasketState.Exiting or BasketState.Adding) + if (slot.Busy || slot.State is BasketState.Entering or BasketState.Exiting or BasketState.Adding || slot.State.IsPending()) { + if (slot.State.IsPending()) + { + slot.Intent = $"IN ATTESA dell'esito della gamba {(slot.State == BasketState.PendingA ? "A" : "B")} (ordine nel registro dal {slot.Pending?.DecidedUtc:HH:mm:ss} UTC)"; + } + return; } @@ -716,7 +778,6 @@ public sealed class BasketEngine : IEngine slot.LastDecision = d; slot.LastEvaluation = d.Evaluation; - slot.Features = ctx.MinutesToNextHigh == int.MaxValue ? slot.Features : slot.Features; if (isBarClose) { @@ -744,7 +805,7 @@ public sealed class BasketEngine : IEngine if (isBarClose) { Log.Info($"[{slot.Name}] {slot.Intent}"); - await HandleEntryAsync(slot, ctx, d, now, ct).ConfigureAwait(false); + await ExecuteEntryAsync(slot, ctx, d, ct).ConfigureAwait(false); } break; @@ -753,7 +814,7 @@ public sealed class BasketEngine : IEngine if (isBarClose) { Log.Info($"[{slot.Name}] {slot.Intent}"); - await HandleAddAsync(slot, ctx, d, now, ct).ConfigureAwait(false); + await ExecuteAddAsync(slot, ctx, d, ct).ConfigureAwait(false); } break; @@ -765,7 +826,7 @@ public sealed class BasketEngine : IEngine } Log.Info($"[{slot.Name}] {slot.Intent}"); - await HandleExitAsync(slot, ctx, d, now, ct).ConfigureAwait(false); + await ExecuteExitAsync(slot, ctx, d, d.Motivazione, d.ReasonCodes.FirstOrDefault() ?? "exit", ct).ConfigureAwait(false); break; } } @@ -785,7 +846,7 @@ public sealed class BasketEngine : IEngine double usdA = PipMath.QuoteToUsd(PipMath.QuoteCurrency(slot.A.Symbol), MidOf) ?? double.NaN; double usdB = PipMath.QuoteToUsd(PipMath.QuoteCurrency(slot.B.Symbol), MidOf) ?? double.NaN; - bool sameCrossOpen = _slots.Any(o => o != slot && o.Cross.Symbol == slot.Cross.Symbol && (o.Position is not null || o.State == BasketState.Entering)); + bool sameCrossOpen = _slots.Any(o => o != slot && o.Cross.Symbol == slot.Cross.Symbol && (o.Position is not null || o.State == BasketState.Entering || o.State.IsPending())); double dailyPnl = _todayRealized + _account.UnrealizedPnl; bool dailyLossHit = _dayStartEquity > 0 && dailyPnl <= -_dayStartEquity * _strategy.DailyLossPct / 100.0; @@ -798,9 +859,9 @@ public sealed class BasketEngine : IEngine A = slot.A, B = slot.B, Equity = _account.Equity, - PeakEquity = _peakEquity, + PeakEquity = _equity.PeakEquity, DailyPnlUsd = dailyPnl, - OpenBaskets = _slots.Count(static s => s.Position is not null || s.State == BasketState.Entering), + OpenBaskets = _slots.Count(static s => s.Position is not null || s.State == BasketState.Entering || s.State.IsPending()), SameCrossOpen = sameCrossOpen, DailyLossHit = dailyLossHit, EquityStopped = _equityStopped, @@ -889,9 +950,6 @@ public sealed class BasketEngine : IEngine // The bot trades by itself in every mode (D-20): an entry, an add or an exit decided // by the decider is executed at once. The only human gates left are the start of the // live mode, the kill-switch and the reset after an equity stop. - private Task HandleEntryAsync(BasketSlot slot, BasketContext ctx, BasketDecision d, DateTime now, CancellationToken ct) => - ExecuteEntryAsync(slot, ctx, d, ct); - private async Task ExecuteEntryAsync(BasketSlot slot, BasketContext ctx, BasketDecision d, CancellationToken ct) { if (slot.Position is not null || slot.State != BasketState.Idle) @@ -904,44 +962,8 @@ public sealed class BasketEngine : IEngine try { string basketId = string.Create(CultureInfo.InvariantCulture, $"B{DateTime.UtcNow:yyyyMMddHHmmss}-{slot.Name.Replace("/", string.Empty)}"); - EntryOutcome outcome = await _executor.OpenAsync(ctx, d, _decider.Preset, ct).ConfigureAwait(false); - if (outcome.Ok && outcome.Position is { } position) - { - slot.Position = position; - slot.PositionBasketId = basketId; - if (slot.LastFeatures is { } fx) - { - _learning.RememberEntry(basketId, fx); - } - - Transition(slot, BasketState.Open); - _knownPositions.UnionWith(position.A.AllPositionIds); - _knownPositions.UnionWith(position.B.AllPositionIds); - slot.Intent = "IN POSIZIONE — " + position.Describe(); - Log.Info(string.Create(CultureInfo.InvariantCulture, - $"[{slot.Name}] APERTO {basketId}: {position.Describe()}; slippage A {outcome.SlippagePipsA:+0.0;-0.0} pip, B {outcome.SlippagePipsB:+0.0;-0.0} pip, {outcome.LatencyMs:F0} ms")); - _ledger.Decision(_runId, ModeLabel, PresetLabel, _configHash, ctx with { Position = position }, d, "ingresso", basketId, - string.Create(CultureInfo.InvariantCulture, $"eseguito: A @ {position.A.EntryPrice} (slippage {outcome.SlippagePipsA:+0.0;-0.0} pip), B @ {position.B.EntryPrice} (slippage {outcome.SlippagePipsB:+0.0;-0.0} pip), latenza {outcome.LatencyMs:F0} ms")); - SaveState(); - } - else - { - Transition(slot, BasketState.Idle); - string evento = outcome.Unwound ? "leg_risk_unwind" : "rifiuto"; - slot.Intent = $"NON APERTO — {outcome.Error}"; - Log.Warn($"[{slot.Name}] {slot.Intent}"); - _ledger.Decision(_runId, ModeLabel, PresetLabel, _configHash, ctx, d, evento, basketId, outcome.Error); - if (outcome.Unwound || outcome.Error.Contains("NON richiusa", StringComparison.Ordinal)) - { - slot.DisabledUntilUtc = DateTime.UtcNow.AddHours(1); - slot.DisabledReason = "leg-risk: seconda gamba non eseguita"; - if (outcome.Error.Contains("NON richiusa", StringComparison.Ordinal)) - { - Transition(slot, BasketState.Error); - _entriesBlocked = $"gamba orfana su {slot.Name}: serve una riconciliazione"; - } - } - } + EntryOutcome outcome = await _executor.OpenAsync(ctx, d, _decider.Preset, basketId, ct).ConfigureAwait(false); + ApplyEntryOutcome(slot, ctx, d, outcome, basketId); } catch (Exception ex) when (ex is not OperationCanceledException) { @@ -955,8 +977,88 @@ public sealed class BasketEngine : IEngine } } - private Task HandleAddAsync(BasketSlot slot, BasketContext ctx, BasketDecision d, DateTime now, CancellationToken ct) => - ExecuteAddAsync(slot, ctx, d, ct); + /// What an entry attempt (first try, or the continuation after a pending leg) did to the slot. + private void ApplyEntryOutcome(BasketSlot slot, BasketContext ctx, BasketDecision? d, EntryOutcome outcome, string basketId) + { + if (outcome.Ok && outcome.Position is { } position) + { + slot.Position = position; + slot.Pending = null; + slot.PositionBasketId = basketId; + if (slot.LastFeatures is { } fx) + { + _learning.RememberEntry(basketId, fx); + } + + Transition(slot, BasketState.Open); + _knownPositions.UnionWith(position.A.AllPositionIds); + _knownPositions.UnionWith(position.B.AllPositionIds); + slot.Intent = "IN POSIZIONE — " + position.Describe(); + string fill = string.Create(CultureInfo.InvariantCulture, $"eseguito: A @ {position.A.EntryPrice} (slippage {outcome.SlippagePipsA:+0.0;-0.0} pip), B @ {position.B.EntryPrice} (slippage {outcome.SlippagePipsB:+0.0;-0.0} pip), latenza {outcome.LatencyMs:F0} ms"); + Log.Info($"[{slot.Name}] APERTO {basketId}: {position.Describe()}; {fill}"); + if (d is not null) + { + _ledger.Decision(_runId, ModeLabel, PresetLabel, _configHash, ctx with { Position = position }, d, "ingresso", basketId, fill); + } + else + { + _ledger.Note(_runId, "ingresso", basketId, $"{position.Describe()} — {fill}", w => w.WriteString("basket", slot.Name)); + } + + SaveState(); + return; + } + + if (outcome.IsPending && outcome.Pending is { } plan) + { + slot.Pending = plan; + slot.PositionBasketId = basketId; + Transition(slot, outcome.PendingLeg == PendingLeg.A ? BasketState.PendingA : BasketState.PendingB); + if (plan.LegA is { PositionId: > 0 } legA) + { + _knownPositions.Add(legA.PositionId); + } + + slot.Intent = $"IN ATTESA — {outcome.Error}"; + Log.Warn($"[{slot.Name}] {slot.Intent}"); + _ledger.Note(_runId, "pending", basketId, outcome.Error, w => + { + w.WriteString("basket", slot.Name); + w.WriteString("leg", outcome.PendingLeg.ToString()); + w.WriteString("client_ref", outcome.PendingOrder?.ClientRef ?? string.Empty); + w.WriteNumber("order_id", outcome.PendingOrder?.OrderId ?? 0); + }); + SaveState(); + return; + } + + slot.Pending = null; + Transition(slot, BasketState.Idle); + string evento = outcome.Unwound ? "leg_risk_unwind" : "rifiuto"; + slot.Intent = $"NON APERTO — {outcome.Error}"; + Log.Warn($"[{slot.Name}] {slot.Intent}"); + if (d is not null) + { + _ledger.Decision(_runId, ModeLabel, PresetLabel, _configHash, ctx, d, evento, basketId, outcome.Error); + } + else + { + _ledger.Note(_runId, evento, basketId, outcome.Error, w => w.WriteString("basket", slot.Name)); + } + + if (outcome.Unwound || outcome.Error.Contains("NON richiusa", StringComparison.Ordinal)) + { + slot.DisabledUntilUtc = DateTime.UtcNow.AddHours(1); + slot.DisabledReason = "leg-risk: seconda gamba non eseguita"; + if (outcome.Error.Contains("NON richiusa", StringComparison.Ordinal)) + { + Transition(slot, BasketState.Error); + _entriesBlocked = $"gamba orfana su {slot.Name}: serve una riconciliazione"; + } + } + + SaveState(); + } private async Task ExecuteAddAsync(BasketSlot slot, BasketContext ctx, BasketDecision d, CancellationToken ct) { @@ -996,11 +1098,6 @@ public sealed class BasketEngine : IEngine } } - private static readonly HashSet ProtectiveExits = ["stop_z", "stop_max_loss", "spread_anomaly", "time_stop", "rho_break", "kill_switch", "equity_stop"]; - - private Task HandleExitAsync(BasketSlot slot, BasketContext ctx, BasketDecision d, DateTime now, CancellationToken ct) => - ExecuteExitAsync(slot, ctx, d, d.Motivazione, d.ReasonCodes.FirstOrDefault() ?? "exit", ct); - private async Task ExecuteExitAsync(BasketSlot slot, BasketContext ctx, BasketDecision? d, string reason, string reasonCode, CancellationToken ct) { if (slot.Position is not { } p || slot.State is not (BasketState.Open or BasketState.Error)) @@ -1062,6 +1159,10 @@ public sealed class BasketEngine : IEngine _ledger.Decision(_runId, ModeLabel, PresetLabel, _configHash, ctx, d, "uscita", slot.PositionBasketId, string.Create(CultureInfo.InvariantCulture, $"chiuso: netto {x.RealizedPnlUsd:+0.00;-0.00} USD, {x.PipsTotal:+0.0;-0.0} pip, slippage {x.SlippagePipsA + x.SlippagePipsB:+0.0;-0.0} pip")); } + else + { + _ledger.Note(_runId, "uscita", slot.PositionBasketId, string.Create(CultureInfo.InvariantCulture, $"{reason}: netto {x.RealizedPnlUsd:+0.00;-0.00} USD, {x.PipsTotal:+0.0;-0.0} pip"), w => w.WriteString("basket", slot.Name)); + } slot.Position = null; slot.PositionBasketId = string.Empty; @@ -1071,7 +1172,12 @@ public sealed class BasketEngine : IEngine SaveState(); } - /// The learning stack sees every close: the shadow model learns, the bandit is rewarded. + /// + /// The learning stack sees every close: the shadow model learns, the bandit is + /// rewarded and its proposal is logged. The proposal is not applied (5.0, + /// D-30): a parameter changed by the bot on its own is exactly what the rules forbid; + /// the path to a different preset is a proposal in knowledge/proposals.csv. + /// private void ObserveOutcome(BasketSlot slot, int label) { double sigma = slot.Vol.Count > 0 ? slot.Vol.Forecast() : double.NaN; @@ -1087,14 +1193,7 @@ public sealed class BasketEngine : IEngine int context = ThompsonBandit.VolatilityContext(sigma, _volHistory); _learning.Observe(slot.PositionBasketId, label, context, _decider.Preset.Name); (PresetName proposed, string text) = _learning.Propose(context); - Log.Info($"bandit: {text}"); - if (proposed != _decider.Preset.Name && !_mode.IsLive()) - { - // Paper and demo apply the proposal automatically (§8.6); live never does. - _decider.SetPreset(proposed); - _ledger.Correction(_runId, string.Empty, $"bandit: preset → {proposed} ({text})"); - Log.Info($"bandit: preset applicato automaticamente in {_mode}: {proposed.ToString().ToUpperInvariant()}"); - } + Log.Info($"bandit (solo proposta, non applicata): {text}{(proposed != _decider.Preset.Name ? $" — il preset resta {PresetLabel}" : string.Empty)}"); } private void Transition(BasketSlot slot, BasketState to) @@ -1107,179 +1206,6 @@ public sealed class BasketEngine : IEngine slot.State = to; } - // ----------------------------------------------------------------------- - // Account, reconciliation, safety - // ----------------------------------------------------------------------- - - private async Task RefreshAccountAsync(CancellationToken ct) - { - try - { - _account = await _broker.GetAccountAsync(ct).ConfigureAwait(false); - if (_account.Equity > _peakEquity) - { - _peakEquity = _account.Equity; - } - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - Log.Warn($"conto non letto: {ex.Message}"); - } - } - - /// Positions on the venue against the local state: a leg that vanished closes its sibling; a stranger is reported and left alone. - private async Task ReconcileAsync(CancellationToken ct) - { - IReadOnlyList positions; - try - { - positions = await _broker.GetPositionsAsync(ct).ConfigureAwait(false); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - Log.Warn($"riconciliazione non riuscita: {ex.Message}"); - return; - } - - HashSet onVenue = [.. positions.Select(static p => p.PositionId)]; - - foreach (BasketSlot slot in _slots) - { - if (slot.Position is not { } p || slot.Busy) - { - continue; - } - - bool aAlive = p.A.AllPositionIds.Any(onVenue.Contains); - bool bAlive = p.B.AllPositionIds.Any(onVenue.Contains); - if (aAlive && bAlive) - { - // Refresh accrued overnight from the venue's own fee field when it has one. - double fees = 0; - foreach (BrokerPosition bp in positions) - { - if (p.A.AllPositionIds.Contains(bp.PositionId) || p.B.AllPositionIds.Contains(bp.PositionId)) - { - fees += bp.Fees; - } - } - - if (fees > 0) - { - p.AccruedFeesUsd = Math.Max(0, fees - p.A.EntryFeesUsd - p.B.EntryFeesUsd); - } - - continue; - } - - if (!aAlive && !bAlive) - { - Log.Warn($"[{slot.Name}] entrambe le gambe sono sparite dal conto (chiuse dal broker o a mano): registro la chiusura"); - await RecordExternalCloseAsync(slot, p, "chiuso dal broker", ct).ConfigureAwait(false); - continue; - } - - string alive = aAlive ? p.A.Symbol : p.B.Symbol; - string gone = aAlive ? p.B.Symbol : p.A.Symbol; - Log.Warn($"[{slot.Name}] la gamba {gone} non è più sul conto (stop nativo o chiusura manuale): chiudo subito {alive} (leg_risk)"); - if (slot.A.HasQuote && slot.B.HasQuote) - { - BasketContext ctx = await BuildContextAsync(slot, DateTime.UtcNow, false, ct).ConfigureAwait(false); - await ExecuteExitAsync(slot, ctx, null, $"gamba {gone} chiusa dal broker", "leg_closed_by_broker", ct).ConfigureAwait(false); - } - } - - // Positions we do not know: say so once, touch nothing. - foreach (BrokerPosition bp in positions) - { - if (!_knownPositions.Contains(bp.PositionId) && _foreignPositions.Add(bp.PositionId)) - { - string symbol = _seriesById.TryGetValue(bp.InstrumentId, out SymbolSeries? s) ? s.Symbol : bp.InstrumentId.ToString(CultureInfo.InvariantCulture); - Log.Warn(string.Create(CultureInfo.InvariantCulture, $"posizione {bp.PositionId} su {symbol} ({(bp.IsBuy ? "long" : "short")} {bp.Units:0.##} @ {bp.OpenRate}) non appartiene a nessun basket: la lascio com'è")); - } - } - } - - private async Task RecordExternalCloseAsync(BasketSlot slot, BasketPosition p, string reason, CancellationToken ct) - { - double pnl = 0; - try - { - IReadOnlyList closed = await _broker.GetClosedTradesAsync(p.OpenedUtc.AddDays(-1), ct).ConfigureAwait(false); - HashSet ids = [.. p.A.AllPositionIds.Concat(p.B.AllPositionIds)]; - pnl = closed.Where(c => ids.Contains(c.PositionId)).Sum(static c => c.NetProfit); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - Log.Warn($"[{slot.Name}] storico chiusure non letto: {ex.Message}"); - } - - BasketOutcomeRow row = new(slot.PositionBasketId, _runId, slot.Name, ModeLabel, PresetLabel, p.OpenedUtc, DateTime.UtcNow, p.BuyCross, p.EntryZ, slot.LastEvaluation.Z, - pnl, pnl, double.NaN, p.EntryCostPips, double.NaN, double.NaN, p.Adds, p.BarsHeld, "closed_by_broker", p.EquityAtEntry, slot.PMl, reason); - _ledger.Basket(row); - _todayRealized += pnl; - foreach (long id in p.A.AllPositionIds.Concat(p.B.AllPositionIds)) - { - _knownPositions.Remove(id); - } - - slot.Position = null; - slot.PositionBasketId = string.Empty; - slot.State = BasketState.Idle; - slot.Intent = $"chiuso dal broker: {pnl:+0.00;-0.00} USD"; - SaveState(); - } - - private async Task CheckEquityStopAsync(CancellationToken ct) - { - if (_equityStopped || _peakEquity <= 0) - { - return; - } - - double dd = (_peakEquity - _account.Equity) / _peakEquity; - if (dd >= _strategy.EquityStopPct / 100.0) - { - _equityStopped = true; - _haltReason = string.Create(CultureInfo.InvariantCulture, $"equity {_account.Equity:F2} a {dd:P2} dal picco {_peakEquity:F2} (soglia {_strategy.EquityStopPct:0.#} %)"); - Log.Error($"EQUITY STOP: {_haltReason}. Chiudo tutto e mi blocco: serve un reset manuale con motivazione.", null); - _ledger.Correction(_runId, string.Empty, $"equity stop: {_haltReason}"); - await CloseAllAsync("equity stop", ct).ConfigureAwait(false); - SaveState(); - } - } - - private async Task CheckStopFileAsync(CancellationToken ct) - { - bool present = File.Exists(_stopFile); - if (present && !_killSwitched) - { - Log.Warn("file STOP trovato: KILL-SWITCH"); - await KillAsync("file STOP", ct).ConfigureAwait(false); - } - } - - private async Task KillAsync(string reason, CancellationToken ct) - { - _killSwitched = true; - _haltReason = $"kill-switch ({reason})"; - _ledger.Correction(_runId, string.Empty, $"kill-switch: {reason}"); - await CloseAllAsync(_haltReason, ct).ConfigureAwait(false); - SaveState(); - } - - public async Task CloseAllAsync(string reason, CancellationToken ct) - { - foreach (BasketSlot slot in _slots) - { - if (slot.Position is not null && slot.A.HasQuote && slot.B.HasQuote) - { - BasketContext ctx = await BuildContextAsync(slot, DateTime.UtcNow, false, ct).ConfigureAwait(false); - await ExecuteExitAsync(slot, ctx, null, reason, reason.Contains("equity", StringComparison.OrdinalIgnoreCase) ? "equity_stop" : "kill_switch", ct).ConfigureAwait(false); - } - } - } - private void RollSessionIfNeeded(DateTime now) { DateOnly today = DateOnly.FromDateTime(now); @@ -1289,17 +1215,17 @@ public sealed class BasketEngine : IEngine } _sessionDate = today; - _dayStartEquity = _account.Equity; + _dayStartEquity = _equity.NetEquity(_account.Equity); _todayRealized = 0; - Log.Info(string.Create(CultureInfo.InvariantCulture, $"── nuova giornata {today:yyyy-MM-dd}: equity di partenza {_dayStartEquity:F2} ──")); + Log.Info(string.Create(CultureInfo.InvariantCulture, $"── nuova giornata {today:yyyy-MM-dd}: equity di partenza {_dayStartEquity:F2} (al netto dei movimenti di cassa) ──")); SaveState(); } private void LogStatus() { - double dd = _peakEquity > 0 ? (_peakEquity - _account.Equity) / _peakEquity : 0; + double dd = _equity.Drawdown(_account.Equity); string line = string.Create(CultureInfo.InvariantCulture, - $"[stato] {_mode} · equity {_account.Equity:F2} (saldo {_account.Balance:F2}, aperto {_account.UnrealizedPnl:+0.00;-0.00}, oggi {_todayRealized:+0.00;-0.00}) · DD {dd:P2} dal picco {_peakEquity:F2} · basket aperti {_slots.Count(static s => s.Position is not null)}/{_decider.Preset.MaxBaskets} · API {_apiState} {_apiLatencyMs:F0} ms"); + $"[stato] {_mode} · equity {_account.Equity:F2} (saldo {_account.Balance:F2}, aperto {_account.UnrealizedPnl:+0.00;-0.00}, oggi {_todayRealized:+0.00;-0.00}) · DD {dd:P2} dal picco {_equity.PeakEquity:F2} · basket aperti {_slots.Count(static s => s.Position is not null)}/{_decider.Preset.MaxBaskets}, in attesa {_slots.Count(static s => s.State.IsPending())}, ordini pendenti {_tracker.PendingCount}, orfane {_orphanCount}, esterne {_foreignCount} · margine {_account.UsedMargin:F0}/{_account.Available:F0} · API {_apiState} {_apiLatencyMs:F0} ms"); if (_haltReason is not null) { line += $" · BLOCCO: {_haltReason}"; @@ -1310,6 +1236,11 @@ public sealed class BasketEngine : IEngine line += $" · entrate bloccate: {_entriesBlocked}"; } + if (_unreconciledSince is not null) + { + line += $" · NON RICONCILIATO: {_unreconciledReason}"; + } + Log.Info(line); foreach (BasketSlot s in _slots) { @@ -1319,419 +1250,6 @@ public sealed class BasketEngine : IEngine } } - // ----------------------------------------------------------------------- - // Commands (from the window or the console, executed on the engine's thread) - // ----------------------------------------------------------------------- - - public Task ExecuteAsync(EngineCommand command, CancellationToken ct) - { - ArgumentNullException.ThrowIfNull(command); - TaskCompletionSource tcs = new(TaskCreationOptions.RunContinuationsAsynchronously); - _commands.Enqueue((command, tcs)); - return tcs.Task.WaitAsync(TimeSpan.FromSeconds(60), ct); - } - - private async Task DrainCommandsAsync(CancellationToken ct) - { - while (_commands.TryDequeue(out (EngineCommand Command, TaskCompletionSource Done) item)) - { - CommandResult result; - try - { - result = await RunCommandAsync(item.Command, ct).ConfigureAwait(false); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - Log.Error($"comando {item.Command.Kind} fallito", ex); - result = new CommandResult(false, ex.Message); - } - - item.Done.TrySetResult(result); - } - } - - private async Task RunCommandAsync(EngineCommand c, CancellationToken ct) - { - switch (c.Kind) - { - case EngineCommandKind.Close: - { - BasketSlot? slot = _slots.FirstOrDefault(s => s.Name.Equals(c.Argument, StringComparison.OrdinalIgnoreCase)); - if (slot is null) - { - return new CommandResult(false, $"basket {c.Argument} sconosciuto"); - } - - if (slot.Position is null) - { - return new CommandResult(false, $"{slot.Name} non ha un basket aperto"); - } - - BasketContext ctx = await BuildContextAsync(slot, DateTime.UtcNow, false, ct).ConfigureAwait(false); - await ExecuteExitAsync(slot, ctx, null, c.Reason.Length > 0 ? c.Reason : "chiusura manuale", "manual", ct).ConfigureAwait(false); - return new CommandResult(slot.Position is null, slot.Position is null ? $"{slot.Name} chiuso" : $"chiusura di {slot.Name} non completata"); - } - - case EngineCommandKind.KillSwitch: - await KillAsync(c.Reason.Length > 0 ? c.Reason : "comando", ct).ConfigureAwait(false); - return new CommandResult(true, "kill-switch eseguito: tutto chiuso, nuove entrate bloccate"); - - case EngineCommandKind.SetPreset: - if (!BasketPresets.TryParse(c.Argument, out PresetName preset)) - { - return new CommandResult(false, $"preset {c.Argument} sconosciuto"); - } - - _decider.SetPreset(preset); - Log.Info($"preset cambiato in {preset.ToString().ToUpperInvariant()} ({c.Reason}); i basket aperti non vengono toccati"); - _ledger.Correction(_runId, string.Empty, $"preset → {preset} ({c.Reason})"); - return new CommandResult(true, $"preset {preset.ToString().ToUpperInvariant()} attivo"); - - case EngineCommandKind.ResetEquityStop: - if (!_equityStopped && !_killSwitched) - { - return new CommandResult(false, "nessun blocco attivo"); - } - - if (c.Reason.Trim().Length < 10) - { - return new CommandResult(false, "serve una motivazione scritta (almeno dieci caratteri)"); - } - - if (File.Exists(_stopFile)) - { - return new CommandResult(false, $"rimuovi prima il file {_stopFile}"); - } - - _equityStopped = false; - _killSwitched = false; - _haltReason = null; - _peakEquity = _account.Equity; - _ledger.Correction(_runId, string.Empty, $"reset del blocco: {c.Reason}"); - Log.Warn($"blocco rimosso dall'operatore: {c.Reason}. Nuovo picco di equity {_peakEquity:F2}"); - SaveState(); - return new CommandResult(true, "blocco rimosso; il picco di equity riparte da adesso"); - - default: - return new CommandResult(false, $"comando {c.Kind} non supportato"); - } - } - - // ----------------------------------------------------------------------- - // State on disk - // ----------------------------------------------------------------------- - - private void SaveState() - { - try - { - using MemoryStream ms = new(); - using (Utf8JsonWriter w = new(ms, new JsonWriterOptions { Indented = true })) - { - w.WriteStartObject(); - w.WriteString("savedUtc", DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture)); - w.WriteString("runId", _runId); - w.WriteString("mode", ModeLabel); - w.WriteNumber("peakEquity", _peakEquity); - w.WriteString("sessionDate", _sessionDate.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)); - w.WriteNumber("dayStartEquity", _dayStartEquity); - w.WriteNumber("todayRealized", _todayRealized); - w.WriteBoolean("killSwitched", _killSwitched); - w.WriteBoolean("equityStopped", _equityStopped); - w.WriteString("haltReason", _haltReason ?? string.Empty); - w.WriteStartArray("baskets"); - foreach (BasketSlot s in _slots) - { - if (s.Position is not { } p) - { - continue; - } - - w.WriteStartObject(); - w.WriteString("name", s.Name); - w.WriteString("basketId", s.PositionBasketId); - w.WriteBoolean("buyCross", p.BuyCross); - w.WriteString("openedUtc", p.OpenedUtc.ToString("O", CultureInfo.InvariantCulture)); - w.WriteNumber("entryZ", p.EntryZ); - w.WriteNumber("lastAddZ", p.LastAddZ); - w.WriteNumber("adds", p.Adds); - w.WriteNumber("barsHeld", p.BarsHeld); - w.WriteNumber("entryCostPips", double.IsFinite(p.EntryCostPips) ? p.EntryCostPips : 0); - w.WriteNumber("tpPips", p.TpPips); - w.WriteNumber("maxLossUsd", p.MaxLossUsd); - w.WriteNumber("equityAtEntry", p.EquityAtEntry); - w.WriteNumber("accruedFeesUsd", p.AccruedFeesUsd); - w.WriteString("entryMotivazione", p.EntryMotivazione); - WriteLeg(w, "a", p.A); - WriteLeg(w, "b", p.B); - w.WriteEndObject(); - } - - w.WriteEndArray(); - w.WriteEndObject(); - } - - File.WriteAllBytes(_statePath + ".tmp", ms.ToArray()); - File.Move(_statePath + ".tmp", _statePath, overwrite: true); - } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) - { - Log.Warn($"stato non salvato: {ex.Message}"); - } - - static void WriteLeg(Utf8JsonWriter w, string name, BasketLeg leg) - { - w.WriteStartObject(name); - w.WriteString("symbol", leg.Symbol); - w.WriteNumber("instrumentId", leg.InstrumentId); - w.WriteBoolean("isBuy", leg.IsBuy); - w.WriteNumber("units", leg.Units); - w.WriteNumber("entryPrice", leg.EntryPrice); - w.WriteNumber("positionId", leg.PositionId); - w.WriteString("clientRef", leg.ClientRef); - w.WriteString("openedUtc", leg.OpenedUtc.ToString("O", CultureInfo.InvariantCulture)); - w.WriteNumber("entryFeesUsd", leg.EntryFeesUsd); - w.WriteNumber("stopLossRate", leg.StopLossRate); - w.WriteStartArray("adds"); - foreach ((long id, double units, double price, string clientRef) in leg.Adds) - { - w.WriteStartObject(); - w.WriteNumber("positionId", id); - w.WriteNumber("units", units); - w.WriteNumber("price", price); - w.WriteString("clientRef", clientRef); - w.WriteEndObject(); - } - - w.WriteEndArray(); - w.WriteEndObject(); - } - } - - private void LoadState() - { - if (!File.Exists(_statePath)) - { - return; - } - - try - { - using JsonDocument doc = JsonDocument.Parse(File.ReadAllBytes(_statePath)); - JsonElement root = doc.RootElement; - string savedMode = root.TryGetProperty("mode", out JsonElement m) ? m.GetString() ?? string.Empty : string.Empty; - if (!savedMode.Equals(ModeLabel, StringComparison.OrdinalIgnoreCase) && savedMode.Length > 0) - { - Log.Warn($"stato salvato in modalità {savedMode}, ora {ModeLabel}: le posizioni salvate non vengono riprese (riconciliazione dal conto)"); - return; - } - - _peakEquity = root.TryGetProperty("peakEquity", out JsonElement pe) ? pe.GetDouble() : 0; - _killSwitched = root.TryGetProperty("killSwitched", out JsonElement ks) && ks.GetBoolean(); - _equityStopped = root.TryGetProperty("equityStopped", out JsonElement es) && es.GetBoolean(); - _haltReason = root.TryGetProperty("haltReason", out JsonElement hr) && hr.GetString() is { Length: > 0 } h ? h : null; - if (root.TryGetProperty("sessionDate", out JsonElement sd) && DateOnly.TryParse(sd.GetString(), CultureInfo.InvariantCulture, out DateOnly day) && day == DateOnly.FromDateTime(DateTime.UtcNow)) - { - _dayStartEquity = root.TryGetProperty("dayStartEquity", out JsonElement dse) ? dse.GetDouble() : 0; - _todayRealized = root.TryGetProperty("todayRealized", out JsonElement tr) ? tr.GetDouble() : 0; - } - - if (_equityStopped) - { - Log.Warn($"equity stop ancora attivo dal run precedente: {_haltReason}"); - } - - if (root.TryGetProperty("baskets", out JsonElement arr)) - { - foreach (JsonElement e in arr.EnumerateArray()) - { - string name = e.GetProperty("name").GetString() ?? string.Empty; - BasketSlot? slot = _slots.FirstOrDefault(s => s.Name == name); - if (slot is null) - { - continue; - } - - BasketPosition p = new() - { - BasketId = e.GetProperty("basketId").GetString() ?? string.Empty, - Name = name, - BuyCross = e.GetProperty("buyCross").GetBoolean(), - A = ReadLeg(e.GetProperty("a")), - B = ReadLeg(e.GetProperty("b")), - OpenedUtc = DateTime.Parse(e.GetProperty("openedUtc").GetString()!, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal), - EntryZ = e.GetProperty("entryZ").GetDouble(), - LastAddZ = e.GetProperty("lastAddZ").GetDouble(), - Adds = e.GetProperty("adds").GetInt32(), - BarsHeld = e.GetProperty("barsHeld").GetInt32(), - EntryCostPips = e.GetProperty("entryCostPips").GetDouble(), - TpPips = e.GetProperty("tpPips").GetDouble(), - MaxLossUsd = e.GetProperty("maxLossUsd").GetDouble(), - EquityAtEntry = e.GetProperty("equityAtEntry").GetDouble(), - AccruedFeesUsd = e.GetProperty("accruedFeesUsd").GetDouble(), - EntryMotivazione = e.GetProperty("entryMotivazione").GetString() ?? string.Empty, - }; - slot.Position = p; - slot.PositionBasketId = p.BasketId; - slot.State = BasketState.Open; - slot.Intent = "ripreso dallo stato salvato: " + p.Describe(); - _knownPositions.UnionWith(p.A.AllPositionIds); - _knownPositions.UnionWith(p.B.AllPositionIds); - Log.Info($"[{name}] basket ripreso dallo stato salvato: {p.Describe()}"); - } - } - } - catch (Exception ex) when (ex is IOException or JsonException or KeyNotFoundException or InvalidOperationException or FormatException) - { - Log.Warn($"stato salvato non leggibile ({ex.Message}): riparto dalla riconciliazione con il conto"); - } - - static BasketLeg ReadLeg(JsonElement e) - { - BasketLeg leg = new() - { - Symbol = e.GetProperty("symbol").GetString() ?? string.Empty, - InstrumentId = e.GetProperty("instrumentId").GetInt64(), - IsBuy = e.GetProperty("isBuy").GetBoolean(), - Units = e.GetProperty("units").GetDouble(), - EntryPrice = e.GetProperty("entryPrice").GetDouble(), - PositionId = e.GetProperty("positionId").GetInt64(), - ClientRef = e.GetProperty("clientRef").GetString() ?? string.Empty, - OpenedUtc = DateTime.Parse(e.GetProperty("openedUtc").GetString()!, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal), - EntryFeesUsd = e.GetProperty("entryFeesUsd").GetDouble(), - StopLossRate = e.GetProperty("stopLossRate").GetDouble(), - }; - foreach (JsonElement a in e.GetProperty("adds").EnumerateArray()) - { - leg.Adds.Add((a.GetProperty("positionId").GetInt64(), a.GetProperty("units").GetDouble(), a.GetProperty("price").GetDouble(), a.GetProperty("clientRef").GetString() ?? string.Empty)); - } - - return leg; - } - } - - // ----------------------------------------------------------------------- - // Snapshot - // ----------------------------------------------------------------------- - - public BotSnapshot Snapshot(BotState state, string? error, DateTime? startedUtc, EventRow[] events) - { - ArgumentNullException.ThrowIfNull(events); - double equity = _account.Equity; - double dd = _peakEquity > 0 ? (_peakEquity - equity) / _peakEquity : 0; - double openPnl = 0; - List rows = new(_slots.Count); - DateTime now = DateTime.UtcNow; - - foreach (BasketSlot s in _slots) - { - BasketEvaluation e = s.LastEvaluation; - double pnl = s.Position is not null && double.IsFinite(e.PnlOpenUsd) ? e.PnlOpenUsd : 0; - openPnl += pnl; - int legs = s.Position is { } p ? 2 + (2 * p.Adds) : 0; - string nextEvent = s.Features.NextEventLabel; - rows.Add(new BasketRow(s.Name, s.Definition.A, s.Definition.B, s.Cross.Symbol, s.State.ToString(), legs, pnl, - equity > 0 ? pnl / equity : 0, double.IsFinite(e.PipsOpen) ? e.PipsOpen : 0, s.Position?.TpPips ?? _decider.Preset.TpPips, - e.RhoW, e.RhoShort, e.Z, e.CostPips, s.PMl, _learning.Active, nextEvent, s.Enabled && s.DisabledUntilUtc <= now, - s.DisabledUntilUtc > now ? $"in pausa fino alle {s.DisabledUntilUtc:HH:mm} UTC: {s.DisabledReason}" : s.DisabledReason, - s.Intent, s.Position?.EntryZ ?? 0, s.Position?.BarsHeld ?? 0, s.Position?.Adds ?? 0, s.Position is not null, e.HalfLife)); - } - - List quotes = new(_series.Count); - foreach (SymbolSeries s in _series.Values) - { - quotes.Add(new QuoteRow(s.Symbol, s.HasQuote ? s.Quote.Bid : 0, s.HasQuote ? s.Quote.Ask : 0, s.SpreadPips, s.HasQuote ? s.Quote.TimeUtc : default, - s.QuoteSeenUtc == default ? -1 : (now - s.QuoteSeenUtc).TotalSeconds)); - } - - return new BotSnapshot - { - State = state, - Error = error, - StartedAtUtc = startedUtc, - Uptime = startedUtc is { } st ? now - st : TimeSpan.Zero, - Mode = _mode.Badge(), - EnvironmentKind = _mode.Kind(), - ExecutionMode = _mode.ToString(), - Endpoint = _feed.Endpoint, - Preset = PresetLabel, - StrategyVersion = $"v{Assembly.GetExecutingAssembly().GetName().Version?.ToString(3) ?? "?"} · strategia {_configHash} · run {_runId}", - ApiState = _apiState, - ApiLatencyMs = _apiLatencyMs, - ClockSkewSeconds = _feed.ClockSkew.TotalSeconds, - Equity = equity, - Balance = _account.Balance, - AvailableBalance = _account.Available, - PeakEquity = _peakEquity, - DrawdownPct = dd, - EquityStopPct = _strategy.EquityStopPct / 100.0, - DailyLossPct = _strategy.DailyLossPct / 100.0, - TodayPnl = _todayRealized, - TodayPnlPct = _dayStartEquity > 0 ? _todayRealized / _dayStartEquity : 0, - OpenPnl = openPnl, - OpenPnlPct = equity > 0 ? openPnl / equity : 0, - OpenBaskets = _slots.Count(static s => s.Position is not null), - MaxBaskets = _decider.Preset.MaxBaskets, - Halted = _killSwitched || _equityStopped, - HaltReason = _haltReason, - EquityStopped = _equityStopped, - KillSwitched = _killSwitched, - EntriesBlockedReason = _entriesBlocked, - Counters = string.Create(CultureInfo.InvariantCulture, $"quote/min {_feed.QuotaUsed(EtoroQuota.MarketData)}/110 · ordini/min {_feed.QuotaUsed(EtoroQuota.Trading)}/18 · ultima quotazione {(_lastQuoteUtc == default ? "—" : (now - _lastQuoteUtc).TotalSeconds.ToString("0") + " s fa")}"), - Events = events, - Baskets = rows, - Quotes = quotes, - Context = ContextWithLearning(now), - }; - } - - private ContextRow ContextWithLearning(DateTime now) - { - ContextRow row = _context.Row(now); - BasketSlot? first = _slots.FirstOrDefault(static s => s.Enabled && s.Vol.Count > 0); - string vol = first is null ? "in attesa di barre" : $"{first.Name}: {first.Vol.Describe()}"; - return row with { VolForecast = vol, MlState = _learning.Describe(), BanditProposal = _learning.BanditText }; - } - - /// The picture while the engine is not running. - public static BotSnapshot IdleSnapshot(BotConfig config, BotState state, string? error, EventRow[] events) - { - ArgumentNullException.ThrowIfNull(config); - ExecutionMode mode = config.Run.Mode; - List rows = []; - try - { - BasketStrategyConfig strategy = File.Exists(config.Run.StrategyPath) - ? BasketStrategyConfig.Load(config.Run.StrategyPath, out _) - : BasketStrategyConfig.ParseText(BasketStrategyConfig.DefaultJson, out _); - foreach (BasketDefinition d in strategy.Baskets) - { - string cross = SyntheticCross.TryDerive(d.A, d.B, out SyntheticCross? c) ? c!.Symbol : "?"; - rows.Add(new BasketRow(d.Name, d.A, d.B, cross, "fermo", 0, 0, 0, 0, strategy.Effective().TpPips, double.NaN, double.NaN, double.NaN, double.NaN, double.NaN, false, "—", - d.Enabled, d.Enabled ? string.Empty : "disattivato in strategy.json", "il bot è fermo", 0, 0, 0, false, double.NaN)); - } - } - catch (Exception ex) when (ex is IOException or InvalidOperationException or JsonException) - { - Log.Warn($"strategy.json non leggibile: {ex.Message}"); - } - - return new BotSnapshot - { - State = state, - Error = error, - Mode = mode.Badge(), - EnvironmentKind = mode.Kind(), - ExecutionMode = mode.ToString(), - Endpoint = config.Etoro.BaseUrl, - Preset = "—", - ApiState = "fermo", - Events = events, - Baskets = rows, - Context = new ContextRow([], [], "—", "—", "—", "—", "—"), - }; - } - public async ValueTask DisposeAsync() { SaveState(); diff --git a/Encelado/src/Encelado.Bot/Baskets/HeadlessRunner.cs b/Encelado/src/Encelado.Bot/Baskets/HeadlessRunner.cs index 9dc975d..4a3db3e 100644 --- a/Encelado/src/Encelado.Bot/Baskets/HeadlessRunner.cs +++ b/Encelado/src/Encelado.Bot/Baskets/HeadlessRunner.cs @@ -13,8 +13,9 @@ namespace Encelado.Bot.Baskets; /// or a long unattended test. /// /// Commands: status, close <basket>, kill, preset <nome>, -/// reset <motivazione>, stop. Arguments: --headless, -/// --confirm-live "CONFERMO LIVE", --minutes N (stop by itself after N minutes). +/// reset <motivazione>, bonifica, stop. Arguments: --headless, +/// --confirm-live "CONFERMO LIVE", --minutes N (stop by itself after N minutes), +/// --bonifica (start without closing orphans on its own; list them and ask, one by one). /// /// public static class HeadlessRunner @@ -81,7 +82,8 @@ public static class HeadlessRunner } } - await using BotSupervisor supervisor = new(config) { StartConfirmed = true }; + bool bonifica = args.Any(static a => a.Equals("--bonifica", StringComparison.OrdinalIgnoreCase)); + await using BotSupervisor supervisor = new(config, (c, confirmed) => new BasketEngine(c, confirmed) { OrphanPolicy = bonifica ? OrphanPolicy.Report : OrphanPolicy.Close }) { StartConfirmed = true }; using CancellationTokenSource stopping = new(); Console.CancelKeyPress += (_, e) => { @@ -97,13 +99,18 @@ public static class HeadlessRunner return 5; } - Log.Info($"bot avviato in {mode}. Comandi: status, close , kill, preset , reset , stop"); + Log.Info($"bot avviato in {mode}. Comandi: status, close , kill, preset , reset , bonifica, stop"); if (minutes > 0) { Log.Info($"arresto automatico fra {minutes} minuti"); stopping.CancelAfter(TimeSpan.FromMinutes(minutes)); } + if (bonifica) + { + await BonificaAsync(supervisor, stopping.Token).ConfigureAwait(false); + } + Task input = Task.Run(() => ReadCommandsAsync(supervisor, stopping), stopping.Token); DateTime lastStatus = DateTime.MinValue; @@ -201,8 +208,11 @@ public static class HeadlessRunner case "reset": result = await supervisor.ExecuteAsync(new EngineCommand(EngineCommandKind.ResetEquityStop, string.Empty, arg), CancellationToken.None).ConfigureAwait(false); break; + case "bonifica": + await BonificaAsync(supervisor, stopping.Token).ConfigureAwait(false); + continue; default: - Console.WriteLine("comandi: status, close , kill, preset , reset , stop"); + Console.WriteLine("comandi: status, close , kill, preset , reset , bonifica, stop"); continue; } @@ -210,13 +220,72 @@ public static class HeadlessRunner } } + /// + /// The one-off clean-up (§5.5 of the 5.0 plan): lists the orphan legs of the bot and + /// the foreign positions, asks for every orphan whether to close it, closes it and + /// records it; foreign positions are never touched here. Ends by handing orphan + /// handling back to the engine. + /// + private static async Task BonificaAsync(BotSupervisor supervisor, CancellationToken ct) + { + CommandResult list = await supervisor.ExecuteAsync(new EngineCommand(EngineCommandKind.Bonifica, "list"), ct).ConfigureAwait(false); + Console.WriteLine(); + Console.WriteLine($"── bonifica: {list.Message}"); + if (list.Payload is not IReadOnlyList rows || rows.Count == 0) + { + Console.WriteLine(" niente da bonificare"); + await supervisor.ExecuteAsync(new EngineCommand(EngineCommandKind.Bonifica, "done"), ct).ConfigureAwait(false); + return; + } + + foreach (PositionInfo r in rows) + { + Console.WriteLine(string.Create(CultureInfo.InvariantCulture, + $" {r.Origin,-11} {r.PositionId} {r.Symbol} {(r.IsBuy ? "long" : "short"),-5} {r.Units,14:0.##} aperta {r.OpenedUtc:yyyy-MM-dd HH:mm:ss} UTC P&L {r.UnrealizedPnl,9:+0.00;-0.00} {r.Reason}")); + } + + foreach (PositionInfo r in rows.Where(static r => r.Origin == "orfana-bot")) + { + Console.Write(string.Create(CultureInfo.InvariantCulture, $" chiudere la posizione {r.PositionId} ({r.Symbol} {(r.IsBuy ? "long" : "short")} {r.Units:0.##}, P&L {r.UnrealizedPnl:+0.00;-0.00})? [s/N] ")); + string? answer; + try + { + answer = await Console.In.ReadLineAsync(ct).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + return; + } + + if (answer is null) + { + Console.WriteLine("(nessuna console: la bonifica si ferma qui; le orfane restano da chiudere a mano o con il prossimo avvio senza --bonifica)"); + return; + } + + if (answer.Trim().ToLowerInvariant() is "s" or "si" or "sì" or "y" or "yes") + { + CommandResult r2 = await supervisor.ExecuteAsync(new EngineCommand(EngineCommandKind.Bonifica, $"close:{r.PositionId.ToString(CultureInfo.InvariantCulture)}"), ct).ConfigureAwait(false); + Console.WriteLine((r2.Ok ? " ok: " : " NO: ") + r2.Message); + } + else + { + Console.WriteLine(" lasciata aperta"); + } + } + + CommandResult done = await supervisor.ExecuteAsync(new EngineCommand(EngineCommandKind.Bonifica, "done"), ct).ConfigureAwait(false); + Console.WriteLine($"── {done.Message}: rapporto in reports/bonifica_{DateTime.UtcNow:yyyyMMdd}.csv"); + } + private static void PrintStatus(BotSnapshot s) { Console.WriteLine(); Console.WriteLine(string.Create(CultureInfo.InvariantCulture, $"── {DateTime.UtcNow:HH:mm:ss} UTC · {s.Mode} · preset {s.Preset} · API {s.ApiState} {(double.IsFinite(s.ApiLatencyMs) ? s.ApiLatencyMs.ToString("0") + " ms" : "—")} · skew {s.ClockSkewSeconds:+0.0;-0.0} s")); Console.WriteLine(string.Create(CultureInfo.InvariantCulture, - $" BALANCE {s.Balance:N2} EQUITY {s.Equity:N2} TOTAL {s.OpenPnl:+0.00;-0.00} ({s.OpenPnlPct:P2}) TODAY {s.TodayPnl:+0.00;-0.00} ({s.TodayPnlPct:P2}) DD {s.DrawdownPct:P2} basket {s.OpenBaskets}/{s.MaxBaskets}") + + $" BALANCE {s.Balance:N2} EQUITY {s.Equity:N2} APERTO {s.AccountOpenPnl:+0.00;-0.00} (basket {s.OpenPnl:+0.00;-0.00}, {s.OpenPnlPct:P2}) TODAY {s.TodayPnl:+0.00;-0.00} ({s.TodayPnlPct:P2}) DD {s.DrawdownPct:P2} basket {s.OpenBaskets}/{s.MaxBaskets} attesa {s.PendingBaskets} orfane {s.OrphanLegs} esterne {s.ForeignPositions} margine {s.UsedMargin:N0}/{s.AvailableBalance:N0}") + + (s.Unreconciled ? $" NON RICONCILIATO: {s.UnreconciledReason}" : string.Empty) + (s.Halted ? $" BLOCCO: {s.HaltReason}" : string.Empty) + (s.EntriesBlockedReason is { Length: > 0 } blocked ? $" entrate bloccate: {blocked}" : string.Empty)); Console.WriteLine($" {"Coppie",-14} {"(n)",3} {"$",9} {"%",7} {"Pips",6} {"TP",3} {"ρ",6} {"z",6} {"HL",4} {"Costo",5} {"p_ML",6} Stato"); diff --git a/Encelado/src/Encelado.Bot/Baskets/Ledger.cs b/Encelado/src/Encelado.Bot/Baskets/Ledger.cs index bf9d07e..f4f15cf 100644 --- a/Encelado/src/Encelado.Bot/Baskets/Ledger.cs +++ b/Encelado/src/Encelado.Bot/Baskets/Ledger.cs @@ -3,6 +3,7 @@ using System.Text; using System.Text.Json; using Encelado.Bot.Logging; using Encelado.Core.Baskets; +using Encelado.Core.Baskets.History; namespace Encelado.Bot.Baskets; @@ -83,6 +84,7 @@ public sealed class Ledger : IDisposable private readonly Lock _gate = new(); private StreamWriter? _decisions; private StreamWriter? _baskets; + private StreamWriter? _orders; private string _decisionsMonth = string.Empty; public Ledger(string directory) @@ -96,6 +98,138 @@ public sealed class Ledger : IDisposable public string BasketsPath => Path.Combine(_directory, "baskets.csv"); + /// One line per order sent and per change of its state (append-only, see ). + public string OrdersPath => Path.Combine(_directory, "orders.jsonl"); + + /// Appends an order line. Never throws into the engine. + public void Order(OrderRecord record) + { + ArgumentNullException.ThrowIfNull(record); + try + { + string line = record.ToJson(); + lock (_gate) + { + _orders ??= Open(OrdersPath); + _orders.WriteLine(line); + _orders.Flush(); + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warn($"ledger: riga di ordine non scritta ({ex.Message})"); + } + } + + /// Every order line, oldest first. + public List ReadOrders() + { + List rows = []; + foreach (string line in ReadLines(OrdersPath)) + { + if (OrderRecord.Parse(line) is { } r) + { + rows.Add(r); + } + } + + return rows; + } + + /// + /// The entries the bot decided or sent since , as the + /// positions they may have produced would look (instrument, side, units, time): the + /// signature used to recognise an orphan of ours on the account. + /// + public List ReadEntrySignatures(DateTime sinceUtc) + { + List list = []; + List files = [DecisionsPath]; + string previous = Path.Combine(_directory, $"decisions_{sinceUtc:yyyyMM}.jsonl"); + if (File.Exists(previous)) + { + files.Insert(0, previous); + } + + foreach (string file in files) + { + foreach (string line in ReadLines(file)) + { + try + { + using JsonDocument doc = JsonDocument.Parse(line); + JsonElement r = doc.RootElement; + string evento = r.TryGetProperty("evento", out JsonElement ev) ? ev.GetString() ?? string.Empty : string.Empty; + if (evento is not ("segnale_ingresso" or "rifiuto" or "ingresso" or "pending" or "leg_risk_unwind" or "segnale_aggiunta" or "aggiunta")) + { + continue; + } + + if (!r.TryGetProperty("ts", out JsonElement tsEl) || !DateTime.TryParse(tsEl.GetString(), CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal, out DateTime ts) || ts < sinceUtc) + { + continue; + } + + string basket = r.TryGetProperty("basket", out JsonElement b) ? b.GetString() ?? string.Empty : string.Empty; + string[] pair = basket.Split('/'); + if (pair.Length != 2 || !SyntheticCross.TryDerive(pair[0], pair[1], out SyntheticCross? cross) || cross is null) + { + continue; + } + + bool buyCross = r.TryGetProperty("buy_cross", out JsonElement bc) && bc.ValueKind == JsonValueKind.True; + (bool buyA, bool buyB) = cross.Legs(buyCross); + double unitsA = r.TryGetProperty("unitsA", out JsonElement ua) && ua.ValueKind == JsonValueKind.Number ? ua.GetDouble() : 0; + double unitsB = r.TryGetProperty("unitsB", out JsonElement ub) && ub.ValueKind == JsonValueKind.Number ? ub.GetDouble() : 0; + list.Add(new EntrySignature(ts, 0, pair[0], buyA, unitsA, basket, evento)); + list.Add(new EntrySignature(ts, 0, pair[1], buyB, unitsB, basket, evento)); + } + catch (JsonException) + { + // A damaged line is skipped, never repaired. + } + } + } + + return list; + } + + /// + /// An event line that is not an evaluation: correzione, pending, + /// pending_risolto, orfana_adottata, orfana_chiusa, + /// movimento_di_cassa, kill_switch_avviato… Extra fields through . + /// + public void Note(string runId, string evento, string basketId, string motivazione, Action? extra = null) + { + try + { + using MemoryStream ms = new(); + using (Utf8JsonWriter w = new(ms)) + { + w.WriteStartObject(); + w.WriteString("ts", DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture)); + w.WriteString("run_id", runId); + w.WriteString("evento", evento); + w.WriteString("basket_id", basketId); + extra?.Invoke(w); + w.WriteString("motivazione", motivazione); + w.WriteEndObject(); + } + + lock (_gate) + { + RotateIfNeeded(DateTime.UtcNow); + _decisions ??= Open(DecisionsPath); + _decisions.WriteLine(Encoding.UTF8.GetString(ms.ToArray())); + _decisions.Flush(); + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warn($"ledger: riga '{evento}' non scritta ({ex.Message})"); + } + } + /// Appends one evaluation. Never throws into the engine. public void Decision( string runId, string mode, string preset, string configHash, BasketContext ctx, BasketDecision d, @@ -143,34 +277,7 @@ public sealed class Ledger : IDisposable } /// A correction is a new line, never an edit of an old one. - public void Correction(string runId, string basketId, string what) - { - try - { - using MemoryStream ms = new(); - using (Utf8JsonWriter w = new(ms)) - { - w.WriteStartObject(); - w.WriteString("ts", DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture)); - w.WriteString("run_id", runId); - w.WriteString("evento", "correzione"); - w.WriteString("basket_id", basketId); - w.WriteString("motivazione", what); - w.WriteEndObject(); - } - - lock (_gate) - { - _decisions ??= Open(DecisionsPath); - _decisions.WriteLine(Encoding.UTF8.GetString(ms.ToArray())); - _decisions.Flush(); - } - } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) - { - Log.Warn($"ledger: correzione non scritta ({ex.Message})"); - } - } + public void Correction(string runId, string basketId, string what) => Note(runId, "correzione", basketId, what); /// Reads every closed basket, oldest first: the training set. public List ReadBaskets() @@ -381,8 +488,10 @@ public sealed class Ledger : IDisposable { _decisions?.Dispose(); _baskets?.Dispose(); + _orders?.Dispose(); _decisions = null; _baskets = null; + _orders = null; } } } diff --git a/Encelado/src/Encelado.Bot/Engine/BotSnapshot.cs b/Encelado/src/Encelado.Bot/Engine/BotSnapshot.cs index 0a3e25c..3fdd142 100644 --- a/Encelado/src/Encelado.Bot/Engine/BotSnapshot.cs +++ b/Encelado/src/Encelado.Bot/Engine/BotSnapshot.cs @@ -66,6 +66,8 @@ public sealed record BasketRow( "Open" => "aperto", "Adding" => "aggiunta…", "Exiting" => "chiusura…", + "PendingA" => "attesa gamba A", + "PendingB" => "attesa gamba B", "Error" => "errore", "fermo" => "fermo", _ => State.ToLowerInvariant(), @@ -183,6 +185,31 @@ public sealed record BotSnapshot public int MaxBaskets { get; init; } + /// Baskets waiting for the venue's word on a leg (PendingA/PendingB). + public int PendingBaskets { get; init; } + + /// Orders in the register whose outcome the venue has not given yet. + public int PendingOrders { get; init; } + + /// Positions on the account that carry the bot's signature but belong to no basket: adopted and closed. + public int OrphanLegs { get; init; } + + /// Positions on the account the bot did not open: reported, never touched. + public int ForeignPositions { get; init; } + + /// The account's own unrealised result, all positions included. + public double AccountOpenPnl { get; init; } + + public double UsedMargin { get; init; } + + /// Cash that came in or went out without a trade since the state was first kept; the peak ignores it. + public double CumulativeCashFlow { get; init; } + + /// True when the account and the engine's picture of it have disagreed for more than a minute. + public bool Unreconciled { get; init; } + + public string UnreconciledReason { get; init; } = string.Empty; + public bool Halted { get; init; } public string? HaltReason { get; init; } @@ -206,4 +233,11 @@ public sealed record BotSnapshot public ContextRow? Context { get; init; } } -public readonly record struct CommandResult(bool Ok, string Message); +/// An orphan or a foreign position as the bonifica lists it. +public sealed record PositionInfo(long PositionId, string Symbol, bool IsBuy, double Units, DateTime OpenedUtc, double UnrealizedPnl, string Origin, string Basket, string Reason); + +public readonly record struct CommandResult(bool Ok, string Message) +{ + /// Structured data for the caller, when a message is not enough (the bonifica's list of positions). + public object? Payload { get; init; } +} diff --git a/Encelado/src/Encelado.Bot/Engine/IEngine.cs b/Encelado/src/Encelado.Bot/Engine/IEngine.cs index cea0462..b0ee644 100644 --- a/Encelado/src/Encelado.Bot/Engine/IEngine.cs +++ b/Encelado/src/Encelado.Bot/Engine/IEngine.cs @@ -14,6 +14,14 @@ public enum EngineCommandKind /// Lift the equity stop or the kill-switch. Reason is written to the ledger and must not be empty. ResetEquityStop, + + /// + /// The one-off clean-up of orphan legs (§5.5 of the 5.0 plan). Argument: + /// list returns the orphans and the foreign positions as + /// rows in the payload; close:<positionId> closes one orphan and records it; + /// done switches the engine back to closing orphans on its own. + /// + Bonifica, } public sealed record EngineCommand(EngineCommandKind Kind, string Argument = "", string Reason = ""); diff --git a/Encelado/src/Encelado.Bot/Ui/MainViewModel.cs b/Encelado/src/Encelado.Bot/Ui/MainViewModel.cs index af68db6..4b66e84 100644 --- a/Encelado/src/Encelado.Bot/Ui/MainViewModel.cs +++ b/Encelado/src/Encelado.Bot/Ui/MainViewModel.cs @@ -36,6 +36,12 @@ public sealed class MainViewModel : INotifyPropertyChanged private double _openPnlPct; private int _openBaskets; private int _maxBaskets; + private int _pendingBaskets; + private int _orphanLegs; + private int _foreignPositions; + private double _accountOpenPnl; + private double _usedMargin; + private bool _unreconciled; private bool _equityStopped; private bool _killSwitched; private string _preset = "—"; @@ -144,6 +150,27 @@ public sealed class MainViewModel : INotifyPropertyChanged public string BasketsDisplay => _maxBaskets > 0 ? $"{_openBaskets} / {_maxBaskets}" : _openBaskets.ToString(CultureInfo.CurrentCulture); + public int PendingBaskets { get => _pendingBaskets; private set => Set(ref _pendingBaskets, value); } + + /// Positions of ours on the account that belong to no basket. Red when above zero. + public int OrphanLegs { get => _orphanLegs; private set => Set(ref _orphanLegs, value); } + + public int ForeignPositions { get => _foreignPositions; private set => Set(ref _foreignPositions, value); } + + public bool HasOrphans => _orphanLegs > 0; + + /// The second line of the baskets tile: pending entries, orphans, foreign positions. + public string BasketsSub => string.Create(CultureInfo.CurrentCulture, $"in attesa {_pendingBaskets} · orfane {_orphanLegs} · esterne {_foreignPositions}"); + + /// The account's own unrealised result, all positions included. + public double AccountOpenPnl { get => _accountOpenPnl; private set => Set(ref _accountOpenPnl, value); } + + public double UsedMargin { get => _usedMargin; private set => Set(ref _usedMargin, value); } + + public string OpenPnlSub => string.Create(CultureInfo.CurrentCulture, $"di cui basket {_openPnl:+#,##0.00;-#,##0.00;0.00} · margine {_usedMargin:N0} / disp. {_availableBalance:N0}"); + + public bool Unreconciled { get => _unreconciled; private set => Set(ref _unreconciled, value); } + public bool EquityStopped { get => _equityStopped; private set => Set(ref _equityStopped, value); } public bool KillSwitched { get => _killSwitched; private set => Set(ref _killSwitched, value); } @@ -220,7 +247,16 @@ public sealed class MainViewModel : INotifyPropertyChanged OpenPnlPct = s.OpenPnlPct; OpenBaskets = s.OpenBaskets; MaxBaskets = s.MaxBaskets; + PendingBaskets = s.PendingBaskets; + OrphanLegs = s.OrphanLegs; + ForeignPositions = s.ForeignPositions; + AccountOpenPnl = s.AccountOpenPnl; + UsedMargin = s.UsedMargin; + Unreconciled = s.Unreconciled; Raise(nameof(BasketsDisplay)); + Raise(nameof(BasketsSub)); + Raise(nameof(HasOrphans)); + Raise(nameof(OpenPnlSub)); EquityStopped = s.EquityStopped; KillSwitched = s.KillSwitched; @@ -285,6 +321,14 @@ public sealed class MainViewModel : INotifyPropertyChanged return; } + if (IsRunning && s.Unreconciled) + { + Banner = $"Posizioni non riconciliate: {s.UnreconciledReason}."; + HasBanner = true; + BannerIsWarning = true; + return; + } + if (IsRunning && s.ApiState is "caduta" or "disconnesso") { Banner = "Collegamento a eToro caduto: il motore prova a riconnettersi da solo."; diff --git a/Encelado/src/Encelado.Bot/Ui/Pages/DashboardPage.xaml b/Encelado/src/Encelado.Bot/Ui/Pages/DashboardPage.xaml index c68e497..9d184da 100644 --- a/Encelado/src/Encelado.Bot/Ui/Pages/DashboardPage.xaml +++ b/Encelado/src/Encelado.Bot/Ui/Pages/DashboardPage.xaml @@ -117,13 +117,13 @@ - + - + - + Text="{Binding AccountOpenPnl, StringFormat='{}{0:+#,##0.00;-#,##0.00;0.00}'}" + Foreground="{Binding AccountOpenPnl, Converter={StaticResource PnlBrush}}"/> + @@ -133,11 +133,22 @@ - + - + + + + + diff --git a/Encelado/src/Encelado.Core/Baskets/Backtest/BacktestBroker.cs b/Encelado/src/Encelado.Core/Baskets/Backtest/BacktestBroker.cs index 50f379d..5c468ca 100644 --- a/Encelado/src/Encelado.Core/Baskets/Backtest/BacktestBroker.cs +++ b/Encelado/src/Encelado.Core/Baskets/Backtest/BacktestBroker.cs @@ -198,6 +198,11 @@ public sealed class BacktestBroker : IBroker public Task LookupOrderAsync(string clientRef, CancellationToken ct) => Task.FromResult(_orders.TryGetValue(clientRef, out OrderOutcome? o) ? o : null); + public Task LookupOrderByIdAsync(long orderId, CancellationToken ct) => + Task.FromResult(_orders.Values.FirstOrDefault(o => o.OrderId == orderId)); + + public Task CancelOrderAsync(long orderId, CancellationToken ct) => Task.FromResult(false); + public Task CloseAsync(long positionId, long instrumentId, CancellationToken ct) { if (!_positions.Remove(positionId, out Position? p)) diff --git a/Encelado/src/Encelado.Core/Baskets/BasketExecutor.cs b/Encelado/src/Encelado.Core/Baskets/BasketExecutor.cs index d12f15c..914ccbb 100644 --- a/Encelado/src/Encelado.Core/Baskets/BasketExecutor.cs +++ b/Encelado/src/Encelado.Core/Baskets/BasketExecutor.cs @@ -3,6 +3,14 @@ using Encelado.Core.Broker; namespace Encelado.Core.Baskets; +/// Which leg, if any, is still waiting for the venue's word after an entry attempt. +public enum PendingLeg +{ + None = 0, + A, + B, +} + /// What opening (or adding to) a basket produced. public sealed record EntryOutcome( bool Ok, @@ -11,7 +19,18 @@ public sealed record EntryOutcome( string Error, double SlippagePipsA, double SlippagePipsB, - double LatencyMs); + double LatencyMs) +{ + /// When a leg's outcome is unknown the basket is neither open nor flat: the order register follows it. + public PendingLeg PendingLeg { get; init; } + + /// Everything needed to finish (or undo) the entry once the pending leg resolves. + public PendingEntry? Pending { get; init; } + + public TrackedOrder? PendingOrder { get; init; } + + public bool IsPending => PendingLeg != PendingLeg.None; +} /// What closing a basket produced. public sealed record ExitOutcome( @@ -27,28 +46,50 @@ public sealed record ExitOutcome( IReadOnlyList StuckPositionIds); /// -/// The two-leg execution protocol of §5.7, over any . +/// The two-leg execution protocol of §5.7, over any , with the +/// order register of the 5.0 plan. /// -/// Send leg A at market and wait for its fill. -/// Within two seconds send leg B. If B is rejected or unconfirmed within the leg -/// timeout, close A at once and report leg_risk_unwind. -/// Every order carries a unique client reference; before resending, the venue is -/// asked what became of the reference, so nothing is ever duplicated. +/// Every order is registered before it is sent, with a unique client reference. +/// Send leg A at market and wait for its fill: the venue's own answer, then the +/// lookup by orderId, then the position list. Past the leg timeout the basket +/// becomes PendingA and the register keeps asking; nothing is resent. +/// Leg B is sized on the units leg A really got, then sent. If B is rejected, +/// A is closed at once (leg_risk_unwind); if B is unknown past the timeout the +/// basket becomes PendingB. /// On exit both legs are closed; a leg that fails is retried three times with /// backoff and then reported as stuck. /// /// -public sealed class BasketExecutor(IBroker broker, BasketStrategyConfig config, Func mid, Action log) +public sealed class BasketExecutor { - private readonly IBroker _broker = broker ?? throw new ArgumentNullException(nameof(broker)); - private readonly BasketStrategyConfig _cfg = config ?? throw new ArgumentNullException(nameof(config)); - private readonly Func _mid = mid ?? throw new ArgumentNullException(nameof(mid)); - private readonly Action _log = log ?? (static _ => { }); + private readonly IBroker _broker; + private readonly BasketStrategyConfig _cfg; + private readonly Func _mid; + private readonly Action _log; + private readonly OrderTracker? _tracker; + private readonly string _mode; - public async Task OpenAsync(BasketContext ctx, BasketDecision decision, BasketPreset preset, CancellationToken ct) + public BasketExecutor(IBroker broker, BasketStrategyConfig config, Func mid, Action log, OrderTracker? tracker = null, string mode = "") + { + _broker = broker ?? throw new ArgumentNullException(nameof(broker)); + _cfg = config ?? throw new ArgumentNullException(nameof(config)); + _mid = mid ?? throw new ArgumentNullException(nameof(mid)); + _log = log ?? (static _ => { }); + _tracker = tracker; + _mode = mode ?? string.Empty; + } + + public OrderTracker? Tracker => _tracker; + + public Task OpenAsync(BasketContext ctx, BasketDecision decision, BasketPreset preset, CancellationToken ct) => + OpenAsync(ctx, decision, preset, ctx?.BasketId ?? string.Empty, ct); + + /// Opens a basket: leg A, then leg B. is the instance id written to the ledger. + public async Task OpenAsync(BasketContext ctx, BasketDecision decision, BasketPreset preset, string basketId, CancellationToken ct) { ArgumentNullException.ThrowIfNull(ctx); ArgumentNullException.ThrowIfNull(decision); + ArgumentNullException.ThrowIfNull(preset); if (decision.Kind != DecisionKind.Enter || decision.Sizing is not { Ok: true } sizing) { return new EntryOutcome(false, null, false, "nessuna decisione di ingresso", 0, 0, 0); @@ -59,72 +100,191 @@ public sealed class BasketExecutor(IBroker broker, BasketStrategyConfig config, double quoteA = buyA ? ctx.A.Quote.Ask : ctx.A.Quote.Bid; double quoteB = buyB ? ctx.B.Quote.Ask : ctx.B.Quote.Bid; - OrderRequest reqA = Request(ctx.A, buyA, sizing.UnitsA, quoteA, decision.Motivazione); - OrderOutcome a = await SendAsync(reqA, ct).ConfigureAwait(false); + PendingEntry plan = new() + { + BasketId = basketId, + BuyCross = decision.BuyCross, + EntryZ = decision.Evaluation.Z, + UnitsA = sizing.UnitsA, + UnitsB = sizing.UnitsB, + TpPips = _cfg.TpMode == TpMode.AtrMultiple && double.IsFinite(decision.Evaluation.AtrPipsA) ? Math.Max(1, _cfg.TpAtrMultiple * decision.Evaluation.AtrPipsA) : preset.TpPips, + MaxLossUsd = ctx.Equity * _cfg.MaxLossPerBasketPct / 100.0, + EntryCostPips = decision.Cost?.CostPips ?? double.NaN, + EquityAtEntry = ctx.Equity, + Motivazione = decision.Motivazione, + DecidedUtc = ctx.TimeUtc, + QuoteA = quoteA, + QuoteB = quoteB, + }; + + OrderRequest reqA = Request(ctx.A, buyA, plan.UnitsA, quoteA, decision.Motivazione); + plan.ClientRefA = reqA.ClientRef; + TrackedOrder trackA = Track(reqA, ctx.Name, basketId, OrderLeg.A, quoteA, decision.Motivazione); + OrderOutcome a = await SendAsync(reqA, trackA, ct).ConfigureAwait(false); + if (a.Pending) + { + _log($"[{ctx.Name}] gamba A ({ctx.A.Symbol}) senza esito dopo {_cfg.LegTimeoutSec} s: resta nel registro degli ordini, il basket aspetta ({Describe(a)})"); + return new EntryOutcome(false, null, false, $"gamba A ({ctx.A.Symbol}) in attesa di esito: {Describe(a)}", 0, 0, Environment.TickCount64 - t0) + { + PendingLeg = PendingLeg.A, + Pending = plan, + PendingOrder = trackA, + }; + } + if (!a.Filled) { return new EntryOutcome(false, null, false, $"gamba A ({ctx.A.Symbol}) non eseguita: {Describe(a)}", 0, 0, Environment.TickCount64 - t0); } - OrderRequest reqB = Request(ctx.B, buyB, sizing.UnitsB, quoteB, decision.Motivazione); - OrderOutcome b = await SendAsync(reqB, ct).ConfigureAwait(false); + plan.LegA = LegFrom(ctx.A, buyA, a, plan.UnitsA, quoteA, reqA.ClientRef, reqA.StopLossRate ?? 0); + return await SendLegBAsync(ctx, plan, t0, ct).ConfigureAwait(false); + } + + /// Leg A, sent earlier and left pending, has been filled: carries on with leg B. + public Task ResumeAfterAAsync(BasketContext ctx, PendingEntry plan, OrderOutcome legAOutcome, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(ctx); + ArgumentNullException.ThrowIfNull(plan); + ArgumentNullException.ThrowIfNull(legAOutcome); + (bool buyA, _) = ctx.Cross.Legs(plan.BuyCross); + plan.LegA = LegFrom(ctx.A, buyA, legAOutcome, plan.UnitsA, plan.QuoteA, plan.ClientRefA, 0); + return SendLegBAsync(ctx, plan, Environment.TickCount64, ct); + } + + /// Leg B, sent earlier and left pending, has resolved: completes the basket or undoes leg A. + public async Task CompleteAfterBAsync(BasketContext ctx, PendingEntry plan, OrderOutcome legBOutcome, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(ctx); + ArgumentNullException.ThrowIfNull(plan); + ArgumentNullException.ThrowIfNull(legBOutcome); + if (plan.LegA is not { } legA) + { + return new EntryOutcome(false, null, false, "gamba A non registrata nel piano: impossibile completare", 0, 0, 0); + } + + (_, bool buyB) = ctx.Cross.Legs(plan.BuyCross); + if (!legBOutcome.Filled) + { + return await UnwindAAsync(ctx, plan, legA, $"gamba B non eseguita: {Describe(legBOutcome)}", 0, ct).ConfigureAwait(false); + } + + return Complete(ctx, plan, legA, legBOutcome, buyB, plan.QuoteB, plan.ClientRefB, 0); + } + + private async Task SendLegBAsync(BasketContext ctx, PendingEntry plan, long t0, CancellationToken ct) + { + BasketLeg legA = plan.LegA!; + (_, bool buyB) = ctx.Cross.Legs(plan.BuyCross); + double quoteB = buyB ? ctx.B.Quote.Ask : ctx.B.Quote.Bid; + + // B is sized on what A really got: the venue may have reduced A (observed on + // 2026-09-16), and a full-size B against a reduced A is not the basket that was decided. + double unitsB = plan.UnitsB; + if (plan.UnitsA > 0 && legA.Units > 0 && legA.Units < plan.UnitsA * 0.99) + { + unitsB = Math.Round(plan.UnitsB * legA.Units / plan.UnitsA, 2); + _log(string.Create(CultureInfo.InvariantCulture, $"[{ctx.Name}] la gamba A è stata eseguita per {legA.Units:0.##} unità su {plan.UnitsA:0.##} richieste: la gamba B scende a {unitsB:0.##}")); + } + + OrderRequest reqB = Request(ctx.B, buyB, unitsB, quoteB, plan.Motivazione); + plan.ClientRefB = reqB.ClientRef; + TrackedOrder trackB = Track(reqB, ctx.Name, plan.BasketId, OrderLeg.B, quoteB, plan.Motivazione); + OrderOutcome b = await SendAsync(reqB, trackB, ct).ConfigureAwait(false); + if (b.Pending) + { + _log($"[{ctx.Name}] gamba B ({ctx.B.Symbol}) senza esito dopo {_cfg.LegTimeoutSec} s: resta nel registro, il basket aspetta con la gamba A aperta ({Describe(b)})"); + return new EntryOutcome(false, null, false, $"gamba B ({ctx.B.Symbol}) in attesa di esito: {Describe(b)}", SlipPips(ctx.A, legA.IsBuy, plan.QuoteA, legA.EntryPrice), 0, Environment.TickCount64 - t0) + { + PendingLeg = PendingLeg.B, + Pending = plan, + PendingOrder = trackB, + }; + } + if (!b.Filled) { // Leg risk: A is alone in the market. Undo it now. _log($"[{ctx.Name}] gamba B ({ctx.B.Symbol}) non eseguita ({Describe(b)}): chiudo subito la gamba A (leg_risk_unwind)"); - CloseOutcome undo = await CloseLegAsync(a.PositionId, ctx.A.Instrument.Id, ct).ConfigureAwait(false); - string error = $"gamba B non eseguita: {Describe(b)}; gamba A {(undo.Closed ? "richiusa" : "NON richiusa: " + undo.Error)}"; - return new EntryOutcome(false, null, undo.Closed, error, SlipPips(ctx.A, buyA, quoteA, a.FillRate), 0, Environment.TickCount64 - t0); + return await UnwindAAsync(ctx, plan, legA, $"gamba B non eseguita: {Describe(b)}", t0, ct).ConfigureAwait(false); } - BasketLeg legA = new() - { - Symbol = ctx.A.Symbol, - InstrumentId = ctx.A.Instrument.Id, - IsBuy = buyA, - Units = a.Units > 0 ? a.Units : sizing.UnitsA, - EntryPrice = a.FillRate > 0 ? a.FillRate : quoteA, - PositionId = a.PositionId, - ClientRef = reqA.ClientRef, - OpenedUtc = a.TimeUtc, - EntryFeesUsd = a.Fees, - StopLossRate = reqA.StopLossRate ?? 0, - }; - BasketLeg legB = new() - { - Symbol = ctx.B.Symbol, - InstrumentId = ctx.B.Instrument.Id, - IsBuy = buyB, - Units = b.Units > 0 ? b.Units : sizing.UnitsB, - EntryPrice = b.FillRate > 0 ? b.FillRate : quoteB, - PositionId = b.PositionId, - ClientRef = reqB.ClientRef, - OpenedUtc = b.TimeUtc, - EntryFeesUsd = b.Fees, - StopLossRate = reqB.StopLossRate ?? 0, - }; + return Complete(ctx, plan, legA, b, buyB, quoteB, reqB.ClientRef, t0); + } + private async Task UnwindAAsync(BasketContext ctx, PendingEntry plan, BasketLeg legA, string why, long t0, CancellationToken ct) + { + CloseOutcome undo = await CloseLegAsync(legA.PositionId, legA.InstrumentId, TrackClose(legA, ctx.Name, plan.BasketId, OrderLeg.Unwind, "leg_risk_unwind: " + why), ct).ConfigureAwait(false); + string error = $"{why}; gamba A {(undo.Closed ? "richiusa" : "NON richiusa: " + undo.Error)}"; + return new EntryOutcome(false, null, undo.Closed, error, SlipPips(ctx.A, legA.IsBuy, plan.QuoteA, legA.EntryPrice), 0, Environment.TickCount64 - t0); + } + + /// Closes a lone leg (a pending A whose signal decayed, an orphan): three attempts, verified on the position list. + public Task UnwindLegAsync(BasketLeg leg, string basket, string basketId, string reason, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(leg); + return CloseLegAsync(leg.PositionId, leg.InstrumentId, TrackClose(leg, basket, basketId, OrderLeg.Unwind, reason), ct); + } + + /// Closes one position that is not a leg of any basket (an orphan, or a residue at the kill-switch). + public Task ClosePositionAsync(BrokerPosition position, string symbol, string basket, string reason, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(position); + TrackedOrder track = new() + { + ClientRef = Guid.NewGuid().ToString("D"), + Symbol = symbol, + InstrumentId = position.InstrumentId, + IsBuy = !position.IsBuy, + RequestedUnits = position.Units, + RequestedPrice = position.CurrentRate, + Basket = basket, + Leg = OrderLeg.Unwind, + SentUtc = DateTime.UtcNow, + Mode = _mode, + Motivazione = reason, + }; + return CloseLegAsync(position.PositionId, position.InstrumentId, track, ct); + } + + private EntryOutcome Complete(BasketContext ctx, PendingEntry plan, BasketLeg legA, OrderOutcome b, bool buyB, double quoteB, string clientRefB, long t0) + { + BasketLeg legB = LegFrom(ctx.B, buyB, b, plan.UnitsB, quoteB, clientRefB, 0); BasketPosition position = new() { - BasketId = ctx.BasketId, + BasketId = plan.BasketId, Name = ctx.Name, - BuyCross = decision.BuyCross, + BuyCross = plan.BuyCross, A = legA, B = legB, - OpenedUtc = ctx.TimeUtc, - EntryZ = decision.Evaluation.Z, - LastAddZ = decision.Evaluation.Z, - EntryCostPips = decision.Cost?.CostPips ?? double.NaN, - TpPips = _cfg.TpMode == TpMode.AtrMultiple && double.IsFinite(decision.Evaluation.AtrPipsA) ? Math.Max(1, _cfg.TpAtrMultiple * decision.Evaluation.AtrPipsA) : preset.TpPips, - MaxLossUsd = ctx.Equity * _cfg.MaxLossPerBasketPct / 100.0, - EquityAtEntry = ctx.Equity, - EntryMotivazione = decision.Motivazione, + OpenedUtc = plan.DecidedUtc, + EntryZ = plan.EntryZ, + LastAddZ = plan.EntryZ, + EntryCostPips = plan.EntryCostPips, + TpPips = plan.TpPips, + MaxLossUsd = plan.MaxLossUsd, + EquityAtEntry = plan.EquityAtEntry, + EntryMotivazione = plan.Motivazione, }; return new EntryOutcome(true, position, false, string.Empty, - SlipPips(ctx.A, buyA, quoteA, legA.EntryPrice), SlipPips(ctx.B, buyB, quoteB, legB.EntryPrice), Environment.TickCount64 - t0); + SlipPips(ctx.A, legA.IsBuy, plan.QuoteA, legA.EntryPrice), SlipPips(ctx.B, buyB, quoteB, legB.EntryPrice), Environment.TickCount64 - t0); } + private static BasketLeg LegFrom(SymbolSeries s, bool isBuy, OrderOutcome o, double requestedUnits, double quote, string clientRef, double stop) => new() + { + Symbol = s.Symbol, + InstrumentId = s.Instrument.Id, + IsBuy = isBuy, + Units = o.Units > 0 ? o.Units : requestedUnits, + EntryPrice = o.FillRate > 0 ? o.FillRate : quote, + PositionId = o.PositionId, + ClientRef = clientRef, + OpenedUtc = o.TimeUtc == default ? DateTime.UtcNow : o.TimeUtc, + EntryFeesUsd = o.Fees, + StopLossRate = stop, + }; + /// Adds to both legs of an open basket (a new position per leg on eToro). public async Task AddAsync(BasketContext ctx, BasketDecision decision, CancellationToken ct) { @@ -140,18 +300,34 @@ public sealed class BasketExecutor(IBroker broker, BasketStrategyConfig config, double quoteB = p.B.IsBuy ? ctx.B.Quote.Ask : ctx.B.Quote.Bid; OrderRequest reqA = Request(ctx.A, p.A.IsBuy, sizing.UnitsA, quoteA, decision.Motivazione); - OrderOutcome a = await SendAsync(reqA, ct).ConfigureAwait(false); + OrderOutcome a = await SendAsync(reqA, Track(reqA, ctx.Name, p.BasketId, OrderLeg.Add, quoteA, decision.Motivazione), ct).ConfigureAwait(false); if (!a.Filled) { + // A pending add stays in the register: if it fills later it is an orphan and the reconciliation closes it. return new EntryOutcome(false, p, false, $"aggiunta su A non eseguita: {Describe(a)}", 0, 0, Environment.TickCount64 - t0); } OrderRequest reqB = Request(ctx.B, p.B.IsBuy, sizing.UnitsB, quoteB, decision.Motivazione); - OrderOutcome b = await SendAsync(reqB, ct).ConfigureAwait(false); + OrderOutcome b = await SendAsync(reqB, Track(reqB, ctx.Name, p.BasketId, OrderLeg.Add, quoteB, decision.Motivazione), ct).ConfigureAwait(false); if (!b.Filled) { _log($"[{ctx.Name}] aggiunta su B non eseguita ({Describe(b)}): richiudo l'aggiunta su A (leg_risk_unwind)"); - CloseOutcome undo = await CloseLegAsync(a.PositionId, ctx.A.Instrument.Id, ct).ConfigureAwait(false); + TrackedOrder undoTrack = new() + { + ClientRef = Guid.NewGuid().ToString("D"), + Symbol = ctx.A.Symbol, + InstrumentId = ctx.A.Instrument.Id, + IsBuy = !p.A.IsBuy, + RequestedUnits = a.Units > 0 ? a.Units : sizing.UnitsA, + RequestedPrice = quoteA, + Basket = ctx.Name, + BasketId = p.BasketId, + Leg = OrderLeg.Unwind, + SentUtc = DateTime.UtcNow, + Mode = _mode, + Motivazione = "leg_risk_unwind dell'aggiunta", + }; + CloseOutcome undo = await CloseLegAsync(a.PositionId, ctx.A.Instrument.Id, undoTrack, ct).ConfigureAwait(false); return new EntryOutcome(false, p, undo.Closed, $"aggiunta su B non eseguita: {Describe(b)}", 0, 0, Environment.TickCount64 - t0); } @@ -181,19 +357,17 @@ public sealed class BasketExecutor(IBroker broker, BasketStrategyConfig config, double quoteA = p.A.IsBuy ? ctx.A.Quote.Bid : ctx.A.Quote.Ask; double quoteB = p.B.IsBuy ? ctx.B.Quote.Bid : ctx.B.Quote.Ask; List stuck = []; - double pnl = 0; - double exitA = 0, exitB = 0; DateTime closedUtc = DateTime.UtcNow; - (double priceA, double pnlA, bool okA) = await CloseLegAllAsync(p.A, ctx.A.Instrument.Id, stuck, ct).ConfigureAwait(false); - (double priceB, double pnlB, bool okB) = await CloseLegAllAsync(p.B, ctx.B.Instrument.Id, stuck, ct).ConfigureAwait(false); + (double priceA, double pnlA, bool okA) = await CloseLegAllAsync(p.A, ctx.A.Instrument.Id, ctx.Name, p.BasketId, reason, stuck, ct).ConfigureAwait(false); + (double priceB, double pnlB, bool okB) = await CloseLegAllAsync(p.B, ctx.B.Instrument.Id, ctx.Name, p.BasketId, reason, stuck, ct).ConfigureAwait(false); - exitA = priceA > 0 ? priceA : quoteA; - exitB = priceB > 0 ? priceB : quoteB; + double exitA = priceA > 0 ? priceA : quoteA; + double exitB = priceB > 0 ? priceB : quoteB; // Realised P&L: the venue's number when it reports one, our own otherwise. double own = p.NetPnlUsd(exitA, exitB, _mid); - pnl = okA && okB && (pnlA != 0 || pnlB != 0) ? pnlA + pnlB - p.AccruedFeesUsd : (double.IsNaN(own) ? 0 : own); + double pnl = okA && okB && (pnlA != 0 || pnlB != 0) ? pnlA + pnlB - p.AccruedFeesUsd : (double.IsNaN(own) ? 0 : own); double pips = p.PipsTotal(exitA, exitB, ctx.A.Instrument.Pip, ctx.B.Instrument.Pip); bool ok = okA && okB; @@ -202,16 +376,17 @@ public sealed class BasketExecutor(IBroker broker, BasketStrategyConfig config, ok ? string.Empty : $"gambe non chiuse: {string.Join(", ", stuck)}", closedUtc, stuck); } - private async Task<(double Price, double Pnl, bool Ok)> CloseLegAllAsync(BasketLeg leg, long instrumentId, List stuck, CancellationToken ct) + private async Task<(double Price, double Pnl, bool Ok)> CloseLegAllAsync(BasketLeg leg, long instrumentId, string basket, string basketId, string reason, List stuck, CancellationToken ct) { double weighted = 0, units = 0, pnl = 0; bool ok = true; foreach (long id in leg.AllPositionIds.ToList()) { - CloseOutcome c = await CloseLegAsync(id, instrumentId, ct).ConfigureAwait(false); + double expected = id == leg.PositionId ? leg.Units : leg.Adds.FirstOrDefault(a => a.PositionId == id).Units; + CloseOutcome c = await CloseLegAsync(id, instrumentId, TrackClose(leg, basket, basketId, OrderLeg.Close, reason, id, expected), ct).ConfigureAwait(false); if (c.Closed) { - double u = c.Units > 0 ? c.Units : (id == leg.PositionId ? leg.Units : leg.Adds.FirstOrDefault(a => a.PositionId == id).Units); + double u = c.Units > 0 ? c.Units : expected; weighted += c.CloseRate * u; units += u; pnl += c.RealizedPnl; @@ -227,8 +402,13 @@ public sealed class BasketExecutor(IBroker broker, BasketStrategyConfig config, } /// Three attempts with backoff; a pending outcome is re-checked against the position list. - private async Task CloseLegAsync(long positionId, long instrumentId, CancellationToken ct) + private async Task CloseLegAsync(long positionId, long instrumentId, TrackedOrder? track, CancellationToken ct) { + if (track is not null) + { + _tracker?.Register(track); + } + CloseOutcome last = new(false, false, 0, 0, 0, DateTime.UtcNow, 0, "non tentata"); for (int attempt = 0; attempt < 3; attempt++) { @@ -243,6 +423,7 @@ public sealed class BasketExecutor(IBroker broker, BasketStrategyConfig config, if (last.Closed) { + ApplyClose(track, last); return last; } @@ -261,15 +442,45 @@ public sealed class BasketExecutor(IBroker broker, BasketStrategyConfig config, if (positions.All(x => x.PositionId != positionId)) { // Gone from the account: closed by the venue (our order, or a native stop). - return new CloseOutcome(true, false, last.OrderId, last.CloseRate, last.Units, DateTime.UtcNow, last.RealizedPnl, string.Empty); + last = new CloseOutcome(true, false, last.OrderId, last.CloseRate, last.Units, DateTime.UtcNow, last.RealizedPnl, string.Empty); + ApplyClose(track, last); + return last; } } + ApplyClose(track, last); return last; } - /// Sends one leg. On an unknown outcome the venue is asked by client reference before giving up. - private async Task SendAsync(OrderRequest request, CancellationToken ct) + private void ApplyClose(TrackedOrder? track, CloseOutcome c) + { + if (track is null) + { + return; + } + + OrderOutcome o = new(c.Closed, c.Rejected, c.OrderId, 0, c.CloseRate, c.Units, c.TimeUtc, 0, c.Closed ? "Closed" : c.Rejected ? "Rejected" : "Unknown", c.Error) + { + RequestedUnits = track.RequestedUnits, + Source = "venue", + }; + if (_tracker is not null) + { + _tracker.Apply(track, o, DateTime.UtcNow); + } + else + { + track.Apply(o, DateTime.UtcNow); + } + } + + /// + /// Sends one leg and waits for its outcome up to the leg timeout. The venue's own + /// answer first; then the lookup by orderId (by client reference only when the + /// submit's answer was lost). Never resends. Past the timeout the order is returned + /// pending: it stays in the register, which keeps asking. + /// + private async Task SendAsync(OrderRequest request, TrackedOrder track, CancellationToken ct) { OrderOutcome outcome; try @@ -278,25 +489,32 @@ public sealed class BasketExecutor(IBroker broker, BasketStrategyConfig config, } catch (BrokerException ex) { - outcome = new OrderOutcome(false, false, 0, 0, 0, request.Units, DateTime.UtcNow, 0, "Unknown", ex.Message); + // The request may or may not have reached the venue: unknown, not rejected. + outcome = OrderOutcome.Unknown(0, request.Units, ex.Message); } - if (outcome.Filled || outcome.Rejected) + Apply(track, outcome); + if (!outcome.Pending) { - return outcome; + return track.ToOutcome(); } - // Idempotency: never resend; ask what became of this reference until the leg timeout. DateTime deadline = DateTime.UtcNow.AddSeconds(_cfg.LegTimeoutSec); while (DateTime.UtcNow < deadline) { - await Task.Delay(500, ct).ConfigureAwait(false); + await Task.Delay(700, ct).ConfigureAwait(false); try { - OrderOutcome? looked = await _broker.LookupOrderAsync(request.ClientRef, ct).ConfigureAwait(false); - if (looked is { Pending: false }) + OrderOutcome? looked = track.OrderId > 0 + ? await _broker.LookupOrderByIdAsync(track.OrderId, ct).ConfigureAwait(false) + : await _broker.LookupOrderAsync(request.ClientRef, ct).ConfigureAwait(false); + if (looked is not null) { - return looked; + Apply(track, looked); + if (!looked.Pending) + { + return track.ToOutcome(); + } } } catch (BrokerException) @@ -305,9 +523,63 @@ public sealed class BasketExecutor(IBroker broker, BasketStrategyConfig config, } } - return outcome with { Error = outcome.Error.Length > 0 ? outcome.Error : $"esito sconosciuto dopo {_cfg.LegTimeoutSec} s" }; + if (track.Error.Length == 0) + { + track.Error = $"esito sconosciuto dopo {_cfg.LegTimeoutSec} s"; + } + + return track.ToOutcome(); } + private void Apply(TrackedOrder track, OrderOutcome outcome) + { + if (_tracker is not null) + { + _tracker.Apply(track, outcome, DateTime.UtcNow); + } + else + { + track.Apply(outcome, DateTime.UtcNow); + } + } + + private TrackedOrder Track(OrderRequest req, string basket, string basketId, OrderLeg leg, double quote, string reason) + { + TrackedOrder t = new() + { + ClientRef = req.ClientRef, + Symbol = req.Symbol, + InstrumentId = req.InstrumentId, + IsBuy = req.IsBuy, + RequestedUnits = req.Units, + RequestedPrice = quote, + Basket = basket, + BasketId = basketId, + Leg = leg, + SentUtc = DateTime.UtcNow, + Mode = _mode, + Motivazione = reason.Length > 160 ? reason[..160] : reason, + }; + _tracker?.Register(t); + return t; + } + + private TrackedOrder TrackClose(BasketLeg leg, string basket, string basketId, OrderLeg kind, string reason, long positionId = 0, double units = 0) => new() + { + ClientRef = Guid.NewGuid().ToString("D"), + Symbol = leg.Symbol, + InstrumentId = leg.InstrumentId, + IsBuy = !leg.IsBuy, + RequestedUnits = units > 0 ? units : leg.Units, + RequestedPrice = _mid(leg.Symbol) ?? 0, + Basket = basket, + BasketId = basketId, + Leg = kind, + SentUtc = DateTime.UtcNow, + Mode = _mode, + Motivazione = (positionId > 0 ? string.Create(CultureInfo.InvariantCulture, $"posizione {positionId}: ") : string.Empty) + (reason.Length > 160 ? reason[..160] : reason), + }; + private OrderRequest Request(SymbolSeries s, bool isBuy, double units, double quote, string reason) { // The venue wants a native stop on every short and on every leveraged order: put it diff --git a/Encelado/src/Encelado.Core/Baskets/BasketPosition.cs b/Encelado/src/Encelado.Core/Baskets/BasketPosition.cs index 1d39d51..baecc53 100644 --- a/Encelado/src/Encelado.Core/Baskets/BasketPosition.cs +++ b/Encelado/src/Encelado.Core/Baskets/BasketPosition.cs @@ -12,6 +12,12 @@ public enum BasketState Exiting, Closed, Error, + + /// Leg A was sent and the venue has not said what became of it; nothing else happens on this basket until it does. + PendingA, + + /// Leg A is filled, leg B was sent and the venue has not said what became of it. + PendingB, } public static class BasketLifecycle @@ -22,6 +28,14 @@ public static class BasketLifecycle (BasketState.Entering, BasketState.Open) => true, (BasketState.Entering, BasketState.Idle) => true, // leg-risk unwind, both legs flat again (BasketState.Entering, BasketState.Error) => true, + (BasketState.Entering, BasketState.PendingA) => true, // leg A sent, outcome unknown past the leg timeout + (BasketState.Entering, BasketState.PendingB) => true, // leg A filled, leg B outcome unknown + (BasketState.PendingA, BasketState.Entering) => true, // leg A filled: sending leg B + (BasketState.PendingA, BasketState.Idle) => true, // leg A rejected, or filled and unwound because the signal decayed + (BasketState.PendingA, BasketState.Error) => true, + (BasketState.PendingB, BasketState.Open) => true, // leg B filled + (BasketState.PendingB, BasketState.Idle) => true, // leg B rejected, leg A unwound + (BasketState.PendingB, BasketState.Error) => true, (BasketState.Open, BasketState.Adding) => true, (BasketState.Adding, BasketState.Open) => true, (BasketState.Adding, BasketState.Error) => true, @@ -33,6 +47,9 @@ public static class BasketLifecycle (BasketState.Error, BasketState.Exiting) => true, _ => from == to, }; + + /// A basket waiting for the venue: no evaluation, no new order, until the order register resolves it. + public static bool IsPending(this BasketState state) => state is BasketState.PendingA or BasketState.PendingB; } /// One leg of an open basket, as filled. @@ -82,6 +99,63 @@ public sealed class BasketLeg /// Signed pips from entry at the exit price of this leg (bid for a long, ask for a short). public double Pips(double exitPrice, double pip) => (IsBuy ? exitPrice - EntryPrice : EntryPrice - exitPrice) / pip; + + /// Writes the leg as a named JSON object (the shape of baskets_state.json). + public static void Write(System.Text.Json.Utf8JsonWriter w, string name, BasketLeg leg) + { + ArgumentNullException.ThrowIfNull(w); + ArgumentNullException.ThrowIfNull(leg); + w.WriteStartObject(name); + w.WriteString("symbol", leg.Symbol); + w.WriteNumber("instrumentId", leg.InstrumentId); + w.WriteBoolean("isBuy", leg.IsBuy); + w.WriteNumber("units", leg.Units); + w.WriteNumber("entryPrice", leg.EntryPrice); + w.WriteNumber("positionId", leg.PositionId); + w.WriteString("clientRef", leg.ClientRef); + w.WriteString("openedUtc", leg.OpenedUtc.ToString("O", CultureInfo.InvariantCulture)); + w.WriteNumber("entryFeesUsd", leg.EntryFeesUsd); + w.WriteNumber("stopLossRate", leg.StopLossRate); + w.WriteStartArray("adds"); + foreach ((long id, double units, double price, string clientRef) in leg.Adds) + { + w.WriteStartObject(); + w.WriteNumber("positionId", id); + w.WriteNumber("units", units); + w.WriteNumber("price", price); + w.WriteString("clientRef", clientRef); + w.WriteEndObject(); + } + + w.WriteEndArray(); + w.WriteEndObject(); + } + + public static BasketLeg Read(System.Text.Json.JsonElement e) + { + BasketLeg leg = new() + { + Symbol = e.GetProperty("symbol").GetString() ?? string.Empty, + InstrumentId = e.GetProperty("instrumentId").GetInt64(), + IsBuy = e.GetProperty("isBuy").GetBoolean(), + Units = e.GetProperty("units").GetDouble(), + EntryPrice = e.GetProperty("entryPrice").GetDouble(), + PositionId = e.GetProperty("positionId").GetInt64(), + ClientRef = e.GetProperty("clientRef").GetString() ?? string.Empty, + OpenedUtc = DateTime.Parse(e.GetProperty("openedUtc").GetString()!, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal), + EntryFeesUsd = e.GetProperty("entryFeesUsd").GetDouble(), + StopLossRate = e.GetProperty("stopLossRate").GetDouble(), + }; + if (e.TryGetProperty("adds", out System.Text.Json.JsonElement adds)) + { + foreach (System.Text.Json.JsonElement a in adds.EnumerateArray()) + { + leg.Adds.Add((a.GetProperty("positionId").GetInt64(), a.GetProperty("units").GetDouble(), a.GetProperty("price").GetDouble(), a.GetProperty("clientRef").GetString() ?? string.Empty)); + } + } + + return leg; + } } /// An open (or opening/closing) basket: both legs plus what the decision knew at entry. diff --git a/Encelado/src/Encelado.Core/Baskets/EquityTracker.cs b/Encelado/src/Encelado.Core/Baskets/EquityTracker.cs new file mode 100644 index 0000000..4f498f0 --- /dev/null +++ b/Encelado/src/Encelado.Core/Baskets/EquityTracker.cs @@ -0,0 +1,89 @@ +using System.Globalization; + +namespace Encelado.Core.Baskets; + +/// A deposit, a withdrawal or a virtual credit: cash that moved without a trade. +public sealed record CashMovement(DateTime TimeUtc, double Amount, double BalanceBefore, double BalanceAfter, double ClosedNetInBetween, string Motivazione); + +/// +/// The equity the risk rules look at, kept clean of cash movements (§5.7 of the 5.0 +/// plan). A deposit raises the balance without any trade explaining it; a withdrawal +/// lowers it. Neither is a profit or a loss, so neither may move the peak the equity +/// stop is measured from, nor the day's starting point of the daily-loss rule. +/// +/// Detection: at every account refresh the change of the cash balance is compared with +/// the realised result of the positions closed in between. A residual beyond the +/// tolerance is a cash movement. The tolerance leaves room for overnight fees the venue +/// debits without a close; a 30 000 USD credit is unmistakable. +/// +/// +public sealed class EquityTracker +{ + /// Residuals below this are noise (fees, rounding), never a cash movement. + public double MinimumUsd { get; init; } = 10; + + /// Residuals below this share of the balance are noise. + public double TolerancePct { get; init; } = 0.0025; + + /// Highest net equity seen since the last reset. + public double PeakNetEquity { get; private set; } + + /// Sum of every cash movement seen since the tracker was created or restored. + public double CumulativeCashFlow { get; private set; } + + /// The cash balance at the last observation, NaN before the first. + public double LastBalance { get; private set; } = double.NaN; + + public DateTime LastObservedUtc { get; private set; } + + /// Equity without the cash that moved in or out: the number the drawdown is measured on. + public double NetEquity(double equity) => equity - CumulativeCashFlow; + + /// The peak expressed in today's account terms (net peak plus the cash that came in since). + public double PeakEquity => PeakNetEquity + CumulativeCashFlow; + + public double Drawdown(double equity) => PeakNetEquity > 0 ? Math.Max(0, (PeakNetEquity - NetEquity(equity)) / PeakNetEquity) : 0; + + /// + /// Records an account reading. is the realised + /// net result (profit minus fees) of the positions closed since the previous reading, + /// which is the only legitimate reason for the cash balance to move. + /// + public CashMovement? Observe(DateTime now, double balance, double equity, double closedNetSinceLast) + { + CashMovement? movement = null; + if (double.IsFinite(LastBalance)) + { + double residual = balance - LastBalance - closedNetSinceLast; + double tolerance = Math.Max(MinimumUsd, Math.Abs(balance) * TolerancePct); + if (Math.Abs(residual) > tolerance) + { + CumulativeCashFlow += residual; + movement = new CashMovement(now, residual, LastBalance, balance, closedNetSinceLast, string.Create(CultureInfo.InvariantCulture, + $"{(residual > 0 ? "accredito" : "prelievo")} di {Math.Abs(residual):F2} USD: saldo da {LastBalance:F2} a {balance:F2} con {closedNetSinceLast:+0.00;-0.00} USD di chiusure nel frattempo; picco e drawdown non ne tengono conto")); + } + } + + LastBalance = balance; + LastObservedUtc = now; + double net = NetEquity(equity); + if (net > PeakNetEquity) + { + PeakNetEquity = net; + } + + return movement; + } + + /// After a reset the peak restarts from the current equity. + public void ResetPeak(double equity) => PeakNetEquity = NetEquity(equity); + + /// Restores the persisted state; a peak saved by a version that knew no cash flows is taken as a net peak. + public void Restore(double peakNetEquity, double cumulativeCashFlow, double lastBalance, DateTime lastObservedUtc) + { + PeakNetEquity = Math.Max(0, peakNetEquity); + CumulativeCashFlow = double.IsFinite(cumulativeCashFlow) ? cumulativeCashFlow : 0; + LastBalance = lastBalance; + LastObservedUtc = lastObservedUtc; + } +} diff --git a/Encelado/src/Encelado.Core/Baskets/History/OrderRecord.cs b/Encelado/src/Encelado.Core/Baskets/History/OrderRecord.cs new file mode 100644 index 0000000..3b0c2bf --- /dev/null +++ b/Encelado/src/Encelado.Core/Baskets/History/OrderRecord.cs @@ -0,0 +1,127 @@ +using System.Globalization; +using System.Text; +using System.Text.Json; + +namespace Encelado.Core.Baskets.History; + +/// +/// One line of data/ledger/orders.jsonl: an order as sent, and every change of +/// its state afterwards (one line per change, append-only). The last line for a +/// client_ref is the order's current state; the first is what was asked. +/// +public sealed record OrderRecord( + DateTime Ts, + string RunId, + string Mode, + string Basket, + string BasketId, + string Symbol, + long InstrumentId, + bool IsBuy, + OrderLeg Leg, + double RequestedUnits, + double ExecutedUnits, + double RequestedPrice, + double FillRate, + double SlippagePips, + string Status, + int StatusId, + OrderResolution Resolution, + long OrderId, + long PositionId, + string ClientRef, + double Fees, + string Evento, + string Motivazione) +{ + /// The line for an order's current state. : inviato, stato, risolto. + public static OrderRecord From(TrackedOrder o, string runId, string evento, DateTime? ts = null) + { + ArgumentNullException.ThrowIfNull(o); + double pip = o.Symbol.Length >= 6 ? PipMath.Pip(o.Symbol) : 0.0001; + double slippage = o.RequestedPrice > 0 && o.FillRate > 0 ? (o.IsBuy ? o.FillRate - o.RequestedPrice : o.RequestedPrice - o.FillRate) / pip : double.NaN; + return new OrderRecord(ts ?? DateTime.UtcNow, runId, o.Mode, o.Basket, o.BasketId, o.Symbol, o.InstrumentId, o.IsBuy, o.Leg, + o.RequestedUnits, o.ExecutedUnits, o.RequestedPrice, o.FillRate, slippage, o.LastStatus, o.StatusId, o.Resolution, o.OrderId, o.PositionId, + o.ClientRef, o.Fees, evento, o.Motivazione.Length > 0 ? o.Motivazione : o.Error); + } + + public string ToJson() + { + using MemoryStream ms = new(); + using (Utf8JsonWriter w = new(ms)) + { + w.WriteStartObject(); + w.WriteString("ts", Ts.ToString("O", CultureInfo.InvariantCulture)); + w.WriteString("run_id", RunId); + w.WriteString("mode", Mode); + w.WriteString("basket", Basket); + w.WriteString("basket_id", BasketId); + w.WriteString("strumento", Symbol); + w.WriteNumber("instrument_id", InstrumentId); + w.WriteString("verso", IsBuy ? "long" : "short"); + w.WriteString("leg", Leg.ToString()); + w.WriteNumber("unita_richieste", Math.Round(RequestedUnits, 6)); + w.WriteNumber("unita_eseguite", Math.Round(ExecutedUnits, 6)); + Num(w, "prezzo_richiesto", RequestedPrice); + Num(w, "prezzo_eseguito", FillRate); + Num(w, "slippage_pip", SlippagePips); + w.WriteString("stato", Status); + w.WriteNumber("stato_id", StatusId); + w.WriteString("esito", Resolution.ToString()); + w.WriteNumber("order_id", OrderId); + w.WriteNumber("position_id", PositionId); + w.WriteString("client_ref", ClientRef); + Num(w, "fee", Fees); + w.WriteString("evento", Evento); + w.WriteString("motivazione", Motivazione); + w.WriteEndObject(); + } + + return Encoding.UTF8.GetString(ms.ToArray()); + + static void Num(Utf8JsonWriter w, string name, double v) + { + if (double.IsFinite(v) && v != 0) + { + w.WriteNumber(name, Math.Round(v, 8)); + } + else if (double.IsFinite(v)) + { + w.WriteNumber(name, 0); + } + else + { + w.WriteNull(name); + } + } + } + + public static OrderRecord? Parse(string line) + { + if (string.IsNullOrWhiteSpace(line)) + { + return null; + } + + try + { + using JsonDocument doc = JsonDocument.Parse(line); + JsonElement r = doc.RootElement; + return new OrderRecord( + Time(r, "ts"), S(r, "run_id"), S(r, "mode"), S(r, "basket"), S(r, "basket_id"), S(r, "strumento"), L(r, "instrument_id"), + S(r, "verso") == "long", Enum.TryParse(S(r, "leg"), out OrderLeg leg) ? leg : OrderLeg.A, + D(r, "unita_richieste"), D(r, "unita_eseguite"), D(r, "prezzo_richiesto"), D(r, "prezzo_eseguito"), D(r, "slippage_pip"), + S(r, "stato"), (int)L(r, "stato_id"), Enum.TryParse(S(r, "esito"), out OrderResolution res) ? res : OrderResolution.Pending, + L(r, "order_id"), L(r, "position_id"), S(r, "client_ref"), D(r, "fee"), S(r, "evento"), S(r, "motivazione")); + } + catch (JsonException) + { + return null; + } + + static string S(JsonElement e, string n) => e.TryGetProperty(n, out JsonElement v) && v.ValueKind == JsonValueKind.String ? v.GetString() ?? string.Empty : string.Empty; + static double D(JsonElement e, string n) => e.TryGetProperty(n, out JsonElement v) && v.ValueKind == JsonValueKind.Number ? v.GetDouble() : double.NaN; + static long L(JsonElement e, string n) => e.TryGetProperty(n, out JsonElement v) && v.ValueKind == JsonValueKind.Number ? v.GetInt64() : 0; + static DateTime Time(JsonElement e, string n) => DateTime.TryParse(S(e, n), CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal, out DateTime t) ? t : default; + } +} diff --git a/Encelado/src/Encelado.Core/Baskets/OrderTracker.cs b/Encelado/src/Encelado.Core/Baskets/OrderTracker.cs new file mode 100644 index 0000000..273a9d9 --- /dev/null +++ b/Encelado/src/Encelado.Core/Baskets/OrderTracker.cs @@ -0,0 +1,617 @@ +using System.Globalization; +using System.Text.Json; +using Encelado.Core.Broker; + +namespace Encelado.Core.Baskets; + +/// Which leg of a basket an order belongs to. +public enum OrderLeg +{ + A = 0, + B, + Add, + Close, + Unwind, +} + +/// How an order ended, or that it has not ended yet. +public enum OrderResolution +{ + Pending = 0, + Filled, + Rejected, + Cancelled, +} + +/// +/// One order the bot sent, from the moment before the HTTP call to the moment the +/// venue said what became of it. It is written to disk before the request goes +/// out, so a crash between the send and the answer cannot lose it. +/// +public sealed class TrackedOrder +{ + public required string ClientRef { get; init; } + + /// The venue's id, known once the submit answered. Zero when the answer was lost. + public long OrderId { get; set; } + + public required string Symbol { get; init; } + + public required long InstrumentId { get; init; } + + public required bool IsBuy { get; init; } + + public required double RequestedUnits { get; init; } + + public double ExecutedUnits { get; set; } + + public double RequestedPrice { get; init; } + + public double FillRate { get; set; } + + public double Fees { get; set; } + + /// The basket slot (EURUSD/USDCHF). + public required string Basket { get; init; } + + /// The basket instance (B20260923101500-EURUSDUSDCHF), when there is one. + public string BasketId { get; init; } = string.Empty; + + public required OrderLeg Leg { get; init; } + + public required DateTime SentUtc { get; init; } + + /// The venue's last word: Sent before any answer, then its own status names, Unknown when it answered nothing. + public string LastStatus { get; set; } = "Sent"; + + public int StatusId { get; set; } + + public OrderResolution Resolution { get; set; } + + public DateTime? ResolvedUtc { get; set; } + + public long PositionId { get; set; } + + public string Error { get; set; } = string.Empty; + + /// How many times the venue was asked. + public int Checks { get; set; } + + public DateTime LastCheckUtc { get; set; } + + /// How the resolution was established: venue, lookup, positions. + public string Source { get; set; } = string.Empty; + + public string Mode { get; init; } = string.Empty; + + public string Motivazione { get; init; } = string.Empty; + + public bool IsPending => Resolution == OrderResolution.Pending; + + public TimeSpan Age(DateTime now) => now - SentUtc; + + /// Copies what an outcome says into the record. Returns true when the order is now resolved. + public bool Apply(OrderOutcome outcome, DateTime now) + { + ArgumentNullException.ThrowIfNull(outcome); + if (outcome.OrderId > 0) + { + OrderId = outcome.OrderId; + } + + LastStatus = outcome.Status.Length > 0 ? outcome.Status : LastStatus; + StatusId = outcome.StatusId != 0 ? outcome.StatusId : StatusId; + if (outcome.Error.Length > 0) + { + Error = outcome.Error; + } + + if (outcome.Source.Length > 0) + { + Source = outcome.Source; + } + + if (outcome.Filled) + { + Resolution = OrderResolution.Filled; + ResolvedUtc = outcome.TimeUtc == default ? now : outcome.TimeUtc; + PositionId = outcome.PositionId != 0 ? outcome.PositionId : PositionId; + ExecutedUnits = outcome.Units > 0 ? outcome.Units : RequestedUnits; + FillRate = outcome.FillRate; + Fees = outcome.Fees; + return true; + } + + if (outcome.Rejected) + { + Resolution = outcome.StatusId is 7 or 9 || outcome.Status.Contains("cancel", StringComparison.OrdinalIgnoreCase) + ? OrderResolution.Cancelled + : OrderResolution.Rejected; + ResolvedUtc = now; + return true; + } + + return false; + } + + /// The outcome this record describes, for the code that waits on it. + public OrderOutcome ToOutcome() => new( + Resolution == OrderResolution.Filled, + Resolution is OrderResolution.Rejected or OrderResolution.Cancelled, + OrderId, PositionId, FillRate, ExecutedUnits, ResolvedUtc ?? SentUtc, Fees, LastStatus, Error) + { + RequestedUnits = RequestedUnits, + StatusId = StatusId, + Source = Source, + }; + + public string Describe() => string.Create(CultureInfo.InvariantCulture, + $"{Leg} {Basket} {(IsBuy ? "long" : "short")} {RequestedUnits:0.##} {Symbol} (ordine {OrderId}, rif. {ClientRef[..Math.Min(8, ClientRef.Length)]}): {LastStatus}{(Error.Length > 0 ? " — " + Error : string.Empty)}"); +} + +/// +/// The register of every order the bot sends and of what became of it (§5.1 of the 5.0 +/// plan). An order enters the register before the HTTP call and leaves the pending +/// set only when the venue says filled, rejected or cancelled, or when a position that +/// matches it appears on the account. The register is persisted to +/// data/state/pending_orders.json with atomic writes and reloaded at startup, so +/// nothing sent is ever forgotten across a restart. +/// +public sealed class OrderTracker +{ + /// A position opened this close to the send time, on the same instrument and side, is the order's fill. + public static readonly TimeSpan MatchWindow = TimeSpan.FromSeconds(90); + + /// Resolved orders are kept this long so an orphan closed later can still be recognised as ours. + public static readonly TimeSpan Retention = TimeSpan.FromHours(48); + + private readonly string _path; + private readonly Lock _gate = new(); + private readonly Dictionary _orders = new(StringComparer.Ordinal); + + public OrderTracker(string path) + { + _path = path ?? string.Empty; + Load(); + } + + /// Raised on every registration and every change, with the event name: inviato, stato, risolto. + public event Action? Changed; + + public string Path => _path; + + public IReadOnlyList Pending + { + get + { + lock (_gate) + { + return [.. _orders.Values.Where(static o => o.IsPending).OrderBy(static o => o.SentUtc)]; + } + } + } + + public IReadOnlyList All + { + get + { + lock (_gate) + { + return [.. _orders.Values.OrderBy(static o => o.SentUtc)]; + } + } + } + + public int PendingCount + { + get + { + lock (_gate) + { + return _orders.Values.Count(static o => o.IsPending); + } + } + } + + /// Registers an order about to be sent. Persisted before the caller may touch the network. + public TrackedOrder Register(TrackedOrder order) + { + ArgumentNullException.ThrowIfNull(order); + lock (_gate) + { + _orders[order.ClientRef] = order; + Save(); + } + + Changed?.Invoke(order, "inviato"); + return order; + } + + /// Copies an outcome into the order's record and persists it. Returns true when the order is now resolved. + public bool Apply(TrackedOrder order, OrderOutcome outcome, DateTime now) + { + ArgumentNullException.ThrowIfNull(order); + bool resolved; + lock (_gate) + { + resolved = order.Apply(outcome, now); + Save(); + } + + Changed?.Invoke(order, resolved ? "risolto" : "stato"); + return resolved; + } + + public TrackedOrder? Find(string clientRef) + { + lock (_gate) + { + return _orders.GetValueOrDefault(clientRef); + } + } + + public TrackedOrder? FindByOrderId(long orderId) + { + lock (_gate) + { + return orderId > 0 ? _orders.Values.FirstOrDefault(o => o.OrderId == orderId) : null; + } + } + + /// The order that opened this position, when the bot sent it. + public TrackedOrder? FindByPosition(long positionId) + { + lock (_gate) + { + return positionId > 0 ? _orders.Values.FirstOrDefault(o => o.PositionId == positionId && o.Leg is not (OrderLeg.Close or OrderLeg.Unwind)) : null; + } + } + + /// Positions the register knows the bot opened, with the basket they belong to. + public IReadOnlyDictionary OpenedPositions() + { + lock (_gate) + { + Dictionary map = []; + foreach (TrackedOrder o in _orders.Values) + { + if (o.Resolution == OrderResolution.Filled && o.PositionId > 0 && o.Leg is not (OrderLeg.Close or OrderLeg.Unwind)) + { + map[o.PositionId] = o.Basket; + } + } + + return map; + } + } + + /// Drops resolved orders older than . Pending ones are never dropped. + public int Prune(DateTime now) + { + lock (_gate) + { + List gone = [.. _orders.Values.Where(o => !o.IsPending && o.ResolvedUtc is { } r && now - r > Retention).Select(static o => o.ClientRef)]; + foreach (string key in gone) + { + _orders.Remove(key); + } + + if (gone.Count > 0) + { + Save(); + } + + return gone.Count; + } + } + + /// + /// Asks the venue about every pending order that is due for a check: by orderId + /// when the submit answered, by client reference otherwise, and — when the venue has no + /// record of either — by matching the position list (same instrument and side, opened + /// within of the send). Returns the orders resolved by this call. + /// + public async Task> ResolveAsync(IBroker broker, DateTime now, Func? isKnownPosition, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(broker); + List<(TrackedOrder, OrderOutcome)> resolved = []; + IReadOnlyList? positions = null; + HashSet claimed; + lock (_gate) + { + claimed = [.. _orders.Values.Where(static o => o.PositionId > 0).Select(static o => o.PositionId)]; + } + + foreach (TrackedOrder order in Pending) + { + if (!IsDue(order, now)) + { + continue; + } + + order.LastCheckUtc = now; + order.Checks++; + + OrderOutcome? outcome = null; + try + { + if (order.OrderId > 0) + { + outcome = await broker.LookupOrderByIdAsync(order.OrderId, ct).ConfigureAwait(false); + } + + outcome ??= await broker.LookupOrderAsync(order.ClientRef, ct).ConfigureAwait(false); + } + catch (BrokerException) + { + continue; + } + + if (outcome is { Pending: false }) + { + Apply(order, outcome with { Source = outcome.Source.Length > 0 ? outcome.Source : "lookup" }, now); + if (outcome.PositionId > 0) + { + claimed.Add(outcome.PositionId); + } + + resolved.Add((order, order.ToOutcome())); + continue; + } + + if (outcome is not null) + { + Apply(order, outcome, now); + continue; + } + + // The venue has no record of the order under either key. If a position that + // fits it appeared on the account, that position is the fill. + if (now - order.SentUtc < TimeSpan.FromSeconds(2)) + { + continue; + } + + try + { + positions ??= await broker.GetPositionsAsync(ct).ConfigureAwait(false); + } + catch (BrokerException) + { + continue; + } + + BrokerPosition? match = Match(order, positions, id => claimed.Contains(id) || (isKnownPosition?.Invoke(id) ?? false)); + if (match is not null) + { + claimed.Add(match.PositionId); + OrderOutcome matched = new(true, false, order.OrderId, match.PositionId, match.OpenRate, match.Units, match.OpenedUtc, match.Fees, "Filled", + string.Create(CultureInfo.InvariantCulture, $"esito ricostruito dalla posizione {match.PositionId} aperta {(match.OpenedUtc - order.SentUtc).TotalSeconds:+0;-0} s dopo l'invio")) + { + RequestedUnits = order.RequestedUnits, + StatusId = 3, + Source = "positions", + }; + Apply(order, matched, now); + resolved.Add((order, order.ToOutcome())); + } + else + { + Apply(order, OrderOutcome.Unknown(order.OrderId, order.RequestedUnits, string.Create(CultureInfo.InvariantCulture, $"nessuna traccia dopo {order.Checks} verifiche")), now); + } + } + + return resolved; + } + + /// + /// The position that fits an order: same instrument, same side, opened within the + /// window around the send, not already accounted for. When several fit, the one + /// closest in time. Units are reported in the reason but not required to agree: the + /// venue may have reduced the order (observed on 2026-09-16, see the post-mortem). + /// + public static BrokerPosition? Match(TrackedOrder order, IReadOnlyList positions, Func excluded) + { + ArgumentNullException.ThrowIfNull(order); + ArgumentNullException.ThrowIfNull(positions); + ArgumentNullException.ThrowIfNull(excluded); + BrokerPosition? best = null; + double bestDistance = double.MaxValue; + foreach (BrokerPosition p in positions) + { + if (p.InstrumentId != order.InstrumentId || p.IsBuy != order.IsBuy || excluded(p.PositionId)) + { + continue; + } + + double distance = Math.Abs((p.OpenedUtc - order.SentUtc).TotalSeconds); + if (distance > MatchWindow.TotalSeconds) + { + continue; + } + + if (distance < bestDistance) + { + best = p; + bestDistance = distance; + } + } + + return best; + } + + /// Fresh orders are checked every two seconds; after a minute every ten; after ten minutes every minute. + private static bool IsDue(TrackedOrder order, DateTime now) + { + TimeSpan age = order.Age(now); + TimeSpan interval = age < TimeSpan.FromMinutes(1) ? TimeSpan.FromSeconds(2) + : age < TimeSpan.FromMinutes(10) ? TimeSpan.FromSeconds(10) + : TimeSpan.FromMinutes(1); + return now - order.LastCheckUtc >= interval; + } + + // ----------------------------------------------------------------------- + // Disk + // ----------------------------------------------------------------------- + + private void Save() + { + if (_path.Length == 0) + { + return; + } + + using MemoryStream ms = new(); + using (Utf8JsonWriter w = new(ms, new JsonWriterOptions { Indented = true })) + { + w.WriteStartObject(); + w.WriteString("savedUtc", DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture)); + w.WriteStartArray("orders"); + foreach (TrackedOrder o in _orders.Values.OrderBy(static o => o.SentUtc)) + { + Write(w, o); + } + + w.WriteEndArray(); + w.WriteEndObject(); + } + + string? dir = System.IO.Path.GetDirectoryName(_path); + if (!string.IsNullOrEmpty(dir)) + { + Directory.CreateDirectory(dir); + } + + File.WriteAllBytes(_path + ".tmp", ms.ToArray()); + File.Move(_path + ".tmp", _path, overwrite: true); + } + + private void Load() + { + if (_path.Length == 0 || !File.Exists(_path)) + { + return; + } + + try + { + using JsonDocument doc = JsonDocument.Parse(File.ReadAllBytes(_path)); + if (!doc.RootElement.TryGetProperty("orders", out JsonElement arr)) + { + return; + } + + foreach (JsonElement e in arr.EnumerateArray()) + { + TrackedOrder? o = Read(e); + if (o is not null) + { + _orders[o.ClientRef] = o; + } + } + } + catch (Exception ex) when (ex is IOException or JsonException or KeyNotFoundException or FormatException or InvalidOperationException) + { + // A register that cannot be read is worse than none: keep the file aside for a person to look at. + try + { + File.Move(_path, _path + ".illeggibile-" + DateTime.UtcNow.ToString("yyyyMMddHHmmss", CultureInfo.InvariantCulture), overwrite: true); + } + catch (IOException) + { + // Nothing else to do. + } + } + } + + public static void Write(Utf8JsonWriter w, TrackedOrder o) + { + ArgumentNullException.ThrowIfNull(w); + ArgumentNullException.ThrowIfNull(o); + w.WriteStartObject(); + w.WriteString("clientRef", o.ClientRef); + w.WriteNumber("orderId", o.OrderId); + w.WriteString("symbol", o.Symbol); + w.WriteNumber("instrumentId", o.InstrumentId); + w.WriteBoolean("isBuy", o.IsBuy); + w.WriteNumber("requestedUnits", o.RequestedUnits); + w.WriteNumber("executedUnits", o.ExecutedUnits); + w.WriteNumber("requestedPrice", o.RequestedPrice); + w.WriteNumber("fillRate", o.FillRate); + w.WriteNumber("fees", o.Fees); + w.WriteString("basket", o.Basket); + w.WriteString("basketId", o.BasketId); + w.WriteString("leg", o.Leg.ToString()); + w.WriteString("sentUtc", o.SentUtc.ToString("O", CultureInfo.InvariantCulture)); + w.WriteString("lastStatus", o.LastStatus); + w.WriteNumber("statusId", o.StatusId); + w.WriteString("resolution", o.Resolution.ToString()); + w.WriteString("resolvedUtc", o.ResolvedUtc?.ToString("O", CultureInfo.InvariantCulture) ?? string.Empty); + w.WriteNumber("positionId", o.PositionId); + w.WriteString("error", o.Error); + w.WriteNumber("checks", o.Checks); + w.WriteString("lastCheckUtc", o.LastCheckUtc == default ? string.Empty : o.LastCheckUtc.ToString("O", CultureInfo.InvariantCulture)); + w.WriteString("source", o.Source); + w.WriteString("mode", o.Mode); + w.WriteString("motivazione", o.Motivazione); + w.WriteEndObject(); + } + + public static TrackedOrder? Read(JsonElement e) + { + string clientRef = Str(e, "clientRef"); + if (clientRef.Length == 0) + { + return null; + } + + TrackedOrder o = new() + { + ClientRef = clientRef, + OrderId = Num(e, "orderId"), + Symbol = Str(e, "symbol"), + InstrumentId = Num(e, "instrumentId"), + IsBuy = e.TryGetProperty("isBuy", out JsonElement b) && b.GetBoolean(), + RequestedUnits = Num(e, "requestedUnits"), + RequestedPrice = Num(e, "requestedPrice"), + Basket = Str(e, "basket"), + BasketId = Str(e, "basketId"), + Leg = Enum.TryParse(Str(e, "leg"), out OrderLeg leg) ? leg : OrderLeg.A, + SentUtc = Time(e, "sentUtc") ?? DateTime.UtcNow, + Mode = Str(e, "mode"), + Motivazione = Str(e, "motivazione"), + }; + o.ExecutedUnits = Num(e, "executedUnits"); + o.FillRate = Num(e, "fillRate"); + o.Fees = Num(e, "fees"); + o.LastStatus = Str(e, "lastStatus", "Sent"); + o.StatusId = (int)Num(e, "statusId"); + o.Resolution = Enum.TryParse(Str(e, "resolution"), out OrderResolution r) ? r : OrderResolution.Pending; + o.ResolvedUtc = Time(e, "resolvedUtc"); + o.PositionId = Num(e, "positionId"); + o.Error = Str(e, "error"); + o.Checks = (int)Num(e, "checks"); + o.LastCheckUtc = Time(e, "lastCheckUtc") ?? default; + o.Source = Str(e, "source"); + return o; + + static string Str(JsonElement e, string name, string fallback = "") => + e.TryGetProperty(name, out JsonElement v) && v.ValueKind == JsonValueKind.String ? v.GetString() ?? fallback : fallback; + + static T Num(JsonElement e, string name) where T : struct + { + if (!e.TryGetProperty(name, out JsonElement v) || v.ValueKind != JsonValueKind.Number) + { + return default; + } + + return typeof(T) == typeof(long) ? (T)(object)v.GetInt64() : (T)(object)v.GetDouble(); + } + + static DateTime? Time(JsonElement e, string name) + { + string s = Str(e, name); + return s.Length > 0 && DateTime.TryParse(s, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal, out DateTime t) ? t : null; + } + } +} diff --git a/Encelado/src/Encelado.Core/Baskets/PendingEntry.cs b/Encelado/src/Encelado.Core/Baskets/PendingEntry.cs new file mode 100644 index 0000000..77e4957 --- /dev/null +++ b/Encelado/src/Encelado.Core/Baskets/PendingEntry.cs @@ -0,0 +1,102 @@ +using System.Globalization; +using System.Text.Json; + +namespace Encelado.Core.Baskets; + +/// +/// An entry the decider approved whose execution is not finished: leg A is on its way +/// (state PendingA) or filled while leg B is on its way (PendingB). Holds +/// everything needed to finish the basket later — or to undo leg A — without the +/// original decision object, which does not survive a restart. +/// +public sealed class PendingEntry +{ + public required string BasketId { get; init; } + + public required bool BuyCross { get; init; } + + public required double EntryZ { get; init; } + + public required double UnitsA { get; init; } + + public required double UnitsB { get; init; } + + public required double TpPips { get; init; } + + public required double MaxLossUsd { get; init; } + + public double EntryCostPips { get; init; } = double.NaN; + + public required double EquityAtEntry { get; init; } + + public string Motivazione { get; init; } = string.Empty; + + public required DateTime DecidedUtc { get; init; } + + public string ClientRefA { get; set; } = string.Empty; + + public string ClientRefB { get; set; } = string.Empty; + + /// The quotes seen at decision time, for the slippage of each leg. + public double QuoteA { get; init; } + + public double QuoteB { get; init; } + + /// Leg A as filled, once it is. + public BasketLeg? LegA { get; set; } + + public void Write(Utf8JsonWriter w) + { + ArgumentNullException.ThrowIfNull(w); + w.WriteStartObject("pending"); + w.WriteString("basketId", BasketId); + w.WriteBoolean("buyCross", BuyCross); + w.WriteNumber("entryZ", EntryZ); + w.WriteNumber("unitsA", UnitsA); + w.WriteNumber("unitsB", UnitsB); + w.WriteNumber("tpPips", TpPips); + w.WriteNumber("maxLossUsd", MaxLossUsd); + w.WriteNumber("entryCostPips", double.IsFinite(EntryCostPips) ? EntryCostPips : 0); + w.WriteNumber("equityAtEntry", EquityAtEntry); + w.WriteString("motivazione", Motivazione); + w.WriteString("decidedUtc", DecidedUtc.ToString("O", CultureInfo.InvariantCulture)); + w.WriteString("clientRefA", ClientRefA); + w.WriteString("clientRefB", ClientRefB); + w.WriteNumber("quoteA", QuoteA); + w.WriteNumber("quoteB", QuoteB); + if (LegA is { } leg) + { + BasketLeg.Write(w, "legA", leg); + } + + w.WriteEndObject(); + } + + public static PendingEntry Read(JsonElement e) + { + PendingEntry p = new() + { + BasketId = e.GetProperty("basketId").GetString() ?? string.Empty, + BuyCross = e.GetProperty("buyCross").GetBoolean(), + EntryZ = e.GetProperty("entryZ").GetDouble(), + UnitsA = e.GetProperty("unitsA").GetDouble(), + UnitsB = e.GetProperty("unitsB").GetDouble(), + TpPips = e.GetProperty("tpPips").GetDouble(), + MaxLossUsd = e.GetProperty("maxLossUsd").GetDouble(), + EntryCostPips = e.GetProperty("entryCostPips").GetDouble(), + EquityAtEntry = e.GetProperty("equityAtEntry").GetDouble(), + Motivazione = e.GetProperty("motivazione").GetString() ?? string.Empty, + DecidedUtc = DateTime.Parse(e.GetProperty("decidedUtc").GetString()!, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal), + QuoteA = e.TryGetProperty("quoteA", out JsonElement qa) ? qa.GetDouble() : 0, + QuoteB = e.TryGetProperty("quoteB", out JsonElement qb) ? qb.GetDouble() : 0, + }; + p.ClientRefA = e.TryGetProperty("clientRefA", out JsonElement ca) ? ca.GetString() ?? string.Empty : string.Empty; + p.ClientRefB = e.TryGetProperty("clientRefB", out JsonElement cb) ? cb.GetString() ?? string.Empty : string.Empty; + if (e.TryGetProperty("legA", out JsonElement la)) + { + p.LegA = BasketLeg.Read(la); + } + + return p; + } +} diff --git a/Encelado/src/Encelado.Core/Baskets/PositionClassifier.cs b/Encelado/src/Encelado.Core/Baskets/PositionClassifier.cs new file mode 100644 index 0000000..f03b42b --- /dev/null +++ b/Encelado/src/Encelado.Core/Baskets/PositionClassifier.cs @@ -0,0 +1,104 @@ +using System.Globalization; +using Encelado.Core.Broker; + +namespace Encelado.Core.Baskets; + +/// Whose a position on the account is. +public enum PositionOrigin +{ + /// A leg of a basket the engine knows. + Basket = 0, + + /// Opened by the bot (the order register or the ledger say so) but belonging to no basket: adopted and closed. + OrphanBot, + + /// Opened by someone else: reported, never touched unless the kill-switch is told to. + Foreign, +} + +/// +/// An entry the bot decided or sent, as the ledger recorded it: enough to recognise +/// the position it may have produced (same instrument and side, opened within the +/// window). Comes from the segnale_ingresso, rifiuto, ingresso and +/// pending rows of decisions.jsonl, and from the order register. +/// +public sealed record EntrySignature(DateTime TimeUtc, long InstrumentId, string Symbol, bool IsBuy, double Units, string Basket, string Source); + +public sealed record ClassifiedPosition(BrokerPosition Position, PositionOrigin Origin, string Basket, string Reason); + +/// +/// Sorts the positions on the account into basket, orfana-bot and +/// esterna (§5.4 of the 5.0 plan). Certainty first: a position id that a basket +/// or the order register holds. Then the signature: same instrument, same side, opened +/// within the window of an entry the bot decided. Units within tolerance strengthen the +/// reason but are not required, because the venue may reduce an order (post-mortem). +/// Everything else is foreign. +/// +public static class PositionClassifier +{ + public static readonly TimeSpan DefaultWindow = TimeSpan.FromSeconds(90); + + public static List Classify( + IReadOnlyList positions, + IReadOnlyDictionary basketLegs, + IReadOnlyDictionary trackedLegs, + IReadOnlyList signatures, + Func? symbolOf = null, + TimeSpan? window = null, + double unitsTolerance = 0.01) + { + ArgumentNullException.ThrowIfNull(positions); + ArgumentNullException.ThrowIfNull(basketLegs); + ArgumentNullException.ThrowIfNull(trackedLegs); + ArgumentNullException.ThrowIfNull(signatures); + TimeSpan w = window ?? DefaultWindow; + List result = new(positions.Count); + + foreach (BrokerPosition p in positions) + { + if (basketLegs.TryGetValue(p.PositionId, out string? basket)) + { + result.Add(new ClassifiedPosition(p, PositionOrigin.Basket, basket, "gamba di un basket noto")); + continue; + } + + if (trackedLegs.TryGetValue(p.PositionId, out string? tracked)) + { + result.Add(new ClassifiedPosition(p, PositionOrigin.OrphanBot, tracked, "aperta da un ordine del registro che non appartiene a nessun basket aperto")); + continue; + } + + EntrySignature? best = null; + double bestDistance = double.MaxValue; + foreach (EntrySignature s in signatures) + { + bool sameInstrument = s.InstrumentId != 0 && s.InstrumentId == p.InstrumentId + || (s.InstrumentId == 0 && symbolOf?.Invoke(p.InstrumentId) is { } sym && sym.Equals(s.Symbol, StringComparison.OrdinalIgnoreCase)); + if (!sameInstrument || s.IsBuy != p.IsBuy) + { + continue; + } + + double distance = Math.Abs((p.OpenedUtc - s.TimeUtc).TotalSeconds); + if (distance <= w.TotalSeconds && distance < bestDistance) + { + best = s; + bestDistance = distance; + } + } + + if (best is not null) + { + bool unitsAgree = best.Units > 0 && Math.Abs(p.Units - best.Units) <= unitsTolerance * best.Units; + string reason = string.Create(CultureInfo.InvariantCulture, + $"firma del bot: {best.Source} di {best.Basket} alle {best.TimeUtc:HH:mm:ss} UTC ({bestDistance:0} s), {(unitsAgree ? "unità coerenti" : $"unità {p.Units:0.##} contro {best.Units:0.##} richieste")}"); + result.Add(new ClassifiedPosition(p, PositionOrigin.OrphanBot, best.Basket, reason)); + continue; + } + + result.Add(new ClassifiedPosition(p, PositionOrigin.Foreign, string.Empty, "nessun basket, nessun ordine del registro, nessuna decisione del ledger coerente")); + } + + return result; + } +} diff --git a/Encelado/src/Encelado.Core/Broker/Broker.cs b/Encelado/src/Encelado.Core/Broker/Broker.cs index 9bb4a22..9fe1ee6 100644 --- a/Encelado/src/Encelado.Core/Broker/Broker.cs +++ b/Encelado/src/Encelado.Core/Broker/Broker.cs @@ -96,6 +96,12 @@ public sealed record OrderRequest( double? TakeProfitRate, string Reason); +/// +/// What became of an order. is the venue's own word (or +/// Unknown when the venue said nothing): never a name the venue did not use. +/// are the units the venue executed; +/// what was asked, kept because the venue may reduce an order instead of rejecting it. +/// public sealed record OrderOutcome( bool Filled, bool Rejected, @@ -109,6 +115,19 @@ public sealed record OrderOutcome( string Error) { public bool Pending => !Filled && !Rejected; + + /// Units the order asked for. Zero when not known. + public double RequestedUnits { get; init; } + + /// The venue's numeric status (1 Received … 12 PendingTriggeredRate on eToro), 0 when not known. + public int StatusId { get; init; } + + /// How the outcome was established: lookup, positions (matched on the position list), venue. + public string Source { get; init; } = string.Empty; + + /// The venue has not said what became of the order: it stays pending and tracked, never assumed. + public static OrderOutcome Unknown(long orderId, double requestedUnits, string error) => + new(false, false, orderId, 0, 0, 0, DateTime.UtcNow, 0, "Unknown", error) { RequestedUnits = requestedUnits }; } public sealed record CloseOutcome( @@ -182,6 +201,12 @@ public interface IBroker : IAsyncDisposable /// Asks what became of an order sent with this ClientRef. Null when the venue has no record of it. Task LookupOrderAsync(string clientRef, CancellationToken ct); + /// Asks what became of an order by the venue's own id. Null when the venue has no record of it. + Task LookupOrderByIdAsync(long orderId, CancellationToken ct); + + /// Asks the venue to cancel an order that has not executed yet. True when the request was accepted; the outcome is confirmed by a lookup. + Task CancelOrderAsync(long orderId, CancellationToken ct); + Task CloseAsync(long positionId, long instrumentId, CancellationToken ct); Task UpdateStopsAsync(long positionId, double? stopLoss, double? takeProfit, CancellationToken ct); diff --git a/Encelado/src/Encelado.Core/Broker/PaperBroker.cs b/Encelado/src/Encelado.Core/Broker/PaperBroker.cs index 6eefa4b..b1eb9ce 100644 --- a/Encelado/src/Encelado.Core/Broker/PaperBroker.cs +++ b/Encelado/src/Encelado.Core/Broker/PaperBroker.cs @@ -279,6 +279,17 @@ public sealed class PaperBroker : IBroker } } + public Task LookupOrderByIdAsync(long orderId, CancellationToken ct) + { + lock (_gate) + { + return Task.FromResult(_orders.Values.FirstOrDefault(o => o.OrderId == orderId)); + } + } + + /// The simulator fills at once: there is never an order to cancel. + public Task CancelOrderAsync(long orderId, CancellationToken ct) => Task.FromResult(false); + public Task CloseAsync(long positionId, long instrumentId, CancellationToken ct) { CloseOutcome? outcome = CloseInternal(positionId, "chiusura richiesta"); diff --git a/Encelado/src/Encelado.Etoro/EtoroBroker.cs b/Encelado/src/Encelado.Etoro/EtoroBroker.cs index bc4e501..423bd04 100644 --- a/Encelado/src/Encelado.Etoro/EtoroBroker.cs +++ b/Encelado/src/Encelado.Etoro/EtoroBroker.cs @@ -12,9 +12,13 @@ namespace Encelado.Etoro; /// the OpenAPI document served by the API on 2026-09-16 (v1.379.0); the exact paths live /// in so a change on the venue's side is a one-line fix. /// -/// Orders are asynchronous on the venue: a 200 on the submit means "received". The -/// outcome comes from the lookup route keyed by our own x-request-id, which is why -/// the engine mints one GUID per attempt and keeps it. +/// Orders are asynchronous on the venue: a 200 on the submit means "received" and +/// carries the orderId. The outcome comes from the lookup route keyed by that +/// id. The client reference (x-request-id) is kept for idempotency, but the venue +/// does not register it for v2 orders (verified 2026-09-23: referenceID +/// comes back all zeros), so a lookup by reference only serves when the submit's answer +/// was lost. When the venue has no record under either key, a position that appeared on +/// the same instrument and side right after the send is the fill. /// /// public sealed class EtoroBroker : IBroker @@ -73,6 +77,12 @@ public sealed class EtoroBroker : IBroker public string OrdersLookup => $"api/v2/trading/info/{_d}orders:lookup"; + /// The v1 order-information route, keyed by the venue's id (verified 2026-09-23). + public string OrderById(long orderId) => demo ? $"api/v1/trading/info/demo/orders/{orderId}" : $"api/v1/trading/info/real/orders/{orderId}"; + + /// Cancels an order that has not executed yet (verified 2026-09-23; 200 = request accepted, outcome via lookup). + public string CancelOrder(long orderId) => $"api/v2/trading/execution/{_d}orders/{orderId}"; + public string ClosePosition(long positionId) => $"api/v1/trading/execution/{_d}market-close-orders/positions/{positionId}"; public string CloseOrderInfo(long orderId) => demo ? $"api/v1/trading/info/demo/close-orders/{orderId}" : $"api/v1/trading/info/real/close-orders/{orderId}"; @@ -429,12 +439,26 @@ public sealed class EtoroBroker : IBroker body.Append('}'); + // The positions before the send: one that appears afterwards on this instrument + // and side is this order's fill even when the lookup has no record of it. + HashSet before = []; + try + { + before = [.. (await PortfolioAsync(ct).ConfigureAwait(false)).Positions.Select(static p => p.PositionId)]; + } + catch (BrokerException) + { + // Matching will fall back on the time window alone. + } + + DateTime sentUtc = DateTime.UtcNow; EtoroResponse r = await _http.SendAsync(HttpMethod.Post, P.Orders, body.ToString(), EtoroQuota.Trading, ct, request.ClientRef, retries: 0).ConfigureAwait(false); if (!r.IsSuccess) { string why = Problem(r); OnLog?.Invoke($"ordine {request.Symbol} rifiutato alla sottomissione ({(int)r.Status}): {why}", null); - return new OrderOutcome(false, (int)r.Status is >= 400 and < 500, 0, 0, 0, request.Units, DateTime.UtcNow, 0, "Rejected", why); + bool rejected = (int)r.Status is >= 400 and < 500; + return new OrderOutcome(false, rejected, 0, 0, 0, 0, DateTime.UtcNow, 0, rejected ? "Rejected" : "Unknown", why) { RequestedUnits = request.Units, Source = "venue" }; } long orderId; @@ -443,20 +467,85 @@ public sealed class EtoroBroker : IBroker orderId = Json.Long(doc.RootElement, "orderId"); } - // The venue works the order asynchronously: poll until filled, rejected or timed out. - DateTime deadline = DateTime.UtcNow.AddSeconds(_options.FillTimeoutSeconds); - OrderOutcome? last = null; + // The venue works the order asynchronously: ask by orderId until filled, rejected + // or timed out; after two seconds without a record, look at the positions too. + OrderOutcome last = new(false, false, orderId, 0, 0, 0, DateTime.UtcNow, 0, "Submitted", "accettato dal server, esito non ancora letto") { RequestedUnits = request.Units, Source = "venue" }; + DateTime deadline = sentUtc.AddSeconds(_options.FillTimeoutSeconds); while (DateTime.UtcNow < deadline) { - await Task.Delay(400, ct).ConfigureAwait(false); - last = await LookupOrderAsync(request.ClientRef, ct).ConfigureAwait(false); - if (last is { Pending: false }) + await Task.Delay(600, ct).ConfigureAwait(false); + OrderOutcome? looked = orderId > 0 + ? await LookupOrderByIdAsync(orderId, ct).ConfigureAwait(false) + : await LookupOrderAsync(request.ClientRef, ct).ConfigureAwait(false); + if (looked is not null) { - return last; + last = looked with { RequestedUnits = request.Units, OrderId = looked.OrderId > 0 ? looked.OrderId : orderId }; + if (!looked.Pending) + { + return last; + } + + continue; + } + + if (DateTime.UtcNow - sentUtc < TimeSpan.FromSeconds(2)) + { + continue; + } + + OrderOutcome? matched = await MatchPositionAsync(request, sentUtc, orderId, before, ct).ConfigureAwait(false); + if (matched is not null) + { + return matched; } } - return last ?? new OrderOutcome(false, false, orderId, 0, 0, request.Units, DateTime.UtcNow, 0, "Received", "esito non ancora noto"); + return last with { Error = string.Create(CultureInfo.InvariantCulture, $"esito non noto dopo {_options.FillTimeoutSeconds} s (ultimo stato {last.Status}): resta nel registro degli ordini") }; + } + + /// A position on the instrument and side of the request, opened within 90 s of the send and not there before: the fill. + private async Task MatchPositionAsync(OrderRequest request, DateTime sentUtc, long orderId, HashSet before, CancellationToken ct) + { + IReadOnlyList positions; + try + { + positions = (await PortfolioAsync(ct).ConfigureAwait(false)).Positions; + } + catch (BrokerException) + { + return null; + } + + BrokerPosition? best = null; + double bestDistance = double.MaxValue; + foreach (BrokerPosition p in positions) + { + if (p.InstrumentId != request.InstrumentId || p.IsBuy != request.IsBuy || before.Contains(p.PositionId)) + { + continue; + } + + double distance = Math.Abs((p.OpenedUtc - sentUtc).TotalSeconds); + if (distance <= 90 && distance < bestDistance) + { + best = p; + bestDistance = distance; + } + } + + if (best is null) + { + return null; + } + + OnLog?.Invoke(string.Create(CultureInfo.InvariantCulture, $"ordine {request.Symbol} {orderId}: il server non lo trova per id, ma la posizione {best.PositionId} è comparsa {bestDistance:0} s dopo l'invio: la prendo come esecuzione"), null); + return new OrderOutcome(true, false, orderId, best.PositionId, best.OpenRate, best.Units, best.OpenedUtc == default ? DateTime.UtcNow : best.OpenedUtc, best.Fees, "Filled", + string.Create(CultureInfo.InvariantCulture, $"esito ricostruito dalla posizione {best.PositionId}")) + { + RequestedUnits = request.Units, + StatusId = 3, + Source = "positions", + }; } public async Task LookupOrderAsync(string clientRef, CancellationToken ct) @@ -473,7 +562,67 @@ public sealed class EtoroBroker : IBroker throw Error("esito ordine", r); } - using JsonDocument doc = JsonDocument.Parse(r.Body); + return ParseLookup(r.Body); + } + + /// By the venue's id: the v2 lookup first, the v1 order-information route as a fallback. Null when neither has a record. + public async Task LookupOrderByIdAsync(long orderId, CancellationToken ct) + { + if (orderId <= 0) + { + return null; + } + + EtoroResponse r = await _http.SendAsync(HttpMethod.Get, $"{P.OrdersLookup}?orderId={orderId.ToString(CultureInfo.InvariantCulture)}", null, EtoroQuota.Lookup, ct).ConfigureAwait(false); + if (r.IsSuccess) + { + return ParseLookup(r.Body); + } + + if ((int)r.Status != 404) + { + throw Error("esito ordine per id", r); + } + + EtoroResponse v1 = await _http.SendAsync(HttpMethod.Get, P.OrderById(orderId), null, EtoroQuota.Lookup, ct).ConfigureAwait(false); + if ((int)v1.Status == 404) + { + return null; + } + + if (!v1.IsSuccess) + { + throw Error("esito ordine per id (v1)", v1); + } + + return ParseOrderInfoV1(v1.Body, orderId); + } + + public async Task CancelOrderAsync(long orderId, CancellationToken ct) + { + if (!SupportsTrading || orderId <= 0) + { + return false; + } + + EtoroResponse r = await _http.SendAsync(HttpMethod.Delete, P.CancelOrder(orderId), null, EtoroQuota.Trading, ct, retries: 0).ConfigureAwait(false); + if (!r.IsSuccess) + { + OnLog?.Invoke($"annullamento dell'ordine {orderId} non accettato ({(int)r.Status}): {Problem(r)}", null); + } + + return r.IsSuccess; + } + + /// + /// The v2 lookup body. Status ids (verified on the OpenAPI document, 2026-09-23): + /// 1 Received, 2 Placed, 3 Filled, 4 Rejected, 5 PartiallyFilled, 6 PendingCancel, + /// 7 Canceled, 8 Expired, 9 CanceledPartiallyFilled, 10 RejectedPartiallyFilled, + /// 11 WaitingForMarket, 12 PendingTriggeredRate. + /// + private static OrderOutcome ParseLookup(string body) + { + using JsonDocument doc = JsonDocument.Parse(body); JsonElement root = doc.RootElement; long orderId = Json.Long(root, "orderId"); int statusId = 0; @@ -497,13 +646,13 @@ public sealed class EtoroBroker : IBroker long positionId = 0; double avgPrice = 0; - double units = requestedUnits; + double units = 0; double fees = 0; DateTime time = Json.Time(root, "lastUpdate"); foreach (JsonElement pe in Json.Array(root, "positionExecutions")) { positionId = Json.Long(pe, "positionId"); - units = Json.Double(pe, "remainingUnits", requestedUnits); + units = Json.Double(pe, "remainingUnits", 0); if (Json.TryGet(pe, "openingData", out JsonElement od)) { avgPrice = Json.Double(od, "avgPrice"); @@ -524,7 +673,71 @@ public sealed class EtoroBroker : IBroker break; } - return new OrderOutcome(filled, rejected, orderId, positionId, avgPrice, units, time == default ? DateTime.UtcNow : time, fees, statusName.Length > 0 ? statusName : statusId.ToString(CultureInfo.InvariantCulture), error); + return new OrderOutcome(filled, rejected, orderId, positionId, avgPrice, units > 0 ? units : (filled ? requestedUnits : 0), time == default ? DateTime.UtcNow : time, fees, + statusName.Length > 0 ? statusName : statusId.ToString(CultureInfo.InvariantCulture), error) + { + RequestedUnits = requestedUnits, + StatusId = statusId, + Source = "lookup", + }; + } + + /// The v1 order-information body: statusID, errorCode, positions[] with positionID, rate, units, occurred. + private static OrderOutcome ParseOrderInfoV1(string body, long orderId) + { + using JsonDocument doc = JsonDocument.Parse(body); + JsonElement root = doc.RootElement; + int statusId = Json.Int(root, "statusID"); + int errorCode = Json.Int(root, "errorCode"); + string error = Json.String(root, "errorMessage"); + if (errorCode != 0 && error.Length == 0) + { + error = $"codice {errorCode}"; + } + + double requestedUnits = Json.Double(root, "units"); + bool filled = statusId is 3 or 5; + bool rejected = statusId is 4 or 7 or 8 or 9 or 10; + long positionId = 0; + double rate = 0; + double units = 0; + DateTime time = Json.Time(root, "requestOccurred"); + foreach (JsonElement p in Json.Array(root, "positions")) + { + positionId = Json.Long(p, "positionID"); + rate = Json.Double(p, "rate"); + units = Json.Double(p, "units"); + DateTime t = Json.Time(p, "occurred"); + if (t != default) + { + time = t; + } + + break; + } + + string name = statusId switch + { + 1 => "Received", + 2 => "Placed", + 3 => "Filled", + 4 => "Rejected", + 5 => "PartiallyFilled", + 6 => "PendingCancel", + 7 => "Canceled", + 8 => "Expired", + 9 => "CanceledPartiallyFilled", + 10 => "RejectedPartiallyFilled", + 11 => "WaitingForMarket", + 12 => "PendingTriggeredRate", + _ => statusId.ToString(CultureInfo.InvariantCulture), + }; + return new OrderOutcome(filled, rejected, Json.Long(root, "orderID", orderId), positionId, rate, units > 0 ? units : (filled ? requestedUnits : 0), time == default ? DateTime.UtcNow : time, 0, name, error) + { + RequestedUnits = requestedUnits, + StatusId = statusId, + Source = "lookup-v1", + }; } public async Task CloseAsync(long positionId, long instrumentId, CancellationToken ct) diff --git a/Encelado/tests/Encelado.Tests/BasketTests.cs b/Encelado/tests/Encelado.Tests/BasketTests.cs index 125a76e..984de34 100644 --- a/Encelado/tests/Encelado.Tests/BasketTests.cs +++ b/Encelado/tests/Encelado.Tests/BasketTests.cs @@ -326,6 +326,10 @@ public class LegRiskTests public Task LookupOrderAsync(string clientRef, CancellationToken ct) => Task.FromResult(null); + public Task LookupOrderByIdAsync(long orderId, CancellationToken ct) => Task.FromResult(null); + + public Task CancelOrderAsync(long orderId, CancellationToken ct) => Task.FromResult(false); + public Task CloseAsync(long positionId, long instrumentId, CancellationToken ct) { Events.Add($"close {positionId}"); @@ -458,6 +462,10 @@ public class LegRiskTests public Task LookupOrderAsync(string clientRef, CancellationToken ct) => inner.LookupOrderAsync(clientRef, ct); + public Task LookupOrderByIdAsync(long orderId, CancellationToken ct) => inner.LookupOrderByIdAsync(orderId, ct); + + public Task CancelOrderAsync(long orderId, CancellationToken ct) => inner.CancelOrderAsync(orderId, ct); + public Task CloseAsync(long positionId, long instrumentId, CancellationToken ct) => inner.CloseAsync(positionId, instrumentId, ct); public Task UpdateStopsAsync(long positionId, double? stopLoss, double? takeProfit, CancellationToken ct) => inner.UpdateStopsAsync(positionId, stopLoss, takeProfit, ct); diff --git a/Encelado/tests/Encelado.Tests/ExecutionTests.cs b/Encelado/tests/Encelado.Tests/ExecutionTests.cs new file mode 100644 index 0000000..20a446a --- /dev/null +++ b/Encelado/tests/Encelado.Tests/ExecutionTests.cs @@ -0,0 +1,634 @@ +using System.Diagnostics; +using System.Globalization; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Text.Json; +using Encelado.Bot.Baskets; +using Encelado.Core.Baskets; +using Encelado.Core.Baskets.Data; +using Encelado.Core.Baskets.History; +using Encelado.Core.Broker; +using Encelado.Etoro; + +namespace Encelado.Tests; + +/// Shared scenery for the execution tests: two instruments, live quotes, an entry decision. +internal static class ExecutionScenery +{ + public static readonly Instrument EurUsd = new(1, "EURUSD", "EUR/USD", "Forex", 0.0001, 5, 0.01, 2_000_000, 1000, [1, 2, 5, 10], true, true, 0, 50, ""); + public static readonly Instrument UsdChf = new(6, "USDCHF", "USD/CHF", "Forex", 0.0001, 5, 0.01, 2_000_000, 1000, [1, 2, 5, 10], true, true, 0, 50, ""); + + public static double? Mid(string s) => s == "EURUSD" ? 1.10005 : s == "USDCHF" ? 0.90005 : null; + + public static (BasketContext Ctx, BasketDecision Decision, BasketStrategyConfig Cfg) Entry(int legTimeoutSec = 1) + { + BasketStrategyConfig cfg = BasketStrategyConfig.ParseText(BasketStrategyConfig.DefaultJson, out _); + cfg.LegTimeoutSec = legTimeoutSec; + SymbolSeries a = new(EurUsd, TimeSpan.FromMinutes(15)); + SymbolSeries b = new(UsdChf, TimeSpan.FromMinutes(15)); + DateTime now = DateTime.UtcNow; + a.OnQuote(new QuoteSnapshot(1, now, 1.1000, 1.1001, true), now); + b.OnQuote(new QuoteSnapshot(6, now, 0.9000, 0.9001, true), now); + BasketContext ctx = new() + { + TimeUtc = now, + BasketId = "EURUSD/USDCHF", + Name = "EURUSD/USDCHF", + Cross = SyntheticCross.Derive("EURUSD", "USDCHF"), + A = a, + B = b, + Equity = 10_000, + PipValueUsdA = 0.0001, + PipValueUsdB = 0.0001 / 0.9, + UsdPerQuoteA = 1, + UsdPerQuoteB = 1 / 0.9, + Mid = Mid, + }; + SizingResult sizing = new(true, 10_000, 9_000, 11_000, 10_000, 50, 2.1, "test"); + BasketDecision d = new(DecisionKind.Enter, true, sizing, ["enter"], "test", new BasketEvaluation { Z = -2.2 }, null); + return (ctx, d, cfg); + } +} + +/// +/// A venue that answers the submit with "no idea yet" and only later admits the fill, +/// the way eToro did on 2026-09-16: the shape of the bug the order register exists for. +/// +internal sealed class SlowVenue : IBroker +{ + private long _nextOrder = 100; + private long _nextPosition = 500; + public readonly List Events = []; + public readonly Dictionary Positions = []; + private readonly Dictionary _orders = []; + + /// Which orders the lookup admits to. Empty: the lookup answers 404 to everything. + public HashSet Admitted { get; } = []; + + /// Orders that reach the venue but are rejected asynchronously. + public HashSet RejectSymbols { get; } = []; + + /// Whether the fill shows up on the position list (as it does on eToro) even when the lookup denies it. + public bool FillsAppearOnAccount { get; set; } = true; + + public BrokerEnvironment Environment => BrokerEnvironment.Backtest; + + public string Name => "slow"; + + public bool SupportsTrading => true; + + public TimeSpan ClockSkew => TimeSpan.Zero; + + public Task> GetInstrumentsAsync(IReadOnlyList symbols, CancellationToken ct) => Task.FromResult>([]); + + public Task> GetQuotesAsync(IReadOnlyList instrumentIds, CancellationToken ct) => Task.FromResult>([]); + + public Task> GetCandlesAsync(long instrumentId, TimeSpan interval, int count, CancellationToken ct) => Task.FromResult>([]); + + public Task GetAccountAsync(CancellationToken ct) => Task.FromResult(new AccountSnapshot(DateTime.UtcNow, "USD", 10_000, 10_000, 10_000, 0, 0)); + + public Task> GetPositionsAsync(CancellationToken ct) => Task.FromResult>([.. Positions.Values]); + + public Task OpenAsync(OrderRequest request, CancellationToken ct) + { + long orderId = _nextOrder++; + Events.Add($"open {request.Symbol} {orderId}"); + long positionId = 0; + if (!RejectSymbols.Contains(request.Symbol)) + { + positionId = _nextPosition++; + if (FillsAppearOnAccount) + { + Positions[positionId] = new BrokerPosition(positionId, request.InstrumentId, request.IsBuy, request.Units, request.IsBuy ? 1.1001 : 1.0999, DateTime.UtcNow, 0, 0, request.Leverage, request.Units * 0.11, 0, 0, 1.1); + } + } + + _orders[orderId] = (request, DateTime.UtcNow, positionId); + return Task.FromResult(new OrderOutcome(false, false, orderId, 0, 0, 0, DateTime.UtcNow, 0, "Submitted", "accettato dal server, esito non ancora letto") { RequestedUnits = request.Units, Source = "venue" }); + } + + public Task LookupOrderAsync(string clientRef, CancellationToken ct) => Task.FromResult(null); + + public Task LookupOrderByIdAsync(long orderId, CancellationToken ct) + { + Events.Add($"lookup {orderId}"); + if (!Admitted.Contains(orderId) || !_orders.TryGetValue(orderId, out (OrderRequest Request, DateTime SentUtc, long PositionId) o)) + { + return Task.FromResult(null); + } + + if (o.PositionId == 0) + { + return Task.FromResult(new OrderOutcome(false, true, orderId, 0, 0, 0, DateTime.UtcNow, 0, "Rejected", "margine insufficiente (simulato)") { StatusId = 4, RequestedUnits = o.Request.Units, Source = "lookup" }); + } + + return Task.FromResult(new OrderOutcome(true, false, orderId, o.PositionId, o.Request.IsBuy ? 1.1001 : 1.0999, o.Request.Units, o.SentUtc, 0, "Filled", string.Empty) { StatusId = 3, RequestedUnits = o.Request.Units, Source = "lookup" }); + } + + public Task CancelOrderAsync(long orderId, CancellationToken ct) => Task.FromResult(false); + + public Task CloseAsync(long positionId, long instrumentId, CancellationToken ct) + { + Events.Add($"close {positionId}"); + bool removed = Positions.Remove(positionId); + return Task.FromResult(new CloseOutcome(removed, !removed, _nextOrder++, 1.1, 0, DateTime.UtcNow, removed ? 1.5 : 0, removed ? string.Empty : "inesistente")); + } + + public Task UpdateStopsAsync(long positionId, double? stopLoss, double? takeProfit, CancellationToken ct) => Task.FromResult(true); + + public Task GetCostAsync(OrderRequest request, CancellationToken ct) => Task.FromResult(null); + + public Task> GetClosedTradesAsync(DateTime fromUtc, CancellationToken ct) => Task.FromResult>([]); + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; +} + +/// (o) A leg whose outcome the venue withholds: the basket waits, the register resolves it, the entry finishes or is undone. +public sealed class OrderTrackerTests : IDisposable +{ + private readonly string _dir = Path.Combine(Path.GetTempPath(), $"encelado-tracker-{Guid.NewGuid():N}"); + + public void Dispose() + { + if (Directory.Exists(_dir)) + { + Directory.Delete(_dir, recursive: true); + } + } + + [Fact] + public async Task ALegWithoutAnOutcomeLeavesTheBasketPendingNotRejected() + { + (BasketContext ctx, BasketDecision d, BasketStrategyConfig cfg) = ExecutionScenery.Entry(); + SlowVenue venue = new() { FillsAppearOnAccount = false }; + OrderTracker tracker = new(Path.Combine(_dir, "pending_orders.json")); + BasketExecutor executor = new(venue, cfg, ctx.Mid, static _ => { }, tracker, "Demo"); + + EntryOutcome outcome = await executor.OpenAsync(ctx, d, cfg.Effective(), "B1", CancellationToken.None); + + Assert.False(outcome.Ok); + Assert.Equal(PendingLeg.A, outcome.PendingLeg); + Assert.NotNull(outcome.Pending); + Assert.Single(tracker.Pending); + Assert.Equal(100, tracker.Pending[0].OrderId); + Assert.Equal("Submitted", tracker.Pending[0].LastStatus); + Assert.DoesNotContain(venue.Events, static e => e.StartsWith("open USDCHF", StringComparison.Ordinal)); + + // The register survives a restart with the order still pending. + OrderTracker reloaded = new(Path.Combine(_dir, "pending_orders.json")); + Assert.Single(reloaded.Pending); + Assert.Equal(outcome.Pending!.ClientRefA, reloaded.Pending[0].ClientRef); + } + + [Fact] + public async Task PendingAThenFilledWithTheSignalStillValidSendsLegBAndOpensTheBasket() + { + (BasketContext ctx, BasketDecision d, BasketStrategyConfig cfg) = ExecutionScenery.Entry(); + SlowVenue venue = new() { FillsAppearOnAccount = false }; + OrderTracker tracker = new(Path.Combine(_dir, "pending_orders.json")); + BasketExecutor executor = new(venue, cfg, ctx.Mid, static _ => { }, tracker, "Demo"); + EntryOutcome first = await executor.OpenAsync(ctx, d, cfg.Effective(), "B1", CancellationToken.None); + Assert.Equal(PendingLeg.A, first.PendingLeg); + + // Later the venue admits the fill. + venue.Admitted.Add(100); + foreach (TrackedOrder o in tracker.Pending) + { + o.LastCheckUtc = default; + } + + List<(TrackedOrder Order, OrderOutcome Outcome)> resolved = await tracker.ResolveAsync(venue, DateTime.UtcNow, null, CancellationToken.None); + Assert.Single(resolved); + Assert.True(resolved[0].Outcome.Filled); + Assert.Equal(OrderResolution.Filled, resolved[0].Order.Resolution); + Assert.Empty(tracker.Pending); + + // Leg B goes out and fills at once (admitted from the start). + venue.Admitted.Add(101); + EntryOutcome second = await executor.ResumeAfterAAsync(ctx, first.Pending!, resolved[0].Outcome, CancellationToken.None); + Assert.True(second.Ok, second.Error); + Assert.NotNull(second.Position); + Assert.Equal(500, second.Position!.A.PositionId); + Assert.Equal(501, second.Position.B.PositionId); + Assert.Equal("B1", second.Position.BasketId); + Assert.Contains(venue.Events, static e => e.StartsWith("open USDCHF", StringComparison.Ordinal)); + } + + [Fact] + public async Task PendingAThenFilledWithTheSignalGoneUnwindsLegA() + { + (BasketContext ctx, BasketDecision d, BasketStrategyConfig cfg) = ExecutionScenery.Entry(); + SlowVenue venue = new(); + OrderTracker tracker = new(string.Empty); + BasketExecutor executor = new(venue, cfg, ctx.Mid, static _ => { }, tracker, "Demo"); + EntryOutcome first = await executor.OpenAsync(ctx, d, cfg.Effective(), "B1", CancellationToken.None); + Assert.Equal(PendingLeg.A, first.PendingLeg); + + // No lookup ever answers, but the position is on the account: the register matches it. + foreach (TrackedOrder o in tracker.Pending) + { + o.LastCheckUtc = default; + } + + await Task.Delay(2100); + List<(TrackedOrder Order, OrderOutcome Outcome)> resolved = await tracker.ResolveAsync(venue, DateTime.UtcNow, null, CancellationToken.None); + Assert.Single(resolved); + Assert.Equal("positions", resolved[0].Outcome.Source); + Assert.Equal(500, resolved[0].Outcome.PositionId); + + // The engine finds the signal gone: the lone leg is closed, nothing else is sent. + BasketLeg legA = new() { Symbol = "EURUSD", InstrumentId = 1, IsBuy = true, Units = 10_000, EntryPrice = 1.1001, PositionId = 500, OpenedUtc = DateTime.UtcNow }; + CloseOutcome undo = await executor.UnwindLegAsync(legA, ctx.Name, "B1", "segnale decaduto", CancellationToken.None); + Assert.True(undo.Closed); + Assert.Empty(venue.Positions); + Assert.DoesNotContain(venue.Events, static e => e.StartsWith("open USDCHF", StringComparison.Ordinal)); + Assert.Contains(tracker.All, static o => o.Leg == OrderLeg.Unwind && o.Resolution == OrderResolution.Filled); + } + + [Fact] + public async Task LegBRejectedAfterAPendingResolutionUnwindsLegA() + { + (BasketContext ctx, BasketDecision d, BasketStrategyConfig cfg) = ExecutionScenery.Entry(); + SlowVenue venue = new() { FillsAppearOnAccount = false }; + venue.RejectSymbols.Add("USDCHF"); + OrderTracker tracker = new(string.Empty); + BasketExecutor executor = new(venue, cfg, ctx.Mid, static _ => { }, tracker, "Demo"); + EntryOutcome first = await executor.OpenAsync(ctx, d, cfg.Effective(), "B1", CancellationToken.None); + venue.Admitted.Add(100); + venue.Admitted.Add(101); + venue.Positions[500] = new BrokerPosition(500, 1, true, 10_000, 1.1001, DateTime.UtcNow, 0, 0, 10, 1100, 0, 0, 1.1); + foreach (TrackedOrder o in tracker.Pending) + { + o.LastCheckUtc = default; + } + + List<(TrackedOrder Order, OrderOutcome Outcome)> resolved = await tracker.ResolveAsync(venue, DateTime.UtcNow, null, CancellationToken.None); + EntryOutcome second = await executor.ResumeAfterAAsync(ctx, first.Pending!, resolved[0].Outcome, CancellationToken.None); + + Assert.False(second.Ok); + Assert.True(second.Unwound, second.Error); + Assert.Empty(venue.Positions); + } + + [Fact] + public void MatchingPrefersTheClosestPositionAndIgnoresTheWrongSideOrTheClaimedOnes() + { + DateTime sent = new(2026, 9, 21, 10, 15, 3, DateTimeKind.Utc); + TrackedOrder order = new() { ClientRef = "x", Symbol = "EURAUD", InstrumentId = 12, IsBuy = false, RequestedUnits = 300_000, Basket = "EURAUD/AUDCAD", Leg = OrderLeg.A, SentUtc = sent }; + BrokerPosition sameBar = new(1, 12, false, 17_420.9, 1.61043, sent.AddSeconds(0.3), 0, 0, 10, 2000, 0, 0, 1.61); + BrokerPosition earlierBar = new(2, 12, false, 17_425.3, 1.60939, sent.AddMinutes(-60), 0, 0, 10, 2000, 0, 0, 1.61); + BrokerPosition wrongSide = new(3, 12, true, 17_420.9, 1.61043, sent.AddSeconds(0.2), 0, 0, 10, 2000, 0, 0, 1.61); + + BrokerPosition? match = OrderTracker.Match(order, [earlierBar, wrongSide, sameBar], static _ => false); + Assert.NotNull(match); + Assert.Equal(1, match!.PositionId); + + // Units may differ (the venue reduced the order on 2026-09-16): the time window decides. + Assert.Null(OrderTracker.Match(order, [earlierBar, wrongSide], static _ => false)); + Assert.Null(OrderTracker.Match(order, [sameBar], id => id == 1)); + } + + [Fact] + public void OrderRecordsRoundTripThroughJsonl() + { + TrackedOrder o = new() { ClientRef = "abc", OrderId = 42, Symbol = "EURUSD", InstrumentId = 1, IsBuy = true, RequestedUnits = 10_000, RequestedPrice = 1.1, Basket = "EURUSD/USDCHF", BasketId = "B1", Leg = OrderLeg.A, SentUtc = new DateTime(2026, 9, 23, 8, 0, 0, DateTimeKind.Utc), Mode = "Demo", Motivazione = "test" }; + o.Apply(new OrderOutcome(true, false, 42, 77, 1.1002, 9_000, o.SentUtc.AddSeconds(1), 0.5, "Filled", string.Empty) { StatusId = 3, RequestedUnits = 10_000, Source = "lookup" }, DateTime.UtcNow); + OrderRecord record = OrderRecord.From(o, "run1", "risolto"); + OrderRecord? back = OrderRecord.Parse(record.ToJson()); + + Assert.NotNull(back); + Assert.Equal(42, back!.OrderId); + Assert.Equal(77, back.PositionId); + Assert.Equal(OrderResolution.Filled, back.Resolution); + Assert.Equal(9_000, back.ExecutedUnits); + Assert.Equal(10_000, back.RequestedUnits); + Assert.Equal(2.0, back.SlippagePips, 6); + Assert.Equal("risolto", back.Evento); + } +} + +/// (n) A executed and B rejected: A is closed again within the leg timeout, not five seconds later. +public class LegRiskTimingTests +{ + [Fact] + public async Task TheUnwindOfLegAHappensWithinFiveSeconds() + { + (BasketContext ctx, BasketDecision d, BasketStrategyConfig cfg) = ExecutionScenery.Entry(legTimeoutSec: 5); + SlowVenue venue = new(); + venue.Admitted.Add(100); + venue.Admitted.Add(101); + venue.RejectSymbols.Add("USDCHF"); + BasketExecutor executor = new(venue, cfg, ctx.Mid, static _ => { }, new OrderTracker(string.Empty), "Demo"); + + Stopwatch sw = Stopwatch.StartNew(); + EntryOutcome outcome = await executor.OpenAsync(ctx, d, cfg.Effective(), "B1", CancellationToken.None); + sw.Stop(); + + Assert.False(outcome.Ok); + Assert.True(outcome.Unwound, outcome.Error); + Assert.Empty(venue.Positions); + Assert.True(sw.Elapsed < TimeSpan.FromSeconds(5), $"unwind in {sw.Elapsed.TotalSeconds:F1} s"); + } +} + +/// (p) Whose is a position on the account. +public class PositionClassifierTests +{ + [Fact] + public void ABasketLegAnOrphanWithTheBotsSignatureAndAStrangerAreToldApart() + { + DateTime bar = new(2026, 9, 18, 4, 0, 3, DateTimeKind.Utc); + BrokerPosition basketLeg = new(10, 1, true, 10_000, 1.1, bar.AddHours(-2), 0, 0, 10, 1100, 5, 0, 1.1); + BrokerPosition orphan = new(11, 3, false, 34_883.3, 0.57334, bar.AddSeconds(0.9), 0, 0, 10, 2000, 17.8, 0, 0.5728); + BrokerPosition tracked = new(12, 12, false, 17_420.9, 1.61043, bar.AddMinutes(30), 0, 0, 10, 2000, 12.3, 0, 1.6094); + BrokerPosition stranger = new(13, 1531, true, 12.1, 41.22, bar.AddDays(-100), 0, 0, 1, 500, -69.9, 0, 35.45); + + List signatures = + [ + new(bar, 0, "NZDUSD", false, 300_000, "NZDUSD/EURNZD", "rifiuto"), + new(bar, 0, "EURNZD", false, 150_000, "NZDUSD/EURNZD", "rifiuto"), + ]; + Dictionary baskets = new() { [10] = "EURUSD/USDCHF" }; + Dictionary register = new() { [12] = "EURAUD/AUDCAD" }; + string? SymbolOf(long id) => id switch { 1 => "EURUSD", 3 => "NZDUSD", 12 => "EURAUD", _ => null }; + + List result = PositionClassifier.Classify([basketLeg, orphan, tracked, stranger], baskets, register, signatures, SymbolOf); + + Assert.Equal(PositionOrigin.Basket, result[0].Origin); + Assert.Equal(PositionOrigin.OrphanBot, result[1].Origin); + Assert.Equal("NZDUSD/EURNZD", result[1].Basket); + Assert.Contains("unità", result[1].Reason, StringComparison.Ordinal); + Assert.Equal(PositionOrigin.OrphanBot, result[2].Origin); + Assert.Equal("EURAUD/AUDCAD", result[2].Basket); + Assert.Equal(PositionOrigin.Foreign, result[3].Origin); + } + + [Fact] + public void ASignatureOutsideTheWindowDoesNotClaimThePosition() + { + DateTime bar = new(2026, 9, 18, 4, 0, 3, DateTimeKind.Utc); + BrokerPosition p = new(11, 3, false, 34_883.3, 0.57334, bar.AddMinutes(5), 0, 0, 10, 2000, 0, 0, 0.5728); + List result = PositionClassifier.Classify([p], new Dictionary(), new Dictionary(), + [new EntrySignature(bar, 3, "NZDUSD", false, 34_883.3, "NZDUSD/EURNZD", "rifiuto")]); + Assert.Equal(PositionOrigin.Foreign, result[0].Origin); + } +} + +/// (q) A deposit is cash, not profit: the peak and the drawdown do not move. +public class CashFlowTests +{ + [Fact] + public void ADepositMovesNeitherThePeakNorTheDrawdown() + { + EquityTracker t = new(); + DateTime now = new(2026, 9, 16, 12, 0, 0, DateTimeKind.Utc); + Assert.Null(t.Observe(now, 110_000, 110_000, 0)); + Assert.Null(t.Observe(now.AddHours(1), 110_000, 100_000, 0)); + double ddBefore = t.Drawdown(100_000); + Assert.Equal(10_000.0 / 110_000, ddBefore, 6); + + CashMovement? m = t.Observe(now.AddDays(2), 140_000, 130_000, 0); + + Assert.NotNull(m); + Assert.Equal(30_000, m!.Amount, 2); + Assert.Equal(110_000, t.PeakNetEquity, 2); + Assert.Equal(140_000, t.PeakEquity, 2); + Assert.Equal(ddBefore, t.Drawdown(130_000), 6); + Assert.Equal(100_000, t.NetEquity(130_000), 2); + } + + [Fact] + public void AClosedTradeExplainsTheBalanceChangeAndIsNotACashMovement() + { + EquityTracker t = new(); + DateTime now = DateTime.UtcNow; + t.Observe(now, 10_000, 10_000, 0); + Assert.Null(t.Observe(now.AddMinutes(1), 10_414.56, 10_414.56, 414.56)); + Assert.Equal(0, t.CumulativeCashFlow); + Assert.Equal(10_414.56, t.PeakNetEquity, 2); + } + + [Fact] + public void SmallResidualsAreNoiseNotWithdrawals() + { + EquityTracker t = new(); + DateTime now = DateTime.UtcNow; + t.Observe(now, 10_000, 10_000, 0); + Assert.Null(t.Observe(now.AddMinutes(1), 9_996, 9_996, 0)); + Assert.NotNull(t.Observe(now.AddMinutes(2), 9_000, 9_000, 0)); + Assert.Equal(-996, t.CumulativeCashFlow, 2); + } +} + +/// (m) The venue keeps answering 404 to the lookup while the position is on the account: the broker reports a fill. +public class EtoroBrokerLookupTests +{ + private sealed class ScriptedHandler(Func script) : HttpMessageHandler + { + public readonly List Requests = []; + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + Requests.Add($"{request.Method} {request.RequestUri!.PathAndQuery}"); + return Task.FromResult(script(request)); + } + } + + private static HttpResponseMessage Json(HttpStatusCode status, string body) => new(status) + { + Content = new StringContent(body, Encoding.UTF8, "application/json"), + }; + + private static EtoroOptions Options() => new() { ApiKey = "a", UserKey = "u", Environment = "demo", FillTimeoutSeconds = 4 }; + + private const string LookupFilled = """ + {"accountId":15467847,"orderId":382724150,"action":"open","transaction":"sell","type":"mkt","etoroOrderTypeId":18, + "status":{"id":3,"name":"Filled","errorCode":0}, + "asset":{"symbol":"EURAUD","instrumentId":12,"currency":"AUD","settlementType":"CFD","leverage":10,"side":"short"}, + "orderCurrency":"usd","requestedAmount":2000.0,"requestedUnits":17420.945125,"frozenAmount":2000.0, + "positionExecutions":[{"positionId":3601651531,"state":"closed","marginAccountCurrency":1999.99,"remainingUnits":17420.945125, + "openingData":{"openTime":"2026-09-21T10:15:03.19Z","orderId":382724150,"executionTime":"2026-09-21T10:15:03.293Z","units":17420.945125,"avgPrice":1.61043,"marketSpread":0.12,"markup":0.12,"fees":0.0}}], + "requestTime":"2026-09-21T10:15:03.19Z","lastUpdate":"2026-09-21T10:15:03.36Z","requestType":"byUnits"} + """; + + private const string OrderInfoV1 = """ + {"orderID":381739181,"CID":15467847,"referenceID":"00000000-0000-0000-0000-000000000000","statusID":3,"orderType":18,"errorCode":0,"instrumentID":1, + "amount":62792.34,"units":547214.68,"requestOccurred":"2026-09-16T19:00:47.96Z", + "positions":[{"positionID":3600352679,"orderType":18,"occurred":"2026-09-16T19:00:48.073Z","rate":1.14749,"units":547214.68,"conversionRate":1.0,"amount":62792.33,"isOpen":false}]} + """; + + [Fact] + public async Task TheLookupByOrderIdParsesTheVenuesAnswerIncludingTheReducedUnits() + { + ScriptedHandler handler = new(r => r.RequestUri!.PathAndQuery.Contains("orders:lookup?orderId=382724150", StringComparison.Ordinal) ? Json(HttpStatusCode.OK, LookupFilled) : Json(HttpStatusCode.NotFound, "{}")); + await using EtoroBroker broker = new(Options(), handler); + + OrderOutcome? o = await broker.LookupOrderByIdAsync(382724150, CancellationToken.None); + + Assert.NotNull(o); + Assert.True(o!.Filled); + Assert.Equal(3, o.StatusId); + Assert.Equal(3601651531, o.PositionId); + Assert.Equal(17420.945125, o.Units, 6); + Assert.Equal(17420.945125, o.RequestedUnits, 6); + Assert.Equal(1.61043, o.FillRate, 6); + Assert.Equal("lookup", o.Source); + Assert.Equal(new DateTime(2026, 9, 21, 10, 15, 3, 293, DateTimeKind.Utc), o.TimeUtc); + } + + [Fact] + public async Task WhenTheV2LookupHasNoRecordTheV1RouteIsAsked() + { + ScriptedHandler handler = new(r => r.RequestUri!.PathAndQuery.EndsWith("/orders/381739181", StringComparison.Ordinal) ? Json(HttpStatusCode.OK, OrderInfoV1) : Json(HttpStatusCode.NotFound, "{}")); + await using EtoroBroker broker = new(Options(), handler); + + OrderOutcome? o = await broker.LookupOrderByIdAsync(381739181, CancellationToken.None); + + Assert.NotNull(o); + Assert.True(o!.Filled); + Assert.Equal("Filled", o.Status); + Assert.Equal(3600352679, o.PositionId); + Assert.Equal(547214.68, o.Units, 2); + Assert.Equal(1.14749, o.FillRate, 5); + Assert.Equal("lookup-v1", o.Source); + Assert.Contains(handler.Requests, static r => r.Contains("orders:lookup?orderId=381739181", StringComparison.Ordinal)); + Assert.Contains(handler.Requests, static r => r.EndsWith("/api/v1/trading/info/demo/orders/381739181", StringComparison.Ordinal)); + } + + [Fact] + public async Task APersistent404WithThePositionOnTheAccountIsAFill() + { + bool submitted = false; + DateTime opened = DateTime.UtcNow; + ScriptedHandler handler = new(r => + { + string path = r.RequestUri!.PathAndQuery; + if (r.Method == HttpMethod.Post && path.EndsWith("/demo/orders", StringComparison.Ordinal)) + { + submitted = true; + opened = DateTime.UtcNow; + return Json(HttpStatusCode.OK, """{"orderId":9001,"token":"t"}"""); + } + + if (path.Contains("/demo/pnl", StringComparison.Ordinal)) + { + string positions = submitted + ? "[{\"positionID\":777,\"instrumentID\":1,\"isBuy\":true,\"units\":10000,\"openRate\":1.10012,\"openDateTime\":\"" + opened.ToString("O", CultureInfo.InvariantCulture) + "\",\"amount\":1100,\"leverage\":10,\"unrealizedPnL\":{\"pnL\":0.5,\"closeRate\":1.1002},\"totalFees\":0}]" + : "[]"; + return Json(HttpStatusCode.OK, "{\"clientPortfolio\":{\"credit\":10000,\"bonusCredit\":0,\"unrealizedPnL\":0,\"positions\":" + positions + "}}"); + } + + return Json(HttpStatusCode.NotFound, """{"title":"Order not found"}"""); + }); + await using EtoroBroker broker = new(Options(), handler); + OrderRequest request = new(Guid.NewGuid().ToString("D"), 1, "EURUSD", true, 10_000, 10, 1.05, null, "test"); + + Stopwatch sw = Stopwatch.StartNew(); + OrderOutcome o = await broker.OpenAsync(request, CancellationToken.None); + + Assert.True(o.Filled, o.Error); + Assert.Equal("positions", o.Source); + Assert.Equal(9001, o.OrderId); + Assert.Equal(777, o.PositionId); + Assert.Equal(1.10012, o.FillRate, 5); + Assert.Equal(10_000, o.RequestedUnits); + Assert.True(sw.Elapsed < TimeSpan.FromSeconds(4), $"riconosciuto in {sw.Elapsed.TotalSeconds:F1} s"); + Assert.Contains(handler.Requests, static r => r.Contains("orders:lookup?orderId=9001", StringComparison.Ordinal)); + Assert.DoesNotContain(handler.Requests, static r => r.Contains("referenceId=", StringComparison.Ordinal)); + } + + [Fact] + public async Task APersistent404WithNoPositionIsReportedUnknownNotReceived() + { + ScriptedHandler handler = new(r => + { + string path = r.RequestUri!.PathAndQuery; + if (r.Method == HttpMethod.Post && path.EndsWith("/demo/orders", StringComparison.Ordinal)) + { + return Json(HttpStatusCode.OK, """{"orderId":9002}"""); + } + + if (path.Contains("/demo/pnl", StringComparison.Ordinal)) + { + return Json(HttpStatusCode.OK, """{"clientPortfolio":{"credit":10000,"bonusCredit":0,"unrealizedPnL":0,"positions":[]}}"""); + } + + return Json(HttpStatusCode.NotFound, "{}"); + }); + await using EtoroBroker broker = new(new EtoroOptions { ApiKey = "a", UserKey = "u", Environment = "demo", FillTimeoutSeconds = 1 }, handler); + + OrderOutcome o = await broker.OpenAsync(new OrderRequest(Guid.NewGuid().ToString("D"), 1, "EURUSD", true, 10_000, 10, 1.05, null, "test"), CancellationToken.None); + + Assert.True(o.Pending); + Assert.Equal(9002, o.OrderId); + Assert.NotEqual("Received", o.Status); + Assert.Contains("registro", o.Error, StringComparison.Ordinal); + } +} + +/// The ledger's new files: orders.jsonl and the entry signatures read back from decisions.jsonl. +public sealed class LedgerOrdersTests : IDisposable +{ + private readonly string _dir = Path.Combine(Path.GetTempPath(), $"encelado-ledger2-{Guid.NewGuid():N}"); + + public void Dispose() + { + if (Directory.Exists(_dir)) + { + Directory.Delete(_dir, recursive: true); + } + } + + [Fact] + public void OrdersAreAppendedAndReadBack() + { + using Ledger ledger = new(_dir); + TrackedOrder o = new() { ClientRef = "c1", OrderId = 5, Symbol = "EURUSD", InstrumentId = 1, IsBuy = false, RequestedUnits = 100, Basket = "EURUSD/USDCHF", BasketId = "B1", Leg = OrderLeg.A, SentUtc = DateTime.UtcNow, Mode = "Demo" }; + ledger.Order(OrderRecord.From(o, "run", "inviato")); + o.Apply(new OrderOutcome(false, true, 5, 0, 0, 0, DateTime.UtcNow, 0, "Rejected", "no") { StatusId = 4 }, DateTime.UtcNow); + ledger.Order(OrderRecord.From(o, "run", "risolto")); + + List rows = ledger.ReadOrders(); + Assert.Equal(2, rows.Count); + Assert.Equal("inviato", rows[0].Evento); + Assert.Equal(OrderResolution.Rejected, rows[1].Resolution); + Assert.Equal("c1", rows[1].ClientRef); + } + + [Fact] + public void EntrySignaturesComeFromTheDecisionRows() + { + using Ledger ledger = new(_dir); + DateTime t = new(2026, 9, 18, 4, 0, 3, DateTimeKind.Utc); + Instrument nzd = new(3, "NZDUSD", "NZD/USD", "Forex", 0.0001, 5, 0.01, 0, 1000, [1], true, true, 0, 50, ""); + Instrument eurnzd = new(49, "EURNZD", "EUR/NZD", "Forex", 0.0001, 5, 0.01, 0, 1000, [1], true, true, 0, 50, ""); + BasketContext ctx = new() + { + TimeUtc = t, + BasketId = "NZDUSD/EURNZD", + Name = "NZDUSD/EURNZD", + Cross = SyntheticCross.Derive("NZDUSD", "EURNZD"), + A = new SymbolSeries(nzd, TimeSpan.FromMinutes(15)), + B = new SymbolSeries(eurnzd, TimeSpan.FromMinutes(15)), + Equity = 100_000, + PipValueUsdA = 1, + PipValueUsdB = 1, + UsdPerQuoteA = 1, + UsdPerQuoteB = 1, + Mid = static _ => null, + }; + BasketDecision d = new(DecisionKind.Enter, false, new SizingResult(true, 34_883, 20_000, 20_000, 20_000, 50, 0.4, "ok"), ["enter"], "vendo", new BasketEvaluation { Z = 2.3 }, null); + ledger.Decision("run", "Demo", "AGGRESSIVE", "h", ctx, d, "rifiuto", "B1", "gamba A non eseguita"); + + List sig = ledger.ReadEntrySignatures(t.AddDays(-1)); + + Assert.Equal(2, sig.Count); + Assert.Equal("NZDUSD", sig[0].Symbol); + Assert.False(sig[0].IsBuy); + Assert.Equal(34_883, sig[0].Units); + Assert.Equal("EURNZD", sig[1].Symbol); + Assert.False(sig[1].IsBuy); + Assert.Equal(t, sig[0].TimeUtc); + } +} diff --git a/Encelado/tests/Encelado.Tests/TestSnapshots.cs b/Encelado/tests/Encelado.Tests/TestSnapshots.cs index b69392f..b1e8686 100644 --- a/Encelado/tests/Encelado.Tests/TestSnapshots.cs +++ b/Encelado/tests/Encelado.Tests/TestSnapshots.cs @@ -49,6 +49,14 @@ internal static class TestSnapshots OpenPnlPct = -0.00045, OpenBaskets = 1, MaxBaskets = 3, + PendingBaskets = 0, + PendingOrders = 0, + OrphanLegs = 1, + ForeignPositions = 2, + AccountOpenPnl = -61.20, + UsedMargin = 12_300, + CumulativeCashFlow = 0, + Unreconciled = false, Halted = false, EquityStopped = false, KillSwitched = false,