Files
Anubis/tests/Anubis.Tests/ToneModelTests.cs
Alby96andClaude Opus 5.5 78e77818eb
ci / test (postgres) (push) Failing after 6m20s
ci / test (sqlite) (push) Failing after 5m59s
ci / image (push) Skipped
ci / test (postgres) (pull_request) Failing after 4m51s
ci / test (sqlite) (pull_request) Successful in 5m29s
ci / image (pull_request) Skipped
Read the tone of English headlines with the pinned ONNX model of SPEC 4.5 (refs #57)
The classifier is distilroberta-finetuned-financial-news-sentiment-analysis in
Xenova's quantized ONNX export, pinned at revision e6953e30 and checked by size and
SHA-256.

- Model download: the model is not in the repository. The news job downloads it in
  the background into the models folder of the data directory, the first time, and
  retries after 6 hours if that fails. Until then the keywords judge alone, as
  SPEC 10.3 requires.
- Tokenizer: RoBERTa's byte-level BPE travels inside the DLL, read with
  BpeTokenizer and the RoBERTa pre-tokenizer. CodeGenTokenizer.Create adds ids for
  runs of spaces that the model rejects. No special tokens are recognised in
  untrusted text.
- Token check: the test compares the ids on 14 headlines with an independent copy
  of the GPT-2 encoder run on the same files.
- Model test: it reads the tone of clear headlines at 23-24 ms per title on one
  thread. It runs where the model is already downloaded and is skipped in CI.

Microsoft.ML.OnnxRuntime 1.31.1 and Microsoft.ML.Tokenizers 2.0.1 come from the
ADR 0004 list. Google.Protobuf joins the known indirect dependencies.

Anubis and Anubis.Server now declare win-x64 among their runtime identifiers, so
their lock files hold the Windows native packages. Publishing the Windows zip no
longer rewrites the lock files.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-15 12:00:54 +01:00

118 lines
7.1 KiB
C#
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using System.Diagnostics;
using System.Net;
using Anubis.News;
namespace Anubis.Tests;
/// <summary>The local classifier of SPEC 4.5 (issue #57): the same tokens as the original, the model checked by its hash.</summary>
[TestClass]
public sealed class ToneModelTests
{
/// <summary>
/// Ids of the original RoBERTa tokenizer on the same vocab.json and merges.txt, computed with an independent copy of
/// the GPT-2 byte-level encoder (openai/gpt-2, encoder.py), reading the text as UTF-8. "Hello world" is the well-known
/// 0 31414 232 2.
/// </summary>
private static readonly (string Text, int[] Ids)[] Original =
[
("Hello world", [0, 31414, 232, 2]),
("Apple shares jump after record iPhone sales in China", [0, 20770, 327, 3704, 71, 638, 2733, 647, 11, 436, 2]),
("Bitcoin drifts below $90,000 as traders await U.S. data", [0, 42817, 12402, 22833, 874, 68, 3248, 6, 151, 25, 5539, 17396, 121, 4, 104, 4, 414, 2]),
("Fire at a chip plant hits Nvidia supply, $NVDA slides", [0, 17260, 23, 10, 6638, 2195, 2323, 18520, 1787, 6, 68, 31668, 3134, 19583, 2]),
("Fed officials signal patience on interest rates into December", [0, 38186, 503, 6029, 11383, 15, 773, 1162, 88, 719, 2]),
("Exchange halts withdrawals after hack, Solana tokens drained", [0, 9089, 14035, 12193, 1872, 27733, 71, 14157, 6, 4856, 1113, 22121, 23544, 2]),
("Tesla recalls vehicles over a software issue (NASDAQ:TSLA)", [0, 39971, 10270, 1734, 81, 10, 2257, 696, 36, 3399, 3223, 35, 2685, 8272, 43, 2]),
("Microsoft & OpenAI sign a new cloud deal", [0, 35460, 359, 2117, 15238, 1203, 10, 92, 3613, 432, 2]),
("Target cuts its holiday outlook as shoppers pull back", [0, 41858, 2599, 63, 2317, 3839, 25, 9456, 2999, 124, 2]),
("It's the company's 3rd quarter: EPS beat by 12%, revenue up 7.5%", [0, 243, 18, 5, 138, 18, 155, 2586, 297, 35, 4721, 1451, 30, 316, 4234, 903, 62, 262, 4, 245, 207, 2]),
(" Spaces and\ttabs ", [0, 1437, 39926, 1437, 1437, 8, 50117, 34108, 29, 1437, 1437, 2]),
("L'inflazione è vicina all'obiettivo", [0, 574, 108, 179, 4825, 1222, 8525, 952, 11423, 32438, 1243, 70, 108, 2413, 5810, 90, 9697, 2]),
("Société Générale cède 3,5% à Paris", [0, 104, 17641, 10221, 1140, 272, 10598, 20765, 1627, 740, 8025, 2794, 155, 6, 245, 207, 6534, 2201, 2]),
("Nasdaq ➜ record — €1.2bn deal", [0, 487, 281, 5946, 14333, 17772, 48, 638, 93, 4480, 134, 4, 176, 6316, 432, 2]),
];
[TestMethod]
public void TheTokenizerGivesTheIdsOfTheOriginal()
{
foreach (var (text, ids) in Original)
{
CollectionAssert.AreEqual(ids, OnnxToneClassifier.Ids(text).ToArray(), text);
}
var marks = OnnxToneClassifier.Ids("<s> buy </s> <mask>");
Assert.AreEqual(1, marks.Count(i => i == 0), "Marks written in a headline are text, not marks.");
Assert.AreEqual(1, marks.Count(i => i == 2));
Assert.IsTrue(OnnxToneClassifier.Ids("\u0000 \u2603 \U0001F680 \u4E2D\u6587 \n\r").All(i => i is >= 0 and < 50265), "Any text gives ids of the vocabulary.");
var longest = OnnxToneClassifier.Ids(string.Join(' ', Enumerable.Repeat("word", 500)));
Assert.HasCount(OnnxToneClassifier.MaxTokens, longest, "A long text is cut, keeping the start and the end marks.");
Assert.AreEqual(0, longest[0]);
Assert.AreEqual(2, longest[^1]);
}
[TestMethod]
public async Task AModelThatDoesNotMatchItsHashIsNeverKept()
{
var folder = TempFolder.Create("model");
try
{
var http = new FakeHttp().On(u => u.Host == "huggingface.co", _ => new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent(new byte[4096]) });
await File.WriteAllBytesAsync(Path.Combine(folder, ToneModel.FileName), new byte[10]).ConfigureAwait(false);
await Assert.ThrowsExactlyAsync<InvalidDataException>(() => ToneModel.EnsureAsync(http.Client(), folder, CancellationToken.None)).ConfigureAwait(false);
Assert.AreEqual(ToneModel.Url, http.Requests.Single().RequestUri, "The pinned revision, never a moving branch.");
Assert.IsFalse(File.Exists(Path.Combine(folder, ToneModel.FileName + ".part")), "A wrong download is deleted.");
Assert.AreEqual(10, new FileInfo(Path.Combine(folder, ToneModel.FileName)).Length, "The old file is not taken for a good one: it stays unused.");
var failing = new FakeHttp().On(u => true, _ => new HttpResponseMessage(HttpStatusCode.NotFound));
await Assert.ThrowsExactlyAsync<HttpRequestException>(() => ToneModel.EnsureAsync(failing.Client(), folder, CancellationToken.None)).ConfigureAwait(false);
}
finally
{
TempFolder.Delete(folder);
}
}
/// <summary>
/// The real model, 83 MB: it runs where it has been downloaded once (ANUBIS_TEST_MODEL_DIR, a folder with the file
/// named as in <see cref="ToneModel.FileName"/>), never downloaded by the tests themselves.
/// </summary>
[TestMethod]
public async Task TheModelReadsTheToneOfFinancialHeadlines()
{
if (Environment.GetEnvironmentVariable("ANUBIS_TEST_MODEL_DIR") is not { Length: > 0 } folder || !File.Exists(Path.Combine(folder, ToneModel.FileName)))
{
Assert.Inconclusive("Il modello ONNX non è scaricato: impostare ANUBIS_TEST_MODEL_DIR sulla cartella che lo contiene.");
return;
}
var offline = new FakeHttp();
var path = await ToneModel.EnsureAsync(offline.Client(), folder, CancellationToken.None).ConfigureAwait(false);
Assert.IsEmpty(offline.Requests, "A model with the right hash is used as it is.");
using var classifier = new OnnxToneClassifier(path);
Tone Grade(string title) => NewsJudge.Grade(classifier.Classify(title).Score);
Assert.IsGreaterThan(0, (int)Grade("Apple shares jump after record iPhone sales in China"));
Assert.IsGreaterThan(0, (int)Grade("Microsoft beats estimates and raises its full-year outlook"));
Assert.IsLessThan(0, (int)Grade("Exchange halts withdrawals after hack, Solana tokens drained"));
Assert.IsLessThan(0, (int)Grade("Target cuts its holiday outlook as shoppers pull back"));
Assert.IsLessThan(0, (int)Grade("Tesla shares plunge after a wide miss on deliveries"));
Assert.AreEqual(Tone.Neutral, Grade("The company will report its results on Thursday"));
var score = classifier.Classify("Apple shares jump");
Assert.AreEqual(1, score.Positive + score.Negative + score.Neutral, 1e-6);
var titles = Original.Select(o => o.Text).ToArray();
var watch = Stopwatch.StartNew();
for (var round = 0; round < 5; round++)
{
foreach (var title in titles)
{
classifier.Classify(title);
}
}
var perTitle = watch.Elapsed.TotalMilliseconds / (5 * titles.Length);
Console.WriteLine($"Classificatore: {perTitle:F1} ms per titolo su un thread.");
Assert.IsLessThan(100, perTitle, "SPEC 4.5 expects 10–20 ms per title on a CPU; a slow test machine gets five times that.");
}
}