ci / test (postgres) (push) Failing after 6m20s
ci / test (sqlite) (push) Failing after 5m59s
ci / image (push) Skipped
ci / test (postgres) (pull_request) Failing after 4m51s
ci / test (sqlite) (pull_request) Successful in 5m29s
ci / image (pull_request) Skipped
The classifier is distilroberta-finetuned-financial-news-sentiment-analysis in Xenova's quantized ONNX export, pinned at revision e6953e30 and checked by size and SHA-256. - Model download: the model is not in the repository. The news job downloads it in the background into the models folder of the data directory, the first time, and retries after 6 hours if that fails. Until then the keywords judge alone, as SPEC 10.3 requires. - Tokenizer: RoBERTa's byte-level BPE travels inside the DLL, read with BpeTokenizer and the RoBERTa pre-tokenizer. CodeGenTokenizer.Create adds ids for runs of spaces that the model rejects. No special tokens are recognised in untrusted text. - Token check: the test compares the ids on 14 headlines with an independent copy of the GPT-2 encoder run on the same files. - Model test: it reads the tone of clear headlines at 23-24 ms per title on one thread. It runs where the model is already downloaded and is skipped in CI. Microsoft.ML.OnnxRuntime 1.31.1 and Microsoft.ML.Tokenizers 2.0.1 come from the ADR 0004 list. Google.Protobuf joins the known indirect dependencies. Anubis and Anubis.Server now declare win-x64 among their runtime identifiers, so their lock files hold the Windows native packages. Publishing the Windows zip no longer rewrites the lock files. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
118 lines
7.1 KiB
C#
118 lines
7.1 KiB
C#
using System.Diagnostics;
|
||
using System.Net;
|
||
using Anubis.News;
|
||
|
||
namespace Anubis.Tests;
|
||
|
||
/// <summary>The local classifier of SPEC 4.5 (issue #57): the same tokens as the original, the model checked by its hash.</summary>
|
||
[TestClass]
|
||
public sealed class ToneModelTests
|
||
{
|
||
/// <summary>
|
||
/// Ids of the original RoBERTa tokenizer on the same vocab.json and merges.txt, computed with an independent copy of
|
||
/// the GPT-2 byte-level encoder (openai/gpt-2, encoder.py), reading the text as UTF-8. "Hello world" is the well-known
|
||
/// 0 31414 232 2.
|
||
/// </summary>
|
||
private static readonly (string Text, int[] Ids)[] Original =
|
||
[
|
||
("Hello world", [0, 31414, 232, 2]),
|
||
("Apple shares jump after record iPhone sales in China", [0, 20770, 327, 3704, 71, 638, 2733, 647, 11, 436, 2]),
|
||
("Bitcoin drifts below $90,000 as traders await U.S. data", [0, 42817, 12402, 22833, 874, 68, 3248, 6, 151, 25, 5539, 17396, 121, 4, 104, 4, 414, 2]),
|
||
("Fire at a chip plant hits Nvidia supply, $NVDA slides", [0, 17260, 23, 10, 6638, 2195, 2323, 18520, 1787, 6, 68, 31668, 3134, 19583, 2]),
|
||
("Fed officials signal patience on interest rates into December", [0, 38186, 503, 6029, 11383, 15, 773, 1162, 88, 719, 2]),
|
||
("Exchange halts withdrawals after hack, Solana tokens drained", [0, 9089, 14035, 12193, 1872, 27733, 71, 14157, 6, 4856, 1113, 22121, 23544, 2]),
|
||
("Tesla recalls vehicles over a software issue (NASDAQ:TSLA)", [0, 39971, 10270, 1734, 81, 10, 2257, 696, 36, 3399, 3223, 35, 2685, 8272, 43, 2]),
|
||
("Microsoft & OpenAI sign a new cloud deal", [0, 35460, 359, 2117, 15238, 1203, 10, 92, 3613, 432, 2]),
|
||
("Target cuts its holiday outlook as shoppers pull back", [0, 41858, 2599, 63, 2317, 3839, 25, 9456, 2999, 124, 2]),
|
||
("It's the company's 3rd quarter: EPS beat by 12%, revenue up 7.5%", [0, 243, 18, 5, 138, 18, 155, 2586, 297, 35, 4721, 1451, 30, 316, 4234, 903, 62, 262, 4, 245, 207, 2]),
|
||
(" Spaces and\ttabs ", [0, 1437, 39926, 1437, 1437, 8, 50117, 34108, 29, 1437, 1437, 2]),
|
||
("L'inflazione è vicina all'obiettivo", [0, 574, 108, 179, 4825, 1222, 8525, 952, 11423, 32438, 1243, 70, 108, 2413, 5810, 90, 9697, 2]),
|
||
("Société Générale cède 3,5% à Paris", [0, 104, 17641, 10221, 1140, 272, 10598, 20765, 1627, 740, 8025, 2794, 155, 6, 245, 207, 6534, 2201, 2]),
|
||
("Nasdaq ➜ record — €1.2bn deal", [0, 487, 281, 5946, 14333, 17772, 48, 638, 93, 4480, 134, 4, 176, 6316, 432, 2]),
|
||
];
|
||
|
||
[TestMethod]
|
||
public void TheTokenizerGivesTheIdsOfTheOriginal()
|
||
{
|
||
foreach (var (text, ids) in Original)
|
||
{
|
||
CollectionAssert.AreEqual(ids, OnnxToneClassifier.Ids(text).ToArray(), text);
|
||
}
|
||
|
||
var marks = OnnxToneClassifier.Ids("<s> buy </s> <mask>");
|
||
Assert.AreEqual(1, marks.Count(i => i == 0), "Marks written in a headline are text, not marks.");
|
||
Assert.AreEqual(1, marks.Count(i => i == 2));
|
||
Assert.IsTrue(OnnxToneClassifier.Ids("\u0000 \u2603 \U0001F680 \u4E2D\u6587 \n\r").All(i => i is >= 0 and < 50265), "Any text gives ids of the vocabulary.");
|
||
|
||
var longest = OnnxToneClassifier.Ids(string.Join(' ', Enumerable.Repeat("word", 500)));
|
||
Assert.HasCount(OnnxToneClassifier.MaxTokens, longest, "A long text is cut, keeping the start and the end marks.");
|
||
Assert.AreEqual(0, longest[0]);
|
||
Assert.AreEqual(2, longest[^1]);
|
||
}
|
||
|
||
[TestMethod]
|
||
public async Task AModelThatDoesNotMatchItsHashIsNeverKept()
|
||
{
|
||
var folder = TempFolder.Create("model");
|
||
try
|
||
{
|
||
var http = new FakeHttp().On(u => u.Host == "huggingface.co", _ => new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent(new byte[4096]) });
|
||
await File.WriteAllBytesAsync(Path.Combine(folder, ToneModel.FileName), new byte[10]).ConfigureAwait(false);
|
||
await Assert.ThrowsExactlyAsync<InvalidDataException>(() => ToneModel.EnsureAsync(http.Client(), folder, CancellationToken.None)).ConfigureAwait(false);
|
||
Assert.AreEqual(ToneModel.Url, http.Requests.Single().RequestUri, "The pinned revision, never a moving branch.");
|
||
Assert.IsFalse(File.Exists(Path.Combine(folder, ToneModel.FileName + ".part")), "A wrong download is deleted.");
|
||
Assert.AreEqual(10, new FileInfo(Path.Combine(folder, ToneModel.FileName)).Length, "The old file is not taken for a good one: it stays unused.");
|
||
|
||
var failing = new FakeHttp().On(u => true, _ => new HttpResponseMessage(HttpStatusCode.NotFound));
|
||
await Assert.ThrowsExactlyAsync<HttpRequestException>(() => ToneModel.EnsureAsync(failing.Client(), folder, CancellationToken.None)).ConfigureAwait(false);
|
||
}
|
||
finally
|
||
{
|
||
TempFolder.Delete(folder);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// The real model, 83 MB: it runs where it has been downloaded once (ANUBIS_TEST_MODEL_DIR, a folder with the file
|
||
/// named as in <see cref="ToneModel.FileName"/>), never downloaded by the tests themselves.
|
||
/// </summary>
|
||
[TestMethod]
|
||
public async Task TheModelReadsTheToneOfFinancialHeadlines()
|
||
{
|
||
if (Environment.GetEnvironmentVariable("ANUBIS_TEST_MODEL_DIR") is not { Length: > 0 } folder || !File.Exists(Path.Combine(folder, ToneModel.FileName)))
|
||
{
|
||
Assert.Inconclusive("Il modello ONNX non è scaricato: impostare ANUBIS_TEST_MODEL_DIR sulla cartella che lo contiene.");
|
||
return;
|
||
}
|
||
|
||
var offline = new FakeHttp();
|
||
var path = await ToneModel.EnsureAsync(offline.Client(), folder, CancellationToken.None).ConfigureAwait(false);
|
||
Assert.IsEmpty(offline.Requests, "A model with the right hash is used as it is.");
|
||
using var classifier = new OnnxToneClassifier(path);
|
||
Tone Grade(string title) => NewsJudge.Grade(classifier.Classify(title).Score);
|
||
|
||
Assert.IsGreaterThan(0, (int)Grade("Apple shares jump after record iPhone sales in China"));
|
||
Assert.IsGreaterThan(0, (int)Grade("Microsoft beats estimates and raises its full-year outlook"));
|
||
Assert.IsLessThan(0, (int)Grade("Exchange halts withdrawals after hack, Solana tokens drained"));
|
||
Assert.IsLessThan(0, (int)Grade("Target cuts its holiday outlook as shoppers pull back"));
|
||
Assert.IsLessThan(0, (int)Grade("Tesla shares plunge after a wide miss on deliveries"));
|
||
Assert.AreEqual(Tone.Neutral, Grade("The company will report its results on Thursday"));
|
||
var score = classifier.Classify("Apple shares jump");
|
||
Assert.AreEqual(1, score.Positive + score.Negative + score.Neutral, 1e-6);
|
||
|
||
var titles = Original.Select(o => o.Text).ToArray();
|
||
var watch = Stopwatch.StartNew();
|
||
for (var round = 0; round < 5; round++)
|
||
{
|
||
foreach (var title in titles)
|
||
{
|
||
classifier.Classify(title);
|
||
}
|
||
}
|
||
|
||
var perTitle = watch.Elapsed.TotalMilliseconds / (5 * titles.Length);
|
||
Console.WriteLine($"Classificatore: {perTitle:F1} ms per titolo su un thread.");
|
||
Assert.IsLessThan(100, perTitle, "SPEC 4.5 expects 10–20 ms per title on a CPU; a slow test machine gets five times that.");
|
||
}
|
||
}
|