using System.Diagnostics; using System.Net; using Anubis.News; namespace Anubis.Tests; /// The local classifier of SPEC 4.5 (issue #57): the same tokens as the original, the model checked by its hash. [TestClass] public sealed class ToneModelTests { /// /// Ids of the original RoBERTa tokenizer on the same vocab.json and merges.txt, computed with an independent copy of /// the GPT-2 byte-level encoder (openai/gpt-2, encoder.py), reading the text as UTF-8. "Hello world" is the well-known /// 0 31414 232 2. /// private static readonly (string Text, int[] Ids)[] Original = [ ("Hello world", [0, 31414, 232, 2]), ("Apple shares jump after record iPhone sales in China", [0, 20770, 327, 3704, 71, 638, 2733, 647, 11, 436, 2]), ("Bitcoin drifts below $90,000 as traders await U.S. data", [0, 42817, 12402, 22833, 874, 68, 3248, 6, 151, 25, 5539, 17396, 121, 4, 104, 4, 414, 2]), ("Fire at a chip plant hits Nvidia supply, $NVDA slides", [0, 17260, 23, 10, 6638, 2195, 2323, 18520, 1787, 6, 68, 31668, 3134, 19583, 2]), ("Fed officials signal patience on interest rates into December", [0, 38186, 503, 6029, 11383, 15, 773, 1162, 88, 719, 2]), ("Exchange halts withdrawals after hack, Solana tokens drained", [0, 9089, 14035, 12193, 1872, 27733, 71, 14157, 6, 4856, 1113, 22121, 23544, 2]), ("Tesla recalls vehicles over a software issue (NASDAQ:TSLA)", [0, 39971, 10270, 1734, 81, 10, 2257, 696, 36, 3399, 3223, 35, 2685, 8272, 43, 2]), ("Microsoft & OpenAI sign a new cloud deal", [0, 35460, 359, 2117, 15238, 1203, 10, 92, 3613, 432, 2]), ("Target cuts its holiday outlook as shoppers pull back", [0, 41858, 2599, 63, 2317, 3839, 25, 9456, 2999, 124, 2]), ("It's the company's 3rd quarter: EPS beat by 12%, revenue up 7.5%", [0, 243, 18, 5, 138, 18, 155, 2586, 297, 35, 4721, 1451, 30, 316, 4234, 903, 62, 262, 4, 245, 207, 2]), (" Spaces and\ttabs ", [0, 1437, 39926, 1437, 1437, 8, 50117, 34108, 29, 1437, 1437, 2]), ("L'inflazione è vicina all'obiettivo", [0, 574, 108, 179, 4825, 1222, 8525, 952, 11423, 32438, 1243, 70, 108, 2413, 5810, 90, 9697, 2]), ("Société Générale cède 3,5% à Paris", [0, 104, 17641, 10221, 1140, 272, 10598, 20765, 1627, 740, 8025, 2794, 155, 6, 245, 207, 6534, 2201, 2]), ("Nasdaq ➜ record — €1.2bn deal", [0, 487, 281, 5946, 14333, 17772, 48, 638, 93, 4480, 134, 4, 176, 6316, 432, 2]), ]; [TestMethod] public void TheTokenizerGivesTheIdsOfTheOriginal() { foreach (var (text, ids) in Original) { CollectionAssert.AreEqual(ids, OnnxToneClassifier.Ids(text).ToArray(), text); } var marks = OnnxToneClassifier.Ids(" buy "); Assert.AreEqual(1, marks.Count(i => i == 0), "Marks written in a headline are text, not marks."); Assert.AreEqual(1, marks.Count(i => i == 2)); Assert.IsTrue(OnnxToneClassifier.Ids("\u0000 \u2603 \U0001F680 \u4E2D\u6587 \n\r").All(i => i is >= 0 and < 50265), "Any text gives ids of the vocabulary."); var longest = OnnxToneClassifier.Ids(string.Join(' ', Enumerable.Repeat("word", 500))); Assert.HasCount(OnnxToneClassifier.MaxTokens, longest, "A long text is cut, keeping the start and the end marks."); Assert.AreEqual(0, longest[0]); Assert.AreEqual(2, longest[^1]); } [TestMethod] public async Task AModelThatDoesNotMatchItsHashIsNeverKept() { var folder = TempFolder.Create("model"); try { var http = new FakeHttp().On(u => u.Host == "huggingface.co", _ => new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent(new byte[4096]) }); await File.WriteAllBytesAsync(Path.Combine(folder, ToneModel.FileName), new byte[10]).ConfigureAwait(false); await Assert.ThrowsExactlyAsync(() => ToneModel.EnsureAsync(http.Client(), folder, CancellationToken.None)).ConfigureAwait(false); Assert.AreEqual(ToneModel.Url, http.Requests.Single().RequestUri, "The pinned revision, never a moving branch."); Assert.IsFalse(File.Exists(Path.Combine(folder, ToneModel.FileName + ".part")), "A wrong download is deleted."); Assert.AreEqual(10, new FileInfo(Path.Combine(folder, ToneModel.FileName)).Length, "The old file is not taken for a good one: it stays unused."); var failing = new FakeHttp().On(u => true, _ => new HttpResponseMessage(HttpStatusCode.NotFound)); await Assert.ThrowsExactlyAsync(() => ToneModel.EnsureAsync(failing.Client(), folder, CancellationToken.None)).ConfigureAwait(false); } finally { TempFolder.Delete(folder); } } /// /// The real model, 83 MB: it runs where it has been downloaded once (ANUBIS_TEST_MODEL_DIR, a folder with the file /// named as in ), never downloaded by the tests themselves. /// [TestMethod] public async Task TheModelReadsTheToneOfFinancialHeadlines() { if (Environment.GetEnvironmentVariable("ANUBIS_TEST_MODEL_DIR") is not { Length: > 0 } folder || !File.Exists(Path.Combine(folder, ToneModel.FileName))) { Assert.Inconclusive("Il modello ONNX non è scaricato: impostare ANUBIS_TEST_MODEL_DIR sulla cartella che lo contiene."); return; } var offline = new FakeHttp(); var path = await ToneModel.EnsureAsync(offline.Client(), folder, CancellationToken.None).ConfigureAwait(false); Assert.IsEmpty(offline.Requests, "A model with the right hash is used as it is."); using var classifier = new OnnxToneClassifier(path); Tone Grade(string title) => NewsJudge.Grade(classifier.Classify(title).Score); Assert.IsGreaterThan(0, (int)Grade("Apple shares jump after record iPhone sales in China")); Assert.IsGreaterThan(0, (int)Grade("Microsoft beats estimates and raises its full-year outlook")); Assert.IsLessThan(0, (int)Grade("Exchange halts withdrawals after hack, Solana tokens drained")); Assert.IsLessThan(0, (int)Grade("Target cuts its holiday outlook as shoppers pull back")); Assert.IsLessThan(0, (int)Grade("Tesla shares plunge after a wide miss on deliveries")); Assert.AreEqual(Tone.Neutral, Grade("The company will report its results on Thursday")); var score = classifier.Classify("Apple shares jump"); Assert.AreEqual(1, score.Positive + score.Negative + score.Neutral, 1e-6); var titles = Original.Select(o => o.Text).ToArray(); var watch = Stopwatch.StartNew(); for (var round = 0; round < 5; round++) { foreach (var title in titles) { classifier.Classify(title); } } var perTitle = watch.Elapsed.TotalMilliseconds / (5 * titles.Length); Console.WriteLine($"Classificatore: {perTitle:F1} ms per titolo su un thread."); Assert.IsLessThan(100, perTitle, "SPEC 4.5 expects 10–20 ms per title on a CPU; a slow test machine gets five times that."); } }