From b1167fceb287b314fdcd1c931fb85a6de302b4cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20Hern=C3=A1ndez?= Date: Tue, 8 Sep 2026 22:29:01 -0400 Subject: [PATCH] The README's tool table lost a tool, and two directories copied the gap write_report shipped and its row was never added to the root README, so the table listed nine tools and the sentence under it said "the first seven run entirely on the machine" when eight did. src/SignsOfAI.Mcp/README.md had it right the whole time; only the file that visitors read was wrong. That file is what directory listings copy from. Six weeks later the merged TensorBlock entry and the open punkpeye one both describe six tools and tell people to install from source, in repositories where we have no guard. So the guard reads the server's own [McpServerTool] attributes -- including OpenWorld, which is what actually means a tool leaves the machine -- and requires both READMEs to agree. Verified by reverting the fix: it fails on exactly the two real defects and passes on the correction. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_015PEbbiYSNPw7jE3LrPNhyF --- README.md | 3 +- .../McpToolDocumentationTests.cs | 117 ++++++++++++++++++ 2 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 tests/SignsOfAI.Core.Tests/McpToolDocumentationTests.cs diff --git a/README.md b/README.md index eb4e0a8..f1365dd 100644 --- a/README.md +++ b/README.md @@ -205,10 +205,11 @@ the engine lives in `SignsOfAI.Core` — pure .NET, no browser — the server ju | `inspect_characters` | invisible characters & letters impersonating Latin ones, with line/column | 🔒 on-device | | `check_citations` | where a document contradicts its own bibliography, with the line of each | 🔒 on-device | | `compare_to_baseline` | how a piece sits against the same writer's earlier work, on their own scale | 🔒 on-device | +| `write_report` | the whole analysis as a document to keep, forward, or take to a committee | 🔒 on-device | | `measure_predictability` | perplexity via the optional server | 🌐 server (**opt-in**) | | `check_paraphrase` | reworded/translated matches via EmbeddingGemma | 🌐 server (**opt-in**) | -The first seven run entirely on the machine; the last two disclose that they send text to the server +The first eight run entirely on the machine; the last two disclose that they send text to the server (endpoint via the `SIGNSOFAI_API_ENDPOINT` environment variable). It ships on NuGet as [`SignsOfAI.Mcp`](https://www.nuget.org/packages/SignsOfAI.Mcp), so nothing needs diff --git a/tests/SignsOfAI.Core.Tests/McpToolDocumentationTests.cs b/tests/SignsOfAI.Core.Tests/McpToolDocumentationTests.cs new file mode 100644 index 0000000..f6064c3 --- /dev/null +++ b/tests/SignsOfAI.Core.Tests/McpToolDocumentationTests.cs @@ -0,0 +1,117 @@ +using System.Text.RegularExpressions; +using Xunit; + +namespace SignsOfAI.Core.Tests; + +/// +/// Two READMEs describe the MCP server's tools by hand: the repository's own, which is what a +/// visitor reads and what every directory listing copies from, and the server package's. +/// +/// The root README went stale and nobody measured it. write_report shipped and its row was +/// never added, so the table listed nine tools and the sentence under it said "the first seven run +/// entirely on the machine" when eight did. Two public directory listings — one merged, one open — +/// then copied that table faithfully and were wrong in the same way, six weeks later, in someone +/// else's repository where we have no guard at all. +/// +/// So this reads the server's own attributes and requires both files to agree with them. Adding a +/// tool without documenting it fails here, by name. +/// +public class McpToolDocumentationTests +{ + private static readonly string RepoRoot = FindRepoRoot(); + + /// + /// The tools as the server actually declares them. OpenWorld is the flag that means a + /// tool reaches off the machine, so the on-device/server split is derived, never typed. + /// + private static readonly IReadOnlyList<(string Name, bool ReachesOut)> Tools = ReadTools(); + + private static string FindRepoRoot() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null && !Directory.Exists(Path.Combine(dir.FullName, "src"))) + dir = dir.Parent; + + Assert.NotNull(dir); + return dir!.FullName; + } + + private static IReadOnlyList<(string, bool)> ReadTools() + { + var toolsDir = Path.Combine(RepoRoot, "src", "SignsOfAI.Mcp", "Tools"); + var attribute = new Regex(@"\[McpServerTool\((?[^\]]*)\)", RegexOptions.Compiled); + + var found = new List<(string, bool)>(); + foreach (var file in Directory.EnumerateFiles(toolsDir, "*.cs").OrderBy(f => f, StringComparer.Ordinal)) + foreach (Match match in attribute.Matches(File.ReadAllText(file))) + { + var args = match.Groups["args"].Value; + var name = Regex.Match(args, @"Name\s*=\s*""(?[^""]+)"""); + if (!name.Success) continue; + found.Add((name.Groups["n"].Value, args.Contains("OpenWorld = true", StringComparison.Ordinal))); + } + + Assert.NotEmpty(found); + return found; + } + + // The two files spell their counts out in words, so the guard has to as well. + private static string InWords(int n) => n switch + { + 1 => "one", 2 => "two", 3 => "three", 4 => "four", 5 => "five", + 6 => "six", 7 => "seven", 8 => "eight", 9 => "nine", 10 => "ten", + 11 => "eleven", 12 => "twelve", + _ => throw new ArgumentOutOfRangeException(nameof(n), + $"The server has {n} tools and this guard cannot spell that. Extend it.") + }; + + public static TheoryData DocumentedFiles() => new() + { + "README.md", + Path.Combine("src", "SignsOfAI.Mcp", "README.md"), + }; + + [Theory] + [MemberData(nameof(DocumentedFiles))] + public void Every_tool_the_server_declares_is_documented(string relativePath) + { + var text = File.ReadAllText(Path.Combine(RepoRoot, relativePath)); + + var missing = Tools.Select(t => t.Name) + .Where(name => !text.Contains($"`{name}`", StringComparison.Ordinal)) + .ToList(); + + Assert.True(missing.Count == 0, + $"{relativePath} does not mention {string.Join(", ", missing)}. The server exposes " + + $"{Tools.Count} tools; add the row, or the next directory listing will copy the gap."); + } + + [Fact] + public void The_root_readme_says_how_many_tools_stay_on_the_machine() + { + var onDevice = Tools.Count(t => !t.ReachesOut); + var reachOut = Tools.Count - onDevice; + var text = File.ReadAllText(Path.Combine(RepoRoot, "README.md")); + + var expected = $"The first {InWords(onDevice)} run entirely on the machine; " + + $"the last {InWords(reachOut)} disclose that they send text to the server"; + + Assert.True(text.Contains(expected, StringComparison.Ordinal), + $"README.md no longer says \"{expected}\". {onDevice} of the {Tools.Count} tools carry no " + + "OpenWorld flag; correct the sentence under the table."); + } + + [Fact] + public void The_server_readme_says_how_many_tools_stay_on_the_machine() + { + var onDevice = Tools.Count(t => !t.ReachesOut); + var text = File.ReadAllText(Path.Combine(RepoRoot, "src", "SignsOfAI.Mcp", "README.md")); + + var expected = $"{char.ToUpperInvariant(InWords(onDevice)[0])}{InWords(onDevice)[1..]} " + + $"of the {InWords(Tools.Count)} run"; + + Assert.True(text.Contains(expected, StringComparison.Ordinal), + $"src/SignsOfAI.Mcp/README.md no longer says \"{expected} ...\". The server has " + + $"{Tools.Count} tools, {onDevice} of them on-device."); + } +}