Publish passing benchmark results

.github/languages.json

@@ -1,24 +1,24 @@
 {
   "acton": {"compilers": ["acton"]},
   "c": {"compilers": ["clang", "gcc"]},
-  "chapel": {"image": "chapel/chapel:2.9.0", "compilers": ["chpl"]},
+  "chapel": {"image": "docker.io/chapel/chapel:2.9.0", "compilers": ["chpl"]},
   "codon": {"compilers": ["codon"]},
   "cpp": {"compilers": ["clang++", "g++"]},
   "crystal": {"compilers": ["crystal"]},
   "csharp": {"compilers": ["dotnet:9"]},
   "d": {"compilers": ["ldc2"]},
-  "dart": {"image": "dart:stable", "compilers": ["dart/exe"]},
-  "elixir": {"image": "elixir:1.18-otp-27", "compilers": ["elixir"]},
+  "dart": {"image": "docker.io/dart:stable", "compilers": ["dart/exe"]},
+  "elixir": {"image": "docker.io/elixir:1.18-otp-27", "compilers": ["elixir"]},
   "fortran": {"compilers": ["gfortran"]},
-  "go": {"image": "golang:1.26.8", "compilers": ["go"]},
-  "hacklang": {"image": "hhvm/hhvm:latest", "compilers": ["hhvm"]},
+  "go": {"image": "docker.io/golang:1.26.8", "compilers": ["go"]},
+  "hacklang": {"image": "docker.io/hhvm/hhvm:latest", "compilers": ["hhvm"]},
   "hare": {"compilers": ["hare"]},
   "haskell": {"compilers": ["ghc"]},
   "haxe": {"compilers": ["haxe/cpp"]},
-  "java": {"image": "eclipse-temurin:21-jdk-noble", "compilers": ["openjdk:21", "openjdk/zgc:21"]},
-  "javascript": {"image": "node:24-bookworm", "compilers": ["node:current", "bun"]},
-  "julia": {"image": "julia:1.11-bookworm", "compilers": ["julia"]},
-  "kotlin": {"image": "eclipse-temurin:21-jdk-noble", "compilers": ["kotlin/jvm"]},
+  "java": {"image": "docker.io/eclipse-temurin:21-jdk-noble", "compilers": ["openjdk:21", "openjdk/zgc:21"]},
+  "javascript": {"image": "docker.io/node:24-bookworm", "compilers": ["node:current", "bun"]},
+  "julia": {"image": "docker.io/julia:1.11-bookworm", "compilers": ["julia"]},
+  "kotlin": {"image": "docker.io/eclipse-temurin:21-jdk-noble", "compilers": ["kotlin/jvm"]},
   "lisp": {"compilers": ["sbcl"]},
   "lua": {"compilers": ["lua", "luajit"]},
   "nelua": {"compilers": ["nelua/clang"]},
@@ -32,7 +32,7 @@
   "racket": {"compilers": ["racket"]},
   "ruby": {"compilers": ["ruby", "ruby/yjit"]},
   "rust": {"compilers": ["rustc:stable"]},
-  "swift": {"image": "swift:6.1-noble", "compilers": ["swift"]},
+  "swift": {"image": "docker.io/swift:6.1-noble", "compilers": ["swift"]},
   "typescript": {"compilers": ["deno"]},
   "v": {"compilers": ["v/clang+gc"]},
   "wasm": {"compilers": ["wasmtime"]},

.github/suite.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Plan the suite and reject incomplete benchmark artifacts."""
+"""Plan the suite and publish verified results from successful languages."""
 import json
 import math
 import os
@@ -130,6 +130,35 @@ def plan():
                 print(f'{key}={value}', file=out)
 
 
+def collect_results():
+    results = BENCH / 'build/_results'
+    available = sorted(p.name for p in results.iterdir() if p.is_dir()) if results.exists() else []
+    if set(available) - LANGUAGES.keys():
+        raise ValueError(f'Unexpected result languages: {set(available) - LANGUAGES.keys()}')
+    if not available:
+        raise ValueError('No successful languages to publish')
+    machines = set()
+    for language in available:
+        machines.update(verify(language, 'results'))
+    if len(machines) != 1:
+        raise ValueError('Results from different machines cannot be published together')
+    cpu, runner = next(iter(machines))
+    missing = sorted(LANGUAGES.keys() - set(available))
+    summary = dict(expectedLanguages=sorted(LANGUAGES), publishedLanguages=available,
+        missingLanguages=missing, cpuInfo=cpu, runnerName=runner,
+        githubSha=os.environ['GITHUB_SHA'], githubRunId=os.environ['GITHUB_RUN_ID'],
+        githubRepository=os.environ['GITHUB_REPOSITORY'])
+    (BENCH / 'build/run-summary.json').write_text(json.dumps(summary, indent=2) + '\n')
+    message = f'Publishing results for {len(available)} of {len(LANGUAGES)} languages.'
+    if missing:
+        message += f' Unavailable in this run: {", ".join(missing)}.'
+    print(message)
+    if os.environ.get('GITHUB_STEP_SUMMARY'):
+        with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out:
+            print(message, file=out)
+    return summary
+
+
 if __name__ == '__main__':
     configured = {config['lang'] for _, config in CONFIGS}
     if LANGUAGES.keys() != configured:
@@ -139,6 +168,8 @@ def plan():
     command = sys.argv[1]
     if command == 'plan':
         plan()
+    elif command == 'collect-results':
+        collect_results()
     elif command == 'list':
         for language in LANGUAGES:
             print(f'{language}: {len(programs(language))} programs')

.github/workflows/bench.yml

@@ -86,7 +86,7 @@ jobs:
 
   results:
     needs: [plan, language]
-    if: needs.plan.outputs.publish == 'true'
+    if: ${{ !cancelled() && needs.plan.outputs.publish == 'true' }}
     runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
@@ -96,20 +96,22 @@ jobs:
           pattern: benchmark-results-*
           merge-multiple: true
           path: bench/build
-      - name: Require complete results from this revision and run
-        run: /usr/bin/python3 .github/suite.py verify-results all
+      - name: Verify successful languages and record missing results
+        run: /usr/bin/python3 .github/suite.py collect-results
       - uses: actions/upload-artifact@v4
         with:
           name: benchmark-results
           path: |
             bench/build/_results/
             bench/build/environment-*.txt
+            bench/build/run-summary.json
           if-no-files-found: error
           retention-days: 90
           overwrite: true
 
   publish:
     needs: results
+    if: ${{ !cancelled() && needs.results.result == 'success' }}
     permissions:
       contents: read
       pages: write

README.md

@@ -3,7 +3,7 @@
 Acton-maintained fork of [hanabi1224/Programming-Language-Benchmarks](https://github.com/hanabi1224/Programming-Language-Benchmarks).
 
 [Published results](https://actonlang.github.io/Programming-Language-Benchmarks/)
-compare every upstream language with Acton. Acton implements all 18 problems;
+compare upstream languages with Acton. Acton implements all 18 problems;
 other languages use the implementations enabled in the upstream configurations.
 
 ## Measurements
@@ -24,8 +24,8 @@ Each language uses its own container with the same harness and inputs. The
 container runs on the host CPU without CPU or process-count quotas. The wrapper holds
 `~/.local/state/acton-perf.lock` throughout setup, checking, and measurement.
 Other measurements on the host must use that lock. A manual language selection
-supports troubleshooting; only a complete run publishes the website. A new full
-run cancels an older full run, while normal pushes leave it running.
+supports troubleshooting; full-suite runs publish the languages that pass.
+A new full run cancels an older full run, while normal pushes leave it running.
 
 Container storage is isolated in `/var/lib/acton-bench-containers`. Unused
 benchmark containers are removed before each job, dangling images afterwards,
@@ -48,10 +48,14 @@ absolute path in the runner service environment, then restart the idle service.
 `.github/languages.json` selects all 39 upstream language entries and their
 primary toolchains, including WebAssembly. It does not select every historical
 compiler release or experimental backend. `.github/suite.py` requires every
-selected program to build, pass correctness checks, and produce every expected
-result before publication. A program that passes correctness checks but exceeds
-a measurement time limit is shown as a timeout, without timing or memory values.
-Crashes and incomplete repetitions fail the job. Toolchain versions, the container image ID, source
+selected program in a language to build, pass correctness checks, and produce
+every expected result before publishing that language. A program that passes
+correctness checks but exceeds a measurement time limit is shown as a timeout,
+without timing or memory values.
+Crashes and incomplete repetitions fail the language job. Other languages can
+still publish. The website lists unavailable languages and links to the run;
+the combined artifact includes this inventory in `run-summary.json`.
+Toolchain versions, the container image ID, source
 revision, and Actions run are recorded with the results.
 
 V uses its garbage-collected backend; the experimental autofree backend corrupts
@@ -116,13 +120,15 @@ NODE_OPTIONS=--openssl-legacy-provider SITE_BASE_PATH=/Programming-Language-Benc
 The generated site is in `website/dist`. `SITE_BASE_PATH` defaults to `/` for
 local development. All internal links and assets respect the configured prefix.
 The checked-in upstream data is only for website development and PR build checks;
-publishing replaces it completely with results from the successful benchmark run.
+publishing replaces it completely with verified results from the current run.
 
 The `bench` workflow measures weekly and on manual dispatch. Its
 `publish` job calls `site.yml`, which downloads that run's results, builds the
 website on a GitHub-hosted runner, and deploys it to GitHub Pages. Configure the
-repository's Pages publishing source as GitHub Actions. A failed build or
-measurement job leaves the previous published site in place.
+repository's Pages publishing source as GitHub Actions. Failed language jobs
+remain red in Actions while verified languages publish. If no language passes,
+result verification fails, or the run is cancelled, the previous site stays in
+place.
 
 ## Attribution
 

bench/algorithm/coro-prime-sieve/1.ts

@@ -12,6 +12,8 @@ async function* generate() {
 
 async function* filter(ch: AsyncGenerator, prime: number) {
     while (true) {
+        // Yield before calling upstream so long chains do not exhaust the stack.
+        await Promise.resolve();
         var i = (await ch.next()).value;
         if (i % prime != 0) {
             yield i;

bench/algorithm/mandelbrot/2.cs

@@ -1,12 +1,12 @@
 using System;
 using System.Linq;
-using System.Numerics;
+using System.Runtime.Intrinsics;
 using System.Security.Cryptography;
 using System.Text;
 
 public class MandelBrot
 {
-    private static readonly Vector<double> _threshold = new Vector<double>(4);
+    private static readonly Vector256<double> _threshold = Vector256.Create(4.0);
     public static void Main(string[] args)
     {
         var size = args.Length == 0 ? 200 : int.Parse(args[0]);
@@ -15,7 +15,7 @@ public static void Main(string[] args)
         var inv = 2.0 / size;
         Console.WriteLine($"P4\n{size} {size}");
 
-        var xloc = new (Vector<double>, Vector<double>)[chunkSize];
+        var xloc = new (Vector256<double>, Vector256<double>)[chunkSize];
         Span<double> array = stackalloc double[8];
         for (var i = 0; i < chunkSize; i++)
         {
@@ -24,7 +24,7 @@ public static void Main(string[] args)
             {
                 array[j] = (offset + j) * inv - 1.5;
             }
-            xloc[i] = (new Vector<double>(array.Slice(0, 4)), new Vector<double>(array.Slice(4, 4)));
+            xloc[i] = (Vector256.Create((ReadOnlySpan<double>)array.Slice(0, 4)), Vector256.Create((ReadOnlySpan<double>)array.Slice(4, 4)));
         }
 
         var data = new byte[size * chunkSize];
@@ -47,19 +47,19 @@ public static void Main(string[] args)
         Console.WriteLine(ToHexString(hash));
     }
 
-    static byte mbrot8((Vector<double>, Vector<double>) cr, double civ)
+    static byte mbrot8((Vector256<double>, Vector256<double>) cr, double civ)
     {
-        var ci = new Vector<double>(new[] { civ, civ, civ, civ });
-        var zr0 = new Vector<double>(0);
-        var zr1 = new Vector<double>(0);
-        var zi0 = new Vector<double>(0);
-        var zi1 = new Vector<double>(0);
-        var tr0 = new Vector<double>(0);
-        var tr1 = new Vector<double>(0);
-        var ti0 = new Vector<double>(0);
-        var ti1 = new Vector<double>(0);
-        var absz0 = new Vector<double>(0);
-        var absz1 = new Vector<double>(0);
+        var ci = Vector256.Create(civ);
+        var zr0 = Vector256<double>.Zero;
+        var zr1 = Vector256<double>.Zero;
+        var zi0 = Vector256<double>.Zero;
+        var zi1 = Vector256<double>.Zero;
+        var tr0 = Vector256<double>.Zero;
+        var tr1 = Vector256<double>.Zero;
+        var ti0 = Vector256<double>.Zero;
+        var ti1 = Vector256<double>.Zero;
+        var absz0 = Vector256<double>.Zero;
+        var absz1 = Vector256<double>.Zero;
         for (var _i = 0; _i < 10; _i++)
         {
             for (var _j = 0; _j < 5; _j++)
@@ -80,7 +80,7 @@ static byte mbrot8((Vector<double>, Vector<double>) cr, double civ)
             }
             absz0 = tr0 + ti0;
             absz1 = tr1 + ti1;
-            if (Vector.GreaterThanAll(absz0, _threshold) && Vector.GreaterThanAll(absz1, _threshold))
+            if (Vector256.GreaterThanAll(absz0, _threshold) && Vector256.GreaterThanAll(absz1, _threshold))
             {
                 return 0;
             }

bench/algorithm/nbody/9.cs

@@ -10,7 +10,7 @@ modified by hanabi1224 to use simd-powered Vector
     */
 
     using System;
-    using System.Numerics;
+    using System.Runtime.Intrinsics;
 
     public class NBody
     {
@@ -30,14 +30,14 @@ public static void Main(String[] args)
 
     public class Body
     {
-        public Vector<double> Pos { get; set; }
-        public Vector<double> Velocity { get; set; }
+        public Vector256<double> Pos { get; set; }
+        public Vector256<double> Velocity { get; set; }
         public double Mass { get; }
 
         public Body(double x, double y, double z, double vx, double vy, double vz, double mass)
         {
-            Pos = new Vector<double>(new[] { x, y, z, 0 });
-            Velocity = new Vector<double>(new[] { vx, vy, vz, 0 });
+            Pos = Vector256.Create(x, y, z, 0);
+            Velocity = Vector256.Create(vx, vy, vz, 0);
             Mass = mass;
         }
     }
@@ -113,7 +113,7 @@ public NBodySystem()
 
         public void OffsetMomentum()
         {
-            var p = new Vector<double>(new[] { 0.0, 0.0, 0.0, 0.0 });
+            var p = Vector256<double>.Zero;
             foreach (var b in _bodies)
             {
                 p -= b.Velocity * b.Mass;
@@ -133,7 +133,7 @@ public void Advance(double dt)
                 {
                     var bj = _bodies[j];
                     var dpos = pos - bj.Pos;
-                    double d2 = Vector.Dot(dpos, dpos);
+                    double d2 = Vector256.Dot(dpos, dpos);
                     double mag = dt / (d2 * Math.Sqrt(d2));
                     dpos *= mag;
                     v -= dpos * bj.Mass;
@@ -150,12 +150,12 @@ public double Energy()
             for (int i = 0; i < bodyCount; i++)
             {
                 var bi = _bodies[i];
-                e += 0.5 * bi.Mass * Vector.Dot(bi.Velocity, bi.Velocity);
+                e += 0.5 * bi.Mass * Vector256.Dot(bi.Velocity, bi.Velocity);
                 for (int j = i + 1; j < bodyCount; j++)
                 {
                     var bj = _bodies[j];
                     var dpos = bi.Pos - bj.Pos;
-                    e -= (bi.Mass * bj.Mass) / Math.Sqrt(Vector.Dot(dpos, dpos));
+                    e -= (bi.Mass * bj.Mass) / Math.Sqrt(Vector256.Dot(dpos, dpos));
                 }
             }
             return e;

bench/include/c/app_ffi.rsp

@@ -1 +1 @@
- -pipe -O3 -fomit-frame-pointer -march=broadwell -fopenmp -pthread -Wno-deprecated-declarations -mno-fma -o app app.c -lm -lcrypto
+ -pipe -O3 -fomit-frame-pointer -march=native -fopenmp -pthread -Wno-deprecated-declarations -ffp-contract=off -o app app.c -lm -lcrypto

bench/tests/test_suite.py

@@ -26,9 +26,11 @@ def setUp(self):
             buildLog={'finished': 'today'}, testLog={'finished': 'today'},
             githubSha='current', githubRunId='123', githubRepository='owner/repo')
         for mock in [patch.object(suite, 'BENCH', self.root),
+                     patch.object(suite, 'LANGUAGES', {'fixture': {}, 'missing': {}}),
                      patch.object(suite, 'programs', return_value={'fixture': ('sample', '1.sh', 'sh', '1')}),
                      patch.object(suite, 'PROBLEMS', {'sample': {'tests': [{'input': 'small'}, {'input': 'large'}]}}),
-                     patch.dict(os.environ, GITHUB_SHA='current', GITHUB_RUN_ID='123', GITHUB_REPOSITORY='owner/repo')]:
+                     patch.dict(os.environ, GITHUB_SHA='current', GITHUB_RUN_ID='123',
+                                GITHUB_REPOSITORY='owner/repo', GITHUB_STEP_SUMMARY='')]:
             mock.start()
             self.addCleanup(mock.stop)
         self.write_records()
@@ -76,6 +78,42 @@ def test_ignored_build_failure_is_rejected(self):
         with self.assertRaisesRegex(ValueError, 'Missing build output'):
             suite.verify('fixture', 'build')
 
+    def test_failed_language_does_not_block_verified_results(self):
+        summary = suite.collect_results()
+        self.assertEqual(summary['publishedLanguages'], ['fixture'])
+        self.assertEqual(summary['missingLanguages'], ['missing'])
+        self.assertEqual(summary['githubRunId'], '123')
+        self.assertEqual(json.loads((self.root / 'build/run-summary.json').read_text()), summary)
+
+    def test_partial_artifact_is_still_rejected(self):
+        (self.results / 'fixture_large.json').unlink()
+        with self.assertRaisesRegex(ValueError, 'missing='):
+            suite.collect_results()
+        self.assertFalse((self.root / 'build/run-summary.json').exists())
+
+    def test_partial_publication_rejects_stale_results(self):
+        self.record['githubRunId'] = 'old run'
+        self.write_records()
+        with self.assertRaisesRegex(ValueError, 'Wrong githubRunId'):
+            suite.collect_results()
+
+    def test_no_results_cannot_replace_the_site(self):
+        for path in self.results.iterdir():
+            path.unlink()
+        self.results.rmdir()
+        with self.assertRaisesRegex(ValueError, 'No successful languages'):
+            suite.collect_results()
+
+    def test_publication_rejects_different_language_machines(self):
+        other = self.results.parent / 'missing'
+        other.mkdir()
+        for path in self.results.iterdir():
+            record = json.loads(path.read_text())
+            record.update(lang='missing', runnerName='another runner')
+            (other / path.name).write_text(json.dumps(record))
+        with self.assertRaisesRegex(ValueError, 'different machines'):
+            suite.collect_results()
+
 
 if __name__ == '__main__':
     unittest.main()