diff --git a/.github/languages.json b/.github/languages.json index 40bd1689..5d8f3bf3 100644 --- a/.github/languages.json +++ b/.github/languages.json @@ -1,24 +1,24 @@ { "acton": {"compilers": ["acton"]}, "c": {"compilers": ["clang", "gcc"]}, - "chapel": {"image": "chapel/chapel:2.9.0", "compilers": ["chpl"]}, + "chapel": {"image": "docker.io/chapel/chapel:2.9.0", "compilers": ["chpl"]}, "codon": {"compilers": ["codon"]}, "cpp": {"compilers": ["clang++", "g++"]}, "crystal": {"compilers": ["crystal"]}, "csharp": {"compilers": ["dotnet:9"]}, "d": {"compilers": ["ldc2"]}, - "dart": {"image": "dart:stable", "compilers": ["dart/exe"]}, - "elixir": {"image": "elixir:1.18-otp-27", "compilers": ["elixir"]}, + "dart": {"image": "docker.io/dart:stable", "compilers": ["dart/exe"]}, + "elixir": {"image": "docker.io/elixir:1.18-otp-27", "compilers": ["elixir"]}, "fortran": {"compilers": ["gfortran"]}, - "go": {"image": "golang:1.26.8", "compilers": ["go"]}, - "hacklang": {"image": "hhvm/hhvm:latest", "compilers": ["hhvm"]}, + "go": {"image": "docker.io/golang:1.26.8", "compilers": ["go"]}, + "hacklang": {"image": "docker.io/hhvm/hhvm:latest", "compilers": ["hhvm"]}, "hare": {"compilers": ["hare"]}, "haskell": {"compilers": ["ghc"]}, "haxe": {"compilers": ["haxe/cpp"]}, - "java": {"image": "eclipse-temurin:21-jdk-noble", "compilers": ["openjdk:21", "openjdk/zgc:21"]}, - "javascript": {"image": "node:24-bookworm", "compilers": ["node:current", "bun"]}, - "julia": {"image": "julia:1.11-bookworm", "compilers": ["julia"]}, - "kotlin": {"image": "eclipse-temurin:21-jdk-noble", "compilers": ["kotlin/jvm"]}, + "java": {"image": "docker.io/eclipse-temurin:21-jdk-noble", "compilers": ["openjdk:21", "openjdk/zgc:21"]}, + "javascript": {"image": "docker.io/node:24-bookworm", "compilers": ["node:current", "bun"]}, + "julia": {"image": "docker.io/julia:1.11-bookworm", "compilers": ["julia"]}, + "kotlin": {"image": "docker.io/eclipse-temurin:21-jdk-noble", "compilers": ["kotlin/jvm"]}, "lisp": {"compilers": ["sbcl"]}, "lua": {"compilers": ["lua", "luajit"]}, "nelua": {"compilers": ["nelua/clang"]}, @@ -32,7 +32,7 @@ "racket": {"compilers": ["racket"]}, "ruby": {"compilers": ["ruby", "ruby/yjit"]}, "rust": {"compilers": ["rustc:stable"]}, - "swift": {"image": "swift:6.1-noble", "compilers": ["swift"]}, + "swift": {"image": "docker.io/swift:6.1-noble", "compilers": ["swift"]}, "typescript": {"compilers": ["deno"]}, "v": {"compilers": ["v/clang+gc"]}, "wasm": {"compilers": ["wasmtime"]}, diff --git a/.github/suite.py b/.github/suite.py index dc7b5f78..d3229fb4 100644 --- a/.github/suite.py +++ b/.github/suite.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Plan the suite and reject incomplete benchmark artifacts.""" +"""Plan the suite and publish verified results from successful languages.""" import json import math import os @@ -130,6 +130,35 @@ def plan(): print(f'{key}={value}', file=out) +def collect_results(): + results = BENCH / 'build/_results' + available = sorted(p.name for p in results.iterdir() if p.is_dir()) if results.exists() else [] + if set(available) - LANGUAGES.keys(): + raise ValueError(f'Unexpected result languages: {set(available) - LANGUAGES.keys()}') + if not available: + raise ValueError('No successful languages to publish') + machines = set() + for language in available: + machines.update(verify(language, 'results')) + if len(machines) != 1: + raise ValueError('Results from different machines cannot be published together') + cpu, runner = next(iter(machines)) + missing = sorted(LANGUAGES.keys() - set(available)) + summary = dict(expectedLanguages=sorted(LANGUAGES), publishedLanguages=available, + missingLanguages=missing, cpuInfo=cpu, runnerName=runner, + githubSha=os.environ['GITHUB_SHA'], githubRunId=os.environ['GITHUB_RUN_ID'], + githubRepository=os.environ['GITHUB_REPOSITORY']) + (BENCH / 'build/run-summary.json').write_text(json.dumps(summary, indent=2) + '\n') + message = f'Publishing results for {len(available)} of {len(LANGUAGES)} languages.' + if missing: + message += f' Unavailable in this run: {", ".join(missing)}.' + print(message) + if os.environ.get('GITHUB_STEP_SUMMARY'): + with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out: + print(message, file=out) + return summary + + if __name__ == '__main__': configured = {config['lang'] for _, config in CONFIGS} if LANGUAGES.keys() != configured: @@ -139,6 +168,8 @@ def plan(): command = sys.argv[1] if command == 'plan': plan() + elif command == 'collect-results': + collect_results() elif command == 'list': for language in LANGUAGES: print(f'{language}: {len(programs(language))} programs') diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 88b07655..5ba49151 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -86,7 +86,7 @@ jobs: results: needs: [plan, language] - if: needs.plan.outputs.publish == 'true' + if: ${{ !cancelled() && needs.plan.outputs.publish == 'true' }} runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 @@ -96,20 +96,22 @@ jobs: pattern: benchmark-results-* merge-multiple: true path: bench/build - - name: Require complete results from this revision and run - run: /usr/bin/python3 .github/suite.py verify-results all + - name: Verify successful languages and record missing results + run: /usr/bin/python3 .github/suite.py collect-results - uses: actions/upload-artifact@v4 with: name: benchmark-results path: | bench/build/_results/ bench/build/environment-*.txt + bench/build/run-summary.json if-no-files-found: error retention-days: 90 overwrite: true publish: needs: results + if: ${{ !cancelled() && needs.results.result == 'success' }} permissions: contents: read pages: write diff --git a/README.md b/README.md index e7243cfe..efd83aef 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Acton-maintained fork of [hanabi1224/Programming-Language-Benchmarks](https://github.com/hanabi1224/Programming-Language-Benchmarks). [Published results](https://actonlang.github.io/Programming-Language-Benchmarks/) -compare every upstream language with Acton. Acton implements all 18 problems; +compare upstream languages with Acton. Acton implements all 18 problems; other languages use the implementations enabled in the upstream configurations. ## Measurements @@ -24,8 +24,8 @@ Each language uses its own container with the same harness and inputs. The container runs on the host CPU without CPU or process-count quotas. The wrapper holds `~/.local/state/acton-perf.lock` throughout setup, checking, and measurement. Other measurements on the host must use that lock. A manual language selection -supports troubleshooting; only a complete run publishes the website. A new full -run cancels an older full run, while normal pushes leave it running. +supports troubleshooting; full-suite runs publish the languages that pass. +A new full run cancels an older full run, while normal pushes leave it running. Container storage is isolated in `/var/lib/acton-bench-containers`. Unused benchmark containers are removed before each job, dangling images afterwards, @@ -48,10 +48,14 @@ absolute path in the runner service environment, then restart the idle service. `.github/languages.json` selects all 39 upstream language entries and their primary toolchains, including WebAssembly. It does not select every historical compiler release or experimental backend. `.github/suite.py` requires every -selected program to build, pass correctness checks, and produce every expected -result before publication. A program that passes correctness checks but exceeds -a measurement time limit is shown as a timeout, without timing or memory values. -Crashes and incomplete repetitions fail the job. Toolchain versions, the container image ID, source +selected program in a language to build, pass correctness checks, and produce +every expected result before publishing that language. A program that passes +correctness checks but exceeds a measurement time limit is shown as a timeout, +without timing or memory values. +Crashes and incomplete repetitions fail the language job. Other languages can +still publish. The website lists unavailable languages and links to the run; +the combined artifact includes this inventory in `run-summary.json`. +Toolchain versions, the container image ID, source revision, and Actions run are recorded with the results. V uses its garbage-collected backend; the experimental autofree backend corrupts @@ -116,13 +120,15 @@ NODE_OPTIONS=--openssl-legacy-provider SITE_BASE_PATH=/Programming-Language-Benc The generated site is in `website/dist`. `SITE_BASE_PATH` defaults to `/` for local development. All internal links and assets respect the configured prefix. The checked-in upstream data is only for website development and PR build checks; -publishing replaces it completely with results from the successful benchmark run. +publishing replaces it completely with verified results from the current run. The `bench` workflow measures weekly and on manual dispatch. Its `publish` job calls `site.yml`, which downloads that run's results, builds the website on a GitHub-hosted runner, and deploys it to GitHub Pages. Configure the -repository's Pages publishing source as GitHub Actions. A failed build or -measurement job leaves the previous published site in place. +repository's Pages publishing source as GitHub Actions. Failed language jobs +remain red in Actions while verified languages publish. If no language passes, +result verification fails, or the run is cancelled, the previous site stays in +place. ## Attribution diff --git a/bench/algorithm/coro-prime-sieve/1.ts b/bench/algorithm/coro-prime-sieve/1.ts index 3e3fc1c3..3714dc16 100644 --- a/bench/algorithm/coro-prime-sieve/1.ts +++ b/bench/algorithm/coro-prime-sieve/1.ts @@ -12,6 +12,8 @@ async function* generate() { async function* filter(ch: AsyncGenerator, prime: number) { while (true) { + // Yield before calling upstream so long chains do not exhaust the stack. + await Promise.resolve(); var i = (await ch.next()).value; if (i % prime != 0) { yield i; diff --git a/bench/algorithm/mandelbrot/2.cs b/bench/algorithm/mandelbrot/2.cs index c3336356..2728595c 100644 --- a/bench/algorithm/mandelbrot/2.cs +++ b/bench/algorithm/mandelbrot/2.cs @@ -1,12 +1,12 @@ using System; using System.Linq; -using System.Numerics; +using System.Runtime.Intrinsics; using System.Security.Cryptography; using System.Text; public class MandelBrot { - private static readonly Vector _threshold = new Vector(4); + private static readonly Vector256 _threshold = Vector256.Create(4.0); public static void Main(string[] args) { var size = args.Length == 0 ? 200 : int.Parse(args[0]); @@ -15,7 +15,7 @@ public static void Main(string[] args) var inv = 2.0 / size; Console.WriteLine($"P4\n{size} {size}"); - var xloc = new (Vector, Vector)[chunkSize]; + var xloc = new (Vector256, Vector256)[chunkSize]; Span array = stackalloc double[8]; for (var i = 0; i < chunkSize; i++) { @@ -24,7 +24,7 @@ public static void Main(string[] args) { array[j] = (offset + j) * inv - 1.5; } - xloc[i] = (new Vector(array.Slice(0, 4)), new Vector(array.Slice(4, 4))); + xloc[i] = (Vector256.Create((ReadOnlySpan)array.Slice(0, 4)), Vector256.Create((ReadOnlySpan)array.Slice(4, 4))); } var data = new byte[size * chunkSize]; @@ -47,19 +47,19 @@ public static void Main(string[] args) Console.WriteLine(ToHexString(hash)); } - static byte mbrot8((Vector, Vector) cr, double civ) + static byte mbrot8((Vector256, Vector256) cr, double civ) { - var ci = new Vector(new[] { civ, civ, civ, civ }); - var zr0 = new Vector(0); - var zr1 = new Vector(0); - var zi0 = new Vector(0); - var zi1 = new Vector(0); - var tr0 = new Vector(0); - var tr1 = new Vector(0); - var ti0 = new Vector(0); - var ti1 = new Vector(0); - var absz0 = new Vector(0); - var absz1 = new Vector(0); + var ci = Vector256.Create(civ); + var zr0 = Vector256.Zero; + var zr1 = Vector256.Zero; + var zi0 = Vector256.Zero; + var zi1 = Vector256.Zero; + var tr0 = Vector256.Zero; + var tr1 = Vector256.Zero; + var ti0 = Vector256.Zero; + var ti1 = Vector256.Zero; + var absz0 = Vector256.Zero; + var absz1 = Vector256.Zero; for (var _i = 0; _i < 10; _i++) { for (var _j = 0; _j < 5; _j++) @@ -80,7 +80,7 @@ static byte mbrot8((Vector, Vector) cr, double civ) } absz0 = tr0 + ti0; absz1 = tr1 + ti1; - if (Vector.GreaterThanAll(absz0, _threshold) && Vector.GreaterThanAll(absz1, _threshold)) + if (Vector256.GreaterThanAll(absz0, _threshold) && Vector256.GreaterThanAll(absz1, _threshold)) { return 0; } diff --git a/bench/algorithm/nbody/9.cs b/bench/algorithm/nbody/9.cs index 4b5c89af..e2af4067 100644 --- a/bench/algorithm/nbody/9.cs +++ b/bench/algorithm/nbody/9.cs @@ -10,7 +10,7 @@ modified by hanabi1224 to use simd-powered Vector */ using System; - using System.Numerics; + using System.Runtime.Intrinsics; public class NBody { @@ -30,14 +30,14 @@ public static void Main(String[] args) public class Body { - public Vector Pos { get; set; } - public Vector Velocity { get; set; } + public Vector256 Pos { get; set; } + public Vector256 Velocity { get; set; } public double Mass { get; } public Body(double x, double y, double z, double vx, double vy, double vz, double mass) { - Pos = new Vector(new[] { x, y, z, 0 }); - Velocity = new Vector(new[] { vx, vy, vz, 0 }); + Pos = Vector256.Create(x, y, z, 0); + Velocity = Vector256.Create(vx, vy, vz, 0); Mass = mass; } } @@ -113,7 +113,7 @@ public NBodySystem() public void OffsetMomentum() { - var p = new Vector(new[] { 0.0, 0.0, 0.0, 0.0 }); + var p = Vector256.Zero; foreach (var b in _bodies) { p -= b.Velocity * b.Mass; @@ -133,7 +133,7 @@ public void Advance(double dt) { var bj = _bodies[j]; var dpos = pos - bj.Pos; - double d2 = Vector.Dot(dpos, dpos); + double d2 = Vector256.Dot(dpos, dpos); double mag = dt / (d2 * Math.Sqrt(d2)); dpos *= mag; v -= dpos * bj.Mass; @@ -150,12 +150,12 @@ public double Energy() for (int i = 0; i < bodyCount; i++) { var bi = _bodies[i]; - e += 0.5 * bi.Mass * Vector.Dot(bi.Velocity, bi.Velocity); + e += 0.5 * bi.Mass * Vector256.Dot(bi.Velocity, bi.Velocity); for (int j = i + 1; j < bodyCount; j++) { var bj = _bodies[j]; var dpos = bi.Pos - bj.Pos; - e -= (bi.Mass * bj.Mass) / Math.Sqrt(Vector.Dot(dpos, dpos)); + e -= (bi.Mass * bj.Mass) / Math.Sqrt(Vector256.Dot(dpos, dpos)); } } return e; diff --git a/bench/include/c/app_ffi.rsp b/bench/include/c/app_ffi.rsp index 5a85682a..36f645eb 100644 --- a/bench/include/c/app_ffi.rsp +++ b/bench/include/c/app_ffi.rsp @@ -1 +1 @@ - -pipe -O3 -fomit-frame-pointer -march=broadwell -fopenmp -pthread -Wno-deprecated-declarations -mno-fma -o app app.c -lm -lcrypto + -pipe -O3 -fomit-frame-pointer -march=native -fopenmp -pthread -Wno-deprecated-declarations -ffp-contract=off -o app app.c -lm -lcrypto diff --git a/bench/tests/test_suite.py b/bench/tests/test_suite.py index c1f0a1ef..cb897e36 100644 --- a/bench/tests/test_suite.py +++ b/bench/tests/test_suite.py @@ -26,9 +26,11 @@ def setUp(self): buildLog={'finished': 'today'}, testLog={'finished': 'today'}, githubSha='current', githubRunId='123', githubRepository='owner/repo') for mock in [patch.object(suite, 'BENCH', self.root), + patch.object(suite, 'LANGUAGES', {'fixture': {}, 'missing': {}}), patch.object(suite, 'programs', return_value={'fixture': ('sample', '1.sh', 'sh', '1')}), patch.object(suite, 'PROBLEMS', {'sample': {'tests': [{'input': 'small'}, {'input': 'large'}]}}), - patch.dict(os.environ, GITHUB_SHA='current', GITHUB_RUN_ID='123', GITHUB_REPOSITORY='owner/repo')]: + patch.dict(os.environ, GITHUB_SHA='current', GITHUB_RUN_ID='123', + GITHUB_REPOSITORY='owner/repo', GITHUB_STEP_SUMMARY='')]: mock.start() self.addCleanup(mock.stop) self.write_records() @@ -76,6 +78,42 @@ def test_ignored_build_failure_is_rejected(self): with self.assertRaisesRegex(ValueError, 'Missing build output'): suite.verify('fixture', 'build') + def test_failed_language_does_not_block_verified_results(self): + summary = suite.collect_results() + self.assertEqual(summary['publishedLanguages'], ['fixture']) + self.assertEqual(summary['missingLanguages'], ['missing']) + self.assertEqual(summary['githubRunId'], '123') + self.assertEqual(json.loads((self.root / 'build/run-summary.json').read_text()), summary) + + def test_partial_artifact_is_still_rejected(self): + (self.results / 'fixture_large.json').unlink() + with self.assertRaisesRegex(ValueError, 'missing='): + suite.collect_results() + self.assertFalse((self.root / 'build/run-summary.json').exists()) + + def test_partial_publication_rejects_stale_results(self): + self.record['githubRunId'] = 'old run' + self.write_records() + with self.assertRaisesRegex(ValueError, 'Wrong githubRunId'): + suite.collect_results() + + def test_no_results_cannot_replace_the_site(self): + for path in self.results.iterdir(): + path.unlink() + self.results.rmdir() + with self.assertRaisesRegex(ValueError, 'No successful languages'): + suite.collect_results() + + def test_publication_rejects_different_language_machines(self): + other = self.results.parent / 'missing' + other.mkdir() + for path in self.results.iterdir(): + record = json.loads(path.read_text()) + record.update(lang='missing', runnerName='another runner') + (other / path.name).write_text(json.dumps(record)) + with self.assertRaisesRegex(ValueError, 'different machines'): + suite.collect_results() + if __name__ == '__main__': unittest.main() diff --git a/website/custom.d.ts b/website/custom.d.ts index 89e97fda..50a48b0f 100644 --- a/website/custom.d.ts +++ b/website/custom.d.ts @@ -5,6 +5,17 @@ declare module '*.vue' { type osType = 'linux' | 'osx' | 'windows' +type BenchmarkRun = { + expectedLanguages: string[] + publishedLanguages: string[] + missingLanguages: string[] + runnerName: string + cpuInfo: string + githubRepository: string + githubRunId: string + githubSha: string +} + type BenchResult = { cpuInfo: string lang: string diff --git a/website/layouts/default.vue b/website/layouts/default.vue index 3b56527d..23d4981c 100644 --- a/website/layouts/default.vue +++ b/website/layouts/default.vue @@ -8,6 +8,19 @@ Benchmarks +
+

+ Results for {{ benchmarkRun.publishedLanguages.length }} of + {{ benchmarkRun.expectedLanguages.length }} languages from + this run + on {{ benchmarkRun.runnerName }}. +

+

+ Unavailable in this run: + {{ benchmarkRun.missingLanguages.join(', ') }}. Follow the run link for + failure details. +

+
@@ -17,5 +30,16 @@ import { Component, Vue } from 'nuxt-property-decorator' @Component({ components: {}, }) -export default class DefaultLayout extends Vue {} +export default class DefaultLayout extends Vue { + get benchmarkRun(): BenchmarkRun | null { + return this.$config.benchmarkRun || null + } + + get runUrl(): string { + const run = this.benchmarkRun + return run + ? `https://github.com/${run.githubRepository}/actions/runs/${run.githubRunId}` + : '' + } +} diff --git a/website/nuxt.config.ts b/website/nuxt.config.ts index f3d9a81b..90128411 100644 --- a/website/nuxt.config.ts +++ b/website/nuxt.config.ts @@ -1,14 +1,22 @@ +import { existsSync, readFileSync } from 'fs' +import { resolve } from 'path' import { NuxtConfig } from '@nuxt/types' import { $content } from '@nuxt/content' import _ from 'lodash' import { getLangBenchResults } from './contentUtils' const basePath = process.env.SITE_BASE_PATH || '/' +const runSummaryPath = resolve(__dirname, '../bench/build/run-summary.json') const config: NuxtConfig = { // Target: https://go.nuxtjs.dev/config-target target: 'static', ssr: true, + publicRuntimeConfig: { + benchmarkRun: existsSync(runSummaryPath) + ? JSON.parse(readFileSync(runSummaryPath, 'utf8')) + : null, + }, loading: { color: 'cyan', }, diff --git a/website/pages/index.vue b/website/pages/index.vue index 6132c77c..bc4afe23 100644 --- a/website/pages/index.vue +++ b/website/pages/index.vue @@ -27,13 +27,15 @@ languages and their different compilers or runtime

- All benchmark programs are measured sequentially in one CI job. Each - run records its machine and compiler versions. Compare numbers - within the same run, since GitHub-hosted hardware can change. + Benchmark programs are measured sequentially on a dedicated runner. + Each run records its machine and compiler versions. Compare numbers + within the same run.

- Successful benchmark runs publish this website on GitHub Pages. Runs - happen on changes to main and weekly. + Measurements run weekly or on request. Languages that pass publish + their results on GitHub Pages, even when another language fails. + Missing languages are listed above. Code changes also run + correctness checks on GitHub-hosted runners.

Main goals: