From 1f4d6a38c6104beb12fad39914b565bba822bcbf Mon Sep 17 00:00:00 2001 From: Kristian Larsson Date: Sun, 27 Sep 2026 13:21:00 +0200 Subject: [PATCH 1/3] Benchmark stable Acton alongside tip Acton results currently use tip alone and shorten the version string. Run stable and tip as separate entries with the same sources and release options. Show them as acton and acton tip with full versions. --- .github/containers/setup-language.sh | 10 +++- .github/languages.json | 1 + README.md | 15 ++++-- bench/bench_acton_tip.yaml | 68 ++++++++++++++++++++++++++++ bench/tests/test_suite.py | 24 ++++++++++ website/contentUtils.ts | 11 ++++- website/layouts/default.vue | 10 +++- 7 files changed, 129 insertions(+), 10 deletions(-) create mode 100644 bench/bench_acton_tip.yaml diff --git a/.github/containers/setup-language.sh b/.github/containers/setup-language.sh index 77f1f9fd..5527e81b 100644 --- a/.github/containers/setup-language.sh +++ b/.github/containers/setup-language.sh @@ -21,8 +21,14 @@ rust() { } case "$language" in - acton) - archive https://github.com/actonlang/acton/releases/download/tip/acton-linux-x86_64-tip.tar.xz /opt/acton + acton|acton-tip) + if [[ "$language" == acton ]]; then + url="$(curl --fail --location --retry 3 https://api.github.com/repos/actonlang/acton/releases/latest | + jq -er '.assets[] | select(.name | test("^acton-linux-x86_64-[0-9].*\\.tar\\.xz$")) | .browser_download_url')" + else + url=https://github.com/actonlang/acton/releases/download/tip/acton-linux-x86_64-tip.tar.xz + fi + archive "$url" /opt/acton chown -R "$BENCH_UID" /opt/acton ln -s /opt/acton/bin/acton /usr/local/bin/acton ;; diff --git a/.github/languages.json b/.github/languages.json index 5d8f3bf3..32f25cfa 100644 --- a/.github/languages.json +++ b/.github/languages.json @@ -1,5 +1,6 @@ { "acton": {"compilers": ["acton"]}, + "acton-tip": {"compilers": ["acton"]}, "c": {"compilers": ["clang", "gcc"]}, "chapel": {"image": "docker.io/chapel/chapel:2.9.0", "compilers": ["chpl"]}, "codon": {"compilers": ["codon"]}, diff --git a/README.md b/README.md index efd83aef..4cf0b4d0 100644 --- a/README.md +++ b/README.md @@ -45,9 +45,9 @@ Measurement jobs update the installed hook. For initial setup on `actest1`, copy `ACTIONS_RUNNER_HOOK_JOB_STARTED` and `ACTIONS_RUNNER_HOOK_JOB_COMPLETED` to that absolute path in the runner service environment, then restart the idle service. -`.github/languages.json` selects all 39 upstream language entries and their -primary toolchains, including WebAssembly. It does not select every historical -compiler release or experimental backend. `.github/suite.py` requires every +`.github/languages.json` selects 40 entries, including separate stable and tip +Acton builds, and their primary toolchains, including WebAssembly. It does not +select every historical compiler release or experimental backend. `.github/suite.py` requires every selected program in a language to build, pass correctness checks, and produce every expected result before publishing that language. A program that passes correctness checks but exceeds a measurement time limit is shown as a timeout, @@ -62,8 +62,13 @@ V uses its garbage-collected backend; the experimental autofree backend corrupts big-integer calculations in the digit benchmarks. Native compiler optimizations target the CPU running the benchmark. Acton uses -release optimization and the current tip compiler. Go uses one 1.26 toolchain -throughout the job; dependencies cannot silently select another compiler. +release optimization for both the latest stable release ("acton") and the latest +published development build ("acton tip"). Both use the same source files and +inputs in separate jobs, with full compiler versions shown on the site. +A failure in one build does not prevent the other from publishing. +Use `acton-tip` to select the development build in workflow inputs or local +commands. Go uses one 1.26 toolchain throughout the job; dependencies cannot +silently select another compiler. ## Running locally diff --git a/bench/bench_acton_tip.yaml b/bench/bench_acton_tip.yaml new file mode 100644 index 00000000..6e07f196 --- /dev/null +++ b/bench/bench_acton_tip.yaml @@ -0,0 +1,68 @@ +lang: acton-tip +problems: + - name: binarytrees + source: + - 1.act + - name: edigits + source: + - 1.act + - name: helloworld + source: + - 1.act + - name: merkletrees + source: + - 1.act + - name: nsieve + source: + - 1.act + - name: pidigits + source: + - 1.act + - name: nbody + source: + - 1.act + - name: spectral-norm + source: + - 1.act + - name: mandelbrot + source: + - 1.act + - name: fannkuch-redux + source: + - 1.act + - name: fasta + source: + - 1.act + - name: knucleotide + source: + - 1.act + - name: regex-redux + source: + - 1.act + - name: json-serde + source: + - 1.act + - name: coro-prime-sieve + source: + - 1.act + - name: http-server + source: + - 1.act + - name: lru + source: + - 1.act + - name: secp256k1 + source: + - 1.act +source_rename_to: app.act +environments: + - os: linux + compiler: acton + compiler_version_command: acton version + version: tip + compiler_options_text: ReleaseFast + build: acton --release app.act + after_build: + - mv app out + out_dir: out + run_cmd: app diff --git a/bench/tests/test_suite.py b/bench/tests/test_suite.py index cb897e36..6e0af146 100644 --- a/bench/tests/test_suite.py +++ b/bench/tests/test_suite.py @@ -14,6 +14,30 @@ spec.loader.exec_module(suite) +class ActonChannelTests(unittest.TestCase): + def test_channels_have_separate_results_for_the_same_programs(self): + stable = suite.programs('acton') + tip = suite.programs('acton-tip') + self.assertTrue(stable) + self.assertTrue(stable.keys().isdisjoint(tip)) + self.assertEqual({p[:3] for p in stable.values()}, {p[:3] for p in tip.values()}) + self.assertEqual({p[3] for p in stable.values()}, {'latest'}) + self.assertEqual({p[3] for p in tip.values()}, {'tip'}) + + def test_source_change_checks_both_channels(self): + output = io.StringIO() + with patch.dict(os.environ, GITHUB_EVENT_NAME='pull_request', + BASE_SHA='a' * 40, HEAD_SHA='b' * 40, + REQUESTED_LANGUAGES='', REQUESTED_MODE='', GITHUB_OUTPUT=''), \ + patch.object(suite.subprocess, 'check_output', + return_value='bench/algorithm/nsieve/1.act\n'), \ + contextlib.redirect_stdout(output): + suite.plan() + plan = json.loads(output.getvalue()) + self.assertEqual(json.loads(plan['languages']), ['acton', 'acton-tip']) + self.assertEqual(plan['measure'], 'false') + + class SuiteTests(unittest.TestCase): def setUp(self): self.directory = tempfile.TemporaryDirectory() diff --git a/website/contentUtils.ts b/website/contentUtils.ts index 47579167..f0445a71 100644 --- a/website/contentUtils.ts +++ b/website/contentUtils.ts @@ -3,12 +3,18 @@ import _ from 'lodash' import findVersions from 'find-versions' const lang2Display: { [key: string]: string } = { + acton: 'acton', + 'acton-tip': 'acton tip', csharp: 'C#', cpp: 'C++', hacklang: 'Hack', ocaml: 'OCaml', } +export function getLangDisplayName(lang: string): string { + return lang2Display[lang] ?? _.capitalize(lang) +} + export async function getLangBenchResults($content: contentFunc) { const pages = (await $content('/', { deep: true, @@ -39,7 +45,7 @@ export function mergeLangBenchResults( console.log(`${k}: ${benches.length} benchmark results`) r.push({ lang: k, - langDisplay: lang2Display[k] ?? _.capitalize(k), + langDisplay: getLangDisplayName(k), benchmarks: benches, }) } @@ -54,6 +60,9 @@ export function getFullCompilerVersion(i: BenchResult) { } export function getRealShortCompilerVersion(i: BenchResult) { + if (i.lang === 'acton' || i.lang === 'acton-tip') { + return i.buildLog?.compilerVersion?.trim() || 'unknown' + } const full = getFullCompilerVersion(i) const versions = findVersions(full, { loose: true }) if (versions && versions.length > 0) { diff --git a/website/layouts/default.vue b/website/layouts/default.vue index 23d4981c..3b7b6c9c 100644 --- a/website/layouts/default.vue +++ b/website/layouts/default.vue @@ -17,8 +17,7 @@

Unavailable in this run: - {{ benchmarkRun.missingLanguages.join(', ') }}. Follow the run link for - failure details. + {{ missingLanguages }}. Follow the run link for failure details.

@@ -27,6 +26,7 @@