Benchmark stable Acton alongside tip

.github/containers/Dockerfile

@@ -13,6 +13,11 @@ ENV DEBIAN_FRONTEND=noninteractive \
     MIX_ENV=prod
 COPY --from=dotnet /usr/share/dotnet /opt/dotnet
 ENV PATH=/opt/dotnet:/opt/bench-home/.cargo/bin:/opt/bench-home/.nimble/bin:/opt/bench-home/.bun/bin:/opt/bench-home/.deno/bin:/opt/bench-home/.wasmtime/bin:/opt/bench-home/.codon/bin:/opt/bench-home/.local/share/ponyup/bin:${PATH}
+ARG BENCH_LANGUAGE
+# HHVM is already installed. Its obsolete apt source has an expired certificate.
+RUN if [ "$BENCH_LANGUAGE" = hacklang ]; then \
+      sed -i '\|https://dl.hhvm.com/ubuntu|d' /etc/apt/sources.list; \
+    fi
 RUN apt-get update && apt-get install -y --no-install-recommends \
     bash build-essential ca-certificates clang cmake curl file git jq \
     libgc-dev libgmp-dev libicu-dev libomp-dev libssl-dev libunwind-dev \
@@ -22,7 +27,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     && chmod 1777 /opt/bench-home \
     && echo 'ALL ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/benchmark \
     && chmod 440 /etc/sudoers.d/benchmark
-ARG BENCH_LANGUAGE
 ARG BENCH_UID=1000
 LABEL org.actonlang.benchmarks=true
 COPY setup-language.sh /opt/setup-language.sh

.github/containers/setup-language.sh

@@ -21,8 +21,14 @@ rust() {
 }
 
 case "$language" in
-  acton)
-    archive https://github.com/actonlang/acton/releases/download/tip/acton-linux-x86_64-tip.tar.xz /opt/acton
+  acton|acton-tip)
+    if [[ "$language" == acton ]]; then
+      url="$(curl --fail --location --retry 3 https://api.github.com/repos/actonlang/acton/releases/latest |
+        jq -er '.assets[] | select(.name | test("^acton-linux-x86_64-[0-9].*\\.tar\\.xz$")) | .browser_download_url')"
+    else
+      url=https://github.com/actonlang/acton/releases/download/tip/acton-linux-x86_64-tip.tar.xz
+    fi
+    archive "$url" /opt/acton
     chown -R "$BENCH_UID" /opt/acton
     ln -s /opt/acton/bin/acton /usr/local/bin/acton
     ;;

.github/languages.json

@@ -1,5 +1,6 @@
 {
   "acton": {"compilers": ["acton"]},
+  "acton-tip": {"compilers": ["acton"]},
   "c": {"compilers": ["clang", "gcc"]},
   "chapel": {"image": "docker.io/chapel/chapel:2.9.0", "compilers": ["chpl"]},
   "codon": {"compilers": ["codon"]},

README.md

@@ -45,9 +45,9 @@ Measurement jobs update the installed hook. For initial setup on `actest1`, copy
 `ACTIONS_RUNNER_HOOK_JOB_STARTED` and `ACTIONS_RUNNER_HOOK_JOB_COMPLETED` to that
 absolute path in the runner service environment, then restart the idle service.
 
-`.github/languages.json` selects all 39 upstream language entries and their
-primary toolchains, including WebAssembly. It does not select every historical
-compiler release or experimental backend. `.github/suite.py` requires every
+`.github/languages.json` selects 40 entries, including separate stable and tip
+Acton builds, and their primary toolchains, including WebAssembly. It does not
+select every historical compiler release or experimental backend. `.github/suite.py` requires every
 selected program in a language to build, pass correctness checks, and produce
 every expected result before publishing that language. A program that passes
 correctness checks but exceeds a measurement time limit is shown as a timeout,
@@ -62,8 +62,13 @@ V uses its garbage-collected backend; the experimental autofree backend corrupts
 big-integer calculations in the digit benchmarks.
 
 Native compiler optimizations target the CPU running the benchmark. Acton uses
-release optimization and the current tip compiler. Go uses one 1.26 toolchain
-throughout the job; dependencies cannot silently select another compiler.
+release optimization for both the latest stable release ("acton") and the latest
+published development build ("acton tip"). Both use the same source files and
+inputs in separate jobs, with full compiler versions shown on the site.
+A failure in one build does not prevent the other from publishing.
+Use `acton-tip` to select the development build in workflow inputs or local
+commands. Go uses one 1.26 toolchain throughout the job; dependencies cannot
+silently select another compiler.
 
 ## Running locally
 

bench/algorithm/json-serde/1.act

@@ -4,16 +4,20 @@ import crypto.hash.md5
 import file
 import json
 
+def checksum(data: bytes) -> str:
+    # TODO: Use bytes.hex() once stable Acton includes its string terminator fix.
+    return "".join([hex(b)[2:].zfill(2) for b in crypto.hash.md5.hash(data)])
+
 actor main(env):
     name = env.argv[1] if len(env.argv) > 1 else "sample"
     n = int(env.argv[2]) if len(env.argv) > 2 else 3
     source = file.ReadFile(file.ReadFileCap(file.FileCap(env.cap)), name + ".json")
     text = source.read().decode().lstrip("\ufeff")
     source.close()
     obj = json.decode(text)
-    print(crypto.hash.md5.hash(json.encode(obj).encode()).hex())
+    print(checksum(json.encode(obj).encode()))
     objects: list[?value] = []
     for _ in range(n):
         objects.append(json.decode(text))
-    print(crypto.hash.md5.hash(json.encode_list(objects).encode()).hex())
+    print(checksum(json.encode_list(objects).encode()))
     env.exit(0)

bench/bench_acton_tip.yaml

@@ -0,0 +1,68 @@
+lang: acton-tip
+problems:
+  - name: binarytrees
+    source:
+      - 1.act
+  - name: edigits
+    source:
+      - 1.act
+  - name: helloworld
+    source:
+      - 1.act
+  - name: merkletrees
+    source:
+      - 1.act
+  - name: nsieve
+    source:
+      - 1.act
+  - name: pidigits
+    source:
+      - 1.act
+  - name: nbody
+    source:
+      - 1.act
+  - name: spectral-norm
+    source:
+      - 1.act
+  - name: mandelbrot
+    source:
+      - 1.act
+  - name: fannkuch-redux
+    source:
+      - 1.act
+  - name: fasta
+    source:
+      - 1.act
+  - name: knucleotide
+    source:
+      - 1.act
+  - name: regex-redux
+    source:
+      - 1.act
+  - name: json-serde
+    source:
+      - 1.act
+  - name: coro-prime-sieve
+    source:
+      - 1.act
+  - name: http-server
+    source:
+      - 1.act
+  - name: lru
+    source:
+      - 1.act
+  - name: secp256k1
+    source:
+      - 1.act
+source_rename_to: app.act
+environments:
+  - os: linux
+    compiler: acton
+    compiler_version_command: acton version
+    version: tip
+    compiler_options_text: ReleaseFast
+    build: acton --release app.act
+    after_build:
+      - mv app out
+    out_dir: out
+    run_cmd: app

bench/tests/test_suite.py

@@ -14,6 +14,30 @@
 spec.loader.exec_module(suite)
 
 
+class ActonChannelTests(unittest.TestCase):
+    def test_channels_have_separate_results_for_the_same_programs(self):
+        stable = suite.programs('acton')
+        tip = suite.programs('acton-tip')
+        self.assertTrue(stable)
+        self.assertTrue(stable.keys().isdisjoint(tip))
+        self.assertEqual({p[:3] for p in stable.values()}, {p[:3] for p in tip.values()})
+        self.assertEqual({p[3] for p in stable.values()}, {'latest'})
+        self.assertEqual({p[3] for p in tip.values()}, {'tip'})
+
+    def test_source_change_checks_both_channels(self):
+        output = io.StringIO()
+        with patch.dict(os.environ, GITHUB_EVENT_NAME='pull_request',
+                        BASE_SHA='a' * 40, HEAD_SHA='b' * 40,
+                        REQUESTED_LANGUAGES='', REQUESTED_MODE='', GITHUB_OUTPUT=''), \
+                patch.object(suite.subprocess, 'check_output',
+                             return_value='bench/algorithm/nsieve/1.act\n'), \
+                contextlib.redirect_stdout(output):
+            suite.plan()
+        plan = json.loads(output.getvalue())
+        self.assertEqual(json.loads(plan['languages']), ['acton', 'acton-tip'])
+        self.assertEqual(plan['measure'], 'false')
+
+
 class SuiteTests(unittest.TestCase):
     def setUp(self):
         self.directory = tempfile.TemporaryDirectory()

website/contentUtils.ts

@@ -3,12 +3,18 @@ import _ from 'lodash'
 import findVersions from 'find-versions'
 
 const lang2Display: { [key: string]: string } = {
+  acton: 'acton',
+  'acton-tip': 'acton tip',
   csharp: 'C#',
   cpp: 'C++',
   hacklang: 'Hack',
   ocaml: 'OCaml',
 }
 
+export function getLangDisplayName(lang: string): string {
+  return lang2Display[lang] ?? _.capitalize(lang)
+}
+
 export async function getLangBenchResults($content: contentFunc) {
   const pages = (await $content('/', {
     deep: true,
@@ -39,7 +45,7 @@ export function mergeLangBenchResults(
     console.log(`${k}: ${benches.length} benchmark results`)
     r.push({
       lang: k,
-      langDisplay: lang2Display[k] ?? _.capitalize(k),
+      langDisplay: getLangDisplayName(k),
       benchmarks: benches,
     })
   }
@@ -54,6 +60,9 @@ export function getFullCompilerVersion(i: BenchResult) {
 }
 
 export function getRealShortCompilerVersion(i: BenchResult) {
+  if (i.lang === 'acton' || i.lang === 'acton-tip') {
+    return i.buildLog?.compilerVersion?.trim() || 'unknown'
+  }
   const full = getFullCompilerVersion(i)
   const versions = findVersions(full, { loose: true })
   if (versions && versions.length > 0) {

website/layouts/default.vue

@@ -17,8 +17,7 @@
       </p>
       <p v-if="benchmarkRun.missingLanguages.length" class="mt-2">
         Unavailable in this run:
-        {{ benchmarkRun.missingLanguages.join(', ') }}. Follow the run link for
-        failure details.
+        {{ missingLanguages }}. Follow the run link for failure details.
       </p>
     </div>
     <Nuxt />
@@ -27,6 +26,7 @@
 
 <script lang="ts">
 import { Component, Vue } from 'nuxt-property-decorator'
+import { getLangDisplayName } from '~/contentUtils'
 @Component({
   components: {},
 })
@@ -35,6 +35,12 @@ export default class DefaultLayout extends Vue {
     return this.$config.benchmarkRun || null
   }
 
+  get missingLanguages(): string {
+    return (this.benchmarkRun?.missingLanguages || [])
+      .map(getLangDisplayName)
+      .join(', ')
+  }
+
   get runUrl(): string {
     const run = this.benchmarkRun
     return run