diff --git a/.github/containers/Dockerfile b/.github/containers/Dockerfile index 96d932d9..6dcc6014 100644 --- a/.github/containers/Dockerfile +++ b/.github/containers/Dockerfile @@ -13,6 +13,11 @@ ENV DEBIAN_FRONTEND=noninteractive \ MIX_ENV=prod COPY --from=dotnet /usr/share/dotnet /opt/dotnet ENV PATH=/opt/dotnet:/opt/bench-home/.cargo/bin:/opt/bench-home/.nimble/bin:/opt/bench-home/.bun/bin:/opt/bench-home/.deno/bin:/opt/bench-home/.wasmtime/bin:/opt/bench-home/.codon/bin:/opt/bench-home/.local/share/ponyup/bin:${PATH} +ARG BENCH_LANGUAGE +# HHVM is already installed. Its obsolete apt source has an expired certificate. +RUN if [ "$BENCH_LANGUAGE" = hacklang ]; then \ + sed -i '\|https://dl.hhvm.com/ubuntu|d' /etc/apt/sources.list; \ + fi RUN apt-get update && apt-get install -y --no-install-recommends \ bash build-essential ca-certificates clang cmake curl file git jq \ libgc-dev libgmp-dev libicu-dev libomp-dev libssl-dev libunwind-dev \ @@ -22,7 +27,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && chmod 1777 /opt/bench-home \ && echo 'ALL ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/benchmark \ && chmod 440 /etc/sudoers.d/benchmark -ARG BENCH_LANGUAGE ARG BENCH_UID=1000 LABEL org.actonlang.benchmarks=true COPY setup-language.sh /opt/setup-language.sh diff --git a/.github/containers/setup-language.sh b/.github/containers/setup-language.sh index 77f1f9fd..5527e81b 100644 --- a/.github/containers/setup-language.sh +++ b/.github/containers/setup-language.sh @@ -21,8 +21,14 @@ rust() { } case "$language" in - acton) - archive https://github.com/actonlang/acton/releases/download/tip/acton-linux-x86_64-tip.tar.xz /opt/acton + acton|acton-tip) + if [[ "$language" == acton ]]; then + url="$(curl --fail --location --retry 3 https://api.github.com/repos/actonlang/acton/releases/latest | + jq -er '.assets[] | select(.name | test("^acton-linux-x86_64-[0-9].*\\.tar\\.xz$")) | .browser_download_url')" + else + url=https://github.com/actonlang/acton/releases/download/tip/acton-linux-x86_64-tip.tar.xz + fi + archive "$url" /opt/acton chown -R "$BENCH_UID" /opt/acton ln -s /opt/acton/bin/acton /usr/local/bin/acton ;; diff --git a/.github/languages.json b/.github/languages.json index 5d8f3bf3..32f25cfa 100644 --- a/.github/languages.json +++ b/.github/languages.json @@ -1,5 +1,6 @@ { "acton": {"compilers": ["acton"]}, + "acton-tip": {"compilers": ["acton"]}, "c": {"compilers": ["clang", "gcc"]}, "chapel": {"image": "docker.io/chapel/chapel:2.9.0", "compilers": ["chpl"]}, "codon": {"compilers": ["codon"]}, diff --git a/README.md b/README.md index efd83aef..4cf0b4d0 100644 --- a/README.md +++ b/README.md @@ -45,9 +45,9 @@ Measurement jobs update the installed hook. For initial setup on `actest1`, copy `ACTIONS_RUNNER_HOOK_JOB_STARTED` and `ACTIONS_RUNNER_HOOK_JOB_COMPLETED` to that absolute path in the runner service environment, then restart the idle service. -`.github/languages.json` selects all 39 upstream language entries and their -primary toolchains, including WebAssembly. It does not select every historical -compiler release or experimental backend. `.github/suite.py` requires every +`.github/languages.json` selects 40 entries, including separate stable and tip +Acton builds, and their primary toolchains, including WebAssembly. It does not +select every historical compiler release or experimental backend. `.github/suite.py` requires every selected program in a language to build, pass correctness checks, and produce every expected result before publishing that language. A program that passes correctness checks but exceeds a measurement time limit is shown as a timeout, @@ -62,8 +62,13 @@ V uses its garbage-collected backend; the experimental autofree backend corrupts big-integer calculations in the digit benchmarks. Native compiler optimizations target the CPU running the benchmark. Acton uses -release optimization and the current tip compiler. Go uses one 1.26 toolchain -throughout the job; dependencies cannot silently select another compiler. +release optimization for both the latest stable release ("acton") and the latest +published development build ("acton tip"). Both use the same source files and +inputs in separate jobs, with full compiler versions shown on the site. +A failure in one build does not prevent the other from publishing. +Use `acton-tip` to select the development build in workflow inputs or local +commands. Go uses one 1.26 toolchain throughout the job; dependencies cannot +silently select another compiler. ## Running locally diff --git a/bench/algorithm/json-serde/1.act b/bench/algorithm/json-serde/1.act index acc35025..80f7232d 100644 --- a/bench/algorithm/json-serde/1.act +++ b/bench/algorithm/json-serde/1.act @@ -4,6 +4,10 @@ import crypto.hash.md5 import file import json +def checksum(data: bytes) -> str: + # TODO: Use bytes.hex() once stable Acton includes its string terminator fix. + return "".join([hex(b)[2:].zfill(2) for b in crypto.hash.md5.hash(data)]) + actor main(env): name = env.argv[1] if len(env.argv) > 1 else "sample" n = int(env.argv[2]) if len(env.argv) > 2 else 3 @@ -11,9 +15,9 @@ actor main(env): text = source.read().decode().lstrip("\ufeff") source.close() obj = json.decode(text) - print(crypto.hash.md5.hash(json.encode(obj).encode()).hex()) + print(checksum(json.encode(obj).encode())) objects: list[?value] = [] for _ in range(n): objects.append(json.decode(text)) - print(crypto.hash.md5.hash(json.encode_list(objects).encode()).hex()) + print(checksum(json.encode_list(objects).encode())) env.exit(0) diff --git a/bench/bench_acton_tip.yaml b/bench/bench_acton_tip.yaml new file mode 100644 index 00000000..6e07f196 --- /dev/null +++ b/bench/bench_acton_tip.yaml @@ -0,0 +1,68 @@ +lang: acton-tip +problems: + - name: binarytrees + source: + - 1.act + - name: edigits + source: + - 1.act + - name: helloworld + source: + - 1.act + - name: merkletrees + source: + - 1.act + - name: nsieve + source: + - 1.act + - name: pidigits + source: + - 1.act + - name: nbody + source: + - 1.act + - name: spectral-norm + source: + - 1.act + - name: mandelbrot + source: + - 1.act + - name: fannkuch-redux + source: + - 1.act + - name: fasta + source: + - 1.act + - name: knucleotide + source: + - 1.act + - name: regex-redux + source: + - 1.act + - name: json-serde + source: + - 1.act + - name: coro-prime-sieve + source: + - 1.act + - name: http-server + source: + - 1.act + - name: lru + source: + - 1.act + - name: secp256k1 + source: + - 1.act +source_rename_to: app.act +environments: + - os: linux + compiler: acton + compiler_version_command: acton version + version: tip + compiler_options_text: ReleaseFast + build: acton --release app.act + after_build: + - mv app out + out_dir: out + run_cmd: app diff --git a/bench/tests/test_suite.py b/bench/tests/test_suite.py index cb897e36..6e0af146 100644 --- a/bench/tests/test_suite.py +++ b/bench/tests/test_suite.py @@ -14,6 +14,30 @@ spec.loader.exec_module(suite) +class ActonChannelTests(unittest.TestCase): + def test_channels_have_separate_results_for_the_same_programs(self): + stable = suite.programs('acton') + tip = suite.programs('acton-tip') + self.assertTrue(stable) + self.assertTrue(stable.keys().isdisjoint(tip)) + self.assertEqual({p[:3] for p in stable.values()}, {p[:3] for p in tip.values()}) + self.assertEqual({p[3] for p in stable.values()}, {'latest'}) + self.assertEqual({p[3] for p in tip.values()}, {'tip'}) + + def test_source_change_checks_both_channels(self): + output = io.StringIO() + with patch.dict(os.environ, GITHUB_EVENT_NAME='pull_request', + BASE_SHA='a' * 40, HEAD_SHA='b' * 40, + REQUESTED_LANGUAGES='', REQUESTED_MODE='', GITHUB_OUTPUT=''), \ + patch.object(suite.subprocess, 'check_output', + return_value='bench/algorithm/nsieve/1.act\n'), \ + contextlib.redirect_stdout(output): + suite.plan() + plan = json.loads(output.getvalue()) + self.assertEqual(json.loads(plan['languages']), ['acton', 'acton-tip']) + self.assertEqual(plan['measure'], 'false') + + class SuiteTests(unittest.TestCase): def setUp(self): self.directory = tempfile.TemporaryDirectory() diff --git a/website/contentUtils.ts b/website/contentUtils.ts index 47579167..f0445a71 100644 --- a/website/contentUtils.ts +++ b/website/contentUtils.ts @@ -3,12 +3,18 @@ import _ from 'lodash' import findVersions from 'find-versions' const lang2Display: { [key: string]: string } = { + acton: 'acton', + 'acton-tip': 'acton tip', csharp: 'C#', cpp: 'C++', hacklang: 'Hack', ocaml: 'OCaml', } +export function getLangDisplayName(lang: string): string { + return lang2Display[lang] ?? _.capitalize(lang) +} + export async function getLangBenchResults($content: contentFunc) { const pages = (await $content('/', { deep: true, @@ -39,7 +45,7 @@ export function mergeLangBenchResults( console.log(`${k}: ${benches.length} benchmark results`) r.push({ lang: k, - langDisplay: lang2Display[k] ?? _.capitalize(k), + langDisplay: getLangDisplayName(k), benchmarks: benches, }) } @@ -54,6 +60,9 @@ export function getFullCompilerVersion(i: BenchResult) { } export function getRealShortCompilerVersion(i: BenchResult) { + if (i.lang === 'acton' || i.lang === 'acton-tip') { + return i.buildLog?.compilerVersion?.trim() || 'unknown' + } const full = getFullCompilerVersion(i) const versions = findVersions(full, { loose: true }) if (versions && versions.length > 0) { diff --git a/website/layouts/default.vue b/website/layouts/default.vue index 23d4981c..3b7b6c9c 100644 --- a/website/layouts/default.vue +++ b/website/layouts/default.vue @@ -17,8 +17,7 @@

Unavailable in this run: - {{ benchmarkRun.missingLanguages.join(', ') }}. Follow the run link for - failure details. + {{ missingLanguages }}. Follow the run link for failure details.

@@ -27,6 +26,7 @@