diff --git a/ci.jsonnet b/ci.jsonnet index cc9cf3fd93..19103ee41e 100644 --- a/ci.jsonnet +++ b/ci.jsonnet @@ -5,7 +5,7 @@ (import "ci/python-gate.libsonnet") + (import "ci/python-bench.libsonnet") + { - overlay: "acd356580b18f9d67b0213c12da04076e6a4d493", + overlay: "3b59b9262eafc3b513fb258297ca791a17108027", specVersion: "8", // Until buildbot issues around CI tiers are resolved, we cannot use them // tierConfig: self.tierConfig, @@ -328,6 +328,29 @@ "tox-example": gpgate_ee + $.cpython_runtime + require(GPYEE_NATIVE_STANDALONE) + platform_spec(no_jobs) + platform_spec({ "linux:amd64:jdk-latest" : tier3 + $.ol8, }), + "python-graalos-resources": gpgate + internet_access_env + task_spec({ + run: [["mx", "--java-home=lookup:default", "python-graalos-resources", "pd-layouts-linux-amd64-musl-swcfi.tgz"]], + environment+: { + GRAALPY_GRAALOS_ARTIFACT_BASE_URL: $.overlay_imports.GRAALPY_GRAALOS_ARTIFACT_BASE_URL, + }, + publishArtifacts+: [{ + name: "graalpy-graalos-resources", + patterns: ["pd-layouts-linux-amd64-musl-swcfi.tgz"], + }], + }) + platform_spec(no_jobs) + platform_spec({ + "linux:amd64:jdk-latest": tier3 + $.ol8, + }), + "python-resource-variants": gpgate + task_spec({ + requireArtifacts+: [{name: "graalpy-graalos-resources", autoExtract: true}], + run: [ + ["mx", "--java-home=lookup:default", "python-jvm"], + ["mx", "--java-home=lookup:default", "restore-pd-layouts", "pd-layouts-linux-amd64-musl-swcfi.tgz"], + ["mx", "--java-home=lookup:default", "--multi-platform-layout-directories=linux-amd64,linux-amd64-musl-swcfi", "build", "--targets=GRAALPYTHON_RESOURCES,GRAALPYTHON_INTEGRATION_UNIT_TESTS"], + ["mx", "--java-home=lookup:default", "python-resource-variant-tests"], + ], + }) + platform_spec(no_jobs) + platform_spec({ + "linux:amd64:jdk-latest": tier3 + $.ol8, + }), "python-svm-graalos-standalone-build": gpgate_ee + internet_access_env + platform_spec(no_jobs) + platform_spec({ "linux:amd64:jdk-latest": tier3 + $.ol8 + task_spec({ capabilities+: ["mpk", "!fast", "!x82", "!x82_16_367"], diff --git a/ci/graal/ci/ci_common/ci-resources.libsonnet b/ci/graal/ci/ci_common/ci-resources.libsonnet index 01849d45bf..f2d21351ad 100644 --- a/ci/graal/ci/ci_common/ci-resources.libsonnet +++ b/ci/graal/ci/ci_common/ci-resources.libsonnet @@ -5,7 +5,8 @@ benchmarking_config_repo: "", notify_releaser_service: [""], notify_indexer_service(java_version, edition): [""], - nexus_base_url: "" + nexus_base_url: "", + graalos_artifact_base_url: "", }, // Public defaults are intentionally empty. Build definitions compose only the // repository environments they require; internal CI overrides the values. diff --git a/ci/graal/common.json b/ci/graal/common.json index 581d322b45..d54f2650d4 100644 --- a/ci/graal/common.json +++ b/ci/graal/common.json @@ -4,7 +4,7 @@ "Jsonnet files should not include this file directly but use ci/common.jsonnet instead." ], - "mx_version": "7.86.0", + "mx_version": "7.87.0", "COMMENT.jdks": "When adding or removing JDKs keep in sync with JDKs in ci/common.jsonnet", "jdks": { diff --git a/docs/user/Platform-Support.md b/docs/user/Platform-Support.md index 3be1b485bd..dd2355edcf 100644 --- a/docs/user/Platform-Support.md +++ b/docs/user/Platform-Support.md @@ -6,7 +6,7 @@ The main operating system is Oracle Linux, the CPU architectures are AMD64 and A As macOS and other platforms are not prioritized, some GraalPy features may not work on these platforms. See [Test Tiers](../user/Test-Tiers.md) for a detailed breakdown. -## Windows +#### Windows GraalPy standalone builds on Windows use a native OS backend by default. The backend maps Python's OS interface to the Windows CRT, Win32, and Winsock APIs instead of relying on the Java-based backend. @@ -14,3 +14,10 @@ It provides native or emulated Windows implementations for files and paths, sock Filesystem paths are passed to wide-character Windows APIs as UTF-16, while other native interfaces continue to use narrow strings where required. Errors from the CRT, Win32, and Winsock APIs are translated to Python `OSError` instances using CPython-compatible error mappings, with the original Windows error retained as `OSError.winerror`. + +#### GraalOS + +GraalPy supports the GraalOS execution sandbox. +This sandbox requires separate builds of native resources, shipped alongside the host platform resources with a separate libc prefix. +As of this writing, that is `linux/amd64/musl-swcfi`. +Selection of the correct resources can be done via the system property `org.graalvm.python.resources.variant=default|musl-swcfi`. diff --git a/graalpython/com.oracle.graal.python.annotations/pom.xml b/graalpython/com.oracle.graal.python.annotations/pom.xml index 61a7878c0f..c8aa8af9e0 100644 --- a/graalpython/com.oracle.graal.python.annotations/pom.xml +++ b/graalpython/com.oracle.graal.python.annotations/pom.xml @@ -69,7 +69,7 @@ SOFTWARE. - ../../mxbuild/jdk25/com.oracle.graal.python.annotations/src_gen + ../../mxbuild/graalpython/com.oracle.graal.python.annotations/src_gen diff --git a/graalpython/com.oracle.graal.python.bouncycastle/pom.xml b/graalpython/com.oracle.graal.python.bouncycastle/pom.xml index 0cbbfabd7b..3ae5fa6978 100644 --- a/graalpython/com.oracle.graal.python.bouncycastle/pom.xml +++ b/graalpython/com.oracle.graal.python.bouncycastle/pom.xml @@ -92,7 +92,7 @@ SOFTWARE. - ../../mxbuild/jdk25/com.oracle.graal.python.bouncycastle/src_gen + ../../mxbuild/graalpython/com.oracle.graal.python.bouncycastle/src_gen diff --git a/graalpython/com.oracle.graal.python.pegparser/pom.xml b/graalpython/com.oracle.graal.python.pegparser/pom.xml index fda67b50dd..018c72982d 100644 --- a/graalpython/com.oracle.graal.python.pegparser/pom.xml +++ b/graalpython/com.oracle.graal.python.pegparser/pom.xml @@ -76,7 +76,7 @@ SOFTWARE. - ../../mxbuild/jdk25/com.oracle.graal.python.pegparser/src_gen + ../../mxbuild/graalpython/com.oracle.graal.python.pegparser/src_gen diff --git a/graalpython/com.oracle.graal.python.processor/pom.xml b/graalpython/com.oracle.graal.python.processor/pom.xml index 0b5beff6a7..9712c166b7 100644 --- a/graalpython/com.oracle.graal.python.processor/pom.xml +++ b/graalpython/com.oracle.graal.python.processor/pom.xml @@ -76,7 +76,7 @@ SOFTWARE. - ../../mxbuild/jdk25/com.oracle.graal.python.processor/src_gen + ../../mxbuild/graalpython/com.oracle.graal.python.processor/src_gen diff --git a/graalpython/com.oracle.graal.python.resources/pom.xml b/graalpython/com.oracle.graal.python.resources/pom.xml index 421355051c..fd91192705 100644 --- a/graalpython/com.oracle.graal.python.resources/pom.xml +++ b/graalpython/com.oracle.graal.python.resources/pom.xml @@ -90,7 +90,7 @@ SOFTWARE. - ../../mxbuild/jdk25/com.oracle.graal.python.resources/src_gen + ../../mxbuild/graalpython/com.oracle.graal.python.resources/src_gen diff --git a/graalpython/com.oracle.graal.python.resources/src/com/oracle/graal/python/resources/GraalPyPlatformInfoProviderImpl.java b/graalpython/com.oracle.graal.python.resources/src/com/oracle/graal/python/resources/GraalPyPlatformInfoProviderImpl.java index 8af29c24cb..b4d8d8cfe5 100644 --- a/graalpython/com.oracle.graal.python.resources/src/com/oracle/graal/python/resources/GraalPyPlatformInfoProviderImpl.java +++ b/graalpython/com.oracle.graal.python.resources/src/com/oracle/graal/python/resources/GraalPyPlatformInfoProviderImpl.java @@ -73,7 +73,7 @@ static VersionsInfo getVersionsInfo() { } static VersionsInfo readVersionsInfo(OS os, CPUArchitecture cpuArchitecture) { - String resource = "/META-INF/resources/" + os + "/" + cpuArchitecture + "/graalpy_versions"; + String resource = "/META-INF/resources/" + PythonResourcePlatform.resourceDirectory(os, cpuArchitecture) + "/graalpy_versions"; try (InputStream stream = GraalPyPlatformInfoProviderImpl.class.getResourceAsStream(resource)) { if (stream == null) { throw new IllegalStateException("Missing GraalPy platform metadata resource: " + resource); diff --git a/graalpython/com.oracle.graal.python.resources/src/com/oracle/graal/python/resources/PythonResource.java b/graalpython/com.oracle.graal.python.resources/src/com/oracle/graal/python/resources/PythonResource.java index 8d14da26c3..ffe4dac0a1 100644 --- a/graalpython/com.oracle.graal.python.resources/src/com/oracle/graal/python/resources/PythonResource.java +++ b/graalpython/com.oracle.graal.python.resources/src/com/oracle/graal/python/resources/PythonResource.java @@ -81,7 +81,7 @@ public final class PythonResource implements InternalResource { public void unpackFiles(Env env, Path targetDirectory) throws IOException { OS os = env.getOS(); CPUArchitecture cpuArchitecture = env.getCPUArchitecture(); - Path osArch = Path.of(os.toString()).resolve(cpuArchitecture.toString()); + Path osArch = Path.of(PythonResourcePlatform.resourceDirectory(os, cpuArchitecture)); GraalPyPlatformInfoProviderImpl.VersionsInfo versionsInfo = GraalPyPlatformInfoProviderImpl.getVersionsInfo(); ResourcesFilter filter = new ResourcesFilter(); if (os.equals(OS.WINDOWS)) { @@ -209,8 +209,9 @@ public String versionHash(Env env) { CPUArchitecture cpuArchitecture = env.getCPUArchitecture(); if (!os.equals(OS.UNSUPPORTED) && !cpuArchitecture.equals(CPUArchitecture.UNSUPPORTED)) { al = new ArrayList<>(al); - al.add(Path.of(os.toString()).resolve(cpuArchitecture.toString()).resolve(VERSIONS_SHA256)); - al.add(Path.of(os.toString()).resolve(cpuArchitecture.toString()).resolve(NATIVE_SHA256)); + Path platform = Path.of(PythonResourcePlatform.resourceDirectory(os, cpuArchitecture)); + al.add(platform.resolve(VERSIONS_SHA256)); + al.add(platform.resolve(NATIVE_SHA256)); } for (var s : al) { try { @@ -219,6 +220,6 @@ public String versionHash(Env env) { throw CompilerDirectives.shouldNotReachHere(e); } } - return sb.toString(); + return sb.append(PythonResourcePlatform.cacheSuffix()).toString(); } } diff --git a/graalpython/com.oracle.graal.python.resources/src/com/oracle/graal/python/resources/PythonResourcePlatform.java b/graalpython/com.oracle.graal.python.resources/src/com/oracle/graal/python/resources/PythonResourcePlatform.java new file mode 100644 index 0000000000..0e8314de84 --- /dev/null +++ b/graalpython/com.oracle.graal.python.resources/src/com/oracle/graal/python/resources/PythonResourcePlatform.java @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package com.oracle.graal.python.resources; + +import com.oracle.truffle.api.InternalResource.CPUArchitecture; +import com.oracle.truffle.api.InternalResource.OS; + +/** Shared by extraction, cache identity and ABI metadata, including launcher-less embeddings. */ +final class PythonResourcePlatform { + static final String VARIANT_PROPERTY = "org.graalvm.python.resources.variant"; + private static final String SWCFI = "musl-swcfi"; + + /* + * Read the property once when platform metadata is initialized, so extraction, cache identity + * and ABI metadata all use the same selection. + */ + private static final String VARIANT = selectVariant(); + + private PythonResourcePlatform() { + } + + static String resourceDirectory(OS os, CPUArchitecture arch) { + if (!VARIANT.isEmpty() && (os != OS.LINUX || arch != CPUArchitecture.AMD64)) { + throw new IllegalStateException("GraalPy resource variant " + VARIANT + " is not supported on " + os + "/" + arch); + } + return os + "/" + arch + (VARIANT.isEmpty() ? "" : "/" + VARIANT); + } + + static String cacheSuffix() { + return VARIANT.isEmpty() ? "" : "-" + VARIANT; + } + + private static String selectVariant() { + String variant = System.getProperty(VARIANT_PROPERTY, ""); + switch (variant) { + case "": + case "default": + return ""; + case SWCFI: + return SWCFI; + default: + throw new IllegalArgumentException("Unsupported " + VARIANT_PROPERTY + ": " + variant); + } + } +} diff --git a/graalpython/com.oracle.graal.python.shell/pom.xml b/graalpython/com.oracle.graal.python.shell/pom.xml index 803ae9a4b6..2482b656a6 100644 --- a/graalpython/com.oracle.graal.python.shell/pom.xml +++ b/graalpython/com.oracle.graal.python.shell/pom.xml @@ -88,7 +88,7 @@ SOFTWARE. - ../../mxbuild/jdk25/com.oracle.graal.python.shell/src_gen + ../../mxbuild/graalpython/com.oracle.graal.python.shell/src_gen diff --git a/graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/advanced/ResourceVariantProbe.java b/graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/advanced/ResourceVariantProbe.java new file mode 100644 index 0000000000..5e889d0566 --- /dev/null +++ b/graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/advanced/ResourceVariantProbe.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package com.oracle.graal.python.test.integration.advanced; + +import java.nio.file.Files; +import java.nio.file.Path; + +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.Engine; +import org.graalvm.polyglot.io.IOAccess; + +/** Runs in a fresh VM for each variant; deliberately does not use the Python launcher or a home. */ +public final class ResourceVariantProbe { + private ResourceVariantProbe() { + } + + public static void main(String[] args) throws Exception { + if (System.getProperty("org.graalvm.language.python.home") != null || System.getenv("GRAAL_PYTHONHOME") != null) { + throw new AssertionError("This probe must use JAR-backed resources, not a configured Python home"); + } + boolean graalos = args[0].equals("musl-swcfi"); + Path target = Path.of(args[1]); + if (!Engine.copyResources(target, "python")) { + throw new AssertionError("Python resources were not copied"); + } + try (var files = Files.walk(target)) { + if (files.noneMatch(p -> p.getFileName().toString().equals("pyconfig.h"))) { + throw new AssertionError("Missing native header in extracted resources"); + } + } + // The host JVM cannot load SW-CFI native libraries. Metadata and extraction must still + // agree without loading them, and without a launcher having set any system properties. + try (Context context = Context.newBuilder("python").allowExperimentalOptions(true).option("python.PosixModuleBackend", "java").allowNativeAccess(false).allowIO(IOAccess.ALL).build()) { + String multiarch = context.eval("python", "import sys; sys.implementation._multiarch").asString(); + if (graalos != multiarch.equals("x86_64-graalos")) { + throw new AssertionError("Wrong resource ABI: " + multiarch); + } + String sysconfigMultiarch = context.eval("python", "import sysconfig; sysconfig.get_config_var('MULTIARCH')").asString(); + if (!multiarch.equals(sysconfigMultiarch)) { + throw new AssertionError("Metadata/sysconfig mismatch: " + multiarch + " != " + sysconfigMultiarch); + } + } + } +} diff --git a/graalpython/com.oracle.graal.python/pom.xml b/graalpython/com.oracle.graal.python/pom.xml index 9b849cbe36..4b3598c2b3 100644 --- a/graalpython/com.oracle.graal.python/pom.xml +++ b/graalpython/com.oracle.graal.python/pom.xml @@ -116,7 +116,7 @@ SOFTWARE. - ../../mxbuild/jdk25/com.oracle.graal.python/src_gen + ../../mxbuild/graalpython/com.oracle.graal.python/src_gen diff --git a/mx.graalpython/mx_graalpython.py b/mx.graalpython/mx_graalpython.py index fb2fae1650..91c156d200 100644 --- a/mx.graalpython/mx_graalpython.py +++ b/mx.graalpython/mx_graalpython.py @@ -190,6 +190,18 @@ def _is_graalos_build(): return "musl" in _libc() +def graalpy_resource_platform(): + platform = f"{mx.get_os()}/{mx.get_arch()}" + libc = _libc() + if libc == "musl-swcfi": + if platform != "linux/amd64": + mx.abort(f"Unsupported GraalPy resource target: {platform}/{libc}") + return f"{platform}/{libc}" + if "musl" in libc: + mx.abort(f"No GraalPy resource variant for libc: {libc}") + return platform + + def _with_bouncycastle(): return GRAALPY_WITH_BOUNCYCASTLE and not _is_graalos_build() @@ -2732,6 +2744,8 @@ def abi_version(): mx_subst.results_substitutions.register_no_arg('graalpy_abiflags', graalpy_abiflags) mx_subst.path_substitutions.register_no_arg('graalpy_soabi', graalpy_soabi) mx_subst.results_substitutions.register_no_arg('graalpy_soabi', graalpy_soabi) +mx_subst.path_substitutions.register_no_arg('graalpy_resource_platform', graalpy_resource_platform) +mx_subst.results_substitutions.register_no_arg('graalpy_resource_platform', graalpy_resource_platform) mx_subst.path_substitutions.register_no_arg('graalpy_multiarch', graalpy_multiarch) mx_subst.results_substitutions.register_no_arg('graalpy_multiarch', graalpy_multiarch) mx_subst.path_substitutions.register_no_arg('graalpy_sysconfigdata', graalpy_sysconfigdata) diff --git a/mx.graalpython/mx_graalpython_graalos.py b/mx.graalpython/mx_graalpython_graalos.py index a7339008f7..9346f6e2cb 100644 --- a/mx.graalpython/mx_graalpython_graalos.py +++ b/mx.graalpython/mx_graalpython_graalos.py @@ -302,6 +302,87 @@ def _upload_graalos_standalone_artifact(standalone_home: Path, work_dir: Path): run(upload_cmd) +def _prepare_graalos_toolchain(work_dir: Path, artifact_name, on_fail=mx.abort): + work_dir.mkdir(parents=True, exist_ok=True) + tarball = work_dir / "graalvm.tar.gz" + graalvm_home = work_dir / "graalvm" + _download_graalos_standalone_artifact(artifact_name, tarball, on_fail=on_fail) + _extract_tarball(tarball, graalvm_home, strip_components=1, on_fail=on_fail) + musl_toolchain = graalvm_home / "lib" / "toolchains" / "musl-swcfi" + if not (graalvm_home / "bin" / mx.exe_suffix("java")).is_file(): + on_fail(f"Extracted GraalOS toolchain artifact does not contain bin/java: {graalvm_home}") + if not musl_toolchain.is_dir(): + on_fail(f"Extracted GraalOS toolchain artifact does not contain musl-swcfi toolchain: {musl_toolchain}") + return graalvm_home, musl_toolchain + + +@mx.command('graalpython', 'python-graalos-resources', '') +def graalpy_graalos_resources(args): + """Build a strict, resource-only Maven input in a separate single-target mx invocation.""" + if len(args) != 1: + mx.abort('Usage: mx python-graalos-resources ') + if (mx.get_os(), mx.get_arch()) != ('linux', 'amd64'): + mx.abort('GraalOS resources must be built on linux-amd64') + from mx_graalpython import extend_os_env, run_mx + versions = load_graalos_versions() + work_dir = Path(SUITE.dir) / 'mxbuild' / 'graalos-resources-ci' + graalvm_home, musl_toolchain = _prepare_graalos_toolchain(work_dir, versions['toolchain']) + env = extend_os_env( + BOOTSTRAP_GRAALVM=str(graalvm_home), + MUSL_TOOLCHAIN=str(musl_toolchain), + GRAALOS_TOOLCHAIN_PATH=str(musl_toolchain), + ) + # Isolation is intentional: substitutions/toolchain registration depend on startup options. + run_mx(['-p', SUITE.dir, '--java-home=lookup:default', '--multitarget=linux-amd64-musl-swcfi', + 'python-graalos-resources-build', str(Path(args[0]).resolve())], env=env) + + +@mx.command('graalpython', 'python-resource-variant-tests', '[--default-only]') +def graalpy_resource_variant_tests(args): + """Check extraction and ABI metadata in fresh, launcher-less JVMs using the resource JAR.""" + if args not in ([], ['--default-only']): + mx.abort('Usage: mx python-resource-variant-tests [--default-only]') + vm_args = mx.get_runtime_jvm_args(['GRAALPYTHON_RESOURCES', 'GRAALPYTHON_INTEGRATION_UNIT_TESTS'], force_cp=True) + main = 'com.oracle.graal.python.test.integration.advanced.ResourceVariantProbe' + env = dict(os.environ) + env.pop('GRAAL_PYTHONHOME', None) + cases = [ + ('default', None, None), + ('default', '', None), + ('default', 'default', None), + ('default', 'invalid', 'Unsupported org.graalvm.python.resources.variant'), + ('musl-swcfi', 'musl-swcfi', 'musl-swcfi' if args else None), + ] + for expected, prop, error in cases: + with tempfile.TemporaryDirectory(prefix='graalpy-resource-variant-') as directory: + output = mx.OutputCapture() + options = ['-Dpolyglot.engine.WarnInterpreterOnly=false', '-Dpolyglot.engine.userResourceCache=' + directory] + if prop is not None: + options.append('-Dorg.graalvm.python.resources.variant=' + prop) + result = mx.run_java(vm_args + options + [main, expected, str(Path(directory) / 'copied')], + env=env, out=output, err=output, nonZeroIsFatal=False) + if (error is None and result != 0) or (error is not None and (result == 0 or error not in output.data)): + mx.abort(f'Resource variant probe failed ({expected=}, {prop=}):\n{output.data}') + mx.log(f'Resource variant probe passed: {expected=}, {prop=}') + + +@mx.command('graalpython', 'python-graalos-resources-build', '') +def graalpy_graalos_resources_build(args): + """Build/export with an already configured GraalOS bootstrap toolchain.""" + from mx_graalpython import PYTHON_NATIVE_PROJECTS, graalpy_resource_platform + if len(args) != 1 or graalpy_resource_platform() != 'linux/amd64/musl-swcfi': + mx.abort('Expected one archive path and --multitarget=linux-amd64-musl-swcfi') + target = 'linux-amd64-musl-swcfi' + # mx can silently fall back to default toolchains. Never publish such a build as GraalOS. + for name in PYTHON_NATIVE_PROJECTS + ['graalpy-versions', 'graalpy-pyconfig']: + toolchains = mx.project(name).toolchains + if len(toolchains) != 1 or toolchains[0].spec.target.name != target: + mx.abort(f'{name} did not select exactly the {target} toolchain') + distributions = 'GRAALPYTHON_NATIVE_RESOURCES,GRAALPYTHON_VERSIONS_RES' + mx.command_function('build')(['--targets=' + distributions]) + mx.command_function('archive-pd-layouts')(['--platform-id=' + target, '--only=' + distributions, args[0]]) + + def graalpy_graalos_standalone_build_and_test(report=None, on_fail=mx.abort): artifact_base_url = os.environ.get("GRAALPY_GRAALOS_ARTIFACT_BASE_URL") if not artifact_base_url: @@ -314,18 +395,9 @@ def graalpy_graalos_standalone_build_and_test(report=None, on_fail=mx.abort): shutil.rmtree(work_dir) work_dir.mkdir(parents=True, exist_ok=True) - graalvm_tarball = work_dir / "graalvm.tar.gz" runtime_tarball = work_dir / "graalos-runtime.tar.gz" - graalvm_home = work_dir / "graalvm" runtime_root = work_dir / "runtime" - - _download_graalos_standalone_artifact(versions["toolchain"], graalvm_tarball, on_fail=on_fail) - _extract_tarball(graalvm_tarball, graalvm_home, strip_components=1, on_fail=on_fail) - musl_toolchain = graalvm_home / "lib" / "toolchains" / "musl-swcfi" - if not (graalvm_home / "bin" / mx.exe_suffix("java")).is_file(): - on_fail(f"Extracted GraalOS toolchain artifact does not contain bin/java: {graalvm_home}") - if not musl_toolchain.is_dir(): - on_fail(f"Extracted GraalOS toolchain artifact does not contain musl-swcfi toolchain: {musl_toolchain}") + graalvm_home, musl_toolchain = _prepare_graalos_toolchain(work_dir, versions["toolchain"], on_fail=on_fail) _download_graalos_standalone_artifact(versions["runtime"], runtime_tarball, on_fail=on_fail) _extract_tarball(runtime_tarball, runtime_root, on_fail=on_fail) @@ -336,6 +408,7 @@ def graalpy_graalos_standalone_build_and_test(report=None, on_fail=mx.abort): from mx_graalpython import extend_os_env, run_mx, run_python_unittests, _graalpy_launcher env = extend_os_env( JAVA_HOME=str(graalvm_home), + BOOTSTRAP_GRAALVM=str(graalvm_home), MUSL_TOOLCHAIN=str(musl_toolchain), GRAALOS_TOOLCHAIN_PATH=str(musl_toolchain), GRAALOS_RUNTIME_HOME=str(graalos_runtime_home), diff --git a/mx.graalpython/mx_pominit.py b/mx.graalpython/mx_pominit.py index b3e7329924..6b6957bd03 100644 --- a/mx.graalpython/mx_pominit.py +++ b/mx.graalpython/mx_pominit.py @@ -58,8 +58,8 @@ LOCAL_GROUP_ID = "${project.groupId}" LOCAL_VERSION = "${project.version}" GRAALVM_VERSION = "${graalvm.version}" -DEFAULT_GRAALVM_VERSION = "25.3.4.1" -CURRENT_GRAALVM_VERSION = "25.4.4.1.1" +DEFAULT_GRAALVM_VERSION = "25.4.4.1.1" +CURRENT_GRAALVM_VERSION = "25.5.5" XML_UPL_HEADER = """