ci: make pixi lockfile updates predictable (#2298)

.github/workflows/ci-pixi-lockfile-freshness-check.yml

@@ -0,0 +1,119 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Fails when a committed pixi.lock is out of date with its pixi.toml, or when
+# its bytes are not what the pinned pixi version generates.
+# Remediation for both: regenerate and commit the lockfile with that pixi
+# version, e.g.
+#   pixi lock --manifest-path <path>
+# See #2298.
+
+name: "CI: pixi lockfile freshness check"
+
+concurrency:
+  # Keyed on the event as well as the ref so a manual dispatch and a push to
+  # the same branch do not cancel each other.
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+    # `pyproject.toml` is a lockfile input too: the manifests consume sibling
+    # packages (including cuda_python_test_helpers) as path dependencies, so
+    # their metadata can stale a lock without any pixi.toml edit. Matched by
+    # glob rather than by name so the filter cannot drift as packages move.
+    paths: &lockfile_inputs
+      - "**/pixi.toml"
+      - "**/pixi.lock"
+      - "**/pyproject.toml"
+      - "ci/tools/list_pixi_workspaces.py"
+      - ".github/workflows/ci-pixi-lockfile-freshness-check.yml"
+  push:
+    # `pull_request` already covers PRs, including those from forks: this check
+    # needs no secrets or GPU runner. Watching copy-pr-bot's `pull-request/N`
+    # mirror too would run the whole matrix a second time per PR.
+    branches:
+      - "main"
+    paths: *lockfile_inputs
+  workflow_dispatch: {}
+
+defaults:
+  run:
+    shell: bash --noprofile --norc -xeuo pipefail {0}
+
+env:
+  # Keep in sync with ci-pixi-source-test.yml. Must be >=0.71.0 so the
+  # committed lockfile format (v7) matches what this job checks against.
+  PIXI_VERSION: "v0.73.0"
+
+permissions: {}
+
+jobs:
+  # The workspace list is derived from the committed manifests so this matrix
+  # cannot silently skip a newly added workspace (#2298).
+  plan:
+    name: Discover pixi workspaces
+    if: ${{ github.repository_owner == 'nvidia' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+    outputs:
+      matrix: ${{ steps.discover.outputs.matrix }}
+    steps:
+      - name: Checkout ${{ github.event.repository.name }}
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          fetch-depth: 1
+          persist-credentials: false
+
+      - name: Discover workspaces
+        id: discover
+        run: |
+          workspaces="$(python3 ci/tools/list_pixi_workspaces.py)"
+          echo "matrix=$(jq -c '{include: .}' <<<"${workspaces}")" >> "${GITHUB_OUTPUT}"
+
+  lockfile-fresh:
+    name: "pixi lock --check (${{ matrix.manifest }})"
+    needs: plan
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    permissions:
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.plan.outputs.matrix) }}
+    steps:
+      - name: Checkout ${{ github.event.repository.name }}
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          fetch-depth: 1
+          persist-credentials: false
+
+      - name: Setup pixi
+        # Pinned to a commit SHA; install logic lives in the action and is
+        # auditable/pinned (vs. a curl|bash of an unverified installer).
+        uses: prefix-dev/setup-pixi@5185adfbffb4bd703da3010310260805d89ebb11 # v0.9.6
+        with:
+          pixi-version: ${{ env.PIXI_VERSION }}
+          run-install: false
+
+      - name: Check lockfile is current with its manifest
+        run: |
+          if ! pixi lock --check --manifest-path "${{ matrix.manifest }}"; then
+            echo "::error::Lockfile is stale for '${{ matrix.manifest }}'. Regenerate with: pixi lock --manifest-path ${{ matrix.manifest }}"
+            exit 1
+          fi
+
+      # `pixi lock --check` exits 0 on a semantically current lock even when it
+      # rewrites the file into the pinned version's canonical form (lockfile
+      # format upgrades, platform alias renames). Without this guard that drift
+      # is invisible: the check prints "Updated lock file" and still passes,
+      # while every later pixi run keeps rewriting the committed file (#2298).
+      - name: Check lockfile is byte-for-byte canonical
+        run: |
+          if ! git diff --exit-code -- "${{ matrix.lockfile }}"; then
+            echo "::error::pixi ${PIXI_VERSION} rewrote ${{ matrix.lockfile }} during the check, so the committed bytes are not what it generates. Regenerate with pixi ${PIXI_VERSION}: pixi lock --manifest-path ${{ matrix.manifest }}"
+            exit 1
+          fi

.github/workflows/ci-pixi-lockfile-refresh.yml

@@ -0,0 +1,177 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Scheduled (and on-demand) pixi lockfile refresh. Each workspace gets its own
+# PR so a benchmark or docs environment cannot drag unrelated package churn
+# into review. See #2298.
+
+name: "CI: pixi lockfile refresh"
+
+on:
+  schedule:
+    # Monday 03:17 Eastern. GitHub handles the DST offset for the IANA zone,
+    # and a non-zero minute avoids the start-of-hour window where GitHub
+    # documents scheduled runs being delayed or dropped.
+    - cron: "17 3 * * 1"
+      timezone: "America/New_York"
+  workflow_dispatch:
+    inputs:
+      package:
+        description: >
+          Pixi workspace id to refresh, or "all". Valid ids come from
+          ci/tools/list_pixi_workspaces.py; an unknown id fails the run.
+        type: string
+        default: all
+
+defaults:
+  run:
+    shell: bash --noprofile --norc -xeuo pipefail {0}
+
+env:
+  # Keep in sync with ci-pixi-source-test.yml.
+  PIXI_VERSION: "v0.73.0"
+
+permissions: {}
+
+jobs:
+  # Job-level `if` cannot use `matrix` (evaluated before expansion), and the
+  # workspace list must not drift from the committed manifests, so resolve the
+  # requested workspaces here and pass them to refresh as a dynamic matrix.
+  plan:
+    name: Select workspaces
+    if: ${{ github.repository_owner == 'nvidia' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+    outputs:
+      matrix: ${{ steps.select.outputs.matrix }}
+    steps:
+      - name: Checkout ${{ github.event.repository.name }}
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          fetch-depth: 1
+          persist-credentials: false
+
+      - name: Select workspaces to refresh
+        id: select
+        env:
+          PACKAGE: ${{ inputs.package || 'all' }}
+        run: |
+          workspaces="$(python3 ci/tools/list_pixi_workspaces.py --select "${PACKAGE}")"
+          echo "matrix=$(jq -c '{include: .}' <<<"${workspaces}")" >> "${GITHUB_OUTPUT}"
+
+  refresh:
+    name: "pixi update (${{ matrix.id }})"
+    needs: plan
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: write
+      pull-requests: write
+    concurrency:
+      group: pixi-lock-refresh-${{ matrix.id }}
+      cancel-in-progress: false
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.plan.outputs.matrix) }}
+    steps:
+      - name: Checkout ${{ github.event.repository.name }}
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.repository.default_branch }}
+
+      # GitHub suppresses workflow-triggering events for pushes and PRs made
+      # with GITHUB_TOKEN, so a refresh PR opened with the default token cannot
+      # reach a green required-check state on its own. An App token restores
+      # normal event delivery; without one the PR needs a manual nudge, which
+      # the PR body and a run warning spell out.
+      - name: Mint App token for the refresh PR
+        id: app-token
+        if: ${{ vars.PIXI_LOCK_REFRESH_APP_ID != '' }}
+        uses: actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349 # v2.2.2
+        with:
+          app-id: ${{ vars.PIXI_LOCK_REFRESH_APP_ID }}
+          private-key: ${{ secrets.PIXI_LOCK_REFRESH_APP_PRIVATE_KEY }}
+
+      - name: Setup pixi
+        # Pinned to a commit SHA; install logic lives in the action and is
+        # auditable/pinned (vs. a curl|bash of an unverified installer).
+        uses: prefix-dev/setup-pixi@5185adfbffb4bd703da3010310260805d89ebb11 # v0.9.6
+        with:
+          pixi-version: ${{ env.PIXI_VERSION }}
+          run-install: false
+
+      - name: Update lockfile without installing
+        run: pixi update --no-install --manifest-path "${{ matrix.manifest }}"
+
+      - name: Compose PR body
+        id: pr-body
+        env:
+          WORKSPACE_ID: ${{ matrix.id }}
+          MANIFEST: ${{ matrix.manifest }}
+          HAS_APP_TOKEN: ${{ steps.app-token.outputs.token != '' }}
+        run: |
+          body_path="${RUNNER_TEMP}/pixi-lock-refresh-body.md"
+          {
+            echo "Automated lockfile refresh for \`${WORKSPACE_ID}\`" \
+              "(\`pixi update --no-install --manifest-path ${MANIFEST}\`)."
+            echo
+            echo "This PR is workspace-scoped so other packages are not forced to review"
+            echo "unrelated solver churn. See #2298."
+            echo
+            if [ "${HAS_APP_TOKEN}" != "true" ]; then
+              echo "> [!IMPORTANT]"
+              echo "> No refresh App token is configured, so this PR was opened with"
+              echo "> \`GITHUB_TOKEN\` and GitHub will not deliver its \`pull_request\` events."
+              echo "> Required checks stay pending until a maintainer nudges the branch, by"
+              echo "> closing and reopening the PR or pushing an empty commit to it."
+              echo
+            fi
+            echo "If the diff looks wrong, close this PR and leave the branch deleted; the"
+            echo "next scheduled run opens a fresh one if an update is still needed."
+          } > "${body_path}"
+          echo "path=${body_path}" >> "${GITHUB_OUTPUT}"
+
+          if [ "${HAS_APP_TOKEN}" != "true" ]; then
+            echo "::warning::Opened with GITHUB_TOKEN; the refresh PR's checks must be triggered manually."
+          fi
+
+      - name: Open or update a lockfile-only PR
+        id: cpr
+        uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
+        with:
+          token: ${{ steps.app-token.outputs.token || github.token }}
+          add-paths: ${{ matrix.lockfile }}
+          commit-message: |
+            chore: refresh pixi.lock for ${{ matrix.id }}
+
+            Scheduled pixi update --no-install for this workspace only.
+          signoff: true
+          sign-commits: true
+          branch: ci/pixi-lock-refresh/${{ matrix.id }}
+          delete-branch: true
+          title: "chore: refresh pixi.lock (${{ matrix.id }})"
+          labels: |
+            CI/CD
+            dependencies
+          body-path: ${{ steps.pr-body.outputs.path }}
+          author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
+          committer: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
+
+      - name: Summarize
+        run: |
+          {
+            echo "### pixi lock refresh (${{ matrix.id }})"
+            echo ""
+            echo "- Manifest: \`${{ matrix.manifest }}\`"
+            echo "- Lockfile: \`${{ matrix.lockfile }}\`"
+            echo "- Operation: \`${{ steps.cpr.outputs.pull-request-operation }}\`"
+            if [ -n "${{ steps.cpr.outputs.pull-request-url }}" ]; then
+              echo "- PR: ${{ steps.cpr.outputs.pull-request-url }}"
+            else
+              echo "- No lockfile diff; PR not opened."
+            fi
+          } >> "$GITHUB_STEP_SUMMARY"

.github/workflows/ci-pixi-source-test.yml

@@ -38,6 +38,9 @@ on:
     paths:
       - "**/pixi.toml"
       - "**/pixi.lock"
+      # Path-dependency metadata (including cuda_python_test_helpers) feeds the
+      # solve, so it can break the source build without a pixi.toml edit.
+      - "**/pyproject.toml"
       - "cuda_bindings/build_hooks.py"
       - "cuda_core/build_hooks.py"
       - "cuda_bindings/cuda/bindings/**" # generated bindings sources
@@ -61,7 +64,13 @@ env:
   # older pixi re-ran the editable source build on every `pixi run`, recompiling
   # all Cython extensions (#2138). The fix (content-addressed source-build cache,
   # prefix-dev/pixi#6285 + #6123) also bumps the pixi.lock format to v7.
+  # Keep in sync with ci-pixi-lockfile-refresh.yml and
+  # ci-pixi-lockfile-freshness-check.yml.
   PIXI_VERSION: "v0.73.0"
+  # Install from the committed lockfile only. Covers nested `pixi run` calls
+  # from root pixi.toml tasks; a stale lock must fail rather than being
+  # refreshed in CI (#2298). Equivalent to `pixi run --locked`.
+  PIXI_LOCKED: "true"
 
 jobs:
   # ── PR guard: CPU-only build + import + placement smoke ──
@@ -84,7 +93,7 @@ jobs:
       - name: Setup pixi
         # Pinned to a commit SHA; install logic lives in the action and is
         # auditable/pinned (vs. a curl|bash of an unverified installer).
-        uses: prefix-dev/setup-pixi@d3f436a425481402e6a95a1d1fc10331c708cd9e # v0.10.2
+        uses: prefix-dev/setup-pixi@5185adfbffb4bd703da3010310260805d89ebb11 # v0.9.6
         with:
           pixi-version: ${{ env.PIXI_VERSION }}
           run-install: false
@@ -145,7 +154,7 @@ jobs:
       - name: Setup pixi
         # Pinned to a commit SHA; install logic lives in the action and is
         # auditable/pinned (vs. a curl|bash of an unverified installer).
-        uses: prefix-dev/setup-pixi@d3f436a425481402e6a95a1d1fc10331c708cd9e # v0.10.2
+        uses: prefix-dev/setup-pixi@5185adfbffb4bd703da3010310260805d89ebb11 # v0.9.6
         with:
           pixi-version: ${{ env.PIXI_VERSION }}
           run-install: false
@@ -210,7 +219,7 @@ jobs:
       - name: Setup pixi
         # Pinned to a commit SHA; install logic lives in the action and is
         # auditable/pinned (vs. a curl|bash of an unverified installer).
-        uses: prefix-dev/setup-pixi@d3f436a425481402e6a95a1d1fc10331c708cd9e # v0.10.2
+        uses: prefix-dev/setup-pixi@5185adfbffb4bd703da3010310260805d89ebb11 # v0.9.6
         with:
           pixi-version: ${{ env.PIXI_VERSION }}
           run-install: false

.github/workflows/ci-workflow-health.yml

@@ -13,6 +13,8 @@ on:
       - "CI: Coverage"
       - "CI: Nightly optional-deps"
       - "CI: pixi run test (source build)"
+      - "CI: pixi lockfile refresh"
+      - "CI: pixi lockfile freshness check"
       - "Security Suite (Pulse + CodeQL)"
       - "Static Analysis: Bandit Scan"
     types:

.github/workflows/pr-metadata-check.yml

@@ -32,7 +32,11 @@ jobs:
           PR_NUMBER: ${{ github.event.pull_request.number }}
           GH_REPO: ${{ github.repository }}
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          IS_BOT: ${{ github.actor == 'dependabot[bot]' || github.actor == 'pre-commit-ci[bot]' || github.actor == 'copy-pr-bot[bot]' }}
+          # Keyed on the PR author, not the actor: a maintainer labelling a bot
+          # PR re-triggers this workflow as themselves. Any bot author is
+          # exempt because none of them can set an assignee or milestone
+          # (e.g. the App identity used by the pixi lockfile refresh).
+          IS_BOT: ${{ github.event.pull_request.user.type == 'Bot' || github.actor == 'dependabot[bot]' || github.actor == 'pre-commit-ci[bot]' || github.actor == 'copy-pr-bot[bot]' || github.actor == 'github-actions[bot]' }}
           IS_DRAFT: ${{ github.event.pull_request.draft }}
         run: |
           if [ "$IS_BOT" = "true" ] || [ "$IS_DRAFT" = "true" ]; then

.gitignore

@@ -176,8 +176,10 @@ cython_debug/
 # Dont ignore
 !.github/actions/build/
 # pixi environments
-.pixi/*
-!.pixi/config.toml
+# Per-workspace commands such as `pixi lock --manifest-path cuda_bindings`
+# create a .pixi/ next to that manifest, so match at any depth.
+**/.pixi/*
+!**/.pixi/config.toml
 
 # Cursor
 .cursorrules