diff --git a/.docker/lyrical.amd64.dockerfile b/.docker/lyrical.amd64.dockerfile index 6fad561b..b7be5c02 100644 --- a/.docker/lyrical.amd64.dockerfile +++ b/.docker/lyrical.amd64.dockerfile @@ -15,6 +15,19 @@ COPY extras /tmp/dave-extras RUN DAVE_EXTRAS_DIR=/tmp/dave-extras \ bash /tmp/dave-extras/ros-lyrical-gz-jetty-install.sh +# docker run/exec do not necessarily start an interactive shell. Keep these +# paths in the image environment, not just the installer's ~/.bashrc hook. +ENV PATH=/opt/ardusub_ws/ardupilot/build/sitl/bin:/opt/ardusub_ws/ardupilot/Tools/autotest:${PATH} +ENV GZ_SIM_SYSTEM_PLUGIN_PATH=/opt/ardusub_ws/ardupilot_gazebo/build +ENV GZ_SIM_RESOURCE_PATH=/opt/ardusub_ws/ardupilot_gazebo/models:/opt/ardusub_ws/ardupilot_gazebo/worlds +ENV GEOGRAPHICLIB_GEOID_PATH=/usr/share/GeographicLib/geoids +ENV POSIM_BRIDGE_UNDERLAY=/opt/posim_bridge_ws +ENV POSIM_MAVROS_UNDERLAY=/opt/posim_mavros_ws +ENV POSIM_TRANSPORT_UNDERLAY=/opt/posim_transport_ws +RUN bash /tmp/dave-extras/build-image-transport.sh +RUN bash /tmp/dave-extras/build-image-mavros.sh +RUN bash /tmp/dave-extras/build-image-bridge.sh + # Install QGroundControl. RUN mkdir -p /opt/QGC && cd /opt/QGC && \ wget -O QGroundControl-x86_64.AppImage \ @@ -46,7 +59,7 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* WORKDIR $DAVE_WS -RUN . "/opt/ros/${ROS_DISTRO}/setup.sh" && \ +RUN . "$POSIM_BRIDGE_UNDERLAY/install/setup.sh" && \ colcon build --merge-install --executor sequential --symlink-install RUN echo "source /opt/ros/${ROS_DISTRO}/setup.bash" >> /root/.bashrc && \ @@ -60,6 +73,15 @@ RUN touch /root/.dave_entrypoint && \ WORKDIR /root +# Resolve the Quickstart Fuel dependency closure during the build, not startup. +ENV GZ_FUEL_CACHE_PATH=/opt/posim_fuel/cache +COPY extras/fuel /opt/posim_fuel +COPY extras/prepare-image-assets.py /opt/posim_fuel/prepare-image-assets.py +RUN . "/opt/ros/${ROS_DISTRO}/setup.sh" && \ + python3 /opt/posim_fuel/prepare-image-assets.py \ + --cache "$GZ_FUEL_CACHE_PATH" --lock /opt/posim_fuel/quickstart-assets.lock.json \ + --receipt /opt/posim_fuel/build-receipt.json + LABEL org.opencontainers.image.title="POSIM" \ org.opencontainers.image.description="Platform for Ocean Simulation" \ org.opencontainers.image.source="https://github.com/IOES-Lab/POSIM" \ diff --git a/.docker/lyrical.arm64v8.dockerfile b/.docker/lyrical.arm64v8.dockerfile index bf86df81..97580749 100644 --- a/.docker/lyrical.arm64v8.dockerfile +++ b/.docker/lyrical.arm64v8.dockerfile @@ -109,8 +109,24 @@ RUN export ROS_APT_SOURCE_VERSION=$(curl -s https://api.github.com/repos/ros-inf apt install -y --no-install-recommends \ ros-${ROS_DISTRO}-desktop ros-${ROS_DISTRO}-ros-gz \ ros-${ROS_DISTRO}-image-view \ + ros-${ROS_DISTRO}-mavros ros-${ROS_DISTRO}-mavros-msgs \ python3-rosdep python3-vcstool python3-colcon-common-extensions +ENV POSIM_BRIDGE_UNDERLAY=/opt/posim_bridge_ws +ENV POSIM_MAVROS_UNDERLAY=/opt/posim_mavros_ws +ENV POSIM_TRANSPORT_UNDERLAY=/opt/posim_transport_ws +COPY extras/build-image-transport.sh /tmp/build-image-transport.sh +COPY extras/build-image-bridge.sh /tmp/build-image-bridge.sh +COPY extras/build-image-mavros.sh /tmp/build-image-mavros.sh +COPY extras/patches /tmp/patches +COPY extras/ci/bridge_ownership /tmp/ci/bridge_ownership +COPY extras/ci/mavconn_self_close /tmp/ci/mavconn_self_close +COPY extras/ci/mavros_ownership /tmp/ci/mavros_ownership +COPY extras/ci/transport_shutdown /tmp/ci/transport_shutdown +RUN bash /tmp/build-image-transport.sh +RUN bash /tmp/build-image-mavros.sh +RUN bash /tmp/build-image-bridge.sh + # --- DAVE workspace --- # Build the exact checked-out revision supplied as the Docker build context. ENV DAVE_UNDERLAY=/home/$USER/dave_ws @@ -132,7 +148,7 @@ RUN apt-get update && \ USER $USER WORKDIR $DAVE_UNDERLAY -RUN . "/opt/ros/${ROS_DISTRO}/setup.sh" && \ +RUN . "$POSIM_BRIDGE_UNDERLAY/install/setup.sh" && \ colcon build --merge-install --executor sequential --symlink-install # --- ArduSub SITL (BlueROV2) — Python 3.14 compatibility shims required, see notes/ardusub-sitl-setup.md --- @@ -141,9 +157,23 @@ USER root ARG ARDUSUB_COMMIT="30257f01185471ab4c1ac544e47d1b4437e44c98" ARG ARDUPILOT_GAZEBO_COMMIT="082a0fe231f6e63bc8d1598f1cba461d9e2ea7f5" WORKDIR /home/$USER -RUN git clone --recurse-submodules https://github.com/ArduPilot/ardupilot.git && \ - cd ardupilot && git fetch --tags && git checkout --detach "$ARDUSUB_COMMIT" && \ - git submodule update --init --recursive +# Fetch the pinned tree directly: cloning current HEAD and all history first +# caused HTTP/2 early-EOF failures on the ARM64 runner. +RUN set -eu; \ + retry_git() { \ + for attempt in 1 2 3; do \ + if git -c http.version=HTTP/1.1 "$@"; then return 0; fi; \ + echo "Git transfer attempt $attempt/3 failed" >&2; \ + sleep 5; \ + done; \ + return 1; \ + }; \ + git init ardupilot && cd ardupilot && \ + git remote add origin https://github.com/ArduPilot/ardupilot.git && \ + retry_git fetch --depth 1 origin "$ARDUSUB_COMMIT" && \ + git checkout --detach FETCH_HEAD && \ + test "$(git rev-parse HEAD)" = "$ARDUSUB_COMMIT" && \ + retry_git submodule update --init --recursive --depth 1 --jobs 2 RUN mkdir -p /home/$USER/imp_shim && \ printf 'import types\ndef new_module(name):\n return types.ModuleType(name)\n' > /home/$USER/imp_shim/imp.py && \ @@ -228,6 +258,16 @@ COPY extras/docker-arm64-entrypoint.sh /usr/local/bin/dave-rdp-entrypoint RUN chmod 0755 /usr/local/bin/dave-rdp-entrypoint CMD ["/usr/local/bin/dave-rdp-entrypoint"] +# Shared by the root Quickstart session and the unprivileged RDP desktop user. +ENV GZ_FUEL_CACHE_PATH=/opt/posim_fuel/cache +COPY extras/fuel /opt/posim_fuel +COPY extras/prepare-image-assets.py /opt/posim_fuel/prepare-image-assets.py +RUN . "/opt/ros/${ROS_DISTRO}/setup.sh" && \ + python3 /opt/posim_fuel/prepare-image-assets.py \ + --cache "$GZ_FUEL_CACHE_PATH" --lock /opt/posim_fuel/quickstart-assets.lock.json \ + --receipt /opt/posim_fuel/build-receipt.json && \ + chown -R $USER:$USER "$GZ_FUEL_CACHE_PATH" + LABEL org.opencontainers.image.title="POSIM" \ org.opencontainers.image.description="Platform for Ocean Simulation" \ org.opencontainers.image.source="https://github.com/IOES-Lab/POSIM" \ diff --git a/.github/workflows/docker-amd64.yml b/.github/workflows/docker-amd64.yml index a9113cf9..7150ce68 100644 --- a/.github/workflows/docker-amd64.yml +++ b/.github/workflows/docker-amd64.yml @@ -23,6 +23,7 @@ on: env: IMAGE_NAME: ioeslab/posim ROS_DISTRO: lyrical + VALIDATION_IMAGE: posim-validation:amd64-${{ github.run_id }}-${{ github.run_attempt }} concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -48,13 +49,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Log in to Docker Hub - if: vars.POSIM_PUBLISH_IMAGES == 'true' && github.event_name != 'pull_request' - uses: docker/login-action@v4 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Docker metadata id: meta uses: docker/metadata-action@v5 @@ -73,10 +67,52 @@ jobs: uses: docker/build-push-action@v5 with: context: . - push: ${{ vars.POSIM_PUBLISH_IMAGES == 'true' && github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} + push: false + load: true + tags: ${{ env.VALIDATION_IMAGE }} labels: ${{ steps.meta.outputs.labels }} file: .docker/lyrical.amd64.dockerfile platforms: linux/amd64 build-args: | ROS_DISTRO=${{ env.ROS_DISTRO }} + + - name: Validate installed image and Quickstarts + # 14 initial paths + 63 additional trials, including two SITL vehicles. + timeout-minutes: 75 + run: | + bash extras/ci/docker_quickstarts.sh "$VALIDATION_IMAGE" linux/amd64 \ + "$RUNNER_TEMP/posim-validation-${{ github.run_id }}-${{ github.run_attempt }}" + + - name: Preserve runtime evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: posim-amd64-runtime-${{ github.run_attempt }} + path: ${{ runner.temp }}/posim-validation-${{ github.run_id }}-${{ github.run_attempt }} + if-no-files-found: warn + retention-days: 14 + + - name: Log in to Docker Hub + if: >- + vars.POSIM_PUBLISH_IMAGES == 'true' && github.event_name != 'pull_request' && + (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Publish the tested image + if: >- + vars.POSIM_PUBLISH_IMAGES == 'true' && github.event_name != 'pull_request' && + (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) + env: + PUBLISH_TAGS: ${{ steps.meta.outputs.tags }} + run: | + results="$RUNNER_TEMP/posim-validation-${{ github.run_id }}-${{ github.run_attempt }}" + tested_id="$(cat "$results/tested-image-id.txt")" + test "$(docker image inspect --format '{{.Id}}' "$VALIDATION_IMAGE")" = "$tested_id" + while IFS= read -r tag; do + test -n "$tag" || continue + docker tag "$tested_id" "$tag" + docker push "$tag" + done <<< "$PUBLISH_TAGS" diff --git a/.github/workflows/docker-arm64v8.yml b/.github/workflows/docker-arm64v8.yml index 6be2ec32..b0679fd5 100644 --- a/.github/workflows/docker-arm64v8.yml +++ b/.github/workflows/docker-arm64v8.yml @@ -23,6 +23,7 @@ on: env: IMAGE_NAME: ioeslab/posim ROS_DISTRO: lyrical + VALIDATION_IMAGE: posim-validation:arm64-${{ github.run_id }}-${{ github.run_attempt }} concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -43,6 +44,13 @@ jobs: steps: - name: Configure non-interactive Docker credentials run: | + # Preserve Desktop's user socket before isolating credentials. + docker_host="${DOCKER_HOST:-$(docker context inspect "$(docker context show)" \ + --format '{{.Endpoints.docker.Host}}')}" + test -n "$docker_host" + echo "DOCKER_HOST=$docker_host" >> "$GITHUB_ENV" + echo "DOCKER_CONTEXT=" >> "$GITHUB_ENV" + docker --host "$docker_host" info --format '{{.ServerVersion}} {{.Architecture}}' docker_config="$RUNNER_TEMP/docker-config" mkdir -p "$docker_config" printf '{}\n' > "$docker_config/config.json" @@ -60,28 +68,6 @@ jobs: # ARM64 build and works with Docker Desktop's containerd image store. driver: docker - - name: Configure Docker Hub authentication - if: vars.POSIM_PUBLISH_IMAGES == 'true' && github.event_name != 'pull_request' - env: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - run: | - # Avoid Docker Desktop's interactive macOS Keychain helper. BuildKit - # reads this short-lived Docker config directly during the push. - python3 - <<'PY' - import base64 - import json - import os - from pathlib import Path - - raw = f"{os.environ['DOCKERHUB_USERNAME']}:{os.environ['DOCKERHUB_TOKEN']}" - auth = base64.b64encode(raw.encode()).decode() - config = {"auths": {"https://index.docker.io/v1/": {"auth": auth}}} - path = Path(os.environ["DOCKER_CONFIG"]) / "config.json" - path.write_text(json.dumps(config), encoding="utf-8") - path.chmod(0o600) - PY - - name: Docker metadata id: meta uses: docker/metadata-action@v5 @@ -100,14 +86,75 @@ jobs: uses: docker/build-push-action@v5 with: context: . - push: ${{ vars.POSIM_PUBLISH_IMAGES == 'true' && github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} + push: false + load: true + tags: ${{ env.VALIDATION_IMAGE }} labels: ${{ steps.meta.outputs.labels }} file: .docker/lyrical.arm64v8.dockerfile platforms: linux/arm64/v8 build-args: | ROS_DISTRO=${{ env.ROS_DISTRO }} + - name: Validate installed image and Quickstarts + # 14 initial paths + 63 additional trials, including two SITL vehicles. + timeout-minutes: 75 + run: | + bash extras/ci/docker_quickstarts.sh "$VALIDATION_IMAGE" linux/arm64/v8 \ + "$RUNNER_TEMP/posim-validation-${{ github.run_id }}-${{ github.run_attempt }}" + + - name: Preserve runtime evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: posim-arm64-runtime-${{ github.run_attempt }} + path: ${{ runner.temp }}/posim-validation-${{ github.run_id }}-${{ github.run_attempt }} + if-no-files-found: warn + retention-days: 14 + + - name: Configure Docker Hub authentication + if: >- + vars.POSIM_PUBLISH_IMAGES == 'true' && github.event_name != 'pull_request' && + (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + run: | + # Avoid Docker Desktop's interactive macOS Keychain helper. BuildKit + # reads this short-lived Docker config directly during the push. + python3 - <<'PY' + import base64 + import json + import os + from pathlib import Path + + raw = f"{os.environ['DOCKERHUB_USERNAME']}:{os.environ['DOCKERHUB_TOKEN']}" + auth = base64.b64encode(raw.encode()).decode() + config = {"auths": {"https://index.docker.io/v1/": {"auth": auth}}} + path = Path(os.environ["DOCKER_CONFIG"]) / "config.json" + path.write_text(json.dumps(config), encoding="utf-8") + path.chmod(0o600) + PY + + - name: Publish the tested image + if: >- + vars.POSIM_PUBLISH_IMAGES == 'true' && github.event_name != 'pull_request' && + (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) + env: + PUBLISH_TAGS: ${{ steps.meta.outputs.tags }} + run: | + results="$RUNNER_TEMP/posim-validation-${{ github.run_id }}-${{ github.run_attempt }}" + tested_id="$(cat "$results/tested-image-id.txt")" + test "$(docker image inspect --format '{{.Id}}' "$VALIDATION_IMAGE")" = "$tested_id" + while IFS= read -r tag; do + test -n "$tag" || continue + docker tag "$tested_id" "$tag" + docker push "$tag" + done <<< "$PUBLISH_TAGS" + - name: Clear Docker Hub credentials - if: always() && vars.POSIM_PUBLISH_IMAGES == 'true' && github.event_name != 'pull_request' - run: printf '{}\n' > "$DOCKER_CONFIG/config.json" + if: >- + always() && vars.POSIM_PUBLISH_IMAGES == 'true' && github.event_name != 'pull_request' && + (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) + run: | + config="$RUNNER_TEMP/docker-config/config.json" + if [[ -f "$config" ]]; then printf '{}\n' > "$config"; fi diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 9302f17b..d258d659 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -46,6 +46,9 @@ jobs: if: ${{ !cancelled() }} run: flake8 --ignore=E203,W503,E501 --exclude=tools/code_check . + - name: Test runtime log failure detection + run: python -m unittest discover -s extras/ci -p 'test_*.py' + - name: Check C and C++ formatting if: ${{ !cancelled() }} run: | diff --git a/.github/workflows/publish-candidate.yml b/.github/workflows/publish-candidate.yml new file mode 100644 index 00000000..b42b41fd --- /dev/null +++ b/.github/workflows/publish-candidate.yml @@ -0,0 +1,166 @@ +--- +name: Publish and re-pull validated PR 5 candidate (manual only) + +# yamllint disable-line rule:truthy +on: + # Register the workflow without granting PR events any publication capability. + pull_request: + branches: [main] + paths: + - ".github/workflows/publish-candidate.yml" + - "tools/candidate-*" + - "tools/check-published-candidate.sh" + - "tools/test_candidate_publication.py" + workflow_dispatch: + inputs: + confirmation: + description: Publish only validation-pr5-abad9d70 tags; never main/latest/release tags + required: true + type: choice + options: [publish-pr5-abad9d70] + architecture: + description: Validated image architecture to publish and recheck + required: true + type: choice + default: both + options: [both, arm64, amd64] + +permissions: + contents: read + actions: read + +jobs: + validate-tooling: + runs-on: ubuntu-26.04 + steps: + - uses: actions/checkout@v4 + - run: python3 -m unittest discover -s tools -p 'test_candidate_publication.py' + + publish: + needs: validate-tooling + if: >- + github.event_name == 'workflow_dispatch' && github.repository == 'IOES-Lab/POSIM' && + github.ref == 'refs/heads/ci/posim-image-validation-20260929' && + inputs.confirmation == 'publish-pr5-abad9d70' + strategy: + fail-fast: false + matrix: + architecture: >- + ${{ fromJSON(inputs.architecture == 'arm64' && '["arm64"]' || + inputs.architecture == 'amd64' && '["amd64"]' || '["arm64","amd64"]') }} + runs-on: >- + ${{ fromJSON(matrix.architecture == 'arm64' && '["self-hosted","ARM64"]' || + '["self-hosted","x64"]') }} + timeout-minutes: 180 + concurrency: + group: posim-validation-tag-${{ matrix.architecture }} + cancel-in-progress: false + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Isolate temporary credentials and preserve Docker endpoint + shell: bash + run: | + set -euo pipefail + docker_host="${DOCKER_HOST:-$(docker context inspect "$(docker context show)" \ + --format '{{.Endpoints.docker.Host}}')}" + test -n "$docker_host" + echo "DOCKER_HOST=$docker_host" >> "$GITHUB_ENV" + echo "DOCKER_CONTEXT=" >> "$GITHUB_ENV" + config="$(mktemp -d "$RUNNER_TEMP/posim-publish-config.XXXXXX")" + chmod 700 "$config" + printf '{}\n' > "$config/config.json" + chmod 600 "$config/config.json" + echo "DOCKER_CONFIG=$config" >> "$GITHUB_ENV" + results="$RUNNER_TEMP/posim-publish-${{ github.run_id }}-${{ matrix.architecture }}" + mkdir -p "$results" + echo "PUB_RESULTS=$results" >> "$GITHUB_ENV" + - name: Download the completed validation artifact + uses: actions/download-artifact@v4 + with: + github-token: ${{ github.token }} + repository: IOES-Lab/POSIM + run-id: ${{ matrix.architecture == 'arm64' && 36670174461 || 36670174383 }} + name: >- + ${{ matrix.architecture == 'arm64' && 'posim-arm64-runtime-2' || + 'posim-amd64-runtime-1' }} + path: ${{ env.PUB_RESULTS }}/prepublication-evidence + - name: Verify source, exact retained image, and all prior acceptance records + env: + GH_TOKEN: ${{ github.token }} + run: | + python3 tools/candidate-publication.py evidence '${{ matrix.architecture }}' \ + --evidence "$PUB_RESULTS/prepublication-evidence" \ + --output "$PUB_RESULTS/publication-gate.json" + python3 tools/candidate-publication.py registry-before '${{ matrix.architecture }}' \ + --output "$PUB_RESULTS/registry-before.json" + - name: Publish only the locked candidate tag + shell: bash + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + CANDIDATE_ARCH: ${{ matrix.architecture }} + run: | + set -euo pipefail + tag="$(python3 -c 'import json,os; + d=json.load(open("tools/candidate-images-20260930.json")); + print(d["repository"]+":"+d["images"][os.environ["CANDIDATE_ARCH"]]["tag"])')" + image="$(python3 -c 'import json,os; + d=json.load(open("tools/candidate-images-20260930.json")); + print(d["images"][os.environ["CANDIDATE_ARCH"]]["image_id"])')" + exists="$(python3 -c 'import json,os; + print(json.load(open(os.environ["PUB_RESULTS"]+"/registry-before.json"))["exists"])')" + if [[ "$exists" == False ]]; then + test -n "$DOCKERHUB_USERNAME" && test -n "$DOCKERHUB_TOKEN" + printf '%s' "$DOCKERHUB_TOKEN" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin + docker tag "$image" "$tag" + docker push "$tag" | tee "$PUB_RESULTS/push.log" + else + echo 'Identical candidate already exists; no overwrite.' | tee "$PUB_RESULTS/push.log" + fi + python3 tools/candidate-publication.py registry-after "$CANDIDATE_ARCH" \ + --output "$PUB_RESULTS/registry-after.json" + - name: Clear credentials before anonymous re-pull + if: always() && env.DOCKER_CONFIG != '' + shell: bash + run: printf '{}\n' > "$DOCKER_CONFIG/config.json" + - name: Pull the public registry digest and verify image identity + shell: bash + run: | + set -euo pipefail + ref="$(python3 -c 'import json,os; + d=json.load(open(os.environ["PUB_RESULTS"]+"/registry-after.json")); + print(d["repository"]+"@"+d["tag_digest"])')" + docker pull --platform 'linux/${{ matrix.architecture }}' "$ref" \ + | tee "$PUB_RESULTS/pull.log" + python3 tools/candidate-publication.py pulled '${{ matrix.architecture }}' \ + --output "$PUB_RESULTS/pulled-image.json" + echo "PULLED_REF=$ref" >> "$GITHUB_ENV" + - name: Revalidate all 14 Quickstarts from the pulled image + shell: bash + timeout-minutes: 45 + run: | + set -euo pipefail + image="$(python3 -c 'import json,os; + print(json.load(open(os.environ["PUB_RESULTS"]+"/pulled-image.json"))["image_id"])')" + config_id="$(python3 -c 'import json,os; + print(json.load(open(os.environ["PUB_RESULTS"]+"/pulled-image.json"))["config_digest"])')" + bash tools/check-published-candidate.sh "$PULLED_REF" '${{ matrix.architecture }}' \ + "$image" "$PUB_RESULTS/repull-quickstarts" "$config_id" + - name: Preserve publication and re-pull evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: posim-candidate-publication-${{ matrix.architecture }}-${{ github.run_attempt }} + path: ${{ env.PUB_RESULTS }} + if-no-files-found: warn + retention-days: 30 + - name: Remove this job's credential directory + if: always() && env.DOCKER_CONFIG != '' + shell: bash + run: | + case "$DOCKER_CONFIG" in + "$RUNNER_TEMP"/posim-publish-config.*) rm -rf -- "$DOCKER_CONFIG" ;; + *) exit 1 ;; + esac diff --git a/.github/workflows/runtime-diagnostics.yml b/.github/workflows/runtime-diagnostics.yml new file mode 100644 index 00000000..fbc8eeb3 --- /dev/null +++ b/.github/workflows/runtime-diagnostics.yml @@ -0,0 +1,59 @@ +--- +name: Diagnose an existing POSIM image (no publication) + +# yamllint disable-line rule:truthy +on: + pull_request: + branches: [main] + paths: + - ".github/workflows/runtime-diagnostics.yml" + - "tools/runtime-diagnostics.sh" + - "tools/instrument-mavros-gdb.py" + workflow_dispatch: + inputs: + architecture: + type: choice + options: [arm64, amd64] + required: true + mode: + description: Diagnostic only; never publishes an image + type: choice + options: [gdb, router-lifetime, gazebo-shutdown, gazebo-segfault, transport-poll, camera-fresh] + default: gdb + required: true + image_id: + description: Exact locally retained sha256 image ID from validation + type: string + required: true + +permissions: + contents: read + +jobs: + diagnose: + # Explicit opt-in, and never execute fork code on the self-hosted runner. + if: >- + (github.event_name == 'workflow_dispatch' || vars.POSIM_ENABLE_RUNTIME_DIAGNOSTICS == 'true') && + (github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository) + runs-on: + - self-hosted + - "${{ (inputs.architecture || vars.POSIM_DIAGNOSTIC_ARCH) != 'amd64' && 'ARM64' || 'x64' }}" + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - name: Run explicitly selected runtime diagnostic + env: + POSIM_DIAGNOSTIC_MODE: ${{ inputs.mode || vars.POSIM_DIAGNOSTIC_MODE }} + IMAGE_ID: ${{ inputs.image_id || vars.POSIM_DIAGNOSTIC_IMAGE_ID }} + ARCHITECTURE: ${{ inputs.architecture || vars.POSIM_DIAGNOSTIC_ARCH || 'arm64' }} + run: | + bash tools/runtime-diagnostics.sh "$IMAGE_ID" "$ARCHITECTURE" \ + "$RUNNER_TEMP/posim-diagnostics-${{ github.run_id }}" + - name: Preserve diagnostic evidence (not acceptance results) + if: always() + uses: actions/upload-artifact@v4 + with: + name: posim-runtime-diagnostics-${{ inputs.architecture || vars.POSIM_DIAGNOSTIC_ARCH || 'arm64' }} + path: ${{ runner.temp }}/posim-diagnostics-${{ github.run_id }} + retention-days: 14 diff --git a/docs/docker.md b/docs/docker.md index 0c675764..321ddac7 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -14,9 +14,24 @@ docker run --rm -it posim:dev-amd64 bash The workspace remains at `/opt/dave_ws`. In an interactive Bash shell its setup is loaded through `.bashrc`. For a non-interactive command, source `/opt/ros/lyrical/setup.bash` and `/opt/dave_ws/install/setup.bash` explicitly. +ArduSub and ArduPilot Gazebo resource/plugin paths are also set in Docker `ENV`, +so they do not depend on an interactive `.bashrc` being loaded. GUI forwarding and GPU passthrough must be configured for the host before using graphical or GPU-dependent scenarios. +Both images build `ros_gz_bridge` from the fixed Lyrical upstream commit +`54a2e78a41c623173608cdd8eef2e049ee3ee3b0` into `/opt/posim_bridge_ws`. +This is the one-commit handle-ownership backport after tag 3.0.10. An additional, +explicit patch in `extras/patches` removes a strong node capture in the ROS +subscription callback and an unnecessary factory pointer in the Gazebo +callback. Its SHA-256 is recorded in the image. A weak-reference probe verifies +node destruction for all three bridge directions. Debian files remain intact. +The DAVE workspace setup chains that +underlay. `ros2 pkg prefix ros_gz_bridge` must resolve to +`/opt/posim_bridge_ws/install`. The image check records the upstream revision +alongside the installed Debian package versions; those version numbers alone +do not identify the bridge executable being used. + ## ARM64 / Apple Silicon ```bash @@ -50,3 +65,71 @@ setup and successful publication. The workflows use architecture-specific tags: Do not assume these tags exist until the corresponding publication has completed. This initial configuration does not create a combined multi-platform manifest. See [maintainer setup](maintainer-setup.md). + +## Installed-image validation + +From the same source revision used to build the image: + +```bash +bash extras/ci/docker_quickstarts.sh posim:dev-arm64-rdp linux/arm64 ./validation-arm64 +# On a native AMD64 host: +bash extras/ci/docker_quickstarts.sh posim:dev-amd64 linux/amd64 ./validation-amd64 +``` + +The check starts a fresh container for each of 14 headless Quickstart paths and +sources only the workspace inside the image. It records installed package and +resource inventories, launch arguments, simulation progress, model presence, +Gazebo payloads, selected ROS payloads, and shutdown status. Logs and JSON/CSV +results are saved beside the tested image ID. Companion repository revisions +and installed ROS package versions are included. + +For spawned models, readiness requires the exact model name in a received pose +message within the original 90-second startup budget. A world control service +alone does not establish model readiness. All pose observations are retained; +the check does not restart the scene or waive missing-model failures. + +The four launch entries default to `wait_for_assets:=true` (Gazebo Jetty's +`--wait-for-assets`). Jetty can otherwise advance an empty world while Fuel +downloads continue in a background thread; shutdown joins that thread. Waiting +for assets keeps normal ready-scene shutdown from racing a cold download. It +does not repair cancellation of an in-progress download or guarantee network +availability. Explicit `wait_for_assets:=false` retains asynchronous startup. +The bimanual-world check now requires its `grabbapole` model and an actual +spherical-coordinate service response, not only a world clock. + +The cases cover the world, object, robot, and sensor launch entries, including +waves, a bimanual scene, REXROV, BlueROV variants, a glider, ocean current, DVL, +camera, USBL, and pressure. CUDA sonar, WGPU, interactive GUI/RDP, joystick input, +and physical sensor accuracy are **not** established by this headless check. +An installed inventory of 18 world files is not an execution test of all 18. + +MAVROS 2.15.1 and libmavconn are rebuilt together with the upstream +`IoContextRunner` self-close lifetime fix (`3a1f39f1a0`, mavlink/mavros#2290). +The base revision, fix revision, and patch checksum are recorded in the image. +The inventory verifies the active package prefixes and dynamic linkage, then +runs 100 self-close/destruction trials under AddressSanitizer and UBSan. This +targets a reproduced use-after-free; it is not a claim that every possible +MAVROS crash has the same cause. + +A separate, hash-recorded local patch makes each Router endpoint's parent a +weak reference. The Router owns the endpoints; a strong reverse link kept its +ROS node and I/O thread alive into middleware-library unloading at process exit. +A native-image GDB trace and a failing ownership reproducer motivated this +patch. Build and inventory checks require release of both the Router and its +endpoints for empty, ROS-only, and ROS-plus-UDP configurations. Passing this +focused probe is not a substitute for the full image runtime matrix. + +BlueROV checks additionally require a received `/mavros/state` message with +`connected=true`, not just a spawned vehicle. The inventory check runs camera +C++ regressions and verifies that ArduSub and its Gazebo plugin resolve in a +non-interactive shell. Seven targeted cases (spherical coordinates, camera, +REXROV/waves, current, pressure, and both BlueROV variants) each run ten times +including their initial matrix trial. All 63 additional trials are saved +individually. The overall CI step budget allows these additional SITL trials; +individual startup and shutdown deadlines are unchanged. +Any failed trial fails the job; these are not retries that select a later pass. +Ten clean trials are regression evidence, not a zero population failure rate. + +CI first loads the built image locally, then runs these checks. Publication uses +the tested image ID without rebuilding it. A failed build or runtime check +prevents publication. PR and non-main branch runs do not publish. diff --git a/docs/maintainer-setup.md b/docs/maintainer-setup.md index cda3f77c..1d741107 100644 --- a/docs/maintainer-setup.md +++ b/docs/maintainer-setup.md @@ -19,15 +19,17 @@ The image build uses the checked-out source as its Docker context. ## Enable publication -After builds work, create or authorize the `ioeslab/posim` Docker Hub repository -and configure these **repository secrets**: +After builds and installed-image Quickstart checks pass, create or authorize +the `ioeslab/posim` Docker Hub repository and configure these **repository secrets**: - `DOCKERHUB_USERNAME` - `DOCKERHUB_TOKEN` with permission to push to `ioeslab/posim` Set repository variable `POSIM_PUBLISH_IMAGES` to `true` to enable login and -publication for non-PR builds. With that variable unset, the workflows build -without logging in or publishing. PR builds never publish images. +publication for successful `main` or version-tag builds. The workflow validates +the local image before logging in and pushing that same image ID. With the +variable unset, it builds and tests without publishing. PR builds never publish +images. Runtime evidence is uploaded as a workflow artifact (14-day retention). The inherited DAVE PR-image publisher is omitted from the initial POSIM setup: the imported Docker build workflows do not produce the image archives that it diff --git a/examples/dave_demos/launch/dave_object.launch.py b/examples/dave_demos/launch/dave_object.launch.py index f85023fa..5d98a561 100755 --- a/examples/dave_demos/launch/dave_object.launch.py +++ b/examples/dave_demos/launch/dave_object.launch.py @@ -35,6 +35,9 @@ def launch_setup(context, *args, **kwargs): run_server_only = ( headless.perform(context).lower() == "true" or gui.perform(context).lower() == "false" ) + # Jetty can advance an empty world while its visual assets still download. + if LaunchConfiguration("wait_for_assets").perform(context).lower() == "true": + gz_args.append(" --wait-for-assets") if run_server_only: gz_args.append(" -s") if paused.perform(context) == "false": @@ -92,6 +95,11 @@ def launch_setup(context, *args, **kwargs): def generate_launch_description(): args = [ + DeclareLaunchArgument( + "wait_for_assets", + default_value="true", + description="Finish world asset downloads before simulation starts (Gazebo Jetty)", + ), DeclareLaunchArgument( "paused", default_value="true", diff --git a/examples/dave_demos/launch/dave_robot.launch.py b/examples/dave_demos/launch/dave_robot.launch.py index ab05d3e6..782b2c91 100644 --- a/examples/dave_demos/launch/dave_robot.launch.py +++ b/examples/dave_demos/launch/dave_robot.launch.py @@ -57,6 +57,9 @@ def launch_setup(context, *args, **kwargs): run_server_only = ( headless.perform(context).lower() == "true" or gui.perform(context).lower() == "false" ) + # Jetty can advance an empty world while its visual assets still download. + if LaunchConfiguration("wait_for_assets").perform(context).lower() == "true": + gz_args.append(" --wait-for-assets") if run_server_only: gz_args.append(" -s") if paused.perform(context) == "false": @@ -130,6 +133,11 @@ def generate_launch_description(): # Declare the launch arguments with default values args = [ + DeclareLaunchArgument( + "wait_for_assets", + default_value="true", + description="Finish world asset downloads before simulation starts (Gazebo Jetty)", + ), DeclareLaunchArgument( "paused", default_value="true", diff --git a/examples/dave_demos/launch/dave_sensor.launch.py b/examples/dave_demos/launch/dave_sensor.launch.py index 98104a4c..9384c884 100644 --- a/examples/dave_demos/launch/dave_sensor.launch.py +++ b/examples/dave_demos/launch/dave_sensor.launch.py @@ -35,6 +35,9 @@ def launch_setup(context, *args, **kwargs): run_server_only = ( headless.perform(context).lower() == "true" or gui.perform(context).lower() == "false" ) + # Jetty can advance an empty world while its visual assets still download. + if LaunchConfiguration("wait_for_assets").perform(context).lower() == "true": + gz_args.append(" --wait-for-assets") if run_server_only: gz_args.append(" -s") if paused.perform(context) == "false": @@ -91,6 +94,11 @@ def launch_setup(context, *args, **kwargs): def generate_launch_description(): args = [ + DeclareLaunchArgument( + "wait_for_assets", + default_value="true", + description="Finish world asset downloads before simulation starts (Gazebo Jetty)", + ), DeclareLaunchArgument( "paused", default_value="true", diff --git a/examples/dave_demos/launch/dave_world.launch.py b/examples/dave_demos/launch/dave_world.launch.py index 8b2c8a46..2d4f4951 100644 --- a/examples/dave_demos/launch/dave_world.launch.py +++ b/examples/dave_demos/launch/dave_world.launch.py @@ -20,6 +20,8 @@ def launch_setup(context, *args, **kwargs): # Gazebo simulation launch gz_args = f"-r {world_path}" + if LaunchConfiguration("wait_for_assets").perform(context).lower() == "true": + gz_args += " --wait-for-assets" if verbose_flag.lower() == "true": gz_args += " --verbose" if headless_flag.lower() == "true": @@ -36,6 +38,11 @@ def launch_setup(context, *args, **kwargs): def generate_launch_description(): return LaunchDescription( [ + DeclareLaunchArgument( + "wait_for_assets", + default_value="true", + description="Finish world asset downloads before simulation starts (Gazebo Jetty)", + ), DeclareLaunchArgument( "world_name", default_value="dave_bimanual_example", diff --git a/extras/ardusub-ubuntu-install.sh b/extras/ardusub-ubuntu-install.sh index 25ac681d..f6dbcba1 100755 --- a/extras/ardusub-ubuntu-install.sh +++ b/extras/ardusub-ubuntu-install.sh @@ -13,10 +13,23 @@ export GZ_VERSION="${GZ_VERSION:-jetty}" ARDUPILOT_COMMIT="${ARDUPILOT_COMMIT:-30257f01185471ab4c1ac544e47d1b4437e44c98}" ARDUPILOT_GAZEBO_COMMIT="${ARDUPILOT_GAZEBO_COMMIT:-082a0fe231f6e63bc8d1598f1cba461d9e2ea7f5}" mkdir -p "/opt/ardusub_ws" && cd "/opt/ardusub_ws" || exit -git clone https://github.com/ArduPilot/ardupilot.git --recurse-submodules +# Fetch only the pinned tree and submodule commits. Bound network retries. +retry_git() { + local attempt + for attempt in 1 2 3; do + if git -c http.version=HTTP/1.1 "$@"; then return 0; fi + echo "Git transfer attempt $attempt/3 failed" >&2 + sleep 5 + done + return 1 +} +git init ardupilot cd "/opt/ardusub_ws/ardupilot" || exit -git checkout --detach "$ARDUPILOT_COMMIT" -git submodule update --init --recursive +git remote add origin https://github.com/ArduPilot/ardupilot.git +retry_git fetch --depth 1 origin "$ARDUPILOT_COMMIT" +git checkout --detach FETCH_HEAD +test "$(git rev-parse HEAD)" = "$ARDUPILOT_COMMIT" +retry_git submodule update --init --recursive --depth 1 --jobs 2 # ArduPilot's waf extras still import Python modules removed in 3.12/3.13. # These minimal compatibility shims were used in the verified Ubuntu 26.04 build. diff --git a/extras/build-image-bridge.sh b/extras/build-image-bridge.sh new file mode 100644 index 00000000..040c6207 --- /dev/null +++ b/extras/build-image-bridge.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Build the Lyrical bridge with the upstream ownership-cycle fix. The published +# 3.0.10 Debian package predates this one-commit backport; do not patch /opt/ros. +set -eo pipefail +ROS_DISTRO="${ROS_DISTRO:-lyrical}" +WS="${POSIM_BRIDGE_UNDERLAY:-/opt/posim_bridge_ws}" +REVISION=54a2e78a41c623173608cdd8eef2e049ee3ee3b0 +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PATCH="$SCRIPT_DIR/patches/ros-gz-bridge-callback-lifetime.patch" +mkdir -p "$WS/src" +git init "$WS/src/ros_gz" +git -C "$WS/src/ros_gz" remote add origin https://github.com/gazebosim/ros_gz.git +git -C "$WS/src/ros_gz" -c http.version=HTTP/1.1 fetch --depth 1 origin "$REVISION" +git -C "$WS/src/ros_gz" checkout --detach FETCH_HEAD +test "$(git -C "$WS/src/ros_gz" rev-parse HEAD)" = "$REVISION" +# The upstream handle fix does not remove the ROS subscription callback's +# shared node capture. Keep the additional patch explicit and hash-recorded. +git -C "$WS/src/ros_gz" apply --check "$PATCH" +git -C "$WS/src/ros_gz" apply "$PATCH" +# shellcheck disable=SC1090 +source "/opt/ros/$ROS_DISTRO/setup.bash" +# Chain the matching MAVROS/libmavconn overlay into subsequent workspaces. +# shellcheck disable=SC1090,SC1091 +source "${POSIM_MAVROS_UNDERLAY:?MAVROS underlay required}/install/setup.bash" +cd "$WS" +# All build dependencies are supplied by the installed ros_gz packages. +export CMAKE_BUILD_PARALLEL_LEVEL=2 MAKEFLAGS=-j2 +colcon build --merge-install --packages-select ros_gz_bridge \ + --executor sequential --cmake-args -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release +printf '%s\n' "$REVISION" > "$WS/upstream-revision.txt" +sha256sum "$PATCH" | cut -d ' ' -f 1 > "$WS/callback-patch.sha256" +# shellcheck disable=SC1091 +source "$WS/install/setup.bash" +cmake -S "$SCRIPT_DIR/ci/bridge_ownership" -B "$WS/probe-build" \ + -DCMAKE_INSTALL_PREFIX="$WS/probe" +cmake --build "$WS/probe-build" --parallel 2 +cmake --install "$WS/probe-build" +"$WS/probe/bin/posim_bridge_ownership_check" diff --git a/extras/build-image-mavros.sh b/extras/build-image-mavros.sh new file mode 100644 index 00000000..f4917fdd --- /dev/null +++ b/extras/build-image-mavros.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Keep MAVROS 2.15.1, backport upstream I/O self-close, and break the Router cycle. +set -eo pipefail +ROS_DISTRO="${ROS_DISTRO:-lyrical}" +WS="${POSIM_MAVROS_UNDERLAY:-/opt/posim_mavros_ws}" +REVISION=22ae5b7cc7cdb4cb9c2070a8213c72dae445a23e +FIX=3a1f39f1a033d39d9e7c34d9ba7cb28cd3dbcd5f +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PATCH="$SCRIPT_DIR/patches/mavconn-self-close-lifetime.patch" +ROUTER_PATCH="$SCRIPT_DIR/patches/mavros-router-parent-lifetime.patch" +mkdir -p "$WS/src" +git init "$WS/src/mavros" +git -C "$WS/src/mavros" remote add origin https://github.com/mavlink/mavros.git +git -C "$WS/src/mavros" -c http.version=HTTP/1.1 fetch --depth 1 origin "$REVISION" "$FIX" +git -C "$WS/src/mavros" checkout --detach "$REVISION" +test "$(git -C "$WS/src/mavros" rev-parse HEAD)" = "$REVISION" +git -C "$WS/src/mavros" apply --check "$PATCH" +git -C "$WS/src/mavros" apply "$PATCH" +# Ensure the backported header is byte-identical to the upstream fixed header. +git -C "$WS/src/mavros" show "$FIX:libmavconn/include/mavconn/io_context_runner.hpp" \ + > "$WS/upstream-fixed-header.hpp" +cmp "$WS/upstream-fixed-header.hpp" \ + "$WS/src/mavros/libmavconn/include/mavconn/io_context_runner.hpp" +# Local fix, distinct from the upstream I/O patch: endpoints must not own Router. +git -C "$WS/src/mavros" apply --check "$ROUTER_PATCH" +git -C "$WS/src/mavros" apply "$ROUTER_PATCH" +# shellcheck disable=SC1090 +source "/opt/ros/$ROS_DISTRO/setup.bash" +# Keep the matching Gazebo Transport overlay in the workspace setup chain. +# shellcheck disable=SC1090,SC1091 +source "${POSIM_TRANSPORT_UNDERLAY:?Transport underlay required}/install/setup.bash" +cd "$WS" +export CMAKE_BUILD_PARALLEL_LEVEL=2 MAKEFLAGS=-j2 +# Router also includes IoContextRunner; rebuild MAVROS, not only libmavconn. +colcon build --merge-install --packages-select libmavconn mavros \ + --executor sequential --cmake-args -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release +printf '%s\n' "$REVISION" > "$WS/upstream-revision.txt" +printf '%s\n' "$FIX" > "$WS/upstream-fix.txt" +sha256sum "$PATCH" | cut -d ' ' -f 1 > "$WS/self-close-patch.sha256" +sha256sum "$ROUTER_PATCH" | cut -d ' ' -f 1 > "$WS/router-parent-patch.sha256" +mkdir -p "$WS/probe/bin" +c++ -std=c++20 -g -O1 -fsanitize=address,undefined -fno-omit-frame-pointer \ + -pthread -I"$WS/install/include" "$SCRIPT_DIR/ci/mavconn_self_close/main.cc" \ + -o "$WS/probe/bin/posim_mavconn_self_close_check" +"$WS/probe/bin/posim_mavconn_self_close_check" +# shellcheck disable=SC1090,SC1091 +source "$WS/install/setup.bash" +cmake -S "$SCRIPT_DIR/ci/mavros_ownership" -B "$WS/ownership-probe-build" \ + -DCMAKE_INSTALL_PREFIX="$WS/probe" +cmake --build "$WS/ownership-probe-build" --parallel 2 +cmake --install "$WS/ownership-probe-build" +"$WS/probe/bin/posim_mavros_ownership_check" diff --git a/extras/build-image-transport.sh b/extras/build-image-transport.sh new file mode 100755 index 00000000..65cbac34 --- /dev/null +++ b/extras/build-image-transport.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Keep the ROS vendor ABI and install the poll fix in a separate ament overlay. +# Never overwrite /opt/ros or treat the churn probe as full-image acceptance. +set -eo pipefail +ROS_DISTRO="${ROS_DISTRO:-lyrical}" +WS="${POSIM_TRANSPORT_UNDERLAY:-/opt/posim_transport_ws}" +VENDOR_REVISION=bc048aec33d25d73e651b86e2858339885dfab87 +REVISION=82b10bdff114f77655c7f0cc856835179a674d6d +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PATCH="$SCRIPT_DIR/patches/gz-transport-poll-serialization.patch" +VENDOR_PATCH="$SCRIPT_DIR/patches/gz-transport-vendor-poll.patch" +apt-get update +apt-get install -y --no-install-recommends "ros-$ROS_DISTRO-ament-cmake-vendor-package" +mkdir -p "$WS/src" +git init "$WS/src/gz_transport_vendor" +git -C "$WS/src/gz_transport_vendor" remote add origin \ + https://github.com/gazebo-release/gz_transport_vendor.git +git -C "$WS/src/gz_transport_vendor" -c http.version=HTTP/1.1 fetch --depth 1 origin "$VENDOR_REVISION" +git -C "$WS/src/gz_transport_vendor" checkout --detach FETCH_HEAD +test "$(git -C "$WS/src/gz_transport_vendor" rev-parse HEAD)" = "$VENDOR_REVISION" +git -C "$WS/src/gz_transport_vendor" apply --check "$VENDOR_PATCH" +git -C "$WS/src/gz_transport_vendor" apply "$VENDOR_PATCH" +cp "$PATCH" "$WS/src/gz_transport_vendor/poll-serialization.patch" +# shellcheck disable=SC1090 +source "/opt/ros/$ROS_DISTRO/setup.bash" +cd "$WS" +export CMAKE_BUILD_PARALLEL_LEVEL=2 MAKEFLAGS=-j2 +colcon build --merge-install --packages-select gz_transport_vendor \ + --executor sequential --cmake-args \ + -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release -DFORCE_BUILD_VENDOR_PKG=ON \ + -DVENDOR_FROM_LIB_VCS_REF=ON "-DLIB_VCS_REF=$REVISION" +# Match the installed vendor's public configuration (including Zenoh disabled). +PREFIX="$WS/install/opt/gz_transport_vendor" +cmp "/opt/ros/$ROS_DISTRO/opt/gz_transport_vendor/include/gz/transport15/gz/transport/config.hh" \ + "$PREFIX/include/gz/transport15/gz/transport/config.hh" +test "$(git -C "$WS/build/gz_transport_vendor/gz_transport_vendor-prefix/src/gz_transport_vendor" rev-parse HEAD)" = "$REVISION" +printf '%s\n' "$VENDOR_REVISION" > "$WS/vendor-revision.txt" +printf '%s\n' "$REVISION" > "$WS/upstream-revision.txt" +sha256sum "$PATCH" | cut -d ' ' -f 1 > "$WS/poll-patch.sha256" +sha256sum "$VENDOR_PATCH" | cut -d ' ' -f 1 > "$WS/vendor-patch.sha256" +sha256sum "$PREFIX/lib/libgz-transport.so.15" | cut -d ' ' -f 1 > "$WS/library.sha256" +# shellcheck disable=SC1091 +source "$WS/install/setup.bash" +cmake -S "$SCRIPT_DIR/ci/transport_shutdown" -B "$WS/probe-build" \ + -DCMAKE_BUILD_TYPE=Release +cmake --build "$WS/probe-build" --parallel 2 +mkdir -p "$WS/probe/bin" +cp "$WS/probe-build/posim_transport_churn" "$WS/probe/bin/" +ldd "$WS/probe/bin/posim_transport_churn" > "$WS/probe-linkage.txt" +grep -F "libgz-transport.so.15 => $PREFIX/lib/libgz-transport.so.15" "$WS/probe-linkage.txt" +python3 "$SCRIPT_DIR/ci/transport_shutdown/run_pairs.py" \ + --executable "$WS/probe/bin/posim_transport_churn" --variant patched: \ + --trials 5 --seconds 20 --output "$WS/build-probe" diff --git a/extras/ci/bridge_ownership/CMakeLists.txt b/extras/ci/bridge_ownership/CMakeLists.txt new file mode 100644 index 00000000..4918a4bb --- /dev/null +++ b/extras/ci/bridge_ownership/CMakeLists.txt @@ -0,0 +1,7 @@ +cmake_minimum_required(VERSION 3.16) +project(posim_bridge_ownership LANGUAGES CXX) +set(CMAKE_CXX_STANDARD 17) +find_package(ros_gz_bridge REQUIRED) +add_executable(posim_bridge_ownership_check main.cc) +target_link_libraries(posim_bridge_ownership_check ${ros_gz_bridge_TARGETS}) +install(TARGETS posim_bridge_ownership_check DESTINATION bin) diff --git a/extras/ci/bridge_ownership/main.cc b/extras/ci/bridge_ownership/main.cc new file mode 100644 index 00000000..a7ee10a6 --- /dev/null +++ b/extras/ci/bridge_ownership/main.cc @@ -0,0 +1,32 @@ +#include +#include +#include +#include +#include + +int main(int argc, char ** argv) +{ + rclcpp::init(argc, argv, rclcpp::InitOptions(), rclcpp::SignalHandlerOptions::None); + using ros_gz_bridge::BridgeDirection; + int failures = 0; + int index = 0; + for (auto direction : + {BridgeDirection::GZ_TO_ROS, BridgeDirection::ROS_TO_GZ, BridgeDirection::BIDIRECTIONAL}) + { + auto node = std::make_shared(); + ros_gz_bridge::BridgeConfig config; + config.ros_topic_name = "/posim_ownership_" + std::to_string(index++); + config.gz_topic_name = config.ros_topic_name; + config.ros_type_name = "std_msgs/msg/Float64"; + config.gz_type_name = "gz.msgs.Double"; + config.direction = direction; + config.is_lazy = false; + node->add_bridge(config); + std::weak_ptr weak = node; + node.reset(); + std::cout << "direction=" << index << " node_expired=" << weak.expired() << std::endl; + failures += !weak.expired(); + } + rclcpp::shutdown(); + return failures ? 1 : 0; +} diff --git a/extras/ci/docker_quickstarts.sh b/extras/ci/docker_quickstarts.sh new file mode 100644 index 00000000..9159a382 --- /dev/null +++ b/extras/ci/docker_quickstarts.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# Run each scene in a fresh container; never mount a host source/install overlay. +set -euo pipefail +IMAGE="${1:?image tag or ID required}" +PLATFORM="${2:?linux/arm64 or linux/amd64 required}" +RESULTS="${3:?results directory required}" +CHECKS="$(cd "$(dirname "$0")" && pwd)" +mkdir -p "$RESULTS" +RESULTS="$(cd "$RESULTS" && pwd)" +chmod 777 "$RESULTS" +docker image inspect "$IMAGE" > "$RESULTS/image-inspect.json" +IMAGE_ID="$(docker image inspect --format '{{.Id}}' "$IMAGE")" +EXPECTED_ARCH="${PLATFORM#linux/}" +EXPECTED_ARCH="${EXPECTED_ARCH%%/*}" +test "$(docker image inspect --format '{{.Architecture}}' "$IMAGE_ID")" = "$EXPECTED_ARCH" +printf '%s\n' "$IMAGE_ID" > "$RESULTS/tested-image-id.txt" +printf 'case,status\n' > "$RESULTS/summary.csv" +FAILED=0 +CURRENT_CONTAINER="" +cleanup() { + if [[ -n "$CURRENT_CONTAINER" ]]; then + docker rm -f "$CURRENT_CONTAINER" >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT +trap 'exit 130' INT TERM +CASES="inventory $(awk -F '\t' '!/^#/ && NF {print $1}' "$CHECKS/quickstarts.tsv")" +# Ten trials per previously failing lifecycle case, counting the matrix run. +# Every trial is retained; a later pass never cancels an earlier failure. +for REPEAT in $(seq 2 10); do + for CASE in spherical_world camera rexrov_waves ocean_current sea_pressure bluerov2 bluerov2_heavy; do + CASES="$CASES ${CASE}:r${REPEAT}" + done +done +for RECORD in $CASES; do + CASE="${RECORD%%:*}" + echo "=== POSIM image validation: $CASE ===" + RECORD="${RECORD//:/-}" + CURRENT_CONTAINER="posim-smoke-${GITHUB_RUN_ID:-$$}-${GITHUB_RUN_ATTEMPT:-1}-$RECORD" + if docker run --rm --init --name "$CURRENT_CONTAINER" --platform "$PLATFORM" --shm-size=1g \ + --entrypoint bash -e ROS_DOMAIN_ID=121 -e GZ_IP=127.0.0.1 \ + -e LIBGL_ALWAYS_SOFTWARE=1 -e QT_QPA_PLATFORM=offscreen \ + -e "CASE=$CASE" -e "RECORD=$RECORD" \ + -v "$CHECKS:/checks:ro" -v "$CHECKS/../patches:/patches:ro" \ + -v "$CHECKS/../fuel:/fuel:ro" -v "$RESULTS:/results" \ + "$IMAGE_ID" -c ' + set -eo pipefail + source /opt/ros/lyrical/setup.bash + if [[ -n "${DAVE_WS:-}" ]]; then + export POSIM_WORKSPACE="$DAVE_WS" + else + export POSIM_WORKSPACE="${DAVE_UNDERLAY:?installed workspace missing}" + fi + source "$POSIM_WORKSPACE/install/setup.bash" + cd /tmp + exec python3 /checks/image_smoke.py "$CASE" --record "$RECORD" + '; then + printf '%s,PASS\n' "$RECORD" >> "$RESULTS/summary.csv" + else + printf '%s,FAIL\n' "$RECORD" >> "$RESULTS/summary.csv" + FAILED=1 + fi +done +# Additional acceptance evidence, not retries of connected camera failures. +printf 'case,status\n' > "$RESULTS/offline-summary.csv" +for REPEAT in $(seq 1 5); do + RECORD="camera-offline-$REPEAT" + mkdir -p "$RESULTS/$RECORD" + chmod 777 "$RESULTS/$RECORD" + CURRENT_CONTAINER="posim-offline-${GITHUB_RUN_ID:-$$}-${GITHUB_RUN_ATTEMPT:-1}-$REPEAT" + docker create --init --name "$CURRENT_CONTAINER" --platform "$PLATFORM" --shm-size=1g \ + --network none --entrypoint bash -e ROS_DOMAIN_ID=121 -e GZ_IP=127.0.0.1 \ + -e LIBGL_ALWAYS_SOFTWARE=1 -e QT_QPA_PLATFORM=offscreen -e "RECORD=$RECORD" \ + -v "$CHECKS:/checks:ro" -v "$RESULTS:/results" "$IMAGE_ID" -c ' + set -eo pipefail + source /opt/ros/lyrical/setup.bash + source "${DAVE_WS:-$DAVE_UNDERLAY}/install/setup.bash" + cd /tmp + exec python3 /checks/image_smoke.py camera --record "$RECORD" + ' > "$RESULTS/$RECORD/container-id.txt" + docker inspect --format '{{json .HostConfig.NetworkMode}}' "$CURRENT_CONTAINER" \ + > "$RESULTS/$RECORD/docker-network-mode.json" + test "$(docker inspect --format '{{.HostConfig.NetworkMode}}' "$CURRENT_CONTAINER")" = none + docker start -a "$CURRENT_CONTAINER" || true + CODE="$(docker inspect --format '{{.State.ExitCode}}' "$CURRENT_CONTAINER")" + STATE="$(docker inspect --format '{{.State.Status}}' "$CURRENT_CONTAINER")" + printf '%s\n' "$CODE" > "$RESULTS/$RECORD/container-exit-code.txt" + printf '%s\n' "$STATE" > "$RESULTS/$RECORD/container-state.txt" + if [[ "$CODE" == 0 && "$STATE" == exited ]] && \ + python3 -c 'import json,sys; sys.exit(json.load(open(sys.argv[1]))["status"] != "PASS")' \ + "$RESULTS/$RECORD/result.json"; then + printf '%s,PASS\n' "$RECORD" >> "$RESULTS/offline-summary.csv" + else + printf '%s,FAIL\n' "$RECORD" >> "$RESULTS/offline-summary.csv" + FAILED=1 + fi + cleanup + CURRENT_CONTAINER="" +done +cat "$RESULTS/summary.csv" +cat "$RESULTS/offline-summary.csv" +cleanup +trap - EXIT +exit "$FAILED" diff --git a/extras/ci/image_smoke.py b/extras/ci/image_smoke.py new file mode 100644 index 00000000..7310ffe2 --- /dev/null +++ b/extras/ci/image_smoke.py @@ -0,0 +1,561 @@ +#!/usr/bin/env python3 +"""Check installed POSIM resources and headless Quickstarts inside an image.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import platform +import re +import shlex +import signal +import shutil +import subprocess +import time + + +PACKAGES = ( + "dave_interfaces dave_demos dave_gz_model_plugins dave_gz_sensor_plugins " + "dave_gz_world_plugins dave_ros_gz_plugins dave_object_models dave_robot_models " + "dave_sensor_models dave_worlds multibeam_sonar multibeam_sonar_system " + "dave_multibeam_sonar_demo" +).split() + + +def fatal_log_lines(log_text): + """Find child failures even when the launch/gz wrapper exits on SIGINT.""" + faults = re.findall( + r".*(?:Segmentation fault|exit code (?:-11|-6|134|139)\b|" + r"\bAborted\b|terminate called|rclcpp::exceptions::RCLError|" + r"Traceback \(most recent call last\)|" + r"Failed to load system plugin|error while loading shared libraries).*", + log_text, + ) + for line in log_text.splitlines(): + match = re.search(r"process has died .*exit code (-?\d+)\b", line) + if match and int(match[1]) not in (0, 130, -signal.SIGINT) and line not in faults: + faults.append(line) + return faults + + +def command(args, timeout=15): + try: + result = subprocess.run(args, capture_output=True, text=True, timeout=timeout) + return result.returncode, result.stdout, result.stderr + except subprocess.TimeoutExpired: + return 124, "", f"Timed out after {timeout}s: {shlex.join(args)}" + + +def capture(out, name, args, timeout=15): + code, stdout, stderr = command(args, timeout) + # Camera payloads can be large. Retain a bounded excerpt and its full digest. + (out / f"{name}.txt").write_text(stdout[:65536]) + (out / f"{name}.stderr").write_text(stderr[:65536]) + (out / f"{name}.json").write_text( + json.dumps( + { + "command": args, + "returncode": code, + "stdout_bytes": len(stdout.encode()), + "stdout_sha256": hashlib.sha256(stdout.encode()).hexdigest(), + }, + indent=2, + ) + ) + return code, stdout + + +def wait_for_entity(out, world, entity, proc, deadline): + """Observe the spawned model within the original 90-second startup budget. + + The world control service can appear before the create service/model. A + fixed sleep followed by one pose sample tests that race, not model loading. + Every observation is retained; this does not restart a failed scene. + """ + attempts = [] + present = False + while proc.poll() is None: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + code, poses = capture( + out, + "poses", + ["gz", "topic", "-e", "-t", f"/world/{world}/pose/info", "-n", "1"], + min(8, remaining), + ) + present = ( + code == 0 and re.search(r'name:\s*"' + re.escape(entity) + r'"', poses) is not None + ) + attempts.append({"returncode": code, "entity_present": present}) + for suffix in ("txt", "stderr", "json"): + source = out / f"poses.{suffix}" + if source.exists(): + shutil.copyfile(source, out / f"poses-attempt-{len(attempts):03d}.{suffix}") + if present: + break + time.sleep(min(1, max(0, deadline - time.monotonic()))) + (out / "entity_readiness.json").write_text( + json.dumps({"entity": entity, "present": present, "attempts": attempts}, indent=2) + ) + return present + + +def linked_library(ldd_output, name): + """Require an absolute resolved path, not an unresolved or partial match.""" + entries = re.findall(r"^\s*" + re.escape(name) + r"\s+=>\s+(/\S+)\s", ldd_output, re.M) + if len(entries) != 1: + raise RuntimeError(f"Missing or ambiguous linkage for {name}") + return Path(entries[0]).resolve() + + +def transport_inventory(out, get_package_prefix): + ws = Path(os.environ["POSIM_TRANSPORT_UNDERLAY"]) + if Path(get_package_prefix("gz_transport_vendor")) != ws / "install": + raise RuntimeError("The pinned Gazebo Transport overlay is not active") + for filename, expected in ( + ("upstream-revision.txt", "82b10bdff114f77655c7f0cc856835179a674d6d"), + ("vendor-revision.txt", "bc048aec33d25d73e651b86e2858339885dfab87"), + ): + actual = (ws / filename).read_text().strip() + if actual != expected: + raise RuntimeError(f"Unexpected transport provenance: {filename}={actual}") + (out / f"transport_{filename}").write_text(actual + "\n") + patches = Path(__file__).resolve().parent.parent / "patches" + for patch_name, recorded_name in ( + ("gz-transport-poll-serialization.patch", "poll-patch.sha256"), + ("gz-transport-vendor-poll.patch", "vendor-patch.sha256"), + ): + expected = hashlib.sha256((patches / patch_name).read_bytes()).hexdigest() + if (ws / recorded_name).read_text().strip() != expected: + raise RuntimeError(f"Transport patch differs from validation source: {patch_name}") + (out / f"transport_{recorded_name}").write_text(expected + "\n") + prefix = ws / "install/opt/gz_transport_vendor" + library = (prefix / "lib/libgz-transport.so.15").resolve() + expected_hash = (ws / "library.sha256").read_text().strip() + if hashlib.sha256(library.read_bytes()).hexdigest() != expected_hash: + raise RuntimeError("Installed Transport library differs from the build record") + (out / "transport_library.sha256").write_text(expected_hash + "\n") + config_path = "include/gz/transport15/gz/transport/config.hh" + vendor_prefix = Path("/opt/ros/lyrical/opt/gz_transport_vendor") + if (prefix / config_path).read_bytes() != (vendor_prefix / config_path).read_bytes(): + raise RuntimeError("Transport public build configuration differs from the ROS vendor") + # Check both the regression binary and the installed Gazebo simulator DSO. + probe = ws / "probe/bin/posim_transport_churn" + sim_prefix = Path(get_package_prefix("gz_sim_vendor")) / "opt/gz_sim_vendor" + for name, binary in ( + ("transport_probe_linkage", probe), + ("sim_transport_linkage", sim_prefix / "lib/libgz-sim.so.10"), + ): + code, dependencies = capture(out, name, ["ldd", str(binary)]) + if code or linked_library(dependencies, "libgz-transport.so.15") != library: + raise RuntimeError(f"{name} is not using the patched Transport overlay") + # Do not force LD_LIBRARY_PATH here: the installed setup chain must work. + code, _ = capture( + out, + "transport_churn", + [ + "python3", + str(Path(__file__).parent / "transport_shutdown/run_pairs.py"), + "--executable", + str(probe), + "--variant", + "patched:", + "--trials", + "5", + "--seconds", + "20", + "--output", + str(out / "transport_churn"), + ], + 240, + ) + if code: + raise RuntimeError("Installed Transport churn regression failed") + + +def inventory(out): + from ament_index_python.packages import get_package_prefix, get_package_share_directory + + shares = {name: Path(get_package_share_directory(name)) for name in PACKAGES} + for share in shares.values(): + if not (share / "package.xml").is_file(): + raise RuntimeError(f"Installed package.xml missing: {share}") + launches = [ + shares["dave_demos"] / "launch" / f"dave_{kind}.launch.py" + for kind in ("world", "robot", "sensor", "object") + ] + for launch in launches: + if not launch.is_file(): + raise RuntimeError(f"Installed launch missing: {launch}") + code, launch_args = capture( + out, launch.stem, ["ros2", "launch", "dave_demos", launch.name, "--show-args"], 30 + ) + if code: + raise RuntimeError(f"Cannot resolve launch arguments: {launch.name}") + if "wait_for_assets" not in launch_args: + raise RuntimeError(f"Asset readiness argument missing: {launch.name}") + worlds = sorted((shares["dave_worlds"] / "worlds").glob("*.world")) + robots = sorted((shares["dave_robot_models"] / "description").glob("*/model.sdf")) + sensors = sorted((shares["dave_sensor_models"] / "description").glob("*/model.sdf")) + counts = {"worlds": len(worlds), "robots": len(robots), "sensors": len(sensors)} + if counts != {"worlds": 18, "robots": 5, "sensors": 11}: + raise RuntimeError(f"Unexpected installed resource inventory: {counts}") + capture(out, "deb_versions", ["dpkg-query", "-W", "ros-lyrical-*"], 30) + capture(out, "gazebo_version", ["gz", "sim", "--versions"]) + fuel = Path("/opt/posim_fuel") + expected_lock = Path(__file__).resolve().parent.parent / "fuel/quickstart-assets.lock.json" + if ( + hashlib.sha256(expected_lock.read_bytes()).digest() + != hashlib.sha256((fuel / "quickstart-assets.lock.json").read_bytes()).digest() + ): + raise RuntimeError("Installed Fuel asset lock differs from validation source") + code, _ = capture( + out, + "fuel_assets", + [ + "python3", + str(fuel / "prepare-image-assets.py"), + "--verify", + "--cache", + os.environ["GZ_FUEL_CACHE_PATH"], + "--lock", + str(fuel / "quickstart-assets.lock.json"), + "--receipt", + str(out / "fuel-assets-receipt.json"), + ], + 60, + ) + if code: + raise RuntimeError("Installed Fuel asset integrity/dependency verification failed") + transport_inventory(out, get_package_prefix) + ws = Path(os.environ["POSIM_WORKSPACE"]) + bridge_ws = Path(os.environ["POSIM_BRIDGE_UNDERLAY"]) + if Path(get_package_prefix("ros_gz_bridge")) != bridge_ws / "install": + raise RuntimeError("The pinned bridge overlay is not active") + revision = (bridge_ws / "upstream-revision.txt").read_text().strip() + if revision != "54a2e78a41c623173608cdd8eef2e049ee3ee3b0": + raise RuntimeError(f"Unexpected bridge source revision: {revision}") + (out / "bridge_source_revision.txt").write_text(revision + "\n") + patch = ( + Path(__file__).resolve().parent.parent / "patches/ros-gz-bridge-callback-lifetime.patch" + ) + expected_patch = hashlib.sha256(patch.read_bytes()).hexdigest() + if (bridge_ws / "callback-patch.sha256").read_text().strip() != expected_patch: + raise RuntimeError("Bridge callback patch does not match the validation source") + (out / "bridge_callback_patch.sha256").write_text(expected_patch + "\n") + code, _ = capture( + out, "bridge_ownership", [str(bridge_ws / "probe/bin/posim_bridge_ownership_check")], 60 + ) + if code: + raise RuntimeError("Bridge ownership regression check failed") + mavros_ws = Path(os.environ["POSIM_MAVROS_UNDERLAY"]) + for package in ("mavros", "libmavconn"): + if Path(get_package_prefix(package)) != mavros_ws / "install": + raise RuntimeError(f"The pinned {package} overlay is not active") + for filename, expected in ( + ("upstream-revision.txt", "22ae5b7cc7cdb4cb9c2070a8213c72dae445a23e"), + ("upstream-fix.txt", "3a1f39f1a033d39d9e7c34d9ba7cb28cd3dbcd5f"), + ): + actual = (mavros_ws / filename).read_text().strip() + if actual != expected: + raise RuntimeError(f"Unexpected MAVROS provenance: {filename}={actual}") + (out / f"mavros_{filename}").write_text(actual + "\n") + mavconn_patch = patch.with_name("mavconn-self-close-lifetime.patch") + expected_mavconn_patch = hashlib.sha256(mavconn_patch.read_bytes()).hexdigest() + if (mavros_ws / "self-close-patch.sha256").read_text().strip() != expected_mavconn_patch: + raise RuntimeError("MAVConn patch does not match the validation source") + (out / "mavconn_patch.sha256").write_text(expected_mavconn_patch + "\n") + router_patch = patch.with_name("mavros-router-parent-lifetime.patch") + expected_router_patch = hashlib.sha256(router_patch.read_bytes()).hexdigest() + if (mavros_ws / "router-parent-patch.sha256").read_text().strip() != expected_router_patch: + raise RuntimeError("MAVROS Router patch does not match the validation source") + (out / "mavros_router_patch.sha256").write_text(expected_router_patch + "\n") + code, _ = capture( + out, + "mavros_ownership", + [str(mavros_ws / "probe/bin/posim_mavros_ownership_check")], + 60, + ) + if code: + raise RuntimeError("MAVROS Router ownership regression failed") + code, dependencies = capture( + out, "mavros_linkage", ["ldd", str(mavros_ws / "install/lib/mavros/mavros_node")] + ) + if code or str(mavros_ws / "install/lib/libmavconn.so") not in dependencies: + raise RuntimeError("MAVROS is not linked to the patched MAVConn overlay") + code, _ = capture( + out, + "mavconn_self_close", + [str(mavros_ws / "probe/bin/posim_mavconn_self_close_check")], + 60, + ) + if code: + raise RuntimeError("MAVConn self-close sanitizer regression failed") + if not shutil.which("ardusub"): + raise RuntimeError("ArduSub is not available in the noninteractive image PATH") + plugin_paths = os.environ.get("GZ_SIM_SYSTEM_PLUGIN_PATH", "").split(":") + if not any((Path(p) / "libArduPilotPlugin.so").is_file() for p in plugin_paths if p): + raise RuntimeError("ArduPilotPlugin is not on GZ_SIM_SYSTEM_PLUGIN_PATH") + code, _ = capture( + out, + "camera_unit_tests", + [str(ws / "build/dave_gz_sensor_plugins/test_underwater_camera")], + 60, + ) + if code: + raise RuntimeError("Camera C++ regression tests failed") + for companion in ("dockwater", "rocker"): + code, _ = capture( + out, + f"revision_{companion}", + [ + "git", + "-c", + f"safe.directory={ws / 'src' / companion}", + "-C", + str(ws / "src" / companion), + "rev-parse", + "HEAD", + ], + ) + if code: + raise RuntimeError(f"Cannot record companion revision: {companion}") + return {"shares": {k: str(v) for k, v in shares.items()}, "counts": counts} + + +def camera_payload(out): + import rclpy + from rclpy.qos import qos_profile_sensor_data + from sensor_msgs.msg import Image + + rclpy.init() + node = rclpy.create_node("posim_camera_image_check") + messages = [] + node.create_subscription( + Image, "/underwater_camera/simulated_image", messages.append, qos_profile_sensor_data + ) + try: + deadline = time.monotonic() + 45 + while not messages and time.monotonic() < deadline: + rclpy.spin_once(node, timeout_sec=1) + if not messages: + return False + msg = messages[0] + payload = bytes(msg.data) + (out / "ros_image.json").write_text( + json.dumps( + { + "topic": "/underwater_camera/simulated_image", + "width": msg.width, + "height": msg.height, + "step": msg.step, + "encoding": msg.encoding, + "data_bytes": len(payload), + "data_sha256": hashlib.sha256(payload).hexdigest(), + }, + indent=2, + ) + ) + return ( + msg.width > 0 + and msg.height > 0 + and msg.step > 0 + and len(payload) == msg.step * msg.height + ) + finally: + node.destroy_node() + rclpy.shutdown() + + +def mavros_connected(out): + import rclpy + from mavros_msgs.msg import State + from rclpy.qos import qos_profile_sensor_data + + rclpy.init() + node = rclpy.create_node("posim_mavros_connection_check") + states = [] + node.create_subscription(State, "/mavros/state", states.append, qos_profile_sensor_data) + try: + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + rclpy.spin_once(node, timeout_sec=1) + if any(state.connected for state in states): + break + connected = any(state.connected for state in states) + (out / "mavros_state.json").write_text( + json.dumps({"topic": "/mavros/state", "connected": connected, "messages": len(states)}) + ) + return connected + finally: + node.destroy_node() + rclpy.shutdown() + + +def exercise(out, case): + _, launch_command, entity, topic_pattern = case + checks = {} + proc = None + forced = False + returncode = None + with (out / "launch.log").open("w") as log: + try: + proc = subprocess.Popen( + shlex.split(launch_command), + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + deadline = time.monotonic() + 90 + world = None + while time.monotonic() < deadline and proc.poll() is None: + code, services, _ = command(["gz", "service", "-l"], 8) + match = re.search(r"^/world/([^/]+)/control$", services, re.M) + if code == 0 and match: + world = match[1] + break + time.sleep(1) + checks["world_ready"] = world is not None + if world: + checks["entity_present"] = entity == "__NONE__" or wait_for_entity( + out, world, entity, proc, deadline + ) + time.sleep(8) + code, stats = capture( + out, + "world_stats", + ["gz", "topic", "-e", "-t", f"/world/{world}/stats", "-n", "2"], + 20, + ) + iterations = [int(n) for n in re.findall(r"iterations:\s*(\d+)", stats)] + checks["simulation_advances"] = ( + code == 0 and len(iterations) >= 2 and iterations[-1] > iterations[0] + ) + if case[0] == "spherical_world": + code, origin = capture( + out, + "spherical_origin", + [ + "ros2", + "service", + "call", + "/gz/get_origin_spherical_coordinates", + "dave_interfaces/srv/GetOriginSphericalCoord", + "{}", + ], + 20, + ) + coords = re.search( + r"latitude_deg=([0-9.eE+-]+), longitude_deg=([0-9.eE+-]+)", origin + ) + checks["spherical_service"] = ( + code == 0 + and coords is not None + and abs(float(coords[1]) - 35.074823) < 1e-6 + and abs(float(coords[2]) - 129.084798) < 1e-6 + ) + # A topic listing is not sufficient: obtain a real message. + if case[0] == "camera": + # This plugin publishes its transformed image to ROS, not Gazebo. + checks["ros_image_payload"] = camera_payload(out) + elif topic_pattern.startswith("/world/") and topic_pattern.endswith("/"): + checks["expected_world"] = ( + re.search(topic_pattern, f"/world/{world}/stats") is not None + ) + else: + payload = "" + payload_code = 1 + deadline = time.monotonic() + 60 + while time.monotonic() < deadline and proc.poll() is None: + _, topics, _ = command(["gz", "topic", "-l"], 8) + choices = [t for t in topics.splitlines() if re.search(topic_pattern, t)] + if choices: + payload_code, payload = capture( + out, + "gz_payload", + ["gz", "topic", "-e", "-t", choices[0], "-n", "1"], + 15, + ) + if payload_code == 0 and payload.strip(): + break + time.sleep(1) + checks["gazebo_payload"] = payload_code == 0 and bool(payload.strip()) + if case[0] in ("rexrov_empty", "rexrov_waves", "dvl"): + ros_topic = "/dvl/velocity" if case[0] == "dvl" else "/model/rexrov/odometry" + code, payload = capture( + out, + "ros_payload", + [ + "ros2", + "topic", + "echo", + ros_topic, + "--once", + "--qos-reliability", + "best_effort", + ], + 40, + ) + checks["ros_payload"] = code == 0 and "header:" in payload + if case[0] in ("bluerov2", "bluerov2_heavy"): + checks["mavros_connected"] = mavros_connected(out) + capture(out, "ros_topics", ["ros2", "topic", "list", "-t"], 20) + checks["launch_alive"] = proc.poll() is None + finally: + if proc is not None: + # Limit shutdown signals to this test's new process group. + try: + os.killpg(proc.pid, signal.SIGINT) + except ProcessLookupError: + pass + try: + returncode = proc.wait(timeout=25) + except subprocess.TimeoutExpired: + forced = True + os.killpg(proc.pid, signal.SIGKILL) + returncode = proc.wait(timeout=10) + checks["clean_shutdown"] = not forced and returncode in (0, 130, -signal.SIGINT) + log_text = (out / "launch.log").read_text(errors="replace") + faults = fatal_log_lines(log_text) + checks["no_fatal_log"] = not faults + return { + "checks": checks, + "shutdown_returncode": returncode, + "faults": faults, + "command": shlex.split(launch_command), + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("case") + parser.add_argument("--record", help="Unique evidence directory for a repeat trial") + args = parser.parse_args() + out = Path("/results") / (args.record or args.case) + out.mkdir(parents=True, exist_ok=True) + result = {"case": args.case, "architecture": platform.machine(), "status": "FAIL"} + try: + if args.case == "inventory": + result.update(inventory(out)) + else: + cases = [ + line.split("\t") + for line in Path(__file__).with_name("quickstarts.tsv").read_text().splitlines() + if line and not line.startswith("#") + ] + case = next(c for c in cases if c[0] == args.case) + result.update(exercise(out, case)) + if not all(result["checks"].values()): + raise RuntimeError("One or more runtime checks failed") + result["status"] = "PASS" + except Exception as error: + result["error"] = str(error) + (out / "result.json").write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result), flush=True) + return 0 if result["status"] == "PASS" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/extras/ci/mavconn_self_close/main.cc b/extras/ci/mavconn_self_close/main.cc new file mode 100644 index 00000000..a13962f9 --- /dev/null +++ b/extras/ci/mavconn_self_close/main.cc @@ -0,0 +1,42 @@ +#include +#include +#include +#include +#include +#include + +struct Completion +{ + std::promise done; + ~Completion() { done.set_value(); } +}; + +int main() +{ + for (int i = 0; i < 100; ++i) + { + auto owner = std::make_shared(); + auto * io = &owner->io(); + std::promise release; + auto gate = release.get_future().share(); + auto completion = std::make_shared(); + auto done = completion->done.get_future(); + asio::post( + *io, + [owned = owner, gate]() mutable + { + gate.wait(); + owned->shutdown_owned(); + owned.reset(); + }); + owner->start([io, completion]() { io->run(); }); + completion.reset(); + owner.reset(); + release.set_value(); + if (done.wait_for(std::chrono::seconds(5)) != std::future_status::ready) + { + throw std::runtime_error("I/O worker did not finish after self-close"); + } + } + std::cout << "100 self-close/destruction trials completed" << std::endl; +} diff --git a/extras/ci/mavros_ownership/CMakeLists.txt b/extras/ci/mavros_ownership/CMakeLists.txt new file mode 100644 index 00000000..a8ef6c95 --- /dev/null +++ b/extras/ci/mavros_ownership/CMakeLists.txt @@ -0,0 +1,17 @@ +cmake_minimum_required(VERSION 3.16) +project(posim_mavros_ownership LANGUAGES CXX) +set(CMAKE_CXX_STANDARD 20) +find_package(mavros REQUIRED) +find_package(rclcpp REQUIRED) +find_package(libmavconn REQUIRED) +find_package(diagnostic_updater REQUIRED) +find_package(mavros_msgs REQUIRED) +find_package(mavlink REQUIRED) +find_package(Eigen3 REQUIRED) +add_executable(posim_mavros_ownership_check main.cc) +target_include_directories(posim_mavros_ownership_check PRIVATE + ${mavros_INCLUDE_DIRS} ${libmavconn_INCLUDE_DIRS} ${mavlink_INCLUDE_DIRS}) +target_link_libraries(posim_mavros_ownership_check PRIVATE + ${mavros_LIBRARIES} ${libmavconn_LIBRARIES} ${mavros_msgs_TARGETS} + rclcpp::rclcpp diagnostic_updater::diagnostic_updater Eigen3::Eigen) +install(TARGETS posim_mavros_ownership_check DESTINATION bin) diff --git a/extras/ci/mavros_ownership/main.cc b/extras/ci/mavros_ownership/main.cc new file mode 100644 index 00000000..c0564992 --- /dev/null +++ b/extras/ci/mavros_ownership/main.cc @@ -0,0 +1,61 @@ +#include +#include +#include +#include +#include + +#include + +namespace mavros::router +{ +// Use the upstream test friend, not timers or a remote autopilot, to establish +// exactly which endpoints the Router owns before releasing its external owner. +class TestRouter +{ +public: + static std::weak_ptr add(const Router::SharedPtr & router, bool ros) + { + auto request = std::make_shared(); + auto response = std::make_shared(); + request->type = ros ? request->TYPE_UAS : request->TYPE_FCU; + request->url = ros ? "/posim_ownership" : "udp://127.0.0.1:19998@127.0.0.1:19999"; + router->add_endpoint(request, response); + if (!response->successful) + { + throw std::runtime_error(response->reason); + } + return router->endpoints.at(response->id); + } +}; +} // namespace mavros::router + +int main(int argc, char ** argv) +{ + rclcpp::init(argc, argv, rclcpp::InitOptions(), rclcpp::SignalHandlerOptions::None); + int failures = 0; + for (int mode = 0; mode < 3; ++mode) + { + auto router = std::make_shared("ownership_" + std::to_string(mode)); + std::vector> endpoints; + if (mode >= 1) + { + endpoints.push_back(mavros::router::TestRouter::add(router, true)); + } + if (mode >= 2) + { + endpoints.push_back(mavros::router::TestRouter::add(router, false)); + } + std::weak_ptr weak = router; + router.reset(); + bool released = weak.expired(); + for (const auto & endpoint : endpoints) + { + released = released && endpoint.expired(); + } + std::cout << "mode=" << mode << " router_expired=" << weak.expired() + << " all_endpoints_expired=" << released << std::endl; + failures += !released; + } + rclcpp::shutdown(); + return failures ? 1 : 0; +} diff --git a/extras/ci/quickstarts.tsv b/extras/ci/quickstarts.tsv new file mode 100644 index 00000000..12a6e07f --- /dev/null +++ b/extras/ci/quickstarts.tsv @@ -0,0 +1,15 @@ +# idlaunch commandexpected entity regexexpected topic regex +world_ocean ros2 launch dave_demos dave_world.launch.py world_name:=dave_ocean_waves headless:=true __NONE__ /world/oceans_waves/ +spherical_world ros2 launch dave_demos dave_world.launch.py world_name:=dave_bimanual_example headless:=true grabbapole /world/dave_bimanual_example/ +object_mossy ros2 launch dave_demos dave_object.launch.py namespace:=mossy_cinder_block paused:=false gui:=false headless:=true mossy_cinder_block /world/empty/ +bluerov2 ros2 launch dave_demos dave_robot.launch.py z:=-0.5 namespace:=bluerov2 world_name:=dave_ocean_waves paused:=false gui:=false headless:=true use_teleop:=false use_web_joystick:=false open_qgc:=false open_virtual_joystick:=false bluerov2 /world/oceans_waves/model/bluerov2/link/base_link/sensor/imu_sensor/imu|/model/bluerov2/imu +bluerov2_heavy ros2 launch dave_demos dave_robot.launch.py z:=-0.5 namespace:=bluerov2_heavy world_name:=dave_ocean_waves paused:=false gui:=false headless:=true use_teleop:=false use_web_joystick:=false open_qgc:=false open_virtual_joystick:=false bluerov2_heavy /world/oceans_waves/model/bluerov2_heavy/link/base_link/sensor/imu_sensor/imu +rexrov_empty ros2 launch dave_demos dave_robot.launch.py z:=2.0 namespace:=rexrov world_name:=empty.sdf paused:=false gui:=false headless:=true use_teleop:=false use_web_joystick:=false rexrov /model/rexrov/odometry +rexrov_waves ros2 launch dave_demos dave_robot.launch.py z:=-5 namespace:=rexrov world_name:=dave_ocean_waves paused:=false gui:=false headless:=true use_teleop:=false use_web_joystick:=false rexrov /model/rexrov/odometry +glider_empty ros2 launch dave_demos dave_robot.launch.py z:=0.2 namespace:=glider_slocum world_name:=empty.sdf paused:=false gui:=false headless:=true use_teleop:=false use_web_joystick:=false glider_slocum /model/glider_slocum/odometry$ +glider_waves ros2 launch dave_demos dave_robot.launch.py x:=4 z:=-1.5 namespace:=glider_slocum world_name:=dave_ocean_waves paused:=false gui:=false headless:=true use_teleop:=false use_web_joystick:=false glider_slocum /model/glider_slocum/odometry$ +ocean_current ros2 launch dave_demos dave_robot.launch.py z:=-5 namespace:=rexrov world_name:=ocean_current_plugin paused:=false gui:=false headless:=true use_teleop:=false use_web_joystick:=false rexrov hydrodynamics/currentVelocityTopic|/ocean_current +dvl ros2 launch dave_demos dave_sensor.launch.py namespace:=nortek_dvl500_300 world_name:=dvl_world paused:=false gui:=false headless:=true z:=-30 nortek_dvl500_300 dvl/velocity +camera ros2 launch dave_demos dave_sensor.launch.py namespace:=underwater_camera world_name:=camera_tutorial paused:=false gui:=false headless:=true x:=10 z:=-93.5 pitch:=0.3 yaw:=3.14 underwater_camera underwater_camera/simulated_image +usbl ros2 launch dave_demos dave_world.launch.py world_name:=usbl_tutorial headless:=true __NONE__ USBL/ +sea_pressure ros2 launch dave_demos dave_robot.launch.py z:=-5 namespace:=rexrov world_name:=dave_ocean_waves paused:=false gui:=false headless:=true use_teleop:=false use_web_joystick:=false rexrov /model/rexrov/sea_pressure diff --git a/extras/ci/test_fuel_assets.py b/extras/ci/test_fuel_assets.py new file mode 100644 index 00000000..6be71d52 --- /dev/null +++ b/extras/ci/test_fuel_assets.py @@ -0,0 +1,115 @@ +"""A cache is complete only when locked files AND nested resources are present.""" + +import hashlib +import importlib.util +import json +from pathlib import Path +from tempfile import TemporaryDirectory +import unittest +from unittest.mock import patch + + +SCRIPT = Path(__file__).resolve().parents[1] / "prepare-image-assets.py" +SPEC = importlib.util.spec_from_file_location("fuel_assets", SCRIPT) +assets = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(assets) + + +class FuelAssetTests(unittest.TestCase): + def setUp(self): + self.tmp = TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.cache = Path(self.tmp.name) + self.url = "https://fuel.gazebosim.org/1.0/Owner/models/Test/2" + self.relative = str(assets.model_path(self.url)) + self.root = self.cache / self.relative + self.root.mkdir(parents=True) + (self.root / "model.sdf").write_text('') + (self.root / "model.config").write_text("test") + self.entry = { + "url": self.url, + "cache_path": self.relative, + "license": {"url": "https://creativecommons.org/licenses/by/4.0/"}, + "files": assets.hashes(self.root), + } + + def test_complete_cache(self): + assets.verify_asset(self.cache, self.entry) + assets.verify_dependencies(self.cache, [self.entry]) + + def test_corrupt_file_rejected(self): + (self.root / "model.sdf").write_text("changed") + with self.assertRaisesRegex(RuntimeError, "integrity mismatch"): + assets.verify_asset(self.cache, self.entry) + + def test_missing_and_extra_files_rejected(self): + (self.root / "extra.txt").write_text("unexpected") + with self.assertRaises(RuntimeError): + assets.verify_asset(self.cache, self.entry) + (self.root / "extra.txt").unlink() + (self.root / "model.config").unlink() + with self.assertRaises(RuntimeError): + assets.verify_asset(self.cache, self.entry) + + def test_symlink_rejected(self): + (self.root / "escape").symlink_to(self.cache) + with self.assertRaisesRegex(RuntimeError, "symlink"): + assets.verify_asset(self.cache, self.entry) + + def test_tip_version_and_unsafe_urls_rejected(self): + for url in [ + self.url[:-1] + "tip", + self.url.rsplit("/", 1)[0], + self.url + "?x=1", + self.url.replace("/Owner/", "/../"), + self.url.replace("https:", "http:"), + ]: + with self.subTest(url=url), self.assertRaises(ValueError): + assets.model_path(url) + + def test_wrong_cached_version_rejected(self): + self.entry["cache_path"] = self.relative[:-1] + "3" + with self.assertRaisesRegex(ValueError, "mismatch"): + assets.verify_asset(self.cache, self.entry) + + def test_nested_dependency_requires_separate_pinned_model(self): + (self.root / "model.sdf").write_text( + "" + "https://fuel.gazebosim.org/1.0/Other/models/Texture/3/files/map.png" + "" + ) + with self.assertRaisesRegex(RuntimeError, "Unpinned nested"): + assets.verify_dependencies(self.cache, [self.entry]) + + def test_nested_file_is_required_even_with_model_present(self): + (self.root / "model.sdf").write_text(f"{self.url}/files/missing.png") + with self.assertRaisesRegex(RuntimeError, "Missing nested"): + assets.verify_dependencies(self.cache, [self.entry]) + + def test_existing_cache_never_downloads(self): + with patch.object(assets.subprocess, "run") as run: + assets.prepare_asset(self.cache, self.entry) + run.assert_not_called() + + def test_bad_existing_cache_is_not_silently_repaired(self): + (self.root / "model.config").unlink() + with patch.object(assets.subprocess, "run") as run, self.assertRaises(RuntimeError): + assets.prepare_asset(self.cache, self.entry) + run.assert_not_called() + + def test_lock_has_critical_version_three_dependency_and_valid_hashes(self): + lock = json.loads((SCRIPT.parent / "fuel/quickstart-assets.lock.json").read_text()) + self.assertEqual(len(lock["assets"]), 12) + paths = {a["cache_path"] for a in lock["assets"]} + self.assertEqual(len(paths), 12) + self.assertIn("fuel.ignitionrobotics.org/cole/models/sunken vase/3", paths) + self.assertIn("fuel.ignitionrobotics.org/cole/models/sunken vase/4", paths) + for asset in lock["assets"]: + self.assertEqual(str(assets.model_path(asset["url"])), asset["cache_path"]) + self.assertTrue(asset["license"]["url"]) + for digest in asset["files"].values(): + self.assertEqual(len(bytes.fromhex(digest)), hashlib.sha256().digest_size) + + +if __name__ == "__main__": + unittest.main() diff --git a/extras/ci/test_gazebo_diagnostic_wrapper.py b/extras/ci/test_gazebo_diagnostic_wrapper.py new file mode 100644 index 00000000..da0ba443 --- /dev/null +++ b/extras/ci/test_gazebo_diagnostic_wrapper.py @@ -0,0 +1,53 @@ +"""A debugger must not corrupt the version probe required by gz's Ruby CLI.""" + +import importlib.util +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + +path = Path(__file__).resolve().parents[2] / "tools/instrument-gazebo-gdb.py" +spec = importlib.util.spec_from_file_location("gazebo_gdb", path) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +class GazeboDiagnosticWrapperTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.binary = self.root / "gz-sim-main" + self.binary.write_text('#!/bin/bash\nprintf "10.5.0\\n"\n') + self.binary.chmod(0o755) + debugger = self.root / "gdb" + debugger.write_text('#!/bin/bash\nprintf "debugger:%s\\n" "$@"\n') + debugger.chmod(0o755) + self.env = {**os.environ, "PATH": str(self.root) + os.pathsep + os.environ["PATH"]} + + def run_binary(self, *args): + return subprocess.check_output([str(self.binary), *args], env=self.env, text=True) + + def test_version_stdout_is_byte_identical(self): + before = self.run_binary("--version") + module.instrument(self.binary) + self.assertEqual(self.run_binary("--version"), before) + + def test_simulation_still_runs_under_debugger(self): + module.instrument(self.binary) + output = self.run_binary("-s", "world with spaces.sdf") + self.assertIn("debugger:set debuginfod enabled off\n", output) + self.assertIn("debugger:" + str(self.binary) + ".real\n", output) + self.assertIn("debugger:world with spaces.sdf\n", output) + + def test_refuses_double_instrumentation(self): + module.instrument(self.binary) + before = self.binary.read_bytes() + with self.assertRaises(RuntimeError): + module.instrument(self.binary) + self.assertEqual(before, self.binary.read_bytes()) + + +if __name__ == "__main__": + unittest.main() diff --git a/extras/ci/test_image_smoke.py b/extras/ci/test_image_smoke.py new file mode 100644 index 00000000..902f118a --- /dev/null +++ b/extras/ci/test_image_smoke.py @@ -0,0 +1,121 @@ +"""Regression tests for failures hidden by launch or Gazebo wrapper exit codes.""" + +import unittest +import json +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import Mock, patch + +from image_smoke import fatal_log_lines, linked_library, wait_for_entity + + +class LibraryLinkageTests(unittest.TestCase): + def test_resolved_path(self): + self.assertEqual( + linked_library( + " libgz-transport.so.15 => /opt/patched/lib.so (0x123)\n", "libgz-transport.so.15" + ), + Path("/opt/patched/lib.so").resolve(), + ) + + def test_missing_library(self): + with self.assertRaises(RuntimeError): + linked_library("libgz-transport.so.15 => not found\n", "libgz-transport.so.15") + + def test_wrong_soname(self): + with self.assertRaises(RuntimeError): + linked_library( + "libgz-transport.so.150 => /opt/wrong.so (0x123)\n", "libgz-transport.so.15" + ) + + def test_ambiguous_library(self): + with self.assertRaises(RuntimeError): + linked_library( + "libgz-transport.so.15 => /opt/a.so (0x123)\n" * 2, "libgz-transport.so.15" + ) + + +class FatalLogTests(unittest.TestCase): + def test_normal_sigint_is_not_a_crash(self): + log = ( + "[INFO] signal_handler(signum=2)\n" + "[ERROR] [gazebo-1]: process has died [exit code -2, cmd='gz']\n" + ) + self.assertEqual(fatal_log_lines(log), []) + + def test_gazebo_abort_is_not_hidden_by_wrapper_sigint(self): + log = "[gazebo-1] Aborted\n[ERROR] [gazebo-1]: exit code -2\n" + self.assertEqual(fatal_log_lines(log), ["[gazebo-1] Aborted"]) + + def test_uncaught_ros_context_error(self): + log = ( + "[gazebo-1] terminate called after throwing an instance of " + "'rclcpp::exceptions::RCLError'\n" + ) + self.assertTrue(fatal_log_lines(log)) + + def test_signal_and_shell_exit_conventions(self): + for code in (-11, -6, 134, 139): + with self.subTest(code=code): + self.assertTrue(fatal_log_lines(f"process has died [exit code {code}, cmd='gz']")) + + def test_loading_and_python_failures(self): + for message in ( + "Failed to load system plugin", + "error while loading shared libraries", + "Traceback (most recent call last):", + "Segmentation fault", + ): + with self.subTest(message=message): + self.assertTrue(fatal_log_lines(message)) + + def test_other_abnormal_child_exits_are_failures(self): + for code in (1, 2, 127, 137, 143, -9): + with self.subTest(code=code): + self.assertTrue(fatal_log_lines(f"process has died [exit code {code}, cmd='gz']")) + + +class EntityReadinessTests(unittest.TestCase): + def test_waits_for_model_not_just_world_control(self): + proc = Mock() + proc.poll.return_value = None + with TemporaryDirectory() as directory: + out = Path(directory) + with ( + patch("image_smoke.time.monotonic", return_value=1), + patch("image_smoke.time.sleep"), + patch( + "image_smoke.capture", + side_effect=[(124, ""), (0, 'name: "ground"'), (0, 'name: "camera"')], + ) as read, + ): + self.assertTrue(wait_for_entity(out, "world", "camera", proc, 90)) + self.assertEqual(read.call_count, 3) + result = json.loads((out / "entity_readiness.json").read_text()) + self.assertEqual(len(result["attempts"]), 3) + self.assertFalse(result["attempts"][0]["entity_present"]) + + def test_absent_model_still_fails_at_original_deadline(self): + proc = Mock() + proc.poll.return_value = None + with TemporaryDirectory() as directory: + with ( + patch("image_smoke.time.monotonic", side_effect=[89, 90, 90]), + patch("image_smoke.time.sleep"), + patch("image_smoke.capture", return_value=(0, 'name: "other"')) as read, + ): + self.assertFalse(wait_for_entity(Path(directory), "world", "camera", proc, 90)) + self.assertEqual(read.call_count, 1) + self.assertEqual(read.call_args.args[-1], 1) + + def test_terminated_launch_is_not_retried(self): + proc = Mock() + proc.poll.return_value = 1 + with TemporaryDirectory() as directory: + with patch("image_smoke.capture") as read: + self.assertFalse(wait_for_entity(Path(directory), "world", "camera", proc, 90)) + read.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/extras/ci/test_transport_churn.py b/extras/ci/test_transport_churn.py new file mode 100644 index 00000000..8a18c3cb --- /dev/null +++ b/extras/ci/test_transport_churn.py @@ -0,0 +1,46 @@ +"""Test that diagnostic outcome classification cannot hide crashes or idle runs.""" + +import importlib.util +from pathlib import Path +import unittest + +SPEC = importlib.util.spec_from_file_location( + "transport_pairs", Path(__file__).parent / "transport_shutdown" / "run_pairs.py" +) +PAIRS = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(PAIRS) + + +class TransportChurnTests(unittest.TestCase): + publisher = "rounds=100 anchor_received=0 churn_received=0" + subscriber = "rounds=10 anchor_received=100 churn_received=200" + + def test_passing_counts(self): + self.assertTrue(PAIRS.trial_passed(0, 0, [], self.publisher, self.subscriber)) + + def test_subscriber_crash(self): + self.assertFalse(PAIRS.trial_passed(0, -11, [], self.publisher, self.subscriber)) + + def test_publisher_crash(self): + self.assertFalse(PAIRS.trial_passed(-11, 0, [], self.publisher, self.subscriber)) + + def test_timeout_is_not_a_pass(self): + self.assertFalse(PAIRS.trial_passed(0, 0, ["subscriber"], self.publisher, self.subscriber)) + + def test_missing_output(self): + self.assertFalse(PAIRS.trial_passed(0, 0, [], self.publisher, "")) + + def test_no_churn_delivery(self): + self.assertFalse( + PAIRS.trial_passed( + 0, + 0, + [], + self.publisher, + self.subscriber.replace("churn_received=200", "churn_received=0"), + ) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/extras/ci/transport_shutdown/CMakeLists.txt b/extras/ci/transport_shutdown/CMakeLists.txt new file mode 100644 index 00000000..4e4098ff --- /dev/null +++ b/extras/ci/transport_shutdown/CMakeLists.txt @@ -0,0 +1,6 @@ +cmake_minimum_required(VERSION 3.22) +project(posim_transport_shutdown LANGUAGES CXX) +set(CMAKE_CXX_STANDARD 17) +find_package(gz-transport 15 REQUIRED) +add_executable(posim_transport_churn main.cc) +target_link_libraries(posim_transport_churn PRIVATE gz-transport::gz-transport) diff --git a/extras/ci/transport_shutdown/README.md b/extras/ci/transport_shutdown/README.md new file mode 100644 index 00000000..78998d9d --- /dev/null +++ b/extras/ci/transport_shutdown/README.md @@ -0,0 +1,29 @@ +# Transport subscription-churn regression + +The publisher sends real payloads on 24 topics. The subscriber retains one +anchor subscription and repeatedly creates and removes the last subscriber on +the other topics. A passing trial requires nonzero anchor and churn deliveries, +normal termination of both processes, and no timeout. `run_pairs.py` preserves +every trial, the loaded library path/hash, and an available core dump. It never +retries a failed trial to obtain a pass. + +The test exercises the concurrent SUB-socket access implicated in a Gazebo +shutdown failure: `RunReceptionTask -> zmq_poll -> xsub_t::match -> trie_t::check`. +The candidate patch serializes the zero-timeout socket poll with subscription +changes. Blocking waits use only `ZMQ_FD` notification descriptors outside the +node mutex. Socket events are rechecked on every loop, as required by ZeroMQ's +[edge-triggered notification contract](https://libzmq.readthedocs.io/en/latest/zmq_getsockopt.html). + +`tools/diagnose-transport-poll.sh` builds matched unpatched/patched libraries in +a disposable container for five paired 20-second trials. This diagnostic uses +the source defaults and is not final-image acceptance. Release-candidate builds +use `extras/build-image-transport.sh`: a pinned ROS vendor overlay, Zenoh disabled +to match the installed vendor, and a byte-for-byte public configuration check. +No files under `/opt/ros` are replaced. + +Image inventory validation verifies the overlay revision, patch and library +hashes, and the linkage of both the probe and Gazebo simulator. It repeats the +probe without overriding the installed library search path. The separate full +matrix still has to pass all 77 runtime trials on each native architecture. +Neither a green diagnostic job nor zero failures in a finite sample establishes +zero failure probability or authorizes image publication. diff --git a/extras/ci/transport_shutdown/main.cc b/extras/ci/transport_shutdown/main.cc new file mode 100644 index 00000000..b4c5f363 --- /dev/null +++ b/extras/ci/transport_shutdown/main.cc @@ -0,0 +1,104 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +using namespace std::chrono_literals; + +struct Counts +{ + std::atomic anchor{0}; + std::atomic churn{0}; +}; + +int main(int argc, char ** argv) +{ + if (argc != 3) + { + return 2; + } + const std::string mode(argv[1]); + if (mode != "publish" && mode != "subscribe") + { + return 2; + } + const bool publish = mode == "publish"; + const int seconds = std::stoi(argv[2]); + if (seconds < 3 || seconds > 300) + { + return 2; + } + const auto stop = std::chrono::steady_clock::now() + std::chrono::seconds(seconds); + constexpr int topics = 24; + auto topic = [](int i) { return "/posim/churn/topic/" + std::to_string(i); }; + // Queued callbacks retain only the counters, never the Node or stack locals. + const auto received = std::make_shared(); + const std::function anchorCallback = + [received](const gz::msgs::StringMsg &) { ++received->anchor; }; + const std::function churnCallback = + [received](const gz::msgs::StringMsg &) { ++received->churn; }; + unsigned long rounds = 0; + if (publish) + { + gz::transport::Node node; + std::vector publishers; + for (int i = 0; i < topics; ++i) + { + publishers.push_back(node.Advertise(topic(i))); + if (!publishers.back()) + { + return 3; + } + } + gz::msgs::StringMsg msg; + msg.set_data(std::string(2048, 'x')); + while (std::chrono::steady_clock::now() < stop) + { + for (auto & pub : publishers) + { + if (!pub.Publish(msg)) + { + return 4; + } + } + ++rounds; + std::this_thread::sleep_for(100us); + } + } + else + { + // Keep a transport connection while destroying the last subscriber to + // each churn topic. Separate publisher process prevents local delivery. + gz::transport::Node anchor; + if (!anchor.Subscribe(topic(0), anchorCallback)) + { + return 3; + } + std::this_thread::sleep_for(2s); + while (std::chrono::steady_clock::now() < stop) + { + { + gz::transport::Node node; + for (int i = 1; i < topics; ++i) + { + if (!node.Subscribe(topic(i), churnCallback)) + { + return 3; + } + } + std::this_thread::sleep_for(100us); + } + ++rounds; + } + } + std::cout << "rounds=" << rounds << " anchor_received=" << received->anchor + << " churn_received=" << received->churn << std::endl; + return rounds > 0 && (publish || (received->anchor > 0 && received->churn > 0)) ? 0 : 1; +} diff --git a/extras/ci/transport_shutdown/run_pairs.py b/extras/ci/transport_shutdown/run_pairs.py new file mode 100644 index 00000000..825daac5 --- /dev/null +++ b/extras/ci/transport_shutdown/run_pairs.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +"""Collect matched transport churn trials without reclassifying failures as passes.""" + +import argparse +import gzip +import hashlib +import json +import os +from pathlib import Path +import re +import resource +import shutil +import signal +import subprocess +import time +import uuid + + +COUNTS = re.compile(r"rounds=(\d+) anchor_received=(\d+) churn_received=(\d+)") + + +def trial_passed(publisher_rc, subscriber_rc, timeouts, publisher_log, subscriber_log): + pub = COUNTS.search(publisher_log) + sub = COUNTS.search(subscriber_log) + return bool( + publisher_rc == 0 + and subscriber_rc == 0 + and not timeouts + and pub + and sub + and int(pub[1]) > 0 + and all(int(sub[i]) > 0 for i in (1, 2, 3)) + ) + + +def sha256(path): + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def stop_process(process): + if process.poll() is None: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--executable", required=True) + parser.add_argument("--variant", action="append", required=True, help="name:library-directory") + parser.add_argument("--trials", type=int, default=5) + parser.add_argument("--seconds", type=int, default=20) + parser.add_argument("--output", required=True) + args = parser.parse_args() + if not 1 <= args.trials <= 20 or not 3 <= args.seconds <= 300: + parser.error("trials must be 1..20 and seconds 3..300") + variants = [item.split(":", 1) for item in args.variant] + if any(len(v) != 2 or not re.fullmatch(r"[a-zA-Z0-9_-]+", v[0]) for v in variants): + parser.error("each variant must be name:library-directory") + if len({v[0] for v in variants}) != len(variants): + parser.error("variant names must be unique") + root = Path(args.output).resolve() + root.mkdir(parents=True, exist_ok=False) + executable = str(Path(args.executable).resolve(strict=True)) + records = [] + captured = set() + metadata = { + "scope": "Diagnostic comparison only; not final-image acceptance", + "trials_per_variant": args.trials, + "seconds_per_trial": args.seconds, + "architecture": os.uname().machine, + "probe_sha256": sha256(executable), + } + (root / "metadata.json").write_text(json.dumps(metadata, indent=2) + "\n") + for number in range(1, args.trials + 1): + for name, library in variants: + directory = root / f"{name}-{number}" + directory.mkdir() + env = os.environ.copy() + env["GZ_PARTITION"] = "posim-" + uuid.uuid4().hex + env["DEBUGINFOD_URLS"] = "" + if library: + env["LD_LIBRARY_PATH"] = library + ":" + env.get("LD_LIBRARY_PATH", "") + linked = subprocess.check_output(["ldd", executable], env=env, text=True) + (directory / "ldd.txt").write_text(linked) + line = next(x for x in linked.splitlines() if "libgz-transport.so.15 =>" in x) + loaded = line.split("=>")[1].split()[0] + if library and Path(loaded).resolve().parent != Path(library).resolve(): + raise RuntimeError(f"Wrong transport library: {line}") + + def configure_core(): + hard = resource.getrlimit(resource.RLIMIT_CORE)[1] + size = hard if name not in captured else 0 + resource.setrlimit(resource.RLIMIT_CORE, (size, hard)) + + timeouts = [] + with (directory / "publisher.log").open("w") as pub_log, ( + directory / "subscriber.log" + ).open("w") as sub_log: + publisher = subprocess.Popen( + [executable, "publish", str(args.seconds)], + env=env, + stdout=pub_log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + subscriber = None + try: + subscriber = subprocess.Popen( + [executable, "subscribe", str(args.seconds)], + env=env, + stdout=sub_log, + stderr=subprocess.STDOUT, + cwd=directory, + start_new_session=True, + preexec_fn=configure_core, + ) + deadline = time.monotonic() + args.seconds + 10 + for label, process in (("subscriber", subscriber), ("publisher", publisher)): + try: + process.wait(timeout=max(0.1, deadline - time.monotonic())) + except subprocess.TimeoutExpired: + timeouts.append(label) + stop_process(process) + finally: + stop_process(publisher) + if subscriber is not None: + stop_process(subscriber) + record = { + "variant": name, + "trial": number, + "publisher_rc": publisher.returncode, + "subscriber_rc": subscriber.returncode, + "timeouts": timeouts, + "library": loaded, + "library_sha256": sha256(loaded), + "partition": env["GZ_PARTITION"], + } + record["pass"] = trial_passed( + publisher.returncode, + subscriber.returncode, + timeouts, + (directory / "publisher.log").read_text(errors="replace"), + (directory / "subscriber.log").read_text(errors="replace"), + ) + core = directory / "core" + if core.exists(): + captured.add(name) + record["core_sha256"] = sha256(core) + with (directory / "core-stack.log").open("w") as stack: + try: + result = subprocess.run( + [ + "gdb", + "--batch", + "-iex", + "set debuginfod enabled off", + executable, + str(core), + "-ex", + "set pagination off", + "-ex", + "thread apply all bt", + "-ex", + "info sharedlibrary", + ], + env=env, + stdout=stack, + stderr=subprocess.STDOUT, + timeout=30, + ) + record["gdb_returncode"] = result.returncode + except (subprocess.TimeoutExpired, FileNotFoundError) as error: + record["gdb_error"] = str(error) + with core.open("rb") as src, gzip.open(directory / "core.gz", "wb") as dst: + shutil.copyfileobj(src, dst) + core.unlink() + records.append(record) + (root / "results.json").write_text(json.dumps(records, indent=2) + "\n") + print(json.dumps(record), flush=True) + summary = { + name: { + "trials": sum(r["variant"] == name for r in records), + "passed": sum(r["variant"] == name and r["pass"] for r in records), + "sigsegv": sum(r["variant"] == name and r["subscriber_rc"] == -11 for r in records), + } + for name, _ in variants + } + (root / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") + print(json.dumps(summary), flush=True) + if "patched" in summary and summary["patched"]["passed"] != args.trials: + raise SystemExit("Patched diagnostic failed; all trial records were retained") + + +if __name__ == "__main__": + main() diff --git a/extras/fuel/README.md b/extras/fuel/README.md new file mode 100644 index 00000000..f81f105f --- /dev/null +++ b/extras/fuel/README.md @@ -0,0 +1,49 @@ +# Docker Quickstart Fuel cache + +Docker builds prepare the external assets used by the 14 headless Quickstart +paths in `extras/ci/quickstarts.tsv`. Previously every fresh container downloaded +these models while Gazebo started. An AMD64 camera trial exceeded its unchanged +90-second entity-readiness budget. A same-image GDB reproduction waited in Fuel's +HTTP downloader while loading a material. In particular, Sunken Vase Distorted +references the **version-3** Sunken Vase textures even when the scene includes +version 4 of Sunken Vase. Downloading only the top-level models is insufficient. + +`quickstart-assets.lock.json` pins the 12 required model versions and SHA-256 of +all 105 prepared files. `prepare-image-assets.py` stages bounded downloads during +the image build, verifies every file, and checks nested SDF HTTP dependencies. +Existing corrupt/incomplete caches fail verification. Runtime inventory verifies +the installed lock against the validation source and rehashes the cache without +network access. The original runtime acceptance budgets are unchanged. + +The Docker environment uses `GZ_FUEL_CACHE_PATH=/opt/posim_fuel/cache`. Both root +and the ARM64 desktop user can read these assets; the desktop user owns this +cache on ARM64 so other worlds can still download additional models. A caller +who overrides the cache path or masks it with an empty volume must prepare the +assets there. This is not an offline bundle of all 18 worlds, nor a CUDA/WGPU +sonar validation. Native installations are not modified by this Docker change. + +The installed lock and this notice are under `/opt/posim_fuel`. Normal Gazebo Fuel +Tools 11 normalization of `model://` references is preserved. No geometry, +textures, scene parameters or physical models were edited. The cache path format +and download behavior follow the [Fuel tools documentation](https://gazebosim.org/api/fuel_tools/11/cmdline.html). + +## Third-party asset attribution + +These assets retain their own licenses, not POSIM's Apache-2.0 license. The lock +records the exact source URLs, uploaders, model-config author credits, versions, +and license URLs; the original `model.config` files remain beside each model. + +| Models | Fuel uploader | License | +| --- | --- | --- | +| North East Down frame; Sand Heightmap; Sunken Vase Distorted; Sunken Vase with Inertia; mossy_cinder_block | hmoyen | [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/) | +| Coast Water; Waves | OpenRobotics | [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/) | +| Sunken Vase (versions 3 and 4); Coral01 | Cole | [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/) | +| Ground Plane; Sun | OpenRobotics | [CC0 1.0](https://creativecommons.org/publicdomain/zero/1.0/) | + +To change a version, download into an isolated cache, check the source's license, +inspect all nested dependencies and review the new file hashes. Do not regenerate +hashes silently in a release build. Rebuild and rerun both architectures afterward. +The CI also runs five fresh camera containers with `--network none`, retains their +Docker network configuration and actual image payloads, and requires normal +shutdown. These extra offline trials are reported separately from the 77-trial +connected matrix; no failing observation is replaced by a later pass. diff --git a/extras/fuel/quickstart-assets.lock.json b/extras/fuel/quickstart-assets.lock.json new file mode 100644 index 00000000..d7672fed --- /dev/null +++ b/extras/fuel/quickstart-assets.lock.json @@ -0,0 +1,326 @@ +{ + "schema": 1, + "scope": "External Fuel assets used by the 14 headless Quickstart paths, including the version-3 Sunken Vase material dependency. Not all 18 worlds.", + "normalization": "Downloaded with Gazebo Fuel Tools 11. Its normal model:// to Fuel file-URI normalization is retained; geometry and textures are not edited.", + "assets": [ + { + "url": "https://fuel.gazebosim.org/1.0/hmoyen/models/North East Down frame/1", + "cache_path": "fuel.gazebosim.org/hmoyen/models/north east down frame/1", + "title": "North East Down frame", + "uploader": "hmoyen", + "authors": [ + { + "name": "Musa Marcusso", + "email": "musa.marcusso@de.bosch.com" + } + ], + "license": { + "name": "Creative Commons Attribution 4.0 International", + "url": "http://creativecommons.org/licenses/by/4.0/" + }, + "files": { + "model.config": "b1526f2ecadfa50e9a8f3fb6dcbfe036015b991b6d4db06df4273b7cb2fde5e3", + "model.sdf": "e7b7037bfb95d4f7ce527fa833422115063b20a87f09fffbf88181bf106d9a65", + "thumbnails/Screenshot from 2024-06-21 14-41-53.png": "e88b8d04b1ad89ceeffc3e640ad60aad4f49bf1c96b1c7bd49b8555a654fe98b" + } + }, + { + "url": "https://fuel.gazebosim.org/1.0/OpenRobotics/models/Coast Water/3", + "cache_path": "fuel.gazebosim.org/openrobotics/models/coast water/3", + "title": "Coast Water", + "uploader": "OpenRobotics", + "authors": [], + "license": { + "name": "Creative Commons Attribution 4.0 International", + "url": "http://creativecommons.org/licenses/by/4.0/" + }, + "files": { + "materials/programs/GerstnerWaves_fs.metal": "c163ed1c8b628e04dcf49f2ea168819b769824af8c95e51ee4ce5d036dff8f6f", + "materials/programs/GerstnerWaves_fs_330.glsl": "e99714172fe768273512078a7cbab4b1f5439a597105108c1f3a6f8e57ba131a", + "materials/programs/GerstnerWaves_vs.metal": "2f7d864919d28e29da0d9eedcdf7adf3a417bc9eecfde111a9b742cd5075c097", + "materials/programs/GerstnerWaves_vs_330.glsl": "2b9be7918f3793c0453a9627ff8605c7d030c0157dbfde608905f9ccbd36ad7c", + "materials/textures/skybox_lowres.dds": "62cbed0ceaa102620bd0857ec8e17b72ca1c7f364870c407cacdbe99c470fc0c", + "materials/textures/wave_normals.dds": "943b5d189997f460ff8d4a35f5259d36809a5c6669a440d99b4f8b5abe0d8e4b", + "meshes/waterlow.dae": "91e96edd71fde67753adedf6d103b6c803c74011be6ec3cf41202cf963db99b5", + "model.config": "576d467f1d18ba9bef9c018e741ce56f2386f9ba2c5a3a180d49fc209331dd71", + "model.sdf": "a306e63650d1375189f924f8be17d875dbd465b273279078469a6291fd568116", + "thumbnails/1.png": "d8fc5890f032dc0d46970fb2ce2d78352cc497ae998223e5029b0c65b27bcffd" + } + }, + { + "url": "https://fuel.gazebosim.org/1.0/hmoyen/models/Sand Heightmap/1", + "cache_path": "fuel.gazebosim.org/hmoyen/models/sand heightmap/1", + "title": "Sand Heightmap", + "uploader": "hmoyen", + "authors": [ + { + "name": "Sebastian Scherer", + "email": "sebastian.scherer2@de.bosch.com" + } + ], + "license": { + "name": "Creative Commons Attribution 4.0 International", + "url": "http://creativecommons.org/licenses/by/4.0/" + }, + "files": { + "materials/scripts/sand.material": "7c4b5fc22b55abeaa0446aa84a2f82c099a00456ce5d526ac99527a9294d4f13", + "materials/textures/soil_sand_0045_01.jpg": "21915cc33c85f803602d6914b4d4614f5a288559dffe2aef0e89b30615eb5b87", + "meshes/heightmap.dae": "1faf98feef0674dd299c9a17f9f8536e52d0bf2b295ea20bc418c57f23647ef3", + "model.config": "f013d286f5e0bffab09b9483351f52f5b2ad3c536dae0b80df18541a87c249cc", + "model.sdf": "9737b8ae0010b3881428df2184be097ec16fc0fa72c38f5df6423e9558dc533f", + "thumbnails/1.png": "a601cb5d27743c1c171de33014c7a51a364f8ec4542fb040d1d4ce981b0a46f5", + "thumbnails/2.png": "ff06732921aa7635f06dbe672ee6e4cc460060e8c3154c52c908c2cf06f56248" + } + }, + { + "url": "https://fuel.ignitionrobotics.org/1.0/Cole/models/Sunken Vase/4", + "cache_path": "fuel.ignitionrobotics.org/cole/models/sunken vase/4", + "title": "Sunken Vase", + "uploader": "Cole", + "authors": [ + { + "name": "Cole Biesemeyer", + "email": "cole@openrobotics.org" + } + ], + "license": { + "name": "Creative Commons Attribution 4.0 International", + "url": "http://creativecommons.org/licenses/by/4.0/" + }, + "files": { + "materials/scripts/model.material": "953311859570ace387e1ffc16c16e7cae9a394f1bca1123bde0c2fe1975274ee", + "materials/textures/Pot01_Albedo.png": "fae3b9f037b971476e8655503186481190ece486aa62c82c7c4c3cf020a6bc15", + "materials/textures/Pot01_Normal.png": "65221adffadcf157aa9252eb9e8a546e5b44bff55e6846d11dfacdd696061fa2", + "materials/textures/Pot01_Roughness.png": "f50d9a104bc141dfa9e9eda4668fe27dfd2599a1b4f8915f2adb16372fd8ebfa", + "meshes/Pot01.dae": "e36d7bdd21eb3c3df559a71642a98e1ecccb5a8bb573b3e1cc7877a835b22a6e", + "meshes/Pot01.obj": "a31e02092891eb0a5502d5e9de5080c5abe3e0699b108cd30cfb0249883f8430", + "model.config": "288e89a6caaff0e491e73b88389e8c016dfec61ac319a3451c50afc65ddcbce3", + "model.sdf": "2a7959e2a60ccb3a6090974415970add7038fb621e7acb6f894a7f506ccd82cb", + "thumbnails/1.png": "658e8529affee50d23c3627fe9a39aef9a199be71bea08e45679d203cd23d10a", + "thumbnails/2.png": "63b1fafae8bb3e2b59f0fb589983cc949332a9be7dd4310284fc7b980342c440", + "thumbnails/3.png": "c611011e837a8e134a9718298f767919585884d74e8ab75650fc297cb9ac01a7", + "thumbnails/4.png": "fd98f9362660e578b2f831556c18185d10c0e1a3ece6d92c484b2c1f3dc3cc70", + "thumbnails/5.png": "93ff6fdb642a3f9627490a3650e9e99e72b337a4cc1aa94943f2fcb8875e24ac" + } + }, + { + "url": "https://fuel.ignitionrobotics.org/1.0/Cole/models/Sunken Vase/3", + "cache_path": "fuel.ignitionrobotics.org/cole/models/sunken vase/3", + "title": "Sunken Vase", + "uploader": "Cole", + "authors": [ + { + "name": "Cole Biesemeyer", + "email": "cole@openrobotics.org" + } + ], + "license": { + "name": "Creative Commons Attribution 4.0 International", + "url": "http://creativecommons.org/licenses/by/4.0/" + }, + "files": { + "materials/scripts/model.material": "953311859570ace387e1ffc16c16e7cae9a394f1bca1123bde0c2fe1975274ee", + "materials/textures/Pot01_Albedo.png": "fae3b9f037b971476e8655503186481190ece486aa62c82c7c4c3cf020a6bc15", + "materials/textures/Pot01_Normal.png": "65221adffadcf157aa9252eb9e8a546e5b44bff55e6846d11dfacdd696061fa2", + "materials/textures/Pot01_Roughness.png": "f50d9a104bc141dfa9e9eda4668fe27dfd2599a1b4f8915f2adb16372fd8ebfa", + "meshes/Pot01.dae": "e36d7bdd21eb3c3df559a71642a98e1ecccb5a8bb573b3e1cc7877a835b22a6e", + "model.config": "288e89a6caaff0e491e73b88389e8c016dfec61ac319a3451c50afc65ddcbce3", + "model.sdf": "1d5ef4716828ca306f6c4d169960416e4cdfcfa6b06bc61be5687e872a9c4879", + "thumbnails/1.png": "658e8529affee50d23c3627fe9a39aef9a199be71bea08e45679d203cd23d10a", + "thumbnails/2.png": "63b1fafae8bb3e2b59f0fb589983cc949332a9be7dd4310284fc7b980342c440", + "thumbnails/3.png": "c611011e837a8e134a9718298f767919585884d74e8ab75650fc297cb9ac01a7", + "thumbnails/4.png": "fd98f9362660e578b2f831556c18185d10c0e1a3ece6d92c484b2c1f3dc3cc70", + "thumbnails/5.png": "93ff6fdb642a3f9627490a3650e9e99e72b337a4cc1aa94943f2fcb8875e24ac" + } + }, + { + "url": "https://fuel.gazebosim.org/1.0/hmoyen/models/Sunken Vase Distorted/1", + "cache_path": "fuel.gazebosim.org/hmoyen/models/sunken vase distorted/1", + "title": "Sunken Vase Distorted", + "uploader": "hmoyen", + "authors": [ + { + "name": "Cole Biesemeyer", + "email": "cole@openrobotics.org" + } + ], + "license": { + "name": "Creative Commons Attribution 4.0 International", + "url": "http://creativecommons.org/licenses/by/4.0/" + }, + "files": { + "meshes/Pot01_distort.dae": "56c3766df05945922ab4adfe0a463bdac61d041c4ad57b90ef00231f6aa6192b", + "model.config": "288e89a6caaff0e491e73b88389e8c016dfec61ac319a3451c50afc65ddcbce3", + "model.sdf": "3e1dbfeb0118970a7d6af82b38fed57d3c178b56bb779acc44ed3007acec06b9", + "thumbnails/Screenshot from 2024-06-13 18-06-42.png": "88ea8654fca9112f47a3848d7c2448caacdbabc50d31821f04eefb68cbd1f68b", + "thumbnails/Screenshot from 2024-06-13 18-07-31.png": "427e27b8c9cc003d4d0e02ca6314d56964e3675d44ca123e749b0d932cdbfa77", + "thumbnails/Screenshot from 2024-06-13 18-08-24.png": "7ecb698a2dd9ceb46109d7f5c4bb218a54882dde3022a8e49869bb3f085cac6d", + "thumbnails/Screenshot from 2024-06-13 18-09-17.png": "93addef38f4cfa3472c4bf25c7134e41e105cde9a2127aec71c42924ada8b84f" + } + }, + { + "url": "https://fuel.ignitionrobotics.org/1.0/Cole/models/Coral01/2", + "cache_path": "fuel.ignitionrobotics.org/cole/models/coral01/2", + "title": "Coral01", + "uploader": "Cole", + "authors": [ + { + "name": "Cole Biesemeyer", + "email": "cole@openrobotics.org" + } + ], + "license": { + "name": "Creative Commons Attribution 4.0 International", + "url": "http://creativecommons.org/licenses/by/4.0/" + }, + "files": { + "materials/scripts/model.material": "1546be4696f1d11e07b068017afb87782c8de92701dad98c95b4a1078ad2dce2", + "materials/textures/Coral01_Albedo.png": "7afc2afbdb845dbbbebaa33fbbc8b9e8f9cc436d948d796af8aab1d27d88e462", + "materials/textures/Coral01_Normal.png": "1ddb85c44384badb1aca803ffe5afb2fef277f6967e327cd77016a02c40f5da8", + "materials/textures/Coral01_Roughness.png": "0f86650753b125218df8af0278cb63b67134c2a9a52c6a671723e443ca875c07", + "meshes/coral01.dae": "97c0ad01310408082dbd700aeea2daf8fdc53a354a3ba84a190ff9225bc9029b", + "model.config": "d99d9289741899b1039e09c2a7a6683682e8dab421d88e30a34c47a7dbbc3cba", + "model.sdf": "7ffd2ef2949a201de38f411f99a5018cf7ff72191ca24f612b753115bc62042e", + "thumbnails/1.png": "c4b98b4da86e272e47c2e80b2c090c663d470887ed86915d7f51dca05f2dadd8", + "thumbnails/2.png": "9ff1709a4dcff314139b32a7c7d303403f60a4b8a938438688aeaaae62d2ef5d", + "thumbnails/3.png": "206cd69fed2fb17c89bb74804bfca3dff639b2a262035ee64f743b1ff7099569", + "thumbnails/4.png": "3f3ecc78ea11ecda60e5b068b5b0647536751ec756aebd37ddab04860f22a69b", + "thumbnails/5.png": "9683a96a70902799e7f0748899e86e9be796ac2c5a96347720954bc20e90e925" + } + }, + { + "url": "https://fuel.gazebosim.org/1.0/OpenRobotics/models/Waves/5", + "cache_path": "fuel.gazebosim.org/openrobotics/models/waves/5", + "title": "Waves", + "uploader": "OpenRobotics", + "authors": [], + "license": { + "name": "Creative Commons Attribution 4.0 International", + "url": "http://creativecommons.org/licenses/by/4.0/" + }, + "files": { + "materials/programs/GerstnerWaves_fs.metal": "d27bcf731967004ead0c11305820430f140eeb9a81eec00200fb0f0f66ff1b42", + "materials/programs/GerstnerWaves_fs_330.glsl": "44091c3b037f4ff8d36fde0bfe7ed15b34a33163915e053bdf66e10e728e8510", + "materials/programs/GerstnerWaves_vs.metal": "2f7d864919d28e29da0d9eedcdf7adf3a417bc9eecfde111a9b742cd5075c097", + "materials/programs/GerstnerWaves_vs_330.glsl": "2b9be7918f3793c0453a9627ff8605c7d030c0157dbfde608905f9ccbd36ad7c", + "materials/textures/skybox_lowres.dds": "62cbed0ceaa102620bd0857ec8e17b72ca1c7f364870c407cacdbe99c470fc0c", + "materials/textures/wave_normals.dds": "943b5d189997f460ff8d4a35f5259d36809a5c6669a440d99b4f8b5abe0d8e4b", + "meshes/mesh.dae": "91f51d619e7d5efdd725761bbe3d55a4e7968593f9f05d97bf1f62e826638592", + "model.config": "8bf08c240426999c42d3e5b54bb61b43faf558f62fe24f03b8b3fa8f79c3f2aa", + "model.sdf": "df507706daf80dc527ce6ee87a67ccd72ca4333a09a64f228cebffd6c083babb", + "thumbnails/1.png": "186934bc6e5035f91aeb7a095dfae6c9866cde51a756d1bb36bf694ff0e38e61" + } + }, + { + "url": "https://fuel.gazebosim.org/1.0/hmoyen/models/Sunken Vase with Inertia/1", + "cache_path": "fuel.gazebosim.org/hmoyen/models/sunken vase with inertia/1", + "title": "Sunken Vase with Inertia", + "uploader": "hmoyen", + "authors": [ + { + "name": "Cole Biesemeyer", + "email": "cole@openrobotics.org" + } + ], + "license": { + "name": "Creative Commons Attribution 4.0 International", + "url": "http://creativecommons.org/licenses/by/4.0/" + }, + "files": { + "model.config": "288e89a6caaff0e491e73b88389e8c016dfec61ac319a3451c50afc65ddcbce3", + "model.sdf": "648543c10fc4e1dc3ce904df24620bd7687c9e59f7a22a46e3d5f70123468671", + "thumbnails/Screenshot from 2024-06-13 17-59-27.png": "675082297e9d55732d1c5ef78137a37d0ced8a45e91024f1df9cddb11cab4abc", + "thumbnails/Screenshot from 2024-06-13 17-59-52.png": "4c3653840c8cd98bde3ea6762f656c0ccafe143db2b6a5fafa9c122f4f7ae51a", + "thumbnails/Screenshot from 2024-06-13 18-00-55.png": "fe64e9f0bbc64dfd53554327323df7654c261804e8599240817641a2f69679e6", + "thumbnails/Screenshot from 2024-06-13 18-01-42.png": "634c399fb67c04a872ab1be3b1957d59d1bff41357b08498680188b3be3b695c" + } + }, + { + "url": "https://fuel.gazebosim.org/1.0/OpenRobotics/models/Ground Plane/5", + "cache_path": "fuel.gazebosim.org/openrobotics/models/ground plane/5", + "title": "Ground Plane", + "uploader": "OpenRobotics", + "authors": [ + { + "name": "Nate Koenig", + "email": "nate@osrfoundation.org" + } + ], + "license": { + "name": "Creative Commons Zero v1.0 Universal", + "url": "https://creativecommons.org/publicdomain/zero/1.0/" + }, + "files": { + "model-1_2.sdf": "a099ecca53ba1d0af3babacb31a4e802099d896926ea9109d38f2e14e40a28d9", + "model-1_3.sdf": "cc2bf63d600f382bbbe81dbd3cd9363ee0436de6bda1c273cae47dfd8ea65bcf", + "model-1_4.sdf": "de8e04148a6b58735b63ea7500811ca1a1bae174d6da3a20f861c679534ddc99", + "model.config": "4fefe57461ff15dd0b8931e249616eb719cf6760767a85db84762d85186ca94b", + "model.sdf": "73be59778d84e631dd0996a0adc461af5e4c8717b93da7ee9151b9ae43e3ca25", + "thumbnails/1.png": "19a31ae622180052518cf40518579f7bb1ce74cd1a2dd2e38fa260e821dd99c7", + "thumbnails/2.png": "3fa42da84beeb517daa91494f1ef1b18265dbf4c04b387e4c45682d9096be35a", + "thumbnails/3.png": "c423f7a4b11bfaf8db92cbe832f7fcfec1c5869edd9d974ea7e445425680c790", + "thumbnails/4.png": "c423f7a4b11bfaf8db92cbe832f7fcfec1c5869edd9d974ea7e445425680c790", + "thumbnails/5.png": "c423f7a4b11bfaf8db92cbe832f7fcfec1c5869edd9d974ea7e445425680c790" + } + }, + { + "url": "https://fuel.gazebosim.org/1.0/OpenRobotics/models/Sun/3", + "cache_path": "fuel.gazebosim.org/openrobotics/models/sun/3", + "title": "Sun", + "uploader": "OpenRobotics", + "authors": [ + { + "name": "Nate Koenig", + "email": "nate@osrfoundation.org" + } + ], + "license": { + "name": "Creative Commons Zero v1.0 Universal", + "url": "https://creativecommons.org/publicdomain/zero/1.0/" + }, + "files": { + "model-1_2.sdf": "ec06c2561585112e6acdc77f20ce827541ea0cb1bcf1d2eab08040f34ebddc3c", + "model-1_3.sdf": "609613ac8ce6a38dbe6be5e2e1f7a43fa557696c5d4a4a48fa246957573d84a9", + "model-1_4.sdf": "9ecf9281ed55a951e97b2fcb494a6d0e5a7247abff5208b2d53a06acb62b714a", + "model.config": "f6d1f0e4b3919c21fc56ad24763275838944ec9acfc60bb86e2d31f23976688c", + "model.sdf": "b10a6631cfac7340ad09c2fdfeb6d950afd3ec4daf1e7d0262c14adcf612d15a" + } + }, + { + "url": "https://fuel.gazebosim.org/1.0/hmoyen/models/mossy_cinder_block/1", + "cache_path": "fuel.gazebosim.org/hmoyen/models/mossy_cinder_block/1", + "title": "mossy_cinder_block", + "uploader": "hmoyen", + "authors": [ + { + "name": "Cole Biesemeyer", + "email": "cole.bsmr@gmail.com" + }, + { + "name": "Nate Koenig", + "email": "natekoenig@gmail.com" + }, + { + "name": "Duane Davis", + "email": "dtdavi1@nps.edu" + } + ], + "license": { + "name": "Creative Commons Attribution 4.0 International", + "url": "http://creativecommons.org/licenses/by/4.0/" + }, + "files": { + "materials/textures/cinder_block_diffuse.png": "4fcebe7ff087552302039e513bf462f9d46df67acce4ee82939efad66b4c3c38", + "materials/textures/grass.png": "2fd3ccbcb63928ced1477f60e8ccc7046f665bbbd3859d39820781d6f6041fbf", + "meshes/mossy_cinder_block.dae": "8d8789047645f04ae2890b884aa21f521088fa15a2203291256a9b8cee4debe7", + "model-1_4.sdf": "9e136e78ed7ea86c289a6747df1eea3fb67d7c1ef0864a6c4a61095630e1d078", + "model.config": "b73f2d3404c4cbf3caddce46db11a997999b0c62ceb4999c87ac402cf54c8ace", + "model.sdf": "9f164e807bcdf00994010f042d978f51b3137ae0706fb7db20b4986c4a506347", + "thumbnails/Screenshot from 2024-06-07 11-32-21.png": "94b59a0ffbf278b72d392b736f848e303f5a7baf585008b6230596573233e395", + "thumbnails/Screenshot from 2024-06-07 11-32-31.png": "6317a34b73c53f2b96eae74a05fb2a730847476df524a46b38d15fb5c2ee2e1b", + "thumbnails/Screenshot from 2024-06-07 11-32-41.png": "dc59984bbeb48134c6dae2406e9c5f5e1470203e888e9c7ee8dadbd7824429aa", + "thumbnails/Screenshot from 2024-06-07 11-33-02.png": "77e3005cce17dae1771ec93853475136e66da937df84d15d62b150e773653c0e" + } + } + ] +} diff --git a/extras/patches/gz-transport-poll-serialization.patch b/extras/patches/gz-transport-poll-serialization.patch new file mode 100644 index 00000000..efe2ef3a --- /dev/null +++ b/extras/patches/gz-transport-poll-serialization.patch @@ -0,0 +1,59 @@ +--- a/src/NodeShared.cc ++++ b/src/NodeShared.cc +@@ -358,19 +358,47 @@ + ////////////////////////////////////////////////// + void NodeShared::RunReceptionTask() + { ++ // ZMQ socket polling reads the subscription trie. Serialize that access ++ // with subscribe/unsubscribe, but never hold the node mutex while waiting. ++ zmq::pollitem_t items[] = ++ { ++ {static_cast(*this->dataPtr->subscriber), 0, ZMQ_POLLIN, 0}, ++ {static_cast(*this->dataPtr->replier), 0, ZMQ_POLLIN, 0}, ++ {static_cast(*this->dataPtr->responseReceiver), 0, ZMQ_POLLIN, 0} ++ }; ++ constexpr auto count = sizeof(items) / sizeof(items[0]); ++ zmq::pollitem_t notifications[count] = {}; ++ { ++ std::lock_guard lock(this->mutex); ++ for (size_t i = 0; i < count; ++i) ++ { ++ size_t size = sizeof(notifications[i].fd); ++ if (zmq_getsockopt(items[i].socket, ZMQ_FD, ¬ifications[i].fd, ++ &size) != 0) ++ { ++ std::cerr << "Unable to obtain transport notification descriptor" ++ << std::endl; ++ return; ++ } ++ notifications[i].events = ZMQ_POLLIN; ++ } ++ } + while (!this->dataPtr->exit) + { +- // Poll socket for a reply, with timeout. +- zmq::pollitem_t items[] = +- { +- {static_cast(*this->dataPtr->subscriber), 0, ZMQ_POLLIN, 0}, +- {static_cast(*this->dataPtr->replier), 0, ZMQ_POLLIN, 0}, +- {static_cast(*this->dataPtr->responseReceiver), 0, ZMQ_POLLIN, 0} +- }; + try + { +- zmq::poll(&items[0], sizeof(items) / sizeof(items[0]), +- std::chrono::milliseconds(NodeSharedPrivate::Timeout)); ++ { ++ std::lock_guard lock(this->mutex); ++ zmq::poll(items, count, std::chrono::milliseconds(0)); ++ } ++ if (!(items[0].revents || items[1].revents || items[2].revents)) ++ { ++ // These are OS descriptors, not ZMQ sockets. Recheck ZMQ_EVENTS at ++ // the top of every loop to respect the edge-triggered ZMQ_FD API. ++ zmq::poll(notifications, count, ++ std::chrono::milliseconds(NodeSharedPrivate::Timeout)); ++ continue; ++ } + } + catch(...) + { diff --git a/extras/patches/gz-transport-vendor-poll.patch b/extras/patches/gz-transport-vendor-poll.patch new file mode 100644 index 00000000..5dd0ad89 --- /dev/null +++ b/extras/patches/gz-transport-vendor-poll.patch @@ -0,0 +1,20 @@ +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -57,6 +57,8 @@ + VCS_URL https://github.com/gazebosim/${GITHUB_NAME}.git + VCS_VERSION ${LIB_VCS_VER} + GLOBAL_HOOK ++ PATCHES poll-serialization.patch ++ CMAKE_ARGS -DGZ_TRANSPORT_ENABLE_ZENOH=OFF + ) + + find_package(ament_cmake_test REQUIRED) +@@ -78,7 +80,7 @@ + gz_utils_vendor + ) + +-if(NOT ${${LIB_NAME_FULL}_FOUND}) ++if(NOT ${${LIB_NAME_FULL}_FOUND} OR FORCE_BUILD_VENDOR_PKG) + ament_environment_hooks("${CMAKE_CURRENT_SOURCE_DIR}/${PROJECT_NAME}.dsv.in") + # Create a dummy .sh file needed for ament_package to source the .dsv file. + # See https://github.com/ament/ament_package/issues/145 diff --git a/extras/patches/mavconn-self-close-lifetime.patch b/extras/patches/mavconn-self-close-lifetime.patch new file mode 100644 index 00000000..7d1a105c --- /dev/null +++ b/extras/patches/mavconn-self-close-lifetime.patch @@ -0,0 +1,158 @@ +--- a/libmavconn/include/mavconn/io_context_runner.hpp ++++ b/libmavconn/include/mavconn/io_context_runner.hpp +@@ -23,23 +23,24 @@ namespace mavconn + + /** + * @brief Small utility to unify owned/shared io_context lifecycle handling. ++ * ++ * When the runner owns its io_context and I/O thread, the worker thread keeps a ++ * reference to the shared state (io_context, work guard, is_running). This lets ++ * a shutdown initiated from the worker thread itself detach safely: the state ++ * survives even if the owning connection is destroyed before the thread exits, ++ * and the io_context is restarted only after run() has returned. + */ + class IoContextRunner + { + public: + explicit IoContextRunner(asio::io_context * shared_io = nullptr) +- : io_owner_(shared_io ? nullptr : std::make_shared()), +- io_(shared_io ? *shared_io : *io_owner_), +- io_work_(shared_io ? nullptr : +- std::make_unique>( +- asio::make_work_guard(io_))), +- owns_thread_(shared_io == nullptr), +- is_running_(false) ++ : owns_thread_(shared_io == nullptr), ++ state_(std::make_shared(shared_io)) + {} + + [[nodiscard]] asio::io_context & io() + { +- return io_; ++ return state_->io(); + } + + [[nodiscard]] bool owns_thread() const +@@ -49,7 +50,7 @@ class IoContextRunner + + [[nodiscard]] bool is_running() const + { +- return is_running_.load(); ++ return state_->is_running.load(); + } + + template +@@ -59,11 +60,20 @@ class IoContextRunner + return; + } + ++ // The worker captures the shared state so the io_context and is_running ++ // flag outlive the connection when shutdown is initiated from this thread ++ // (self-close) and join_owned() has to detach. ++ auto state = state_; + io_thread_ = std::jthread( +- [this, f = std::forward(fn)]() mutable { +- is_running_ = true; ++ [state, f = std::forward(fn)]() mutable { ++ state->is_running = true; + f(); +- is_running_ = false; ++ // io_context::run() has returned, so it is now safe to restart the ++ // context. This handles self-initiated shutdown: shutdown_owned() ++ // cannot restart while run() is still active on this thread, so the ++ // restart is deferred until here. ++ state->restart(); ++ state->is_running = false; + }); + } + +@@ -74,8 +84,8 @@ class IoContextRunner + } + + io_thread_.request_stop(); +- io_work_.reset(); +- io_.stop(); ++ state_->release_work_guard(); ++ state_->stop(); + } + + void join_owned() +@@ -90,35 +100,64 @@ class IoContextRunner + + if (std::this_thread::get_id() == io_thread_.get_id()) { + // Cannot join from the same thread; detach so destructor can't terminate. ++ // The worker keeps the shared state alive until it completes. + io_thread_.detach(); + return; + } + + io_thread_.join(); + } + +- void reset_owned() +- { +- if (!owns_thread_) { +- return; +- } +- +- io_.restart(); +- } +- + void shutdown_owned() + { + stop_owned(); + join_owned(); +- reset_owned(); ++ // The io_context is restarted by the worker thread once run() has ++ // returned (see start()). On the self-close path join_owned() detaches, ++ // and restarting here while run() is still active on this thread would ++ // be undefined behaviour. + } + + private: +- std::shared_ptr io_owner_; +- asio::io_context & io_; +- std::unique_ptr> io_work_; ++ class State ++ { ++public: ++ explicit State(asio::io_context * shared_io) ++ : io_owner_(shared_io ? nullptr : std::make_shared()), ++ io_ref_(shared_io ? *shared_io : *io_owner_), ++ io_work_(shared_io ? nullptr : ++ std::make_unique>( ++ asio::make_work_guard(io_ref_))) ++ {} ++ ++ [[nodiscard]] asio::io_context & io() ++ { ++ return io_ref_; ++ } ++ ++ void release_work_guard() ++ { ++ io_work_.reset(); ++ } ++ ++ void stop() ++ { ++ io_ref_.stop(); ++ } ++ ++ void restart() ++ { ++ io_ref_.restart(); ++ } ++ ++ std::shared_ptr io_owner_; ++ asio::io_context & io_ref_; ++ std::unique_ptr> io_work_; ++ std::atomic is_running{false}; ++ }; ++ + bool owns_thread_; +- std::atomic is_running_; ++ std::shared_ptr state_; + std::jthread io_thread_; + }; + diff --git a/extras/patches/mavros-router-parent-lifetime.patch b/extras/patches/mavros-router-parent-lifetime.patch new file mode 100644 index 00000000..d7b0fa7f --- /dev/null +++ b/extras/patches/mavros-router-parent-lifetime.patch @@ -0,0 +1,83 @@ +--- a/mavros/include/mavros/mavros_router.hpp ++++ b/mavros/include/mavros/mavros_router.hpp +@@ -94,7 +94,8 @@ + remote_addrs.emplace(broadcast_addr); + } + +- std::shared_ptr parent; ++ // The router owns its endpoints; the reverse link must not keep it alive. ++ std::weak_ptr parent; + + uint32_t id; // id of the endpoint + Type link_type; // class of the endpoint +--- a/mavros/src/lib/mavros_router.cpp ++++ b/mavros/src/lib/mavros_router.cpp +@@ -445,7 +445,10 @@ + void Endpoint::recv_message(const mavlink_message_t * msg, const Framing framing) + { + rcpputils::assert_true(msg, "msg not nullptr"); +- // rcpputils::assert_true(this->parent, "parent not nullptr"); ++ auto nh = this->parent.lock(); ++ if (!nh) { ++ return; ++ } + + const addr_t sysid_addr = msg->sysid << 8; + const addr_t sysid_compid_addr = (msg->sysid << 8) | msg->compid; +@@ -464,7 +467,6 @@ + this->stale_addrs.erase(sysid_compid_addr); + } + +- auto & nh = this->parent; + if (new_sysid_addr || new_sysid_compid_addr) { + // A new remote was learned: flag the router's reverse index so the next + // routed message rebuilds. A relaxed store is enough -- the flag is just a +@@ -494,7 +496,7 @@ + + std::pair MAVConnEndpoint::open() + { +- auto nh = this->parent; ++ auto nh = this->parent.lock(); + if (!nh) { + return {false, "parent not set"}; + } +@@ -599,7 +601,7 @@ + + std::pair ROSEndpoint::open() + { +- auto & nh = this->parent; ++ auto nh = this->parent.lock(); + if (!nh) { + return {false, "parent not set"}; + } +@@ -644,7 +646,11 @@ + return; + } + +- rmsg->header.stamp = this->parent->now(); ++ auto nh = this->parent.lock(); ++ if (!nh) { ++ return; ++ } ++ rmsg->header.stamp = nh->now(); + rmsg->header.frame_id = from_frame_id; + + if (ok) { +@@ -652,7 +658,7 @@ +- } else if (auto & nh = this->parent) { ++ } else { + RCLCPP_ERROR(nh->get_logger(), "message conversion error"); + } + } + + void ROSEndpoint::ros_recv_message(mavros_msgs::msg::Mavlink::UniquePtr rmsg) + { +@@ -668,7 +674,7 @@ +- } else if (auto & nh = this->parent) { ++ } else if (auto nh = this->parent.lock()) { + RCLCPP_ERROR(nh->get_logger(), "message conversion error"); + } + } + + void ROSEndpoint::diag_run(diagnostic_updater::DiagnosticStatusWrapper & stat) + { diff --git a/extras/patches/ros-gz-bridge-callback-lifetime.patch b/extras/patches/ros-gz-bridge-callback-lifetime.patch new file mode 100644 index 00000000..2063b745 --- /dev/null +++ b/extras/patches/ros-gz-bridge-callback-lifetime.patch @@ -0,0 +1,53 @@ +--- a/ros_gz_bridge/src/factory.hpp ++++ b/ros_gz_bridge/src/factory.hpp +@@ -112,10 +112,11 @@ + } + } + ++ auto logger = ros_node->get_logger(); + auto ros_type = ros_type_name_; + auto gz_type = gz_type_name_; + std::function, const rclcpp::MessageInfo &)> fn = +- [self_pub_gids, gz_pub, ros_type, gz_type, ros_node]( ++ [self_pub_gids, gz_pub, ros_type, gz_type, logger]( + std::shared_ptr ros_msg, + const rclcpp::MessageInfo & msg_info) mutable + { +@@ -126,7 +127,7 @@ + return; + } + } +- ros_callback(ros_msg, gz_pub, ros_type, gz_type, ros_node); ++ ros_callback(ros_msg, gz_pub, ros_type, gz_type, logger); + }; + + auto options = rclcpp::SubscriptionOptions(); +@@ -151,9 +152,9 @@ + return; + } + std::function subCb = +- [this, pub, gz_to_ros_parameters](const GZ_T & _msg) +- { +- this->gz_callback(_msg, pub, gz_to_ros_parameters); ++ [pub, gz_to_ros_parameters](const GZ_T & _msg) ++ { ++ Factory::gz_callback(_msg, pub, gz_to_ros_parameters); + }; + + // Ignore messages that are published from this bridge. +@@ -169,13 +170,13 @@ + gz::transport::Node::Publisher & gz_pub, + const std::string & ros_type_name, + const std::string & gz_type_name, +- rclcpp::Node::SharedPtr ros_node) ++ const rclcpp::Logger & logger) + { + GZ_T gz_msg; + convert_ros_to_gz(*ros_msg, gz_msg); + gz_pub.Publish(gz_msg); + RCLCPP_INFO_ONCE( +- ros_node->get_logger(), ++ logger, + "Passing message from ROS %s to Gazebo %s (showing msg only once per type)", + ros_type_name.c_str(), gz_type_name.c_str()); + } diff --git a/extras/prepare-image-assets.py b/extras/prepare-image-assets.py new file mode 100644 index 00000000..559c9f26 --- /dev/null +++ b/extras/prepare-image-assets.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""Prepare a versioned Fuel cache at image build time; verify it offline at runtime.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import shutil +import subprocess +import tempfile +import time +from urllib.parse import unquote, urlsplit +import xml.etree.ElementTree as ET + + +def digest(path): + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def model_path(url): + uri = urlsplit(url) + parts = unquote(uri.path).strip("/").split("/") + if ( + uri.scheme != "https" + or uri.netloc not in ("fuel.gazebosim.org", "fuel.ignitionrobotics.org") + or uri.query + or uri.fragment + or len(parts) != 5 + or parts[0] != "1.0" + or parts[2] != "models" + or not parts[4].isdigit() + or int(parts[4]) < 1 + or any(p in ("", ".", "..") for p in parts) + ): + raise ValueError(f"Expected a pinned public Fuel model URL: {url}") + return PurePosixPath(uri.netloc, parts[1].lower(), "models", parts[3].lower(), parts[4]) + + +def hashes(root): + if not root.is_dir() or root.is_symlink(): + raise RuntimeError(f"Missing asset directory: {root}") + result = {} + for path in sorted(root.rglob("*")): + if path.is_symlink(): + raise RuntimeError(f"Unexpected asset symlink: {path}") + if path.is_file(): + result[path.relative_to(root).as_posix()] = digest(path) + return result + + +def verify_asset(cache, asset): + relative = model_path(asset["url"]) + if str(relative) != asset["cache_path"]: + raise ValueError("Fuel URL/cache path mismatch") + if not asset.get("license", {}).get("url"): + raise ValueError("Missing asset license attribution") + if "model.sdf" not in asset["files"] or "model.config" not in asset["files"]: + raise ValueError("Incomplete asset lock") + actual = hashes(cache / relative) + if actual != asset["files"]: + changed = sorted( + k + for k in actual.keys() | asset["files"].keys() + if actual.get(k) != asset["files"].get(k) + ) + raise RuntimeError(f"Asset integrity mismatch: {relative}: {changed[:8]}") + + +def verify_dependencies(cache, assets): + """Reject missing nested HTTP resources, including versioned material files.""" + locked = {a["cache_path"] for a in assets} + for asset in assets: + for sdf in (cache / asset["cache_path"]).rglob("*.sdf"): + for element in ET.parse(sdf).iter(): + text = (element.text or "").strip() + if not text.startswith(("https://fuel.", "http://fuel.")): + continue + uri = urlsplit(text) + parts = unquote(uri.path).strip("/").split("/") + model_url = f"{uri.scheme}://{uri.netloc}/{'/'.join(parts[:5])}" + relative = model_path(model_url) + if str(relative) not in locked: + raise RuntimeError(f"Unpinned nested dependency: {text}") + if len(parts) > 5: + if parts[5] != "files" or any(p in (".", "..") for p in parts[6:]): + raise RuntimeError(f"Invalid nested resource: {text}") + target = cache / relative / "/".join(parts[6:]) + if not target.exists(): + raise RuntimeError(f"Missing nested resource: {text}") + + +def prepare_asset(cache, asset): + relative = model_path(asset["url"]) + target = cache / relative + if target.exists(): + verify_asset(cache, asset) + return + # Retrying a build-time transfer is not retrying a failed simulation trial. + # Isolated staging prevents a partial download from appearing ready. + for attempt in range(1, 4): + with tempfile.TemporaryDirectory(prefix="fuel-stage-", dir=cache.parent) as tmp: + env = dict(os.environ, GZ_FUEL_CACHE_PATH=tmp) + try: + run = subprocess.run( + ["gz", "fuel", "download", "-v", "1", "-u", asset["url"]], + env=env, + capture_output=True, + text=True, + timeout=300, + ) + if run.returncode: + raise RuntimeError(f"Fuel downloader exit {run.returncode}") + verify_asset(Path(tmp), asset) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree(Path(tmp) / relative, target) + print(f"Prepared {relative}", flush=True) + return + except (subprocess.TimeoutExpired, RuntimeError) as error: + # Do not emit temporary signed referral URLs from Fuel logs. + message = re.sub(r"https?://\S+\?\S+", "[URL query redacted]", str(error)) + print(f"Asset transfer {attempt}/3 failed: {relative}: {message}", flush=True) + if attempt == 3: + raise + time.sleep(5) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cache", required=True, type=Path) + parser.add_argument("--lock", required=True, type=Path) + parser.add_argument("--verify", action="store_true", help="No downloads or writes to cache") + parser.add_argument("--receipt", type=Path) + args = parser.parse_args() + lock = json.loads(args.lock.read_text()) + if lock["schema"] != 1 or not lock["assets"]: + raise ValueError("Unknown or empty Fuel asset lock") + if not args.verify: + args.cache.mkdir(parents=True, exist_ok=True) + for asset in lock["assets"]: + if not args.verify: + prepare_asset(args.cache, asset) + verify_asset(args.cache, asset) + verify_dependencies(args.cache, lock["assets"]) + receipt = { + "status": "PASS", + "lock_sha256": digest(args.lock), + "assets": len(lock["assets"]), + "files": sum(len(a["files"]) for a in lock["assets"]), + "nested_dependencies_present": True, + "verification_downloads": 0, + } + if args.receipt: + args.receipt.write_text(json.dumps(receipt, indent=2) + "\n") + print(json.dumps(receipt), flush=True) + + +if __name__ == "__main__": + main() diff --git a/gazebo/dave_gz_model_plugins/src/OceanCurrentModelPlugin.cc b/gazebo/dave_gz_model_plugins/src/OceanCurrentModelPlugin.cc index 05a34732..aa9b5d4c 100644 --- a/gazebo/dave_gz_model_plugins/src/OceanCurrentModelPlugin.cc +++ b/gazebo/dave_gz_model_plugins/src/OceanCurrentModelPlugin.cc @@ -121,7 +121,9 @@ void OceanCurrentModelPlugin::Configure( // Check if ROS is initialized; if not, initialize it if (!rclcpp::ok()) { - rclcpp::init(0, nullptr); + // Gazebo owns process signals. A plugin must not replace its handlers or + // invalidate ROS while Gazebo is still running update/transport callbacks. + rclcpp::init(0, nullptr, rclcpp::InitOptions(), rclcpp::SignalHandlerOptions::None); } // Initialize the ROS 2 node diff --git a/gazebo/dave_gz_multibeam_sonar/multibeam_sonar/MultibeamSonarSensor.cc b/gazebo/dave_gz_multibeam_sonar/multibeam_sonar/MultibeamSonarSensor.cc index 07ae98db..283a8ecc 100644 --- a/gazebo/dave_gz_multibeam_sonar/multibeam_sonar/MultibeamSonarSensor.cc +++ b/gazebo/dave_gz_multibeam_sonar/multibeam_sonar/MultibeamSonarSensor.cc @@ -400,7 +400,9 @@ bool MultibeamSonarSensor::Implementation::InitializeBeamArrangement(MultibeamSo if (!rclcpp::ok()) { - rclcpp::init(0, nullptr); + // Gazebo owns process signals. A plugin must not replace its handlers or + // invalidate ROS while Gazebo is still running update/transport callbacks. + rclcpp::init(0, nullptr, rclcpp::InitOptions(), rclcpp::SignalHandlerOptions::None); } this->ros_node_ = std::make_shared("multibeam_sonar_node"); diff --git a/gazebo/dave_gz_sensor_plugins/CMakeLists.txt b/gazebo/dave_gz_sensor_plugins/CMakeLists.txt index 06ee9345..acc1a918 100644 --- a/gazebo/dave_gz_sensor_plugins/CMakeLists.txt +++ b/gazebo/dave_gz_sensor_plugins/CMakeLists.txt @@ -103,6 +103,10 @@ ament_environment_hooks( # Testing setup if(BUILD_TESTING) + find_package(ament_cmake_gtest REQUIRED) + ament_add_gtest(test_underwater_camera test/test_underwater_camera.cc) + target_include_directories(test_underwater_camera PRIVATE include) + target_link_libraries(test_underwater_camera UnderwaterCamera) find_package(ament_lint_auto REQUIRED) ament_lint_auto_find_test_dependencies() endif() diff --git a/gazebo/dave_gz_sensor_plugins/include/dave_gz_sensor_plugins/UnderwaterCamera.hh b/gazebo/dave_gz_sensor_plugins/include/dave_gz_sensor_plugins/UnderwaterCamera.hh index 91725041..357eacc0 100644 --- a/gazebo/dave_gz_sensor_plugins/include/dave_gz_sensor_plugins/UnderwaterCamera.hh +++ b/gazebo/dave_gz_sensor_plugins/include/dave_gz_sensor_plugins/UnderwaterCamera.hh @@ -62,6 +62,11 @@ private: struct PrivateData; std::unique_ptr dataPtr; + + // Queued transport callbacks may outlive Unsubscribe(). They must not retain + // an unguarded pointer to the plugin after its destructor starts. + struct CallbackState; + std::shared_ptr callbackState; }; } // namespace dave_gz_sensor_plugins diff --git a/gazebo/dave_gz_sensor_plugins/package.xml b/gazebo/dave_gz_sensor_plugins/package.xml index 2b621e59..9c349318 100644 --- a/gazebo/dave_gz_sensor_plugins/package.xml +++ b/gazebo/dave_gz_sensor_plugins/package.xml @@ -12,6 +12,7 @@ geometry_msgs ament_cmake ament_lint_auto + ament_cmake_gtest dave_interfaces protobuf protobuf @@ -21,4 +22,4 @@ ament_cmake - \ No newline at end of file + diff --git a/gazebo/dave_gz_sensor_plugins/src/UnderwaterCamera.cc b/gazebo/dave_gz_sensor_plugins/src/UnderwaterCamera.cc index 57d5a01e..5b3a59dc 100644 --- a/gazebo/dave_gz_sensor_plugins/src/UnderwaterCamera.cc +++ b/gazebo/dave_gz_sensor_plugins/src/UnderwaterCamera.cc @@ -73,7 +73,7 @@ struct UnderwaterCamera::PrivateData gz::msgs::Image lastImage; /// \brief Depth to range lookup table (LUT) - float * depth2rangeLUT; + float * depth2rangeLUT = nullptr; /// \brief Attenuation constants per channel (RGB) float attenuation[3]; @@ -87,10 +87,34 @@ struct UnderwaterCamera::PrivateData float max_range; }; -UnderwaterCamera::UnderwaterCamera() : dataPtr(std::make_unique()) {} +struct UnderwaterCamera::CallbackState +{ + std::mutex mutex; + UnderwaterCamera * owner = nullptr; +}; + +UnderwaterCamera::UnderwaterCamera() +: dataPtr(std::make_unique()), callbackState(std::make_shared()) +{ + this->callbackState->owner = this; +} UnderwaterCamera::~UnderwaterCamera() { + // Drain an active callback before freeing the LUT, and make already-queued + // callbacks harmless. Do not hold this lock while calling transport APIs. + { + std::lock_guard lock(this->callbackState->mutex); + this->callbackState->owner = nullptr; + } + if (!this->dataPtr->image_topic.empty()) + { + this->dataPtr->gz_node.Unsubscribe(this->dataPtr->image_topic); + } + if (!this->dataPtr->depth_image_topic.empty()) + { + this->dataPtr->gz_node.Unsubscribe(this->dataPtr->depth_image_topic); + } if (this->dataPtr->depth2rangeLUT) { delete[] this->dataPtr->depth2rangeLUT; @@ -128,7 +152,9 @@ void UnderwaterCamera::Configure( if (!rclcpp::ok()) { - rclcpp::init(0, nullptr); + // Gazebo owns process signals. A plugin must not replace its handlers or + // invalidate ROS while Gazebo is still running update/transport callbacks. + rclcpp::init(0, nullptr, rclcpp::InitOptions(), rclcpp::SignalHandlerOptions::None); } std::string rosNodeName = sensorSdf.Name() + "_node"; @@ -276,21 +302,36 @@ void UnderwaterCamera::Configure( this->dataPtr->background[0] = (unsigned char)_sdf->Get("backgroundB"); } - // Gazebo camera subscriber + // Construct the publisher before making callbacks visible to transport. + this->dataPtr->image_pub = this->ros_node_->create_publisher( + this->dataPtr->simulated_image_topic, 1); + + // Capture the lifetime gate, not an unguarded `this` pointer. + const auto state = this->callbackState; std::function camera_callback = - std::bind(&UnderwaterCamera::CameraCallback, this, std::placeholders::_1); + [state](const gz::msgs::Image & image) + { + std::lock_guard lock(state->mutex); + if (state->owner) + { + state->owner->CameraCallback(image); + } + }; this->dataPtr->gz_node.Subscribe(this->dataPtr->image_topic, camera_callback); // Gazebo depth image subscriber std::function depth_callback = - std::bind(&UnderwaterCamera::DepthImageCallback, this, std::placeholders::_1); + [state](const gz::msgs::Image & image) + { + std::lock_guard lock(state->mutex); + if (state->owner) + { + state->owner->DepthImageCallback(image); + } + }; this->dataPtr->gz_node.Subscribe(this->dataPtr->depth_image_topic, depth_callback); - - // ROS2 publisher - this->dataPtr->image_pub = this->ros_node_->create_publisher( - this->dataPtr->simulated_image_topic, 1); } cv::Mat UnderwaterCamera::ConvertGazeboToOpenCV(const gz::msgs::Image & gz_image) @@ -317,10 +358,19 @@ cv::Mat UnderwaterCamera::ConvertGazeboToOpenCV(const gz::msgs::Image & gz_image throw std::runtime_error("Unsupported pixel format"); } - // Create OpenCV Mat header that uses the same memory as the Gazebo image data + const size_t rowBytes = static_cast(gz_image.width()) * CV_ELEM_SIZE(cv_type); + const size_t stride = gz_image.step(); + if ( + gz_image.width() == 0 || gz_image.height() == 0 || stride < rowBytes || + static_cast(stride) * gz_image.height() > gz_image.data().size()) + { + return {}; + } + + // Respect Gazebo's row stride, including padded depth images. cv::Mat cv_image( gz_image.height(), gz_image.width(), cv_type, - const_cast(reinterpret_cast(gz_image.data().data()))); + const_cast(reinterpret_cast(gz_image.data().data())), stride); // Optionally convert color space if needed (e.g., RGB to BGR) if (gz_image.pixel_format_type() == gz::msgs::PixelFormatType::RGB_INT8) @@ -339,6 +389,10 @@ void UnderwaterCamera::CameraCallback(const gz::msgs::Image & msg) { std::lock_guard lock(this->dataPtr->mutex_); + if (!this->ros_node_ || !rclcpp::ok(this->ros_node_->get_node_base_interface()->get_context())) + { + return; + } if (!this->dataPtr->depth2rangeLUT) { gzerr << "Depth2range LUT not initialized" << std::endl; @@ -355,14 +409,29 @@ void UnderwaterCamera::CameraCallback(const gz::msgs::Image & msg) } else { + // RGB and depth arrive independently. Do not index a missing, truncated, + // or differently-sized depth frame (including during startup/teardown). + const auto & depth = this->dataPtr->lastDepth; + if ( + depth.pixel_format_type() != gz::msgs::PixelFormatType::R_FLOAT32 || + depth.width() != this->dataPtr->width || depth.height() != this->dataPtr->height || + msg.width() != this->dataPtr->width || msg.height() != this->dataPtr->height) + { + return; + } // Convert Gazebo image to OpenCV image cv::Mat image = this->ConvertGazeboToOpenCV(msg); // Convert depth image to OpenCV image using the ConvertGazeboToOpenCV function cv::Mat depth_image = this->ConvertGazeboToOpenCV(this->dataPtr->lastDepth); - // Create output image - cv::Mat output_image = this->ConvertGazeboToOpenCV(this->dataPtr->lastImage); + if (image.empty() || image.type() != CV_8UC3 || depth_image.empty()) + { + return; + } + + // Own the output buffer instead of writing through a protobuf const view. + cv::Mat output_image(image.rows, image.cols, CV_8UC3); // Simulate underwater cv::Mat simulated_image = this->SimulateUnderwater(image, depth_image, output_image); @@ -440,9 +509,31 @@ cv::Mat UnderwaterCamera::SimulateUnderwater( void UnderwaterCamera::PostUpdate( const gz::sim::UpdateInfo & _info, const gz::sim::EntityComponentManager & _ecm) { + if (!this->ros_node_) + { + return; + } + const auto context = this->ros_node_->get_node_base_interface()->get_context(); + if (!rclcpp::ok(context)) + { + return; + } if (!_info.paused) { - rclcpp::spin_some(this->ros_node_); + try + { + rclcpp::spin_some(this->ros_node_); + } + catch (const rclcpp::exceptions::RCLError &) + { + // Another embedded ROS component may shut down the shared context. + // Do not mask an error while the context is still valid. + if (rclcpp::ok(context)) + { + throw; + } + return; + } if (_info.iterations % 1000 == 0) { diff --git a/gazebo/dave_gz_sensor_plugins/src/UsblTransceiver.cc b/gazebo/dave_gz_sensor_plugins/src/UsblTransceiver.cc index 4ed1b637..5680fe0f 100644 --- a/gazebo/dave_gz_sensor_plugins/src/UsblTransceiver.cc +++ b/gazebo/dave_gz_sensor_plugins/src/UsblTransceiver.cc @@ -109,7 +109,9 @@ void UsblTransceiver::Configure( if (!rclcpp::ok()) { - rclcpp::init(0, nullptr); + // Gazebo owns process signals. A plugin must not replace its handlers or + // invalidate ROS while Gazebo is still running update/transport callbacks. + rclcpp::init(0, nullptr, rclcpp::InitOptions(), rclcpp::SignalHandlerOptions::None); } this->ros_node_ = std::make_shared("usbl_transceiver_node"); diff --git a/gazebo/dave_gz_sensor_plugins/src/UsblTransponder.cc b/gazebo/dave_gz_sensor_plugins/src/UsblTransponder.cc index 62899fa8..f2adf6f3 100644 --- a/gazebo/dave_gz_sensor_plugins/src/UsblTransponder.cc +++ b/gazebo/dave_gz_sensor_plugins/src/UsblTransponder.cc @@ -91,7 +91,9 @@ void UsblTransponder::Configure( if (!rclcpp::ok()) { - rclcpp::init(0, nullptr); + // Gazebo owns process signals. A plugin must not replace its handlers or + // invalidate ROS while Gazebo is still running update/transport callbacks. + rclcpp::init(0, nullptr, rclcpp::InitOptions(), rclcpp::SignalHandlerOptions::None); } this->dataPtr->ecm = &_ecm; diff --git a/gazebo/dave_gz_sensor_plugins/src/sea_pressure_sensor.cc b/gazebo/dave_gz_sensor_plugins/src/sea_pressure_sensor.cc index 68a1227a..1d6f0620 100644 --- a/gazebo/dave_gz_sensor_plugins/src/sea_pressure_sensor.cc +++ b/gazebo/dave_gz_sensor_plugins/src/sea_pressure_sensor.cc @@ -67,7 +67,9 @@ void SubseaPressureSensorPlugin::Configure( { if (!rclcpp::ok()) { - rclcpp::init(0, nullptr); + // Gazebo owns process signals. A plugin must not replace its handlers or + // invalidate ROS while Gazebo is still running update/transport callbacks. + rclcpp::init(0, nullptr, rclcpp::InitOptions(), rclcpp::SignalHandlerOptions::None); } gzdbg << "dave_gz_sensor_plugins::SubseaPressureSensorPlugin::Configure on entity: " << _entity diff --git a/gazebo/dave_gz_sensor_plugins/test/test_underwater_camera.cc b/gazebo/dave_gz_sensor_plugins/test/test_underwater_camera.cc new file mode 100644 index 00000000..179cc84c --- /dev/null +++ b/gazebo/dave_gz_sensor_plugins/test/test_underwater_camera.cc @@ -0,0 +1,130 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dave_gz_sensor_plugins/UnderwaterCamera.hh" + +using dave_gz_sensor_plugins::UnderwaterCamera; + +TEST(UnderwaterCamera, UnconfiguredUpdateAndDestruction) +{ + UnderwaterCamera camera; + gz::sim::EntityComponentManager ecm; + gz::sim::UpdateInfo info; + camera.PostUpdate(info, ecm); +} + +TEST(UnderwaterCamera, PaddedDepthRows) +{ + UnderwaterCamera camera; + gz::msgs::Image image; + image.set_width(2); + image.set_height(2); + image.set_step(12); + image.set_pixel_format_type(gz::msgs::PixelFormatType::R_FLOAT32); + const float data[] = {1, 2, 99, 3, 4, 99}; + image.set_data(data, sizeof(data)); + const auto mat = camera.ConvertGazeboToOpenCV(image); + ASSERT_FALSE(mat.empty()); + EXPECT_FLOAT_EQ(mat.at(1, 0), 3); + EXPECT_FLOAT_EQ(mat.at(1, 1), 4); +} + +TEST(UnderwaterCamera, TruncatedOrMissingImageIsRejected) +{ + UnderwaterCamera camera; + gz::msgs::Image image; + image.set_pixel_format_type(gz::msgs::PixelFormatType::RGB_INT8); + EXPECT_TRUE(camera.ConvertGazeboToOpenCV(image).empty()); + image.set_width(2); + image.set_height(2); + image.set_step(6); + image.set_data(std::string(11, '\0')); + EXPECT_TRUE(camera.ConvertGazeboToOpenCV(image).empty()); + image.set_step(5); + image.set_data(std::string(12, '\0')); + EXPECT_TRUE(camera.ConvertGazeboToOpenCV(image).empty()); +} + +TEST(UnderwaterCamera, RgbToBgrIsUnchanged) +{ + UnderwaterCamera camera; + gz::msgs::Image image; + image.set_width(1); + image.set_height(1); + image.set_step(3); + image.set_pixel_format_type(gz::msgs::PixelFormatType::RGB_INT8); + const unsigned char data[] = {10, 20, 30}; + image.set_data(data, sizeof(data)); + const auto mat = camera.ConvertGazeboToOpenCV(image); + ASSERT_FALSE(mat.empty()); + EXPECT_EQ(mat.at(0, 0), cv::Vec3b(30, 20, 10)); +} + +TEST(UnderwaterCamera, ConfigurePreservesSignalHandlerAndMissingDepthIsSafe) +{ + // A Gazebo plugin must not take ownership of the enclosing process's SIGINT. + auto handler = +[](int) {}; + auto previous = std::signal(SIGINT, handler); + { + UnderwaterCamera camera; + gz::sim::EntityComponentManager ecm; + gz::sim::EventManager events; + auto world = ecm.CreateEntity(); + ecm.CreateComponent(world, gz::sim::components::World()); + ecm.CreateComponent(world, gz::sim::components::Name("camera_test")); + sdf::Camera config; + config.SetImageWidth(2); + config.SetImageHeight(2); + config.SetNearClip(0.1); + config.SetFarClip(100); + config.SetLensIntrinsicsFx(2); + config.SetLensIntrinsicsFy(2); + config.SetLensIntrinsicsCx(1); + config.SetLensIntrinsicsCy(1); + sdf::Sensor sensor; + sensor.SetName("test_camera"); + sensor.SetType(sdf::SensorType::RGBD_CAMERA); + sensor.SetTopic("/camera_unit_test"); + sensor.SetCameraSensor(config); + auto entity = ecm.CreateEntity(); + ecm.CreateComponent(entity, gz::sim::components::RgbdCamera(sensor)); + auto plugin = std::make_shared(); + plugin->SetName("plugin"); + camera.Configure(entity, plugin, ecm, events); + EXPECT_EQ(std::signal(SIGINT, handler), handler); + + gz::msgs::Image rgb; + rgb.set_width(2); + rgb.set_height(2); + rgb.set_step(6); + rgb.set_pixel_format_type(gz::msgs::PixelFormatType::RGB_INT8); + rgb.set_data(std::string(12, '\x20')); + camera.CameraCallback(rgb); + camera.CameraCallback(rgb); // No depth has arrived; must not index it. + + cv::Mat color(2, 2, CV_8UC3, cv::Scalar(90, 120, 150)); + cv::Mat depth(2, 2, CV_32FC1, cv::Scalar(3)); + cv::Mat output(2, 2, CV_8UC3); + camera.SimulateUnderwater(color, depth, output); + // At the principal point, range equals depth. Default attenuation is 1/30. + for (int channel = 0; channel < 3; ++channel) + { + const auto expected = (90 + 30 * channel) * std::exp(-3.0 / 30.0); + EXPECT_NEAR(output.at(1, 1)[channel], expected, 1.0); + } + + // Explicit shutdown by another component must also be safe for PostUpdate. + rclcpp::shutdown(); + gz::sim::UpdateInfo info; + camera.PostUpdate(info, ecm); + } + std::signal(SIGINT, previous); +} diff --git a/gazebo/dave_ros_gz_plugins/src/DVLBridge.cc b/gazebo/dave_ros_gz_plugins/src/DVLBridge.cc index b15a65b9..b28bf869 100644 --- a/gazebo/dave_ros_gz_plugins/src/DVLBridge.cc +++ b/gazebo/dave_ros_gz_plugins/src/DVLBridge.cc @@ -49,7 +49,9 @@ void DVLBridge::Configure( if (!rclcpp::ok()) { - rclcpp::init(0, nullptr); + // Gazebo owns process signals. A plugin must not replace its handlers or + // invalidate ROS while Gazebo is still running update/transport callbacks. + rclcpp::init(0, nullptr, rclcpp::InitOptions(), rclcpp::SignalHandlerOptions::None); } this->ros_node_ = std::make_shared("dvl_bridge_node"); diff --git a/gazebo/dave_ros_gz_plugins/src/OceanCurrentPlugin.cc b/gazebo/dave_ros_gz_plugins/src/OceanCurrentPlugin.cc index 59841b3b..b9908cef 100644 --- a/gazebo/dave_ros_gz_plugins/src/OceanCurrentPlugin.cc +++ b/gazebo/dave_ros_gz_plugins/src/OceanCurrentPlugin.cc @@ -102,7 +102,9 @@ void OceanCurrentPlugin::Configure( { if (!rclcpp::ok()) { - rclcpp::init(0, nullptr); + // Gazebo owns process signals. A plugin must not replace its handlers or + // invalidate ROS while Gazebo is still running update/transport callbacks. + rclcpp::init(0, nullptr, rclcpp::InitOptions(), rclcpp::SignalHandlerOptions::None); // gzerr << "ROS 2 has not been properly initialized. Please make sure you have initialized your // ROS 2 environment."; } diff --git a/gazebo/dave_ros_gz_plugins/src/SphericalCoords.cc b/gazebo/dave_ros_gz_plugins/src/SphericalCoords.cc index 72aceed2..e9d57f0c 100644 --- a/gazebo/dave_ros_gz_plugins/src/SphericalCoords.cc +++ b/gazebo/dave_ros_gz_plugins/src/SphericalCoords.cc @@ -55,7 +55,9 @@ void SphericalCoords::Configure( if (!rclcpp::ok()) { - rclcpp::init(0, nullptr); + // Gazebo owns process signals. A plugin must not replace its handlers or + // invalidate ROS while Gazebo is still running update/transport callbacks. + rclcpp::init(0, nullptr, rclcpp::InitOptions(), rclcpp::SignalHandlerOptions::None); } this->ros_node_ = std::make_shared("sc_node"); @@ -225,9 +227,29 @@ bool SphericalCoords::SetOriginSphericalCoord( void SphericalCoords::PostUpdate( const gz::sim::UpdateInfo & _info, const gz::sim::EntityComponentManager & _ecm) { + if (!this->ros_node_) + { + return; + } + const auto context = this->ros_node_->get_node_base_interface()->get_context(); + if (!rclcpp::ok(context)) + { + return; + } if (!_info.paused) { - rclcpp::spin_some(this->ros_node_); + try + { + rclcpp::spin_some(this->ros_node_); + } + catch (const rclcpp::exceptions::RCLError &) + { + if (rclcpp::ok(context)) + { + throw; + } + return; + } if (_info.iterations % 1000 == 0) { diff --git a/tools/candidate-images-20260930.json b/tools/candidate-images-20260930.json new file mode 100644 index 00000000..437e27a4 --- /dev/null +++ b/tools/candidate-images-20260930.json @@ -0,0 +1,24 @@ +{ + "repository": "ioeslab/posim", + "source_commit": "abad9d70de843474478de5ef55371c049e40deef", + "image_revision": "32eedbacf781a0fd1e517481571e76bfaab73c51", + "fuel_lock_sha256": "9776a35839f1797769f7ba3a80dfe046366493889838d8e78cd40c16c9ab31fa", + "images": { + "arm64": { + "image_id": "sha256:72179187c96a801184a15ab9cdaf3ca9714d3717ce04e97cd2ec60f34d83edb8", + "run_id": 36670174461, + "attempt": 2, + "tag": "validation-pr5-abad9d70-arm64-rdp", + "artifact": "posim-arm64-runtime-2", + "image_identity": "index_digest" + }, + "amd64": { + "image_id": "sha256:72d00bb8e32725bd8e4fe91b8f21de770d4e4f4753f93d596bce4cc97d018cc5", + "run_id": 36670174383, + "attempt": 1, + "tag": "validation-pr5-abad9d70-amd64", + "artifact": "posim-amd64-runtime-1", + "image_identity": "config_digest" + } + } +} diff --git a/tools/candidate-publication.py b/tools/candidate-publication.py new file mode 100644 index 00000000..b9939331 --- /dev/null +++ b/tools/candidate-publication.py @@ -0,0 +1,315 @@ +#!/usr/bin/env python3 +"""Guard publication of the exact prevalidated PR #5 images to candidate tags only.""" + +import argparse +import csv +import hashlib +import json +import os +from pathlib import Path +import re +import shlex +import subprocess +import sys +import urllib.error +import urllib.parse +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +LOCK = json.loads(Path(__file__).with_name("candidate-images-20260930.json").read_text()) +ACCEPT = ", ".join( + [ + "application/vnd.oci.image.index.v1+json", + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.list.v2+json", + "application/vnd.docker.distribution.manifest.v2+json", + ] +) +sys.path.insert(0, str(ROOT / "extras/ci")) +from image_smoke import fatal_log_lines # noqa: E402 + + +def read(path): + return json.loads(path.read_text()) + + +def require(condition, message): + if not condition: + raise RuntimeError(message) + + +def docker(*args): + return subprocess.check_output(["docker", *args], text=True).strip() + + +def request_json(url, headers=None): + with urllib.request.urlopen( + urllib.request.Request(url, headers=headers or {}), timeout=60 + ) as r: + return json.load(r), dict(r.headers) + + +def registry_manifest(reference, token): + url = f"https://registry-1.docker.io/v2/{LOCK['repository']}/manifests/{reference}" + request = urllib.request.Request( + url, headers={"Authorization": f"Bearer {token}", "Accept": ACCEPT} + ) + with urllib.request.urlopen(request, timeout=60) as response: + raw = response.read() + calculated = "sha256:" + hashlib.sha256(raw).hexdigest() + require( + response.headers.get("Docker-Content-Digest") == calculated, + "Registry manifest digest mismatch", + ) + return json.loads(raw), calculated + + +def registry_image(arch, allow_missing=False): + """Anonymous registry read; do not print or persist the short-lived pull token.""" + target = LOCK["images"][arch] + require( + re.fullmatch(r"validation-pr5-abad9d70-(arm64-rdp|amd64)", target["tag"]), + "Not a candidate tag", + ) + auth, _ = request_json( + "https://auth.docker.io/token?" + + urllib.parse.urlencode( + { + "service": "registry.docker.io", + "scope": f"repository:{LOCK['repository']}:pull", + } + ) + ) + token = auth["token"] + try: + manifest, top_digest = registry_manifest(target["tag"], token) + except urllib.error.HTTPError as error: + if error.code == 404 and allow_missing: + return {"exists": False, "tag": target["tag"]} + raise + platform_digest = top_digest + if "manifests" in manifest: + selected = [ + m + for m in manifest["manifests"] + if m.get("platform", {}).get("os") == "linux" + and m["platform"].get("architecture") == arch + ] + require(len(selected) == 1, "Missing or ambiguous platform in registry index") + platform_digest = selected[0]["digest"] + manifest, actual = registry_manifest(platform_digest, token) + require(actual == platform_digest, "Platform manifest digest mismatch") + config_digest = manifest.get("config", {}).get("digest", "") + require(re.fullmatch(r"sha256:[0-9a-f]{64}", config_digest), "Invalid config digest") + # Docker's containerd store reports the index/manifest digest as image Id; + # the legacy image store reports the config digest. Compare the recorded kind. + if target["image_identity"] == "index_digest": + observed_id = top_digest + elif target["image_identity"] == "config_digest": + observed_id = config_digest + else: + raise RuntimeError("Unknown image identity kind") + require( + observed_id == target["image_id"], + "Registry tag contains another image; refuse overwrite or validation", + ) + return { + "exists": True, + "repository": LOCK["repository"], + "tag": target["tag"], + "tag_digest": top_digest, + "platform_digest": platform_digest, + "image_id": target["image_id"], + "image_identity": target["image_identity"], + "config_digest": config_digest, + "architecture": arch, + "anonymous_read": True, + } + + +def rows(path): + with path.open() as stream: + return list(csv.DictReader(stream)) + + +def verify_evidence(arch, evidence): + image = LOCK["images"][arch] + # This workflow may add publication tooling, never silently change image inputs. + subprocess.run( + [ + "git", + "diff", + "--exit-code", + LOCK["source_commit"], + "--", + ".docker", + ".dockerignore", + "extras", + "gazebo", + "models", + "examples", + "dave_interfaces", + ], + cwd=ROOT, + check=True, + stdout=subprocess.DEVNULL, + ) + run, _ = request_json( + f"https://api.github.com/repos/IOES-Lab/POSIM/actions/runs/{image['run_id']}", + { + "Authorization": "Bearer " + os.environ["GH_TOKEN"], + "Accept": "application/vnd.github+json", + }, + ) + require( + run["head_sha"] == LOCK["source_commit"] + and run["conclusion"] == "success" + and run["run_attempt"] == image["attempt"], + "Validated run provenance changed", + ) + (evidence / "verified-run-metadata.json").write_text( + json.dumps( + {k: run[k] for k in ["id", "head_sha", "conclusion", "run_attempt", "html_url"]}, + indent=2, + ) + + "\n" + ) + require( + (evidence / "tested-image-id.txt").read_text().strip() == image["image_id"], + "Artifact image differs", + ) + saved = read(evidence / "image-inspect.json")[0] + actual = json.loads(docker("image", "inspect", image["image_id"]))[0] + for info in [saved, actual]: + require( + info["Id"] == image["image_id"] and info["Architecture"] == arch, + "Local/artifact image mismatch", + ) + if image["image_identity"] == "index_digest": + require( + info.get("Descriptor", {}).get("digest") == image["image_id"], + "Containerd index identity differs", + ) + require( + info["Config"]["Labels"]["org.opencontainers.image.revision"] + == LOCK["image_revision"], + "Image source label differs", + ) + require(read(evidence / "inventory/result.json")["status"] == "PASS", "Inventory did not pass") + receipt = read(evidence / "inventory/fuel-assets-receipt.json") + require( + receipt + == { + "status": "PASS", + "lock_sha256": LOCK["fuel_lock_sha256"], + "assets": 12, + "files": 105, + "nested_dependencies_present": True, + "verification_downloads": 0, + }, + "Fuel verification differs", + ) + cases = { + line.split("\t")[0]: line.split("\t")[1] + for line in (ROOT / "extras/ci/quickstarts.tsv").read_text().splitlines() + if line and not line.startswith("#") + } + repeat = [ + "spherical_world", + "camera", + "rexrov_waves", + "ocean_current", + "sea_pressure", + "bluerov2", + "bluerov2_heavy", + ] + expected = list(cases) + [f"{case}-r{n}" for n in range(2, 11) for case in repeat] + summary = rows(evidence / "summary.csv") + require( + len(summary) == 78 + and {r["case"] for r in summary} == set(expected + ["inventory"]) + and all(r["status"] == "PASS" for r in summary), + "Connected acceptance incomplete", + ) + offline = rows(evidence / "offline-summary.csv") + extra = [f"camera-offline-{n}" for n in range(1, 6)] + require( + len(offline) == 5 + and {r["case"] for r in offline} == set(extra) + and all(r["status"] == "PASS" for r in offline), + "Offline acceptance incomplete", + ) + for record in expected + extra: + directory = evidence / record + result = read(directory / "result.json") + case = "camera" if record in extra else re.sub(r"-r\d+$", "", record) + require(result["command"] == shlex.split(cases[case]), f"Command mismatch: {record}") + require( + result["status"] == "PASS" + and result["checks"] + and all(v is True for v in result["checks"].values()) + and not result.get("error") + and not result.get("faults"), + f"Failed trial: {record}", + ) + require( + not fatal_log_lines((directory / "launch.log").read_text()), + f"Raw child failure: {record}", + ) + if record in extra: + require( + read(directory / "docker-network-mode.json") == "none" + and (directory / "container-exit-code.txt").read_text().strip() == "0", + "Offline evidence differs", + ) + churn = read(evidence / "inventory/transport_churn/results.json") + require( + len(churn) == 5 + and all( + r["pass"] and r["publisher_rc"] == r["subscriber_rc"] == 0 and not r["timeouts"] + for r in churn + ), + "Transport regression incomplete", + ) + return { + "status": "PASS", + "image_id": image["image_id"], + "architecture": arch, + "connected_trials": 77, + "offline_trials": 5, + "run_id": image["run_id"], + "run_attempt": image["attempt"], + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "mode", choices=["evidence", "registry-before", "registry-after", "pulled"] + ) + parser.add_argument("architecture", choices=["arm64", "amd64"]) + parser.add_argument("--evidence", type=Path) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if args.mode == "evidence": + result = verify_evidence(args.architecture, args.evidence) + else: + result = registry_image(args.architecture, args.mode == "registry-before") + if args.mode == "pulled": + info = json.loads( + docker("image", "inspect", LOCK["repository"] + "@" + result["tag_digest"]) + )[0] + require( + info["Id"] == result["image_id"] and info["Architecture"] == args.architecture, + "Pulled image differs from tested image", + ) + result["pulled_image_matches"] = True + result["layer_cache_note"] = ( + "docker pull resolves the registry digest; pre-existing local layers may be reused." + ) + args.output.write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result)) + + +if __name__ == "__main__": + main() diff --git a/tools/check-published-candidate.sh b/tools/check-published-candidate.sh new file mode 100644 index 00000000..7c2e9592 --- /dev/null +++ b/tools/check-published-candidate.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Fresh containers from the registry-resolved image, not the source checkout. +set -euo pipefail +REF="${1:?repository@digest required}" +ARCH="${2:?architecture required}" +IMAGE_ID="${3:?prevalidated image ID required}" +RESULTS="${4:?results directory required}" +CONFIG_ID="${5:?registry-verified configuration digest required}" +[[ "$REF" =~ ^ioeslab/posim@sha256:[0-9a-f]{64}$ ]] +[[ "$IMAGE_ID" =~ ^sha256:[0-9a-f]{64}$ ]] +[[ "$CONFIG_ID" =~ ^sha256:[0-9a-f]{64}$ ]] +[[ "$ARCH" == arm64 || "$ARCH" == amd64 ]] +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +CHECKS="$ROOT/extras/ci" +mkdir -p "$RESULTS" +RESULTS="$(cd "$RESULTS" && pwd)" +chmod 777 "$RESULTS" +docker image inspect "$REF" > "$RESULTS/image-inspect.json" +test "$(docker image inspect --format '{{.Id}}' "$REF")" = "$IMAGE_ID" +test "$(docker image inspect --format '{{.Architecture}}' "$REF")" = "$ARCH" +printf '%s\n' "$REF" > "$RESULTS/pulled-reference.txt" +printf 'case,status\n' > "$RESULTS/summary.csv" +CURRENT="" +cleanup() { if [[ -n "$CURRENT" ]]; then docker rm -f "$CURRENT" >/dev/null 2>&1 || true; fi; } +trap cleanup EXIT +trap 'exit 130' INT TERM +FAILED=0 +CASES="inventory $(awk -F '\t' '!/^#/ && NF {print $1}' "$CHECKS/quickstarts.tsv") camera-offline" +for RECORD in $CASES; do + CASE="$RECORD" + NETWORK=bridge + if [[ "$RECORD" == camera-offline ]]; then CASE=camera; NETWORK=none; fi + mkdir -p "$RESULTS/$RECORD" + chmod 777 "$RESULTS/$RECORD" + CURRENT="posim-pull-${GITHUB_RUN_ID:-$$}-$ARCH-$RECORD" + docker create --init --name "$CURRENT" --platform "linux/$ARCH" --network "$NETWORK" \ + --shm-size=1g --entrypoint bash -e ROS_DOMAIN_ID=124 -e GZ_IP=127.0.0.1 \ + -e LIBGL_ALWAYS_SOFTWARE=1 -e QT_QPA_PLATFORM=offscreen -e "CASE=$CASE" -e "RECORD=$RECORD" \ + -v "$CHECKS:/checks:ro" -v "$CHECKS/../patches:/patches:ro" \ + -v "$CHECKS/../fuel:/fuel:ro" -v "$RESULTS:/results" "$REF" -c ' + set -eo pipefail + source /opt/ros/lyrical/setup.bash + export POSIM_WORKSPACE="${DAVE_WS:-$DAVE_UNDERLAY}" + source "$POSIM_WORKSPACE/install/setup.bash" + cd /tmp + exec python3 /checks/image_smoke.py "$CASE" --record "$RECORD" + ' > "$RESULTS/$RECORD/container-id.txt" + docker inspect --format '{{json .HostConfig.NetworkMode}}' "$CURRENT" \ + > "$RESULTS/$RECORD/docker-network-mode.json" + container_image="$(docker inspect --format '{{.Image}}' "$CURRENT")" + printf '%s\n' "$container_image" > "$RESULTS/$RECORD/container-image-id.txt" + requested_image="$(docker inspect --format '{{.Config.Image}}' "$CURRENT")" + printf '%s\n' "$requested_image" > "$RESULTS/$RECORD/container-requested-image.txt" + test "$requested_image" = "$REF" + # Both digests were verified against the same registry manifest before create. + [[ "$container_image" == "$IMAGE_ID" || "$container_image" == "$CONFIG_ID" ]] + test "$(docker inspect --format '{{.HostConfig.NetworkMode}}' "$CURRENT")" = "$NETWORK" + docker start -a "$CURRENT" || true + CODE="$(docker inspect --format '{{.State.ExitCode}}' "$CURRENT")" + STATE="$(docker inspect --format '{{.State.Status}}' "$CURRENT")" + printf '%s\n' "$CODE" > "$RESULTS/$RECORD/container-exit-code.txt" + printf '%s\n' "$STATE" > "$RESULTS/$RECORD/container-state.txt" + if [[ "$STATE" == exited && "$CODE" == 0 ]] && \ + python3 -c 'import json,sys; sys.exit(json.load(open(sys.argv[1]))["status"] != "PASS")' \ + "$RESULTS/$RECORD/result.json"; then + printf '%s,PASS\n' "$RECORD" >> "$RESULTS/summary.csv" + else + printf '%s,FAIL\n' "$RECORD" >> "$RESULTS/summary.csv" + FAILED=1 + fi + cleanup + CURRENT="" +done +cat "$RESULTS/summary.csv" +exit "$FAILED" diff --git a/tools/diagnose-camera-readiness.py b/tools/diagnose-camera-readiness.py new file mode 100644 index 00000000..aebad4cd --- /dev/null +++ b/tools/diagnose-camera-readiness.py @@ -0,0 +1,133 @@ +"""Capture a camera startup stall after the original readiness deadline expires. + +Runs only in a disposable diagnostic container. Additional verbosity and GDB +attachment make these diagnostic observations, never acceptance replacements. +""" + +import importlib.util +import json +import os +from pathlib import Path +import signal +import subprocess +import sys + + +def descendants(parent): + records = {} + for folder in Path("/proc").glob("[0-9]*"): + try: + status = (folder / "status").read_text() + ppid = int( + next(line.split()[1] for line in status.splitlines() if line.startswith("PPid:")) + ) + command = ( + (folder / "cmdline").read_bytes().replace(b"\0", b" ").decode(errors="replace") + ) + records[int(folder.name)] = {"ppid": ppid, "command": command} + except (OSError, StopIteration, ProcessLookupError): + continue + found = {parent} + while True: + added = {pid for pid, rec in records.items() if rec["ppid"] in found} - found + if not added: + break + found.update(added) + return {pid: records[pid] for pid in found if pid != parent and pid in records} + + +def capture_stall(parent, output): + processes = descendants(parent) + (output / "processes-at-deadline.json").write_text(json.dumps(processes, indent=2)) + targets = [pid for pid, rec in processes.items() if "gz-sim-main" in rec["command"]] + captures = [] + # Prevent launch from escalating signals while the failed server is inspected. + try: + os.kill(parent, signal.SIGSTOP) + except ProcessLookupError: + return captures + try: + for pid in targets: + for item in ("maps", "status", "wchan"): + try: + (output / f"proc-{pid}-{item}.txt").write_text( + Path(f"/proc/{pid}/{item}").read_text() + ) + except OSError: + pass + with (output / f"gdb-{pid}.log").open("w") as log: + try: + result = subprocess.run( + [ + "gdb", + "-q", + "-batch", + "-iex", + "set debuginfod enabled off", + "-p", + str(pid), + "-ex", + "set pagination off", + "-ex", + "thread apply all bt", + "-ex", + "info sharedlibrary", + "-ex", + "detach", + ], + stdout=log, + stderr=subprocess.STDOUT, + timeout=30, + ) + captures.append({"pid": pid, "returncode": result.returncode}) + except subprocess.TimeoutExpired: + captures.append({"pid": pid, "error": "GDB timed out"}) + finally: + try: + os.kill(parent, signal.SIGCONT) + except ProcessLookupError: + pass + return captures + + +def main(): + spec = importlib.util.spec_from_file_location("image_smoke", "/checks/image_smoke.py") + smoke = importlib.util.module_from_spec(spec) + spec.loader.exec_module(smoke) + original_wait = smoke.wait_for_entity + original_exercise = smoke.exercise + captured = [] + + def traced_wait(out, world, entity, proc, deadline): + ready = original_wait(out, world, entity, proc, deadline) + if not ready and proc.poll() is None: + captured.extend(capture_stall(proc.pid, out)) + return ready + + def verbose_exercise(out, case): + case = list(case) + case[1] += " debug:=true verbosity_level:=4" + return original_exercise(out, case) + + smoke.wait_for_entity = traced_wait + smoke.exercise = verbose_exercise + sys.argv = ["image_smoke.py", "camera", "--record", "camera-readiness"] + status = smoke.main() + cache = Path.home() / ".gz/fuel" + files = [] + if cache.exists(): + for path in cache.rglob("*"): + if path.is_file(): + files.append({"path": str(path.relative_to(cache)), "bytes": path.stat().st_size}) + Path("/results/fuel-cache-files.json").write_text(json.dumps(files, indent=2)) + Path("/results/camera-readiness-summary.json").write_text( + json.dumps( + {"diagnostic_status": status, "captures": captured, "scope": "Not acceptance"}, + indent=2, + ) + ) + return status + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/diagnose-gazebo-shutdown.py b/tools/diagnose-gazebo-shutdown.py new file mode 100644 index 00000000..4e2b75f9 --- /dev/null +++ b/tools/diagnose-gazebo-shutdown.py @@ -0,0 +1,133 @@ +"""Capture a slow Gazebo shutdown in a disposable diagnostic container. + +Pausing launch temporarily prevents its SIGTERM escalation from destroying the +stack while GDB attaches. These instrumented runs are NOT acceptance trials. +""" + +import importlib.util +import json +import os +from pathlib import Path +import signal +import subprocess +import sys +import threading + + +def descendants(parent): + records = {} + for folder in Path("/proc").glob("[0-9]*"): + try: + status = (folder / "status").read_text() + ppid = int( + next(line.split()[1] for line in status.splitlines() if line.startswith("PPid:")) + ) + command = ( + (folder / "cmdline").read_bytes().replace(b"\0", b" ").decode(errors="replace") + ) + records[int(folder.name)] = {"ppid": ppid, "command": command} + except (OSError, StopIteration): + continue + found = {parent} + while True: + added = {pid for pid, rec in records.items() if rec["ppid"] in found} - found + if not added: + break + found.update(added) + return {pid: records[pid] for pid in found if pid != parent and pid in records} + + +original_wait = subprocess.Popen.wait +original_killpg = os.killpg +captured = [] +current_output = None + + +def delayed_stack(parent, done, output): + if done.wait(3): + return + processes = descendants(parent) + targets = [ + pid + for pid, rec in processes.items() + if "gz sim" in rec["command"] or "gz-sim" in rec["command"] + ] + if not targets: + return + try: + os.kill(parent, signal.SIGSTOP) + except ProcessLookupError: + return + try: + output.mkdir(parents=True, exist_ok=True) + (output / "processes-at-3s.json").write_text(json.dumps(processes, indent=2)) + for pid in targets: + with (output / f"gdb-{pid}.log").open("w") as log: + result = subprocess.run( + [ + "gdb", + "-q", + "-batch", + "-p", + str(pid), + "-ex", + "set pagination off", + "-ex", + "thread apply all bt", + "-ex", + "detach", + ], + stdout=log, + stderr=subprocess.STDOUT, + timeout=15, + ) + captured.append( + {"pid": pid, "gdb_status": result.returncode, "record": str(output)} + ) + except subprocess.TimeoutExpired: + (output / "gdb-timeout.txt").write_text("Debugger attachment timed out.\n") + finally: + try: + os.kill(parent, signal.SIGCONT) + except ProcessLookupError: + pass + + +active = {} + + +def traced_killpg(pid, sig): + original_killpg(pid, sig) + if sig == signal.SIGINT and current_output is not None: + done = threading.Event() + worker = threading.Thread(target=delayed_stack, args=(pid, done, current_output)) + active[pid] = (done, worker) + worker.start() + + +def traced_wait(proc, timeout=None): + try: + return original_wait(proc, timeout) + finally: + if proc.pid in active: + done, worker = active.pop(proc.pid) + done.set() + worker.join(timeout=20) + + +spec = importlib.util.spec_from_file_location("image_smoke", "/checks/image_smoke.py") +smoke = importlib.util.module_from_spec(spec) +spec.loader.exec_module(smoke) +os.killpg = traced_killpg +subprocess.Popen.wait = traced_wait +for n in range(1, int(os.environ.get("POSIM_DIAGNOSTIC_TRIALS", "20")) + 1): + record = f"gazebo-shutdown-{n}" + current_output = Path("/results") / record + sys.argv = ["image_smoke.py", "spherical_world", "--record", record] + smoke.main() + if captured: + break +Path("/results/gazebo-stack-captures.json").write_text(json.dumps(captured, indent=2)) +Path("/results/diagnostic-outcome.txt").write_text( + f"Captured {len(captured)} slow-shutdown process stacks; diagnostic only, not acceptance.\n" +) diff --git a/tools/diagnose-router-lifetime.sh b/tools/diagnose-router-lifetime.sh new file mode 100644 index 00000000..f3cb6ed0 --- /dev/null +++ b/tools/diagnose-router-lifetime.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Controlled diagnostic in a disposable existing-image container, not a release. +set -eo pipefail +WS="${POSIM_MAVROS_UNDERLAY:?}" +CHECK=/candidate/extras/ci/mavros_ownership +PATCH=/candidate/extras/patches/mavros-router-parent-lifetime.patch +test "$(cat "$WS/upstream-revision.txt")" = 22ae5b7cc7cdb4cb9c2070a8213c72dae445a23e +printf '%s\n' "$(sha256sum "$PATCH")" > /results/router-patch.sha256 +for phase in before after; do + if [[ "$phase" == after ]]; then + git -C "$WS/src/mavros" apply --check "$PATCH" + git -C "$WS/src/mavros" apply "$PATCH" + ( + cd "$WS" + export CMAKE_BUILD_PARALLEL_LEVEL=2 MAKEFLAGS=-j2 + colcon build --merge-install --packages-select mavros --executor sequential \ + --cmake-args -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release + ) > /results/router-rebuild.log 2>&1 + fi + # shellcheck disable=SC1090,SC1091 + source "$WS/install/setup.bash" + cmake -S "$CHECK" -B "/tmp/probe-$phase" > "/results/$phase-build.log" 2>&1 + cmake --build "/tmp/probe-$phase" --parallel 2 >> "/results/$phase-build.log" 2>&1 + status=0 + timeout 30 "/tmp/probe-$phase/posim_mavros_ownership_check" \ + > "/results/$phase-ownership.log" 2>&1 || status=$? + printf '%s\n' "$status" > "/results/$phase-ownership.exit" + if [[ "$phase" == before ]]; then + # A failing baseline must specifically exhibit both endpoint cycles. + test "$status" != 0 + grep -q 'mode=0 router_expired=1' /results/before-ownership.log + grep -q 'mode=1 router_expired=0' /results/before-ownership.log + grep -q 'mode=2 router_expired=0' /results/before-ownership.log + else + test "$status" = 0 + test "$(grep -c 'router_expired=1 all_endpoints_expired=1' /results/after-ownership.log)" = 3 + fi +done +# Same payload/connection/shutdown classifier; no debugger in these trials. +failures=0 +cd /tmp +for n in $(seq 1 5); do + for model in bluerov2 bluerov2_heavy; do + python3 /checks/image_smoke.py "$model" --record "router-patch-$model-$n" \ + || failures=$((failures + 1)) + done +done +printf 'Candidate-patch diagnostic only: 10 runtime trials, %s failures.\n' "$failures" \ + > /results/diagnostic-outcome.txt +test "$failures" = 0 diff --git a/tools/diagnose-transport-poll.sh b/tools/diagnose-transport-poll.sh new file mode 100644 index 00000000..fe2b96ee --- /dev/null +++ b/tools/diagnose-transport-poll.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Disposable-container diagnostic. Never installs into the release image. +set -eo pipefail +REVISION=82b10bdff114f77655c7f0cc856835179a674d6d +WORK=/tmp/posim-transport-comparison +PATCH=/candidate/extras/patches/gz-transport-poll-serialization.patch +mkdir -p "$WORK" /results/transport-poll +RESULTS=/results/transport-poll +printf '%s\n' "$REVISION" > "$RESULTS/upstream-revision.txt" +sha256sum "$PATCH" > "$RESULTS/patch.sha256" +dpkg-query -W libzmq5 ros-lyrical-gz-transport-vendor > "$RESULTS/packages.txt" +# shellcheck disable=SC1091 +source /opt/ros/lyrical/setup.bash +cmake -S /candidate/extras/ci/transport_shutdown -B "$WORK/probe" \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo > "$RESULTS/probe-build.log" 2>&1 +cmake --build "$WORK/probe" --parallel 2 >> "$RESULTS/probe-build.log" 2>&1 +sha256sum "$WORK/probe/posim_transport_churn" > "$RESULTS/probe.sha256" +git init "$WORK/source" > "$RESULTS/source-fetch.log" 2>&1 +git -C "$WORK/source" remote add origin https://github.com/gazebosim/gz-transport.git +git -C "$WORK/source" -c http.version=HTTP/1.1 fetch --depth 1 origin "$REVISION" \ + >> "$RESULTS/source-fetch.log" 2>&1 +git -C "$WORK/source" checkout --detach FETCH_HEAD >> "$RESULTS/source-fetch.log" 2>&1 +test "$(git -C "$WORK/source" rev-parse HEAD)" = "$REVISION" +for VARIANT in baseline patched; do + cp -a "$WORK/source" "$WORK/$VARIANT-src" + if [[ "$VARIANT" == patched ]]; then + git -C "$WORK/$VARIANT-src" apply --check "$PATCH" + git -C "$WORK/$VARIANT-src" apply "$PATCH" + fi + cmake -S "$WORK/$VARIANT-src" -B "$WORK/$VARIANT-build" \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo -DBUILD_TESTING=OFF -DSKIP_PYBIND11=ON \ + > "$RESULTS/$VARIANT-build.log" 2>&1 + cmake --build "$WORK/$VARIANT-build" --target gz-transport --parallel 2 \ + >> "$RESULTS/$VARIANT-build.log" 2>&1 +done +python3 /checks/transport_shutdown/run_pairs.py \ + --executable "$WORK/probe/posim_transport_churn" \ + --variant "baseline:$WORK/baseline-build/lib" \ + --variant "patched:$WORK/patched-build/lib" \ + --trials 5 --seconds 20 --output "$RESULTS/pairs" +# The driver retains all outcomes. A successful collection is not release acceptance. +printf 'Transport diagnostic completed; inspect transport-poll/pairs/summary.json. Not release acceptance.\n' \ + > /results/diagnostic-outcome.txt diff --git a/tools/instrument-gazebo-gdb.py b/tools/instrument-gazebo-gdb.py new file mode 100644 index 00000000..073a396d --- /dev/null +++ b/tools/instrument-gazebo-gdb.py @@ -0,0 +1,51 @@ +"""Wrap only the disposable diagnostic container's native Gazebo executable.""" + +from pathlib import Path +import shlex + + +def instrument(binary): + real = binary.with_name(binary.name + ".real") + if real.exists(): + raise RuntimeError("Refusing to wrap an already instrumented executable") + binary.rename(real) + # Disable automatic network symbol downloads BEFORE loading the executable. + # Fetch the one suspect library's symbols separately with a bounded timeout. + args = [ + "gdb", + "-q", + "-batch", + "--return-child-result", + "-iex", + "set debuginfod enabled off", + ] + for command in ( + "set pagination off", + "set confirm off", + "handle SIGINT nostop noprint pass", + "handle SIGPIPE nostop noprint pass", + "run", + "info sharedlibrary", + "thread apply all bt full", + ): + args.extend(["-ex", command]) + args.extend(["--args", str(real)]) + # cmdsim.rb requires stdout from --version to match its version exactly. + # GDB chatter in that probe prevents the server from ever being launched. + binary.write_text( + '#!/bin/bash\nif [[ "$#" == 1 && "$1" == --version ]]; then\n' + + " exec " + + shlex.quote(str(real)) + + ' "$@"\nfi\n' + + "exec " + + shlex.join(args) + + ' "$@"\n' + ) + binary.chmod(0o755) + + +if __name__ == "__main__": + matches = list(Path("/opt/ros/lyrical/opt/gz_sim_vendor/libexec/gz").glob("sim*/gz-sim-main")) + if len(matches) != 1: + raise RuntimeError(f"Expected one native Gazebo executable, got {matches}") + instrument(matches[0]) diff --git a/tools/instrument-mavros-gdb.py b/tools/instrument-mavros-gdb.py new file mode 100644 index 00000000..1cab6758 --- /dev/null +++ b/tools/instrument-mavros-gdb.py @@ -0,0 +1,27 @@ +"""Instrument only a disposable diagnostic container, never the production image.""" + +from pathlib import Path +import shlex + +from ament_index_python.packages import get_package_share_directory + +root = Path(get_package_share_directory("dave_robot_models")) +args = ["gdb", "-q", "-batch", "--return-child-result"] +for command in ( + "set pagination off", + "set confirm off", + "handle SIGINT nostop noprint pass", + "handle SIGPIPE nostop noprint pass", + "run", + "thread apply all bt", +): + args.extend(["-ex", command]) +args.append("--args") +prefix = shlex.join(args) +for vehicle in ("bluerov2", "bluerov2_heavy"): + path = root / "config" / vehicle / "robot_config.py" + text = path.read_text() + needle = 'executable="mavros_node",' + if text.count(needle) != 1: + raise RuntimeError(f"Expected one MAVROS node in {path}") + path.write_text(text.replace(needle, needle + "\n prefix=" + repr(prefix) + ",")) diff --git a/tools/runtime-diagnostics.sh b/tools/runtime-diagnostics.sh new file mode 100644 index 00000000..39c249f5 --- /dev/null +++ b/tools/runtime-diagnostics.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +# Diagnostic only: same installed image, with GDB and an instrumented launch. +# A green workflow means evidence was collected, not that runtime tests passed. +set -euo pipefail +IMAGE_ID="${1:?image ID required}" +ARCH="${2:?architecture required}" +RESULTS="${3:?results directory required}" +[[ "$IMAGE_ID" =~ ^sha256:[0-9a-f]{64}$ ]] +[[ "$ARCH" == arm64 || "$ARCH" == amd64 ]] +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +mkdir -p "$RESULTS" +RESULTS="$(cd "$RESULTS" && pwd)" +chmod 777 "$RESULTS" +docker image inspect "$IMAGE_ID" > "$RESULTS/base-image-inspect.json" +test "$(docker image inspect --format '{{.Architecture}}' "$IMAGE_ID")" = "$ARCH" +if [[ "${POSIM_DIAGNOSTIC_MODE:-}" == camera-fresh ]]; then + for n in $(seq 1 10); do + if ! POSIM_DIAGNOSTIC_MODE=camera-readiness \ + bash "$0" "$IMAGE_ID" "$ARCH" "$RESULTS/fresh-$n"; then + printf 'Camera diagnostic failed in fresh container %s; inspect the retained logs and stacks. Not acceptance.\n' "$n" \ + > "$RESULTS/diagnostic-outcome.txt" + exit 1 + fi + done + printf 'No readiness failure reproduced in 10 instrumented fresh containers. The original failure remains unresolved.\n' \ + > "$RESULTS/diagnostic-outcome.txt" + exit 0 +fi +if [[ "${POSIM_DIAGNOSTIC_MODE:-}" == gazebo-fresh ]]; then + for n in $(seq 1 20); do + POSIM_DIAGNOSTIC_MODE=gazebo-shutdown POSIM_DIAGNOSTIC_TRIALS=1 \ + bash "$0" "$IMAGE_ID" "$ARCH" "$RESULTS/fresh-$n" + if grep -q '"pid"' "$RESULTS/fresh-$n/gazebo-stack-captures.json"; then + printf 'Slow-shutdown evidence captured in fresh container %s; diagnostic only.\n' "$n" \ + > "$RESULTS/diagnostic-outcome.txt" + exit 0 + fi + done + printf 'No slow-shutdown stack in 20 fresh containers; earlier failures remain unresolved.\n' \ + > "$RESULTS/diagnostic-outcome.txt" + exit 0 +fi +if [[ "${POSIM_DIAGNOSTIC_MODE:-}" == assets-fresh ]]; then + failures=0 + for n in $(seq 1 10); do + POSIM_DIAGNOSTIC_MODE=asset-ready bash "$0" "$IMAGE_ID" "$ARCH" "$RESULTS/fresh-$n" \ + || failures=$((failures + 1)) + done + printf 'Asset readiness candidate, 10 fresh containers, %s failures; not release acceptance.\n' \ + "$failures" > "$RESULTS/diagnostic-outcome.txt" + test "$failures" = 0 + exit +fi +CONTAINER="posim-diagnose-${GITHUB_RUN_ID:-$$}" +trap 'docker rm -f "$CONTAINER" >/dev/null 2>&1 || true' EXIT +docker run --rm --init --name "$CONTAINER" --platform "linux/$ARCH" \ + --user root --shm-size=1g --cap-add SYS_PTRACE --security-opt seccomp=unconfined \ + --entrypoint bash -e ROS_DOMAIN_ID=123 -e GZ_IP=127.0.0.1 \ + -e POSIM_DIAGNOSTIC_TRIALS="${POSIM_DIAGNOSTIC_TRIALS:-20}" \ + -e POSIM_DIAGNOSTIC_MODE="${POSIM_DIAGNOSTIC_MODE:-gdb}" \ + -v "$ROOT:/candidate:ro" \ + -e LIBGL_ALWAYS_SOFTWARE=1 -e QT_QPA_PLATFORM=offscreen \ + -v "$ROOT/extras/ci:/checks:ro" -v "$ROOT/tools:/diagnostics:ro" -v "$RESULTS:/results" "$IMAGE_ID" -c ' + set -eo pipefail + dpkg-query -W > /results/packages-before.tsv + apt-get update > /results/debugger-install.log 2>&1 + apt-get install -y --no-install-recommends gdb >> /results/debugger-install.log 2>&1 + dpkg-query -W > /results/packages-after.tsv + source /opt/ros/lyrical/setup.bash + source "${DAVE_WS:-$DAVE_UNDERLAY}/install/setup.bash" + if [[ "$POSIM_DIAGNOSTIC_MODE" == camera-readiness ]]; then + cd /tmp + exec python3 /diagnostics/diagnose-camera-readiness.py + fi + if [[ "$POSIM_DIAGNOSTIC_MODE" == transport-poll ]]; then + exec bash /diagnostics/diagnose-transport-poll.sh + fi + if [[ "$POSIM_DIAGNOSTIC_MODE" == router-lifetime ]]; then + exec bash /diagnostics/diagnose-router-lifetime.sh + fi + if [[ "$POSIM_DIAGNOSTIC_MODE" == gazebo-shutdown ]]; then + exec python3 /diagnostics/diagnose-gazebo-shutdown.py + fi + if [[ "$POSIM_DIAGNOSTIC_MODE" == asset-ready ]]; then + share="$(ros2 pkg prefix --share dave_demos)" + cp /candidate/examples/dave_demos/launch/dave_*.launch.py "$share/launch/" + cd /tmp + exec python3 /checks/image_smoke.py spherical_world --record asset-ready + fi + if [[ "$POSIM_DIAGNOSTIC_MODE" == gazebo-segfault ]]; then + # Preserve the exact crashing library and its build ID for symbolization. + zmq="$(ldconfig -p | awk '\''/libzmq.so.5 / {print $NF}'\'')" + cp -L "$zmq" /results/libzmq.so.5 + readelf -n "$zmq" > /results/libzmq-build-id.txt + build_id="$(sed -n '\''s/.*Build ID: //p'\'' /results/libzmq-build-id.txt)" + if [[ "$build_id" =~ ^[0-9a-f]+$ ]] && + curl -fL --connect-timeout 10 --max-time 120 \ + "https://debuginfod.ubuntu.com/buildid/$build_id/debuginfo" \ + -o /results/libzmq.debuginfo 2> /results/symbol-download.log; then + mkdir -p "/usr/lib/debug/.build-id/${build_id:0:2}" + cp /results/libzmq.debuginfo "/usr/lib/debug/.build-id/${build_id:0:2}/${build_id:2}.debug" + else + printf "Symbol download unavailable; use the retained binary/build ID.\n" \ + >> /results/symbol-download.log + fi + unset DEBUGINFOD_URLS + gz sim --version > /results/gazebo-version-before.txt + python3 /diagnostics/instrument-gazebo-gdb.py + timeout 10 gz sim --version > /results/gazebo-version-after.txt + cmp /results/gazebo-version-before.txt /results/gazebo-version-after.txt + cd /tmp + for n in $(seq 1 20); do + record="gazebo-segfault-$n" + python3 /checks/image_smoke.py rexrov_waves --record "$record" || true + if grep -Eq "received signal SIGSEGV|Program terminated with signal SIGSEGV" \ + "/results/$record/launch.log"; then + printf "Captured Gazebo SIGSEGV in %s; diagnostic only.\n" "$record" \ + > /results/diagnostic-outcome.txt + exit 0 + fi + if ! python3 -c '\''import json,sys; sys.exit(not json.load(open(sys.argv[1]))["checks"].get("world_ready", False))'\'' \ + "/results/$record/result.json"; then + printf "Diagnostic invalid: world was not ready in %s; stopping instead of repeating.\n" \ + "$record" > /results/diagnostic-outcome.txt + exit 1 + fi + done + printf "No GDB-captured Gazebo SIGSEGV in 20 trials; original failure remains unresolved.\n" \ + > /results/diagnostic-outcome.txt + exit 0 + fi + python3 /diagnostics/instrument-mavros-gdb.py + cd /tmp + for n in $(seq 1 20); do + record="mavros-gdb-$n" + python3 /checks/image_smoke.py bluerov2_heavy --record "$record" || true + if grep -Eq "received signal SIGSEGV|Program terminated with signal SIGSEGV" \ + "/results/$record/launch.log"; then + printf "Captured SIGSEGV in %s; diagnostic run, not acceptance.\n" "$record" \ + > /results/diagnostic-outcome.txt + exit 0 + fi + done + printf "No GDB-captured SIGSEGV in 20 trials; this does not establish absence.\n" \ + > /results/diagnostic-outcome.txt + ' diff --git a/tools/test_candidate_publication.py b/tools/test_candidate_publication.py new file mode 100644 index 00000000..8fb2ab3c --- /dev/null +++ b/tools/test_candidate_publication.py @@ -0,0 +1,120 @@ +"""Use the recorded store identity; never overwrite another candidate image.""" + +import importlib.util +from pathlib import Path +import unittest +from unittest.mock import patch +import urllib.error + +SPEC = importlib.util.spec_from_file_location( + "candidate_publication", Path(__file__).with_name("candidate-publication.py") +) +publication = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(publication) + + +class CandidatePublicationTests(unittest.TestCase): + def resolve(self, responses, arch="amd64", **kwargs): + with patch.object( + publication, "request_json", return_value=({"token": "test-pull-token"}, {}) + ), patch.object(publication, "registry_manifest", side_effect=responses): + return publication.registry_image(arch, **kwargs) + + def index_responses(self): + top = publication.LOCK["images"]["arm64"]["image_id"] + platform = "sha256:" + "2" * 64 + config = "sha256:" + "4" * 64 + index = { + "manifests": [ + {"digest": platform, "platform": {"os": "linux", "architecture": "arm64"}}, + { + "digest": "sha256:" + "3" * 64, + "platform": {"os": "unknown", "architecture": "unknown"}, + }, + ] + } + return [(index, top), ({"config": {"digest": config}}, platform)] + + def test_tags_are_validation_only(self): + for arch, entry in publication.LOCK["images"].items(): + self.assertTrue(entry["tag"].startswith("validation-pr5-abad9d70-")) + self.assertNotIn("latest", entry["tag"]) + self.assertNotIn("main", entry["tag"]) + self.assertEqual(len(entry["image_id"]), 71) + self.assertIn(arch, ("arm64", "amd64")) + + def test_missing_tag_allowed_before_publication(self): + error = urllib.error.HTTPError("https://registry.example", 404, "missing", {}, None) + self.assertEqual(self.resolve([error], allow_missing=True)["exists"], False) + + def test_missing_tag_rejected_after_publication(self): + error = urllib.error.HTTPError("https://registry.example", 404, "missing", {}, None) + with self.assertRaises(urllib.error.HTTPError): + self.resolve([error]) + + def test_auth_and_network_failures_are_not_absence(self): + for status in (401, 403, 429, 500): + error = urllib.error.HTTPError("https://registry.example", status, "failure", {}, None) + with self.subTest(status=status), self.assertRaises(urllib.error.HTTPError): + self.resolve([error], allow_missing=True) + + def test_matching_legacy_config_permits_idempotent_reuse(self): + expected = publication.LOCK["images"]["amd64"]["image_id"] + r = self.resolve([({"config": {"digest": expected}}, "sha256:" + "1" * 64)]) + self.assertTrue(r["exists"]) + self.assertEqual(r["image_id"], expected) + self.assertEqual(r["config_digest"], expected) + self.assertEqual(r["image_identity"], "config_digest") + + def test_different_legacy_config_refuses_overwrite(self): + with self.assertRaisesRegex(RuntimeError, "another image"): + self.resolve( + [({"config": {"digest": "sha256:" + "0" * 64}}, "sha256:" + "1" * 64)], + allow_missing=True, + ) + + def test_containerd_index_is_not_the_config_digest(self): + r = self.resolve(self.index_responses(), arch="arm64") + self.assertEqual(r["tag_digest"], r["image_id"]) + self.assertNotEqual(r["config_digest"], r["image_id"]) + self.assertEqual(r["image_identity"], "index_digest") + + def test_platform_selection_ignores_attestation(self): + r = self.resolve(self.index_responses(), arch="arm64") + self.assertEqual(r["platform_digest"], "sha256:" + "2" * 64) + + def test_wrong_index_refuses_even_when_config_matches_old_assumption(self): + expected = publication.LOCK["images"]["arm64"]["image_id"] + with self.assertRaisesRegex(RuntimeError, "another image"): + self.resolve( + [({"config": {"digest": expected}}, "sha256:" + "1" * 64)], + arch="arm64", + allow_missing=True, + ) + + def test_ambiguous_platform_is_rejected(self): + descriptor = { + "digest": "sha256:" + "2" * 64, + "platform": {"os": "linux", "architecture": "arm64"}, + } + with self.assertRaisesRegex(RuntimeError, "ambiguous"): + self.resolve( + [({"manifests": [descriptor, descriptor]}, "sha256:" + "1" * 64)], + arch="arm64", + ) + + def test_wrong_platform_digest_is_rejected(self): + responses = self.index_responses() + responses[1] = (responses[1][0], "sha256:" + "3" * 64) + with self.assertRaisesRegex(RuntimeError, "digest mismatch"): + self.resolve(responses, arch="arm64") + + def test_missing_config_digest_is_rejected(self): + responses = self.index_responses() + responses[1] = ({}, responses[1][1]) + with self.assertRaisesRegex(RuntimeError, "Invalid config"): + self.resolve(responses, arch="arm64") + + +if __name__ == "__main__": + unittest.main()