Validate POSIM images and fix Docker runtime shutdown failures

.docker/lyrical.amd64.dockerfile

@@ -15,6 +15,19 @@ COPY extras /tmp/dave-extras
 RUN DAVE_EXTRAS_DIR=/tmp/dave-extras \
     bash /tmp/dave-extras/ros-lyrical-gz-jetty-install.sh
 
+# docker run/exec do not necessarily start an interactive shell. Keep these
+# paths in the image environment, not just the installer's ~/.bashrc hook.
+ENV PATH=/opt/ardusub_ws/ardupilot/build/sitl/bin:/opt/ardusub_ws/ardupilot/Tools/autotest:${PATH}
+ENV GZ_SIM_SYSTEM_PLUGIN_PATH=/opt/ardusub_ws/ardupilot_gazebo/build
+ENV GZ_SIM_RESOURCE_PATH=/opt/ardusub_ws/ardupilot_gazebo/models:/opt/ardusub_ws/ardupilot_gazebo/worlds
+ENV GEOGRAPHICLIB_GEOID_PATH=/usr/share/GeographicLib/geoids
+ENV POSIM_BRIDGE_UNDERLAY=/opt/posim_bridge_ws
+ENV POSIM_MAVROS_UNDERLAY=/opt/posim_mavros_ws
+ENV POSIM_TRANSPORT_UNDERLAY=/opt/posim_transport_ws
+RUN bash /tmp/dave-extras/build-image-transport.sh
+RUN bash /tmp/dave-extras/build-image-mavros.sh
+RUN bash /tmp/dave-extras/build-image-bridge.sh
+
 # Install QGroundControl.
 RUN mkdir -p /opt/QGC && cd /opt/QGC && \
     wget -O QGroundControl-x86_64.AppImage \
@@ -46,7 +59,7 @@ RUN apt-get update && \
     rm -rf /var/lib/apt/lists/*
 
 WORKDIR $DAVE_WS
-RUN . "/opt/ros/${ROS_DISTRO}/setup.sh" && \
+RUN . "$POSIM_BRIDGE_UNDERLAY/install/setup.sh" && \
     colcon build --merge-install --executor sequential --symlink-install
 
 RUN echo "source /opt/ros/${ROS_DISTRO}/setup.bash" >> /root/.bashrc && \
@@ -60,6 +73,15 @@ RUN touch /root/.dave_entrypoint && \
 
 WORKDIR /root
 
+# Resolve the Quickstart Fuel dependency closure during the build, not startup.
+ENV GZ_FUEL_CACHE_PATH=/opt/posim_fuel/cache
+COPY extras/fuel /opt/posim_fuel
+COPY extras/prepare-image-assets.py /opt/posim_fuel/prepare-image-assets.py
+RUN . "/opt/ros/${ROS_DISTRO}/setup.sh" && \
+    python3 /opt/posim_fuel/prepare-image-assets.py \
+      --cache "$GZ_FUEL_CACHE_PATH" --lock /opt/posim_fuel/quickstart-assets.lock.json \
+      --receipt /opt/posim_fuel/build-receipt.json
+
 LABEL org.opencontainers.image.title="POSIM" \
       org.opencontainers.image.description="Platform for Ocean Simulation" \
       org.opencontainers.image.source="https://github.com/IOES-Lab/POSIM" \

.docker/lyrical.arm64v8.dockerfile

@@ -109,8 +109,24 @@ RUN export ROS_APT_SOURCE_VERSION=$(curl -s https://api.github.com/repos/ros-inf
     apt install -y --no-install-recommends \
       ros-${ROS_DISTRO}-desktop ros-${ROS_DISTRO}-ros-gz \
       ros-${ROS_DISTRO}-image-view \
+      ros-${ROS_DISTRO}-mavros ros-${ROS_DISTRO}-mavros-msgs \
       python3-rosdep python3-vcstool python3-colcon-common-extensions
 
+ENV POSIM_BRIDGE_UNDERLAY=/opt/posim_bridge_ws
+ENV POSIM_MAVROS_UNDERLAY=/opt/posim_mavros_ws
+ENV POSIM_TRANSPORT_UNDERLAY=/opt/posim_transport_ws
+COPY extras/build-image-transport.sh /tmp/build-image-transport.sh
+COPY extras/build-image-bridge.sh /tmp/build-image-bridge.sh
+COPY extras/build-image-mavros.sh /tmp/build-image-mavros.sh
+COPY extras/patches /tmp/patches
+COPY extras/ci/bridge_ownership /tmp/ci/bridge_ownership
+COPY extras/ci/mavconn_self_close /tmp/ci/mavconn_self_close
+COPY extras/ci/mavros_ownership /tmp/ci/mavros_ownership
+COPY extras/ci/transport_shutdown /tmp/ci/transport_shutdown
+RUN bash /tmp/build-image-transport.sh
+RUN bash /tmp/build-image-mavros.sh
+RUN bash /tmp/build-image-bridge.sh
+
 # --- DAVE workspace ---
 # Build the exact checked-out revision supplied as the Docker build context.
 ENV DAVE_UNDERLAY=/home/$USER/dave_ws
@@ -132,7 +148,7 @@ RUN apt-get update && \
 
 USER $USER
 WORKDIR $DAVE_UNDERLAY
-RUN . "/opt/ros/${ROS_DISTRO}/setup.sh" && \
+RUN . "$POSIM_BRIDGE_UNDERLAY/install/setup.sh" && \
     colcon build --merge-install --executor sequential --symlink-install
 
 # --- ArduSub SITL (BlueROV2) — Python 3.14 compatibility shims required, see notes/ardusub-sitl-setup.md ---
@@ -141,9 +157,23 @@ USER root
 ARG ARDUSUB_COMMIT="30257f01185471ab4c1ac544e47d1b4437e44c98"
 ARG ARDUPILOT_GAZEBO_COMMIT="082a0fe231f6e63bc8d1598f1cba461d9e2ea7f5"
 WORKDIR /home/$USER
-RUN git clone --recurse-submodules https://github.com/ArduPilot/ardupilot.git && \
-    cd ardupilot && git fetch --tags && git checkout --detach "$ARDUSUB_COMMIT" && \
-    git submodule update --init --recursive
+# Fetch the pinned tree directly: cloning current HEAD and all history first
+# caused HTTP/2 early-EOF failures on the ARM64 runner.
+RUN set -eu; \
+    retry_git() { \
+      for attempt in 1 2 3; do \
+        if git -c http.version=HTTP/1.1 "$@"; then return 0; fi; \
+        echo "Git transfer attempt $attempt/3 failed" >&2; \
+        sleep 5; \
+      done; \
+      return 1; \
+    }; \
+    git init ardupilot && cd ardupilot && \
+    git remote add origin https://github.com/ArduPilot/ardupilot.git && \
+    retry_git fetch --depth 1 origin "$ARDUSUB_COMMIT" && \
+    git checkout --detach FETCH_HEAD && \
+    test "$(git rev-parse HEAD)" = "$ARDUSUB_COMMIT" && \
+    retry_git submodule update --init --recursive --depth 1 --jobs 2
 
 RUN mkdir -p /home/$USER/imp_shim && \
     printf 'import types\ndef new_module(name):\n    return types.ModuleType(name)\n' > /home/$USER/imp_shim/imp.py && \
@@ -228,6 +258,16 @@ COPY extras/docker-arm64-entrypoint.sh /usr/local/bin/dave-rdp-entrypoint
 RUN chmod 0755 /usr/local/bin/dave-rdp-entrypoint
 CMD ["/usr/local/bin/dave-rdp-entrypoint"]
 
+# Shared by the root Quickstart session and the unprivileged RDP desktop user.
+ENV GZ_FUEL_CACHE_PATH=/opt/posim_fuel/cache
+COPY extras/fuel /opt/posim_fuel
+COPY extras/prepare-image-assets.py /opt/posim_fuel/prepare-image-assets.py
+RUN . "/opt/ros/${ROS_DISTRO}/setup.sh" && \
+    python3 /opt/posim_fuel/prepare-image-assets.py \
+      --cache "$GZ_FUEL_CACHE_PATH" --lock /opt/posim_fuel/quickstart-assets.lock.json \
+      --receipt /opt/posim_fuel/build-receipt.json && \
+    chown -R $USER:$USER "$GZ_FUEL_CACHE_PATH"
+
 LABEL org.opencontainers.image.title="POSIM" \
       org.opencontainers.image.description="Platform for Ocean Simulation" \
       org.opencontainers.image.source="https://github.com/IOES-Lab/POSIM" \

.github/workflows/docker-amd64.yml

@@ -23,6 +23,7 @@ on:
 env:
   IMAGE_NAME: ioeslab/posim
   ROS_DISTRO: lyrical
+  VALIDATION_IMAGE: posim-validation:amd64-${{ github.run_id }}-${{ github.run_attempt }}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -48,13 +49,6 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Log in to Docker Hub
-        if: vars.POSIM_PUBLISH_IMAGES == 'true' && github.event_name != 'pull_request'
-        uses: docker/login-action@v4
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
       - name: Docker metadata
         id: meta
         uses: docker/metadata-action@v5
@@ -73,10 +67,52 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          push: ${{ vars.POSIM_PUBLISH_IMAGES == 'true' && github.event_name != 'pull_request' }}
-          tags: ${{ steps.meta.outputs.tags }}
+          push: false
+          load: true
+          tags: ${{ env.VALIDATION_IMAGE }}
           labels: ${{ steps.meta.outputs.labels }}
           file: .docker/lyrical.amd64.dockerfile
           platforms: linux/amd64
           build-args: |
             ROS_DISTRO=${{ env.ROS_DISTRO }}
+
+      - name: Validate installed image and Quickstarts
+        # 14 initial paths + 63 additional trials, including two SITL vehicles.
+        timeout-minutes: 75
+        run: |
+          bash extras/ci/docker_quickstarts.sh "$VALIDATION_IMAGE" linux/amd64 \
+            "$RUNNER_TEMP/posim-validation-${{ github.run_id }}-${{ github.run_attempt }}"
+
+      - name: Preserve runtime evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: posim-amd64-runtime-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/posim-validation-${{ github.run_id }}-${{ github.run_attempt }}
+          if-no-files-found: warn
+          retention-days: 14
+
+      - name: Log in to Docker Hub
+        if: >-
+          vars.POSIM_PUBLISH_IMAGES == 'true' && github.event_name != 'pull_request' &&
+          (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
+        uses: docker/login-action@v4
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Publish the tested image
+        if: >-
+          vars.POSIM_PUBLISH_IMAGES == 'true' && github.event_name != 'pull_request' &&
+          (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
+        env:
+          PUBLISH_TAGS: ${{ steps.meta.outputs.tags }}
+        run: |
+          results="$RUNNER_TEMP/posim-validation-${{ github.run_id }}-${{ github.run_attempt }}"
+          tested_id="$(cat "$results/tested-image-id.txt")"
+          test "$(docker image inspect --format '{{.Id}}' "$VALIDATION_IMAGE")" = "$tested_id"
+          while IFS= read -r tag; do
+            test -n "$tag" || continue
+            docker tag "$tested_id" "$tag"
+            docker push "$tag"
+          done <<< "$PUBLISH_TAGS"

.github/workflows/docker-arm64v8.yml

@@ -23,6 +23,7 @@ on:
 env:
   IMAGE_NAME: ioeslab/posim
   ROS_DISTRO: lyrical
+  VALIDATION_IMAGE: posim-validation:arm64-${{ github.run_id }}-${{ github.run_attempt }}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -43,6 +44,13 @@ jobs:
     steps:
       - name: Configure non-interactive Docker credentials
         run: |
+          # Preserve Desktop's user socket before isolating credentials.
+          docker_host="${DOCKER_HOST:-$(docker context inspect "$(docker context show)" \
+            --format '{{.Endpoints.docker.Host}}')}"
+          test -n "$docker_host"
+          echo "DOCKER_HOST=$docker_host" >> "$GITHUB_ENV"
+          echo "DOCKER_CONTEXT=" >> "$GITHUB_ENV"
+          docker --host "$docker_host" info --format '{{.ServerVersion}} {{.Architecture}}'
           docker_config="$RUNNER_TEMP/docker-config"
           mkdir -p "$docker_config"
           printf '{}\n' > "$docker_config/config.json"
@@ -60,28 +68,6 @@ jobs:
           # ARM64 build and works with Docker Desktop's containerd image store.
           driver: docker
 
-      - name: Configure Docker Hub authentication
-        if: vars.POSIM_PUBLISH_IMAGES == 'true' && github.event_name != 'pull_request'
-        env:
-          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
-          DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
-        run: |
-          # Avoid Docker Desktop's interactive macOS Keychain helper. BuildKit
-          # reads this short-lived Docker config directly during the push.
-          python3 - <<'PY'
-          import base64
-          import json
-          import os
-          from pathlib import Path
-
-          raw = f"{os.environ['DOCKERHUB_USERNAME']}:{os.environ['DOCKERHUB_TOKEN']}"
-          auth = base64.b64encode(raw.encode()).decode()
-          config = {"auths": {"https://index.docker.io/v1/": {"auth": auth}}}
-          path = Path(os.environ["DOCKER_CONFIG"]) / "config.json"
-          path.write_text(json.dumps(config), encoding="utf-8")
-          path.chmod(0o600)
-          PY
-
       - name: Docker metadata
         id: meta
         uses: docker/metadata-action@v5
@@ -100,14 +86,75 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          push: ${{ vars.POSIM_PUBLISH_IMAGES == 'true' && github.event_name != 'pull_request' }}
-          tags: ${{ steps.meta.outputs.tags }}
+          push: false
+          load: true
+          tags: ${{ env.VALIDATION_IMAGE }}
           labels: ${{ steps.meta.outputs.labels }}
           file: .docker/lyrical.arm64v8.dockerfile
           platforms: linux/arm64/v8
           build-args: |
             ROS_DISTRO=${{ env.ROS_DISTRO }}
 
+      - name: Validate installed image and Quickstarts
+        # 14 initial paths + 63 additional trials, including two SITL vehicles.
+        timeout-minutes: 75
+        run: |
+          bash extras/ci/docker_quickstarts.sh "$VALIDATION_IMAGE" linux/arm64/v8 \
+            "$RUNNER_TEMP/posim-validation-${{ github.run_id }}-${{ github.run_attempt }}"
+
+      - name: Preserve runtime evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: posim-arm64-runtime-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/posim-validation-${{ github.run_id }}-${{ github.run_attempt }}
+          if-no-files-found: warn
+          retention-days: 14
+
+      - name: Configure Docker Hub authentication
+        if: >-
+          vars.POSIM_PUBLISH_IMAGES == 'true' && github.event_name != 'pull_request' &&
+          (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
+        env:
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+          DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+        run: |
+          # Avoid Docker Desktop's interactive macOS Keychain helper. BuildKit
+          # reads this short-lived Docker config directly during the push.
+          python3 - <<'PY'
+          import base64
+          import json
+          import os
+          from pathlib import Path
+
+          raw = f"{os.environ['DOCKERHUB_USERNAME']}:{os.environ['DOCKERHUB_TOKEN']}"
+          auth = base64.b64encode(raw.encode()).decode()
+          config = {"auths": {"https://index.docker.io/v1/": {"auth": auth}}}
+          path = Path(os.environ["DOCKER_CONFIG"]) / "config.json"
+          path.write_text(json.dumps(config), encoding="utf-8")
+          path.chmod(0o600)
+          PY
+
+      - name: Publish the tested image
+        if: >-
+          vars.POSIM_PUBLISH_IMAGES == 'true' && github.event_name != 'pull_request' &&
+          (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
+        env:
+          PUBLISH_TAGS: ${{ steps.meta.outputs.tags }}
+        run: |
+          results="$RUNNER_TEMP/posim-validation-${{ github.run_id }}-${{ github.run_attempt }}"
+          tested_id="$(cat "$results/tested-image-id.txt")"
+          test "$(docker image inspect --format '{{.Id}}' "$VALIDATION_IMAGE")" = "$tested_id"
+          while IFS= read -r tag; do
+            test -n "$tag" || continue
+            docker tag "$tested_id" "$tag"
+            docker push "$tag"
+          done <<< "$PUBLISH_TAGS"
+
       - name: Clear Docker Hub credentials
-        if: always() && vars.POSIM_PUBLISH_IMAGES == 'true' && github.event_name != 'pull_request'
-        run: printf '{}\n' > "$DOCKER_CONFIG/config.json"
+        if: >-
+          always() && vars.POSIM_PUBLISH_IMAGES == 'true' && github.event_name != 'pull_request' &&
+          (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
+        run: |
+          config="$RUNNER_TEMP/docker-config/config.json"
+          if [[ -f "$config" ]]; then printf '{}\n' > "$config"; fi

.github/workflows/lint.yml

@@ -46,6 +46,9 @@ jobs:
         if: ${{ !cancelled() }}
         run: flake8 --ignore=E203,W503,E501 --exclude=tools/code_check .
 
+      - name: Test runtime log failure detection
+        run: python -m unittest discover -s extras/ci -p 'test_*.py'
+
       - name: Check C and C++ formatting
         if: ${{ !cancelled() }}
         run: |