CARRY: ci: format CVE scans and add Slack notification

.github/workflows/odh-cargo-deny-advisories.yml

@@ -57,3 +57,35 @@ jobs:
             CARGO="$cargo_bin" \
             PATH="$(dirname "$cargo_bin"):$(dirname "$deny_bin"):$PATH" \
             "$deny_bin" check advisories
+  notify-slack:
+    name: Notify Slack on failure
+    needs: advisories
+    if: >-
+      ${{ always() &&
+          needs.advisories.result == 'failure' &&
+          github.repository == 'red-hat-data-services/openshell' }}
+    runs-on: ubuntu-latest
+    permissions: {}
+    steps:
+      - name: Post failure notification
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          WORKFLOW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          if [ -z "${SLACK_WEBHOOK_URL:-}" ]; then
+            echo "::warning::SLACK_WEBHOOK_URL is not set; skipping Slack notification."
+            exit 0
+          fi
+
+          message="$(printf '%s\n%s' \
+            ':x: ODH Cargo Deny Advisories' \
+            "Run: $WORKFLOW_RUN_URL")"
+
+          jq -nc \
+            --arg text "$message" \
+            '{text: $text}' |
+            curl --fail --silent --show-error \
+              -X POST \
+              -H 'Content-Type: application/json' \
+              --data @- \
+              "$SLACK_WEBHOOK_URL"

.github/workflows/odh-trivy-images.yml

@@ -1,5 +1,5 @@
-# Midstream image CVE scan for the ODH Quay builds. Catch MEDIUM/HIGH/CRITICAL
-# image vulnerabilities early, before a build is released on a GA branch.
+# Midstream image CVE scan for the ODH Quay builds. Catch Critical RPM and
+# Medium/High/Critical Rust crate vulnerabilities before a GA release.
 # Kept separate from trivy-scan.yml so we do not inherit NVIDIA GHCR login
 # or Helm config scans.
 # Rust crate rows appear only after Konflux images are built with
@@ -77,58 +77,213 @@ jobs:
             refs+=("${image}:${tag}")
           done <<<"$DEFAULT_IMAGES"
 
-          severity=MEDIUM,HIGH,CRITICAL
+          rpm_severity=MEDIUM,HIGH,CRITICAL
+          crate_severity=MEDIUM,HIGH,CRITICAL
 
           {
             echo "### ODH Trivy image scan"
             echo
-            echo "Severity gate: \`$severity\`"
+            echo "RPM gate: \`CRITICAL\`"
+            echo "Rust crate gate: \`$crate_severity\`"
             echo
-            echo "Each image prints two tables: RHEL RPMs, then Rust crates. An empty crate table (\`language-specific files num=0\`) means the image has no cargo-auditable metadata; check \`Dockerfile.konflux.*\`."
+            echo "RPM Medium/High findings are reported but do not fail this workflow. They originate in the base image; ProdSec handles remediation on a best-effort basis."
+            echo
+            echo "Missing cargo-auditable crate metadata is a coverage warning only. Check \`Dockerfile.konflux.*\`."
             echo
             echo "Images:"
             for ref in "${refs[@]}"; do
               echo "- \`$ref\`"
             done
+            echo
+            echo "| Image | Scan | Result | Findings | Notes |"
+            echo "| --- | --- | :---: | --- | --- |"
           } >> "$GITHUB_STEP_SUMMARY"
 
+          reports_dir="$(mktemp -d)"
+          trap 'rm -rf "$reports_dir"' EXIT
+
           status=0
+
+          record_result() {
+            local ref="$1"
+            local scan="$2"
+            local result="$3"
+            local findings="$4"
+            local notes="$5"
+
+            echo "$result $scan - $ref: $findings. $notes"
+            printf '| `%s` | %s | %s | %s | %s |\n' \
+              "$ref" "$scan" "$result" "$findings" "$notes" \
+              >> "$GITHUB_STEP_SUMMARY"
+          }
+
           for ref in "${refs[@]}"; do
             echo "::group::RPMs $ref"
+            rpm_report="$reports_dir/rpm-${ref##*/}.json"
             if trivy image \
               --scanners vuln \
               --pkg-types os \
-              --severity "$severity" \
-              --format table \
-              --exit-code 1 \
+              --severity "$rpm_severity" \
+              --format json \
+              --output "$rpm_report" \
               "$ref"; then
-              echo "No $severity RPM vulnerabilities in $ref"
+              if ! trivy convert \
+                --scanners vuln \
+                --format table \
+                --severity "$rpm_severity" \
+                "$rpm_report"; then
+                echo "::error::Trivy could not render the RPM report for $ref"
+                record_result \
+                  "$ref" "RPMs" "❌" "scan error" \
+                  "Trivy could not render the RPM report."
+                status=1
+              else
+                critical_count=$(
+                  jq '[.Results[]?.Vulnerabilities[]? |
+                    select(.Severity == "CRITICAL")] | length' \
+                    "$rpm_report"
+                )
+                medium_high_count=$(
+                  jq '[.Results[]?.Vulnerabilities[]? |
+                    select(.Severity == "MEDIUM" or .Severity == "HIGH")] | length' \
+                    "$rpm_report"
+                )
+                if [ "$critical_count" -gt 0 ]; then
+                  echo "::error::Trivy found Critical RPM vulnerabilities in $ref"
+                  record_result \
+                    "$ref" "RPMs" "❌" \
+                    "$critical_count Critical, $medium_high_count Medium/High" \
+                    "Critical RPM findings fail the workflow."
+                  status=1
+                elif [ "$medium_high_count" -gt 0 ]; then
+                  record_result \
+                    "$ref" "RPMs" "⚠️" \
+                    "0 Critical, $medium_high_count Medium/High" \
+                    "Medium/High are reported only; base-image RPM remediation is ProdSec-owned and best-effort."
+                else
+                  record_result \
+                    "$ref" "RPMs" "✅" \
+                    "0 at MEDIUM/HIGH/CRITICAL" \
+                    "No gated RPM findings."
+                fi
+              fi
             else
-              echo "::error::Trivy found $severity RPM vulnerabilities in $ref"
+              echo "::error::Trivy could not scan RPMs in $ref"
+              record_result \
+                "$ref" "RPMs" "❌" "scan error" \
+                "Trivy could not complete the RPM scan."
               status=1
             fi
-            echo "::endgroup::"
 
+            echo "::endgroup::"
             echo "::group::Rust crates $ref"
-            crate_log=$(mktemp)
+
+            crate_report="$reports_dir/crates-${ref##*/}.json"
             if trivy image \
               --scanners vuln \
               --pkg-types library \
-              --severity "$severity" \
-              --format table \
-              --exit-code 1 \
-              "$ref" 2>&1 | tee "$crate_log"; then
-              if grep -Eq 'language-specific files[[:space:]]+num=0' "$crate_log"; then
-                echo "No Rust crate results for $ref. Check that Dockerfile.konflux.* uses cargo auditable."
+              --severity "$crate_severity" \
+              --list-all-pkgs \
+              --format json \
+              --output "$crate_report" \
+              "$ref"; then
+              if ! trivy convert \
+                --scanners vuln \
+                --format table \
+                --severity "$crate_severity" \
+                "$crate_report"; then
+                echo "::error::Trivy could not render the crate report for $ref"
+                record_result \
+                  "$ref" "Rust crates" "❌" "scan error" \
+                  "Trivy could not render the crate report."
+                status=1
               else
-                echo "No $severity crate vulnerabilities in $ref"
+                crate_count=$(
+                  jq '[.Results[]?.Vulnerabilities[]? |
+                    select(
+                      .Severity == "MEDIUM" or
+                      .Severity == "HIGH" or
+                      .Severity == "CRITICAL"
+                    )] | length' \
+                    "$crate_report"
+                )
+
+                rustbinary_count=$(
+                  jq '[.Results[]? |
+                    select(
+                      .Type == "rustbinary" and
+                      ((.Packages // []) | length > 0)
+                    )] | length' \
+                    "$crate_report"
+                )
+
+                metadata_note=""
+                if [ "$rustbinary_count" -eq 0 ]; then
+                  metadata_note=" No auditable crate metadata detected; check Dockerfile.konflux.* uses cargo auditable."
+                  echo "::warning::No auditable crate metadata detected in $ref. Check Dockerfile.konflux.* uses cargo auditable."
+                fi
+
+                if [ "$crate_count" -gt 0 ]; then
+                  echo "::error::Trivy found Medium, High, or Critical crate vulnerabilities in $ref"
+                  record_result \
+                    "$ref" "Rust crates" "❌" \
+                    "$crate_count at MEDIUM/HIGH/CRITICAL" \
+                    "Crate findings fail the workflow.$metadata_note"
+                  status=1
+                elif [ -n "$metadata_note" ]; then
+                  record_result \
+                    "$ref" "Rust crates" "✅" \
+                    "0 at MEDIUM/HIGH/CRITICAL; metadata missing" \
+                    "Coverage warning:$metadata_note"
+                else
+                  record_result \
+                    "$ref" "Rust crates" "✅" \
+                    "0 at MEDIUM/HIGH/CRITICAL" \
+                    "No gated crate findings."
+                fi
               fi
             else
-              echo "::error::Trivy found $severity crate vulnerabilities in $ref"
+              echo "::error::Trivy could not scan Rust crates in $ref"
+              record_result \
+                "$ref" "Rust crates" "❌" "scan error" \
+                "Trivy could not complete the crate scan."
               status=1
             fi
-            rm -f "$crate_log"
+
             echo "::endgroup::"
           done
 
           exit "$status"
+  notify-slack:
+    name: Notify Slack on failure
+    needs: scan
+    if: >-
+      ${{ always() &&
+          needs.scan.result == 'failure' &&
+          github.repository == 'red-hat-data-services/openshell' }}
+    runs-on: ubuntu-latest
+    permissions: {}
+    steps:
+      - name: Post failure notification
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          WORKFLOW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          if [ -z "${SLACK_WEBHOOK_URL:-}" ]; then
+            echo "::warning::SLACK_WEBHOOK_URL is not set; skipping Slack notification."
+            exit 0
+          fi
+
+          message="$(printf '%s\n%s\n%s' \
+            ':x: ODH Trivy Image Scan' \
+            "Run: $WORKFLOW_RUN_URL" \
+            "See the run's Step Summary for the full RPM and Rust-crate result table.")"
+
+          jq -nc \
+            --arg text "$message" \
+            '{text: $text}' |
+            curl --fail --silent --show-error \
+              -X POST \
+              -H 'Content-Type: application/json' \
+              --data @- \
+              "$SLACK_WEBHOOK_URL"

.github/workflows/release-tag.yml

@@ -777,6 +777,36 @@ jobs:
 
           echo "Published qualification summary to \`${ref}\`." >> "${GITHUB_STEP_SUMMARY}"
 
+  notify-prerelease-failure:
+    name: Notify Prerelease Failure
+    needs: [release-helm, publish-qualification]
+    if: failure() && contains(inputs.tag || github.ref_name, '-pre.')
+    runs-on: ubuntu-latest
+    timeout-minutes: 2
+    permissions: {}
+    steps:
+      - name: Send Slack notification
+        continue-on-error: true
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_OPENSHELL_TRIAGE_WEBHOOK_URL }}
+          SLACK_MENTION: ${{ secrets.SLACK_OPENSHELL_TRIAGE_MENTION }}
+          RUN_URL: ${{ format('{0}/{1}/actions/runs/{2}/attempts/{3}', github.server_url, github.repository, github.run_id, github.run_attempt) }}
+        run: |
+          set -euo pipefail
+          if [[ -z "${SLACK_WEBHOOK_URL}" ]]; then
+            echo "::notice::SLACK_OPENSHELL_TRIAGE_WEBHOOK_URL is unset; skipping Slack notification."
+            exit 0
+          fi
+
+          message=":x: OpenShell prerelease ${RELEASE_TAG} failed (attempt ${GITHUB_RUN_ATTEMPT})."
+          if [[ -n "${SLACK_MENTION}" ]]; then
+            message+=" ${SLACK_MENTION}"
+          fi
+          jq -n --arg text "${message}" --arg run_url "${RUN_URL}" \
+            '{text: ($text + "\n<" + $run_url + "|View failed release run>"), unfurl_links: false, unfurl_media: false}' |
+            curl --fail --silent --show-error --connect-timeout 5 --max-time 15 \
+              --header 'Content-Type: application/json' --data-binary @- "${SLACK_WEBHOOK_URL}"
+
   publish-fern-docs:
     name: Sync and Publish Fern Docs
     needs: [compute-versions, release, publish-sdk-typescript, release-helm, trigger-wheel-publish]

CI.md

@@ -194,6 +194,13 @@ temporarily informational for tagged releases: the existing findings were
 reviewed and accepted for v0.1.0 and will be addressed in 0.1.x releases.
 Scanner failures still fail qualification.
 
+Failed pre-release builds send notifications via Slack using a webhook and
+at-mentioning the triage engineer with a link to the failure. The webhook is
+stored in the repository secret `SLACK_OPENSHELL_TRIAGE_WEBHOOK_URL` and the
+mention is stored in the repository secret `SLACK_OPENSHELL_TRIAGE_MENTION`.
+Notifications are non-blocking and do not affect release results; an unset
+webhook skips sending; an unset mention sends without a mention.
+
 ```shell
 gh workflow run security-scan.yml --ref main \
   -f candidate_ref=v0.1.1-pre.1 \